新增，下载字幕缓存的存储和查询功能

Signed-off-by: allan716 <[email protected]>

.gitignore

@@ -89,3 +89,6 @@ TestData/
 /task_queue
 /internal/pkg/sub_file_hash/Logs
 /internal/logic/sub_supplier/xunlei/task_queue
+/internal/pkg/download_file_cache/ChineseSubFinderSettings.json
+/internal/pkg/download_file_cache/Logs
+/internal/pkg/download_file_cache/download_sub_cache

internal/logic/task_queue/daily_download_limit.go

@@ -1,6 +1,7 @@
 package task_queue
 
 import (
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/badger_err_check"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_util"
 	"github.com/dgraph-io/badger/v3"
 	"time"
@@ -27,7 +28,7 @@ func GetDailyDownloadCount(supplierName string, publicIP string, whichDay ...str
 			e, err := tx.Get(key)
 			if err != nil {
 
-				if IsErrOk(err) == true {
+				if badger_err_check.IsErrOk(err) == true {
 					return nil
 				}
 

internal/logic/task_queue/task_queue.go

@@ -3,6 +3,7 @@ package task_queue
 import (
 	"errors"
 	"fmt"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/badger_err_check"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/settings"
 	"github.com/allanpk716/ChineseSubFinder/internal/types/task_queue"
 	taskQueue2 "github.com/allanpk716/ChineseSubFinder/internal/types/task_queue"
@@ -401,7 +402,7 @@ func (t *TaskQueue) read() {
 				var item *badger.Item
 				item, err = tx.Get(key)
 				if err != nil {
-					if IsErrOk(err) == true {
+					if badger_err_check.IsErrOk(err) == true {
 						return nil
 					}
 					return err

internal/logic/task_queue/badger_err_check.go → internal/pkg/badger_err_check/checker.go

@@ -1,4 +1,4 @@
-package task_queue
+package badger_err_check
 
 import "github.com/dgraph-io/badger/v3"
 

internal/pkg/download_file_cache/download_file_cahce.go

@@ -0,0 +1,93 @@
+package download_file_cache
+
+import (
+	"encoding/json"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/badger_err_check"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/settings"
+	"github.com/allanpk716/ChineseSubFinder/internal/types/supplier"
+	"github.com/dgraph-io/badger/v3"
+	"time"
+)
+
+type DownloadFileCache struct {
+	settings *settings.Settings
+}
+
+func NewDownloadFileCache(settings *settings.Settings) *DownloadFileCache {
+	return &DownloadFileCache{settings: settings}
+}
+
+func (d *DownloadFileCache) Get(fileUrlUID string) (bool, *supplier.SubInfo, error) {
+
+	var subInfo supplier.SubInfo
+	err := GetDb().View(
+		func(tx *badger.Txn) error {
+			var err error
+
+			key := []byte(fileUrlUID)
+			e, err := tx.Get(key)
+			if err != nil {
+
+				if badger_err_check.IsErrOk(err) == true {
+					return nil
+				}
+
+				return err
+			}
+			valCopy, err := e.ValueCopy(nil)
+			if err != nil {
+				return err
+			}
+
+			err = json.Unmarshal(valCopy, &subInfo)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	if err != nil {
+		return false, nil, err
+	}
+
+	if subInfo.GetUID() == "" {
+		return false, nil, nil
+	}
+	return true, &subInfo, nil
+}
+
+// Add 新增一个下载好的文件到缓存种，TTL 时间需要从 Settings 中去配置
+func (d DownloadFileCache) Add(subInfo *supplier.SubInfo) error {
+
+	err := GetDb().Update(
+		func(tx *badger.Txn) error {
+
+			var err error
+			key := []byte(subInfo.GetUID())
+			if err != nil {
+				return err
+			}
+
+			b, err := json.Marshal(subInfo)
+			if err != nil {
+				return err
+			}
+			// 只支持秒或者小时为单位
+			tmpTTL := time.Duration(d.settings.AdvancedSettings.DownloadFileCache.TTL) * time.Second
+			if d.settings.AdvancedSettings.DownloadFileCache.Unit == "hour" {
+				tmpTTL = time.Duration(d.settings.AdvancedSettings.DownloadFileCache.TTL) * time.Hour
+			} else {
+				tmpTTL = time.Duration(d.settings.AdvancedSettings.DownloadFileCache.TTL) * time.Second
+			}
+
+			e := badger.NewEntry(key, b).WithTTL(tmpTTL)
+			err = tx.SetEntry(e)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/pkg/download_file_cache/download_file_cahce_test.go

@@ -0,0 +1,85 @@
+package download_file_cache
+
+import (
+	"bytes"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/settings"
+	"github.com/allanpk716/ChineseSubFinder/internal/types/language"
+	"github.com/allanpk716/ChineseSubFinder/internal/types/supplier"
+	"testing"
+	"time"
+)
+
+func TestDownloadFileCache_Get(t *testing.T) {
+
+	dfc := NewDownloadFileCache(settings.GetSettings())
+	found, _, err := dfc.Get("asd")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if found == true {
+		t.Error("Found sub")
+	}
+}
+
+func TestDownloadFileCache_Add(t *testing.T) {
+
+	dfc := NewDownloadFileCache(settings.GetSettings())
+
+	inSubInfo := supplier.NewSubInfo("local", 1, "testfile", language.ChineseSimple, "2esd12c31c123asecqwec", 3, 0, ".srt", []byte{1, 2, 3, 4})
+	err := dfc.Add(inSubInfo)
+	if err != nil {
+		t.Error(err)
+	}
+
+	fileUID := inSubInfo.GetUID()
+
+	found, gotSubInfo, err := dfc.Get(fileUID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if found == false {
+		t.Error("Not Found sub")
+	}
+
+	if gotSubInfo.GetUID() != fileUID {
+		t.Error("Wrong UID")
+	}
+
+	if gotSubInfo.FileUrl != inSubInfo.FileUrl {
+		t.Error("Wrong FileUrl")
+	}
+
+	if bytes.Equal(gotSubInfo.Data, inSubInfo.Data) == false {
+		t.Error("Wrong Data")
+	}
+
+}
+
+func TestDownloadFileCache_AddTTL(t *testing.T) {
+
+	nowSettings := settings.GetSettings()
+	nowSettings.AdvancedSettings.DownloadFileCache.TTL = 3
+	nowSettings.AdvancedSettings.DownloadFileCache.Unit = "second"
+	dfc := NewDownloadFileCache(nowSettings)
+
+	inSubInfo := supplier.NewSubInfo("local", 1, "testfile", language.ChineseSimple, "2esd12c31c123asecqwec", 3, 0, ".srt", []byte{1, 2, 3, 4})
+	err := dfc.Add(inSubInfo)
+	if err != nil {
+		t.Error(err)
+	}
+
+	fileUID := inSubInfo.GetUID()
+
+	time.Sleep(time.Second * 4)
+
+	found, _, err := dfc.Get(fileUID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if found == true {
+		t.Error("Found sub")
+	}
+}

internal/pkg/download_file_cache/init.go

@@ -0,0 +1,59 @@
+package download_file_cache
+
+import (
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/log_helper"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_folder"
+	"github.com/dgraph-io/badger/v3"
+	"path/filepath"
+	"time"
+)
+
+func GetDb() *badger.DB {
+
+	if dbBase == nil {
+		var err error
+		opt := badger.DefaultOptions(getDbName())
+		// 1MB
+		opt.ValueLogFileSize = 1 << 20
+		// 10 MB
+		opt.MemTableSize = 10 << 20
+		// 这边数据库会自动创建这个目录文件
+		dbBase, err = badger.Open(opt)
+		if err != nil {
+			log_helper.GetLogger().Panicln("sub_cache.GetDb()", err)
+		}
+
+		go badgerGC(dbBase)
+	}
+	return dbBase
+}
+
+func DelDb() error {
+
+	if dbBase != nil {
+		_ = dbBase.Close()
+	}
+	return my_folder.ClearFolder(getDbName())
+}
+
+func getDbName() string {
+	return filepath.Join(my_folder.GetConfigRootDirFPath(), dbFileName)
+}
+
+func badgerGC(_dbBase *badger.DB) {
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+	for range ticker.C {
+	again:
+		err := _dbBase.RunValueLogGC(0.5)
+		if err == nil {
+			goto again
+		}
+	}
+}
+
+const (
+	dbFileName = "download_sub_cache"
+)
+
+var dbBase *badger.DB

internal/pkg/my_util/util_test.go

@@ -33,9 +33,6 @@ func TestGetRestOfDaySec(t *testing.T) {
 
 func TestGetPublicIP(t *testing.T) {
 
-	got, err := GetPublicIP(settings.NewTaskQueue())
-	if err != nil {
-		t.Fatal(err)
-	}
+	got := GetPublicIP(settings.NewTaskQueue())
 	println(got)
 }

internal/pkg/settings/advanced_settings.go

@@ -17,6 +17,7 @@ type AdvancedSettings struct {
 	SuppliersSettings          *SuppliersSettings `json:"suppliers_settings"`             // 每个字幕源的设置
 	ScanLogic                  *ScanLogic         `json:"scan_logic"`                     // 扫描的逻辑
 	TaskQueue                  *TaskQueue         `json:"task_queue"`                     // 任务队列的设置
+	DownloadFileCache          *DownloadFileCache `json:"download_file_cache"`            // 下载文件的缓存
 }
 
 func NewAdvancedSettings() *AdvancedSettings {
@@ -27,5 +28,6 @@ func NewAdvancedSettings() *AdvancedSettings {
 		SuppliersSettings: NewSuppliersSettings(),
 		ScanLogic:         NewScanLogic(false, false),
 		TaskQueue:         NewTaskQueue(),
+		DownloadFileCache: NewDownloadFileCache(),
 	}
 }

internal/pkg/settings/download_file_cache.go

@@ -0,0 +1,10 @@
+package settings
+
+type DownloadFileCache struct {
+	TTL  int    `json:"ttl" default:"4320"`  // 单位需要根据下面的单位转换，默认是小时的单位，然后是半年的过期时间
+	Unit string `json:"unit" default:"hour"` // second, hour, 目前仅仅支持 秒和小时
+}
+
+func NewDownloadFileCache() *DownloadFileCache {
+	return &DownloadFileCache{TTL: 4320, Unit: "hour"}
+}

internal/types/supplier/subinfo.go

@@ -1,22 +1,25 @@
 package supplier
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"github.com/allanpk716/ChineseSubFinder/internal/types/language"
 )
 
 type SubInfo struct {
-	FromWhere    string              `json:"from_where"`     // 从哪个网站下载来的
-	TopN         int64               `json:"top_n"`          // 是 Top 几？
-	Name         string              `json:"name"`           // 字幕的名称，这个比较随意，优先是影片的名称，然后才是从网上下载字幕的对应名称
-	Language     language.MyLanguage `json:"language"`       // 字幕的语言
-	FileUrl      string              `json:"file-url"`       // 字幕文件下载的路径
-	Score        int64               `json:"score"`          // TODO 字幕的评分，需要有一个独立的评价体系。首先是每个网站自己的评价排序，然后再到统一的评分体系
-	Offset       int64               `json:"offset"`         // 字幕的偏移
-	Ext          string              `json:"ext"`            // 字幕文件的后缀名带点，有可能是直接能用的字幕文件，也可能是压缩包
-	Data         []byte              `json:"data"`           // 字幕文件的二进制数据
-	Season       int                 `json:"season"`         // 第几季，默认-1
-	Episode      int                 `json:"episode"`        // 第几集，默认-1
-	IsFullSeason bool                `json:"is_full_season"` // 是否是全季的字幕
+	FromWhere     string              `json:"from_where"`     // 从哪个网站下载来的
+	TopN          int64               `json:"top_n"`          // 是 Top 几？
+	Name          string              `json:"name"`           // 字幕的名称，这个比较随意，优先是影片的名称，然后才是从网上下载字幕的对应名称
+	Language      language.MyLanguage `json:"language"`       // 字幕的语言
+	FileUrl       string              `json:"file-url"`       // 字幕文件下载的路径
+	Score         int64               `json:"score"`          // TODO 字幕的评分，需要有一个独立的评价体系。首先是每个网站自己的评价排序，然后再到统一的评分体系
+	Offset        int64               `json:"offset"`         // 字幕的偏移
+	Ext           string              `json:"ext"`            // 字幕文件的后缀名带点，有可能是直接能用的字幕文件，也可能是压缩包
+	Data          []byte              `json:"data"`           // 字幕文件的二进制数据
+	Season        int                 `json:"season"`         // 第几季，默认-1
+	Episode       int                 `json:"episode"`        // 第几集，默认-1
+	IsFullSeason  bool                `json:"is_full_season"` // 是否是全季的字幕
+	fileUrlSha256 string              // 字幕文件的 FileUrl sha256 值
 }
 
 func NewSubInfo(fromWhere string, topN int64, name string, language language.MyLanguage, fileUrl string,
@@ -30,3 +33,17 @@ func NewSubInfo(fromWhere string, topN int64, name string, language language.MyL
 
 	return &s
 }
+
+// GetUID 通过 FileUrl 获取字幕的唯一标识
+func (s *SubInfo) GetUID() string {
+
+	if s.fileUrlSha256 == "" {
+		if s.FileUrl == "" {
+			return ""
+		}
+		s.fileUrlSha256 = fmt.Sprintf("%x", sha256.Sum256([]byte(s.FileUrl)))
+		return s.fileUrlSha256
+	} else {
+		return s.fileUrlSha256
+	}
+}