保存进度

Signed-off-by: 716 <[email protected]>

cmd/chinesesubfinder/main.go

@@ -47,14 +47,15 @@ func main() {
 	if err != nil {
 		log_helper.GetLogger().Panicln(err)
 	}
-	_, err = scan.GetPlayedItemsSubtitle()
+	bok, err := scan.GetPlayedItemsSubtitle()
 	if err != nil {
 		log_helper.GetLogger().Panicln(err)
 	}
-
-	err = scan.Scan()
-	if err != nil {
-		log_helper.GetLogger().Panicln(err)
+	if bok == true {
+		err = scan.Scan()
+		if err != nil {
+			log_helper.GetLogger().Panicln(err)
+		}
 	}
 
 	// 支持在外部配置特殊的端口号，以防止本地本占用了无法使用

internal/logic/scan_played_video_subinfo/scan_played_video_subinfo.go

@@ -51,6 +51,8 @@ func NewScanPlayedVideoSubInfo(_settings settings.Settings) (*ScanPlayedVideoSub
 	var scanPlayedVideoSubInfo ScanPlayedVideoSubInfo
 	scanPlayedVideoSubInfo.log = log_helper.GetLogger()
 	// 参入设置信息
+	// 最大获取的视频数目设置到 100W
+	_settings.EmbySettings.MaxRequestVideoNumber = 1000000
 	scanPlayedVideoSubInfo.settings = _settings
 	// 检测是否某些参数超出范围
 	scanPlayedVideoSubInfo.settings.Check()
@@ -160,11 +162,13 @@ func (s *ScanPlayedVideoSubInfo) scan(ctx context.Context, inData interface{}) e
 	}
 
 	imdbInfoCache := make(map[string]*models.IMDBInfo)
+	index := 0
 	for videoFPath, orgSubFPath := range scanInputData.Videos {
 
+		index++
 		stage := make(chan interface{}, 1)
 		go func() {
-			s.dealOneVideo(videoFPath, orgSubFPath, videoTypes, shareRootDir, scanInputData.IsMovie, imdbInfoCache)
+			s.dealOneVideo(index, videoFPath, orgSubFPath, videoTypes, shareRootDir, scanInputData.IsMovie, imdbInfoCache)
 			stage <- 1
 		}()
 
@@ -181,16 +185,20 @@ func (s *ScanPlayedVideoSubInfo) scan(ctx context.Context, inData interface{}) e
 	return nil
 }
 
-func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoTypes, shareRootDir string,
+func (s *ScanPlayedVideoSubInfo) dealOneVideo(index int, videoFPath, orgSubFPath, videoTypes, shareRootDir string,
 	isMovie bool,
 	imdbInfoCache map[string]*models.IMDBInfo) {
 
+	s.log.Infoln(index, orgSubFPath)
+
 	if my_util.IsFile(orgSubFPath) == false {
 
 		log_helper.GetLogger().Errorln("Skip", orgSubFPath, "not exist")
 		return
 	}
 
+	s.log.Debugln(0)
+
 	// 通过视频的绝对路径，从本地的视频文件对应的 nfo 获取到这个视频的 IMDB ID,
 	var err error
 	var imdbInfo4Video types.VideoIMDBInfo
@@ -205,6 +213,9 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetImdbInfo", videoFPath, err)
 		return
 	}
+
+	s.log.Debugln(1)
+
 	// 使用 shooter 的技术 hash 的算法，得到视频的唯一 ID
 	fileHash, err := sub_file_hash.Calculate(videoFPath)
 	if err != nil {
@@ -212,6 +223,8 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		return
 	}
 
+	s.log.Debugln(2)
+
 	var imdbInfo *models.IMDBInfo
 	var ok bool
 	if imdbInfo, ok = imdbInfoCache[imdbInfo4Video.ImdbId]; ok == false {
@@ -223,35 +236,53 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		}
 		imdbInfoCache[imdbInfo4Video.ImdbId] = imdbInfo
 	}
+
+	s.log.Debugln(3)
+
 	// 判断找到的关联字幕信息是否已经存在了，不存在则新增关联
-	var exist bool
-	for _, info := range imdbInfo.VideoSubInfos {
+	var skip bool
+	for tt, cacheInfo := range imdbInfo.VideoSubInfos {
 
+		s.log.Debugln(4, tt)
+
+		skip = false
 		// 转绝对路径存储
 		// 首先，这里会进行已有缓存字幕是否存在的判断，把不存在的字幕给删除了
-		if my_util.IsFile(filepath.Join(shareRootDir, info.StoreRPath)) == false {
+		cacheSubFPath := filepath.Join(shareRootDir, cacheInfo.StoreRPath)
+
+		tmpSHA1String, err := my_util.GetFileSHA1String(cacheSubFPath)
+		if err != nil {
+			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, "VideoSubInfos.GetFileSHA1String", videoFPath, err)
+			return
+		}
+
+		if my_util.IsFile(cacheSubFPath) == false {
+			// 如果文件不存在，那么就删除之前的关联
 			// 关联删除了，但是不会删除这些对象，所以后续还需要再次删除
-			err := dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Delete(&info)
+			err := dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Delete(&cacheInfo)
 			if err != nil {
-				s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Delete Association", info.SubName, err)
+				s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Delete Association", cacheInfo.SubName, err)
 				continue
 			}
 			// 继续删除这个对象
-			dao.GetDb().Delete(&info)
-			s.log.Infoln("Delete Not Exist Sub Association", info.SubName, err)
+			dao.GetDb().Delete(&cacheInfo)
+			s.log.Infoln("Delete Not Exist or SHA1 not the same， Sub Association", cacheInfo.SubName)
 			continue
 		}
-		// 文件对应的视频唯一 ID 一致
-		if info.Feature == fileHash {
-			exist = true
+		if cacheInfo.SHA1 == tmpSHA1String {
+			// 如果 SHA1 一样，那么就说明存在了，这里其实并不是很关心文件名是否是一样的
+			skip = true
 			break
 		}
 	}
-	if exist == true {
-		// 存在
+	if skip == true {
+		// 存在，跳过
 		return
 	}
 
+	s.log.Debugln(5)
+
+	// 新增插入
 	// 把现有的字幕 copy 到缓存目录中
 	bok, subCacheFPath := sub_share_center.CopySub2Cache(orgSubFPath, imdbInfo.IMDBID, imdbInfo.Year)
 	if bok == false {
@@ -259,6 +290,8 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		return
 	}
 
+	s.log.Debugln(6)
+
 	// 不存在，插入，建立关系
 	bok, fileInfo, err := s.subParserHub.DetermineFileTypeFromFile(subCacheFPath)
 	if err != nil {
@@ -270,12 +303,10 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		return
 	}
 
+	s.log.Debugln(7)
+
 	// 特指 emby 字幕的情况
-	bok, _, _, _, extraSubPreName := s.subFormatter.IsMatchThisFormat(filepath.Base(subCacheFPath))
-	if bok == false {
-		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".IsMatchThisFormat == false", imdbInfo4Video.ImdbId)
-		return
-	}
+	_, _, _, _, extraSubPreName := s.subFormatter.IsMatchThisFormat(filepath.Base(subCacheFPath))
 	// 转相对路径存储
 	subRelPath, err := filepath.Rel(shareRootDir, subCacheFPath)
 	if err != nil {
@@ -283,6 +314,17 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		return
 	}
 
+	s.log.Debugln(8)
+
+	// 计算需要插入字幕的 sha1
+	saveSHA1String, err := my_util.GetFileSHA1String(subCacheFPath)
+	if err != nil {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, "GetFileSHA1String", videoFPath, err)
+		return
+	}
+
+	s.log.Debugln(9)
+
 	oneVideoSubInfo := models.NewVideoSubInfo(
 		fileHash,
 		filepath.Base(subCacheFPath),
@@ -292,6 +334,7 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		fileInfo.Lang.String(),
 		subRelPath,
 		extraSubPreName,
+		saveSHA1String,
 	)
 
 	if isMovie == false {
@@ -305,6 +348,8 @@ func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoType
 		oneVideoSubInfo.Episode = torrentInfo.Episode
 	}
 
+	s.log.Debugln(10)
+
 	err = dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Append(oneVideoSubInfo)
 	if err != nil {
 		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Append Association", oneVideoSubInfo.SubName, err)

internal/models/video_sub_info.go

@@ -2,7 +2,7 @@ package models
 
 // VideoSubInfo 属于 IMDBInfo，IMDBInfoID 是外键，使用了 GORM 的 HasMany 关联
 type VideoSubInfo struct {
-	Feature      string `gorm:"primaryKey" json:"feature"` // 特征码，这个未必有，比如是蓝光格式，分散成多个视频文件的时候，暂定使用 shooter 的特征提前方式
+	Feature      string `gorm:"primaryKey" json:"feature"` // 特征码，这个未必有，比如是蓝光格式，分散成多个视频文件的时候，暂定使用本程序的特征提前方式
 	SubName      string `json:"sub_name"`                  // 字幕的文件名
 	Season       int    `json:"season"`                    // 如果对应的是电影则可能是 0，没有
 	Episode      int    `json:"episode"`                   // 如果对应的是电影则可能是 0，没有
@@ -12,9 +12,11 @@ type VideoSubInfo struct {
 	MyLanguage   string `json:"my_language"`               // 这个是本程序定义的语言类型，见 my_language.go 文件
 	StoreRPath   string `json:"store_r_path"`              // 字幕存在出本地的哪里相对路径上，cache/CSF-ShareSubCache
 	ExtraPreName string `json:"extra_pre_name"`            // 字幕额外的命名信息，指 Emby 字幕命名格式(简英,subhd)，的 subhd
+	SHA1         string `json:"sha_1"`                     // 当前文件的 sha1 的值
+	IsSend       bool   `json:"is_send"`                   // 是否已经发送
 	IMDBInfoID   string
 }
 
-func NewVideoSubInfo(feature string, subName string, languageISO string, isDouble bool, chineseISO string, myLanguage string, storeFPath string, extraPreName string) *VideoSubInfo {
-	return &VideoSubInfo{Feature: feature, SubName: subName, LanguageISO: languageISO, IsDouble: isDouble, ChineseISO: chineseISO, MyLanguage: myLanguage, StoreRPath: storeFPath, ExtraPreName: extraPreName}
+func NewVideoSubInfo(feature string, subName string, languageISO string, isDouble bool, chineseISO string, myLanguage string, storeFPath string, extraPreName string, sha1String string) *VideoSubInfo {
+	return &VideoSubInfo{Feature: feature, SubName: subName, LanguageISO: languageISO, IsDouble: isDouble, ChineseISO: chineseISO, MyLanguage: myLanguage, StoreRPath: storeFPath, ExtraPreName: extraPreName, SHA1: sha1String}
 }

internal/pkg/imdb_helper/imdb.go

@@ -1,22 +1,47 @@
 package imdb_helper
 
 import (
+	"crypto/tls"
 	"github.com/StalkR/imdb"
 	"github.com/allanpk716/ChineseSubFinder/internal/dao"
 	"github.com/allanpk716/ChineseSubFinder/internal/models"
-	"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_util"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/log_helper"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/notify_center"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/settings"
+	"net/http"
+	"net/url"
 	"strings"
+	"time"
 )
 
 // GetVideoInfoFromIMDB 从 IMDB ID 查询影片的信息
 func GetVideoInfoFromIMDB(imdbID string, _proxySettings ...settings.ProxySettings) (*imdb.Title, error) {
+
+	var cli *http.Client
 	var proxySettings settings.ProxySettings
-	if len(_proxySettings) > 0 {
+	if len(_proxySettings) > 0 && _proxySettings[0].UseHttpProxy == true || len(_proxySettings[0].HttpProxyAddress) > 0 {
 		proxySettings = _proxySettings[0]
+
+		proxy, err := url.Parse(proxySettings.HttpProxyAddress)
+		if err != nil {
+			return nil, err
+		}
+		tr := &http.Transport{
+			Proxy:           http.ProxyURL(proxy),
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+
+		cli = &http.Client{
+			Timeout:   15 * time.Second,
+			Transport: tr,
+		}
+	} else {
+		cli = &http.Client{
+			Timeout: 15 * time.Second,
+		}
 	}
-	t, err := imdb.NewTitle(my_util.NewHttpClient(proxySettings).GetClient(), imdbID)
+
+	t, err := imdb.NewTitle(cli, imdbID)
 	if err != nil {
 		notify_center.Notify.Add("imdb model - imdb.NewTitle :", err.Error())
 		return nil, err
@@ -33,25 +58,35 @@ func GetVideoIMDBInfoFromLocal(imdbID string, _proxySettings ...settings.ProxySe
 		proxySettings = _proxySettings[0]
 	}
 
+	log_helper.GetLogger().Debugln("GetVideoIMDBInfoFromLocal", 0)
+
 	// 首先从数据库中查找是否存在这个 IMDB 信息，如果不存在再使用 Web 查找，且写入数据库
 	var imdbInfos []models.IMDBInfo
 	// 把嵌套关联的 has many 的信息都查询出来
 	dao.GetDb().Preload("VideoSubInfos").Limit(1).Where(&models.IMDBInfo{IMDBID: imdbID}).Find(&imdbInfos)
 
+	log_helper.GetLogger().Debugln("GetVideoIMDBInfoFromLocal", 1)
+
 	if len(imdbInfos) <= 0 {
 		// 没有找到，去网上获取
 		t, err := GetVideoInfoFromIMDB(imdbID, proxySettings)
 		if err != nil {
 			return nil, err
 		}
+		log_helper.GetLogger().Debugln("GetVideoIMDBInfoFromLocal", 2)
+
 		// 存入数据库
 		nowIMDBInfo := models.NewIMDBInfo(imdbID, t.Name, t.Year, t.Description, t.Languages, t.AKA)
 		imdbInfos = make([]models.IMDBInfo, 0)
 		imdbInfos = append(imdbInfos, *nowIMDBInfo)
 		dao.GetDb().Create(nowIMDBInfo)
 
+		log_helper.GetLogger().Debugln("GetVideoIMDBInfoFromLocal", 3)
+
 		return nowIMDBInfo, nil
 	} else {
+
+		log_helper.GetLogger().Debugln("GetVideoIMDBInfoFromLocal", 4)
 		// 找到
 		return &imdbInfos[0], nil
 	}

internal/pkg/my_util/util.go

@@ -1,6 +1,7 @@
 package my_util
 
 import (
+	"crypto/md5"
 	"crypto/sha1"
 	"encoding/hex"
 	"fmt"
@@ -622,3 +623,26 @@ func UrlJoin(hostUrl, subUrl string) (string, error) {
 	u.Path = path.Join(u.Path, subUrl)
 	return u.String(), nil
 }
+
+// GetFileSHA1String 获取文件的 SHA1 字符串
+func GetFileSHA1String(fileFPath string) (string, error) {
+	h := sha1.New()
+
+	fp, err := os.Open(fileFPath)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		_ = fp.Close()
+	}()
+
+	partAll, err := io.ReadAll(fp)
+	if err != nil {
+		return "", err
+	}
+
+	h.Write(partAll)
+	hashBytes := h.Sum(nil)
+
+	return fmt.Sprintf("%x", md5.Sum(hashBytes)), nil
+}