改进字幕共享的细节

Signed-off-by: allan716 <[email protected]>

internal/common/constvalue.go

@@ -7,6 +7,7 @@ import (
 const HTMLTimeOut = 2 * 60 * time.Second // HttpClient 超时时间
 const OneMovieProcessTimeOut = 5 * 60    // 一部电影，最多的处理时间
 const OneSeriesProcessTimeOut = 30 * 60  // 一部连续剧，最多的处理时间
+const ScanPlayedSubTimeOut = 30 * 60     // 扫描已经播放的字幕进行缓存的时间
 const DownloadSubsPerSite = 1            // 默认，每个网站下载一个字幕，允许额外传参调整
 const EmbyApiGetItemsLimitMin = 50
 const EmbyApiGetItemsLimitMax = 50000

internal/logic/scan_played_video_subinfo/scan_played_video_subinfo.go

@@ -1,12 +1,14 @@
 package scan_played_video_subinfo
 
 import (
+	"errors"
+	"fmt"
+	"github.com/allanpk716/ChineseSubFinder/internal/common"
 	"github.com/allanpk716/ChineseSubFinder/internal/dao"
 	"github.com/allanpk716/ChineseSubFinder/internal/ifaces"
 	embyHelper "github.com/allanpk716/ChineseSubFinder/internal/logic/emby_helper"
 	"github.com/allanpk716/ChineseSubFinder/internal/logic/sub_parser/ass"
 	"github.com/allanpk716/ChineseSubFinder/internal/logic/sub_parser/srt"
-	"github.com/allanpk716/ChineseSubFinder/internal/logic/sub_supplier/shooter"
 	"github.com/allanpk716/ChineseSubFinder/internal/models"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/decode"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/imdb_helper"
@@ -14,12 +16,14 @@ import (
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/log_helper"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_util"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/settings"
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/sub_file_hash"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/sub_formatter/emby"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/sub_parser_hub"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/sub_share_center"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/task_control"
 	"github.com/allanpk716/ChineseSubFinder/internal/types"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	"path/filepath"
 	"sync"
 )
@@ -96,28 +100,47 @@ func (s *ScanPlayedVideoSubInfo) GetPlayedItemsSubtitle() (bool, error) {
 
 func (s *ScanPlayedVideoSubInfo) Scan() error {
 
-	err := s.scan(s.movieSubMap, true)
+	// -----------------------------------------------------
+	// 并发控制
+	s.taskControl.SetCtxProcessFunc("ScanSubPlayedPool", s.scan, common.ScanPlayedSubTimeOut)
+	// -----------------------------------------------------
+
+	err := s.taskControl.Invoke(&task_control.TaskData{
+		Index: 0,
+		Count: len(s.movieSubMap),
+		DataEx: ScanInputData{
+			Videos:  s.movieSubMap,
+			IsMovie: true,
+		},
+	})
 	if err != nil {
-		return err
+		s.log.Errorln("ScanPlayedVideoSubInfo.Movie Sub Error", err)
 	}
 
-	err = s.scan(s.seriesSubMap, false)
+	err = s.taskControl.Invoke(&task_control.TaskData{
+		Index: 0,
+		Count: len(s.seriesSubMap),
+		DataEx: ScanInputData{
+			Videos:  s.seriesSubMap,
+			IsMovie: false,
+		},
+	})
 	if err != nil {
-		return err
+		s.log.Errorln("ScanPlayedVideoSubInfo.Series Sub Error", err)
 	}
 
+	s.taskControl.Hold()
+
 	return nil
 }
 
-func (s *ScanPlayedVideoSubInfo) scan(videos map[string]string, isMovie bool) error {
+func (s *ScanPlayedVideoSubInfo) scan(ctx context.Context, inData interface{}) error {
 
-	shareRootDir, err := my_util.GetShareSubRootFolder()
-	if err != nil {
-		return err
-	}
+	taskData := inData.(*task_control.TaskData)
+	scanInputData := taskData.DataEx.(ScanInputData)
 
 	videoTypes := ""
-	if isMovie == true {
+	if scanInputData.IsMovie == true {
 		videoTypes = "Movie"
 	} else {
 		videoTypes = "Series"
@@ -131,133 +154,165 @@ func (s *ScanPlayedVideoSubInfo) scan(videos map[string]string, isMovie bool) er
 	s.log.Infoln("-----------------------------------------------")
 	s.log.Infoln("ScanPlayedVideoSubInfo", videoTypes, "Sub Start...")
 
+	shareRootDir, err := my_util.GetShareSubRootFolder()
+	if err != nil {
+		return err
+	}
+
 	imdbInfoCache := make(map[string]*models.IMDBInfo)
-	for movieFPath, orgSubFPath := range videos {
+	for videoFPath, orgSubFPath := range scanInputData.Videos {
+
+		stage := make(chan interface{}, 1)
+		go func() {
+			s.dealOneVideo(videoFPath, orgSubFPath, videoTypes, shareRootDir, scanInputData.IsMovie, imdbInfoCache)
+			stage <- 1
+		}()
+
+		select {
+		case <-ctx.Done():
+			{
+				return errors.New(fmt.Sprintf("cancel at scan: %s", videoFPath))
+			}
+		case <-stage:
+			break
+		}
+	}
+
+	return nil
+}
 
-		if my_util.IsFile(orgSubFPath) == false {
+func (s *ScanPlayedVideoSubInfo) dealOneVideo(videoFPath, orgSubFPath, videoTypes, shareRootDir string,
+	isMovie bool,
+	imdbInfoCache map[string]*models.IMDBInfo) {
 
-			log_helper.GetLogger().Errorln("Skip", orgSubFPath, "not exist")
-			continue
-		}
+	if my_util.IsFile(orgSubFPath) == false {
 
-		// 通过视频的绝对路径，从本地的视频文件对应的 nfo 获取到这个视频的 IMDB ID,
-		var err error
-		var imdbInfo4Video types.VideoIMDBInfo
+		log_helper.GetLogger().Errorln("Skip", orgSubFPath, "not exist")
+		return
+	}
 
-		if isMovie == true {
-			imdbInfo4Video, err = decode.GetImdbInfo4Movie(movieFPath)
-		} else {
-			imdbInfo4Video, err = decode.GetSeriesImdbInfoFromEpisode(movieFPath)
-		}
-		if err != nil {
-			// 如果找不到当前电影的 IMDB Info 本地文件，那么就跳过
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetImdbInfo4Movie", movieFPath, err)
-			continue
-		}
-		// 使用 shooter 的技术 hash 的算法，得到视频的唯一 ID
-		fileHash, err := shooter.ComputeFileHash(movieFPath)
+	// 通过视频的绝对路径，从本地的视频文件对应的 nfo 获取到这个视频的 IMDB ID,
+	var err error
+	var imdbInfo4Video types.VideoIMDBInfo
+
+	if isMovie == true {
+		imdbInfo4Video, err = decode.GetImdbInfo4Movie(videoFPath)
+	} else {
+		imdbInfo4Video, err = decode.GetSeriesImdbInfoFromEpisode(videoFPath)
+	}
+	if err != nil {
+		// 如果找不到当前电影的 IMDB Info 本地文件，那么就跳过
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetImdbInfo", videoFPath, err)
+		return
+	}
+	// 使用 shooter 的技术 hash 的算法，得到视频的唯一 ID
+	fileHash, err := sub_file_hash.Calculate(videoFPath)
+	if err != nil {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".ComputeFileHash", videoFPath, err)
+		return
+	}
+
+	var imdbInfo *models.IMDBInfo
+	var ok bool
+	if imdbInfo, ok = imdbInfoCache[imdbInfo4Video.ImdbId]; ok == false {
+		// 不存在，那么就去查询和新建缓存
+		imdbInfo, err = imdb_helper.GetVideoIMDBInfoFromLocal(imdbInfo4Video.ImdbId, *s.settings.AdvancedSettings.ProxySettings)
 		if err != nil {
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".ComputeFileHash", movieFPath, err)
-			continue
+			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetVideoIMDBInfoFromLocal", videoFPath, err)
+			return
 		}
-
-		var imdbInfo *models.IMDBInfo
-		var ok bool
-		if imdbInfo, ok = imdbInfoCache[imdbInfo4Video.ImdbId]; ok == false {
-			// 不存在，那么就去查询和新建缓存
-			imdbInfo, err = imdb_helper.GetVideoIMDBInfoFromLocal(imdbInfo4Video.ImdbId, *s.settings.AdvancedSettings.ProxySettings)
+		imdbInfoCache[imdbInfo4Video.ImdbId] = imdbInfo
+	}
+	// 判断找到的关联字幕信息是否已经存在了，不存在则新增关联
+	var exist bool
+	for _, info := range imdbInfo.VideoSubInfos {
+
+		// 转绝对路径存储
+		// 首先，这里会进行已有缓存字幕是否存在的判断，把不存在的字幕给删除了
+		if my_util.IsFile(filepath.Join(shareRootDir, info.StoreRPath)) == false {
+			// 关联删除了，但是不会删除这些对象，所以后续还需要再次删除
+			err := dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Delete(&info)
 			if err != nil {
-				return err
-			}
-			imdbInfoCache[imdbInfo4Video.ImdbId] = imdbInfo
-		}
-		// 判断找到的关联字幕信息是否已经存在了，不存在则新增关联
-		var exist bool
-		for _, info := range imdbInfo.VideoSubInfos {
-
-			// 转绝对路径存储
-			// 首先，这里会进行已有缓存字幕是否存在的判断，把不存在的字幕给删除了
-			if my_util.IsFile(filepath.Join(shareRootDir, info.StoreRPath)) == false {
-				// 关联删除了，但是不会删除这些对象，所以后续还需要再次删除
-				err := dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Delete(&info)
-				if err != nil {
-					s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Delete Association", info.SubName, err)
-					continue
-				}
-				// 继续删除这个对象
-				dao.GetDb().Delete(&info)
-				s.log.Infoln("Delete Not Exist Sub Association", info.SubName, err)
+				s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Delete Association", info.SubName, err)
 				continue
 			}
-			// 文件对应的视频唯一 ID 一致
-			if info.Feature == fileHash {
-				exist = true
-				break
-			}
-		}
-		if exist == true {
-			// 存在
+			// 继续删除这个对象
+			dao.GetDb().Delete(&info)
+			s.log.Infoln("Delete Not Exist Sub Association", info.SubName, err)
 			continue
 		}
-
-		// 把现有的字幕 copy 到缓存目录中
-		bok, subCacheFPath := sub_share_center.CopySub2Cache(orgSubFPath, imdbInfo.Year)
-		if bok == false {
-			continue
+		// 文件对应的视频唯一 ID 一致
+		if info.Feature == fileHash {
+			exist = true
+			break
 		}
+	}
+	if exist == true {
+		// 存在
+		return
+	}
 
-		// 不存在，插入，建立关系
-		bok, fileInfo, err := s.subParserHub.DetermineFileTypeFromFile(subCacheFPath)
-		if err != nil {
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".DetermineFileTypeFromFile", imdbInfo4Video.ImdbId, err)
-			continue
-		}
-		if bok == false {
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".DetermineFileTypeFromFile == false", imdbInfo4Video.ImdbId)
-			continue
-		}
+	// 把现有的字幕 copy 到缓存目录中
+	bok, subCacheFPath := sub_share_center.CopySub2Cache(orgSubFPath, imdbInfo.IMDBID, imdbInfo.Year)
+	if bok == false {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".CopySub2Cache", orgSubFPath, err)
+		return
+	}
 
-		// 特指 emby 字幕的情况
-		bok, _, _, _, extraSubPreName := s.subFormatter.IsMatchThisFormat(filepath.Base(subCacheFPath))
-		if bok == false {
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".IsMatchThisFormat == false", imdbInfo4Video.ImdbId)
-			continue
-		}
-		// 转相对路径存储
-		subRelPath, err := filepath.Rel(shareRootDir, subCacheFPath)
-		if err != nil {
-			return err
-		}
+	// 不存在，插入，建立关系
+	bok, fileInfo, err := s.subParserHub.DetermineFileTypeFromFile(subCacheFPath)
+	if err != nil {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".DetermineFileTypeFromFile", imdbInfo4Video.ImdbId, err)
+		return
+	}
+	if bok == false {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".DetermineFileTypeFromFile == false", imdbInfo4Video.ImdbId)
+		return
+	}
 
-		oneVideoSubInfo := models.NewVideoSubInfo(
-			fileHash,
-			filepath.Base(subCacheFPath),
-			language.MyLang2ISO_639_1_String(fileInfo.Lang),
-			language.IsBilingualSubtitle(fileInfo.Lang),
-			language.MyLang2ChineseISO(fileInfo.Lang),
-			fileInfo.Lang.String(),
-			subRelPath,
-			extraSubPreName,
-		)
-
-		if isMovie == false {
-			// 连续剧的时候，如果可能应该获取是 第几季  第几集
-			torrentInfo, _, err := decode.GetVideoInfoFromFileFullPath(subCacheFPath)
-			if err != nil {
-				s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetVideoInfoFromFileFullPath", imdbInfo4Video.Title, err)
-				continue
-			}
-			oneVideoSubInfo.Season = torrentInfo.Season
-			oneVideoSubInfo.Episode = torrentInfo.Episode
-		}
+	// 特指 emby 字幕的情况
+	bok, _, _, _, extraSubPreName := s.subFormatter.IsMatchThisFormat(filepath.Base(subCacheFPath))
+	if bok == false {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".IsMatchThisFormat == false", imdbInfo4Video.ImdbId)
+		return
+	}
+	// 转相对路径存储
+	subRelPath, err := filepath.Rel(shareRootDir, subCacheFPath)
+	if err != nil {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Rel", imdbInfo4Video.ImdbId, err)
+		return
+	}
 
-		err = dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Append(oneVideoSubInfo)
+	oneVideoSubInfo := models.NewVideoSubInfo(
+		fileHash,
+		filepath.Base(subCacheFPath),
+		language.MyLang2ISO_639_1_String(fileInfo.Lang),
+		language.IsBilingualSubtitle(fileInfo.Lang),
+		language.MyLang2ChineseISO(fileInfo.Lang),
+		fileInfo.Lang.String(),
+		subRelPath,
+		extraSubPreName,
+	)
+
+	if isMovie == false {
+		// 连续剧的时候，如果可能应该获取是 第几季  第几集
+		torrentInfo, _, err := decode.GetVideoInfoFromFileFullPath(subCacheFPath)
 		if err != nil {
-			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Append Association", oneVideoSubInfo.SubName, err)
-			continue
+			s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".GetVideoInfoFromFileFullPath", imdbInfo4Video.Title, err)
+			return
 		}
+		oneVideoSubInfo.Season = torrentInfo.Season
+		oneVideoSubInfo.Episode = torrentInfo.Episode
+	}
 
+	err = dao.GetDb().Model(imdbInfo).Association("VideoSubInfos").Append(oneVideoSubInfo)
+	if err != nil {
+		s.log.Warningln("ScanPlayedVideoSubInfo.Scan", videoTypes, ".Append Association", oneVideoSubInfo.SubName, err)
+		return
 	}
+}
 
-	return nil
+type ScanInputData struct {
+	Videos  map[string]string
+	IsMovie bool
 }

internal/pkg/sub_file_hash/sub_file_hash.go

@@ -0,0 +1,80 @@
+package sub_file_hash
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"fmt"
+	"github.com/allanpk716/ChineseSubFinder/internal/common"
+	"math"
+	"os"
+)
+
+func Calculate(filePath string) (string, error) {
+
+	h := sha1.New()
+	fp, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		_ = fp.Close()
+	}()
+
+	stat, err := fp.Stat()
+	if err != nil {
+		return "", err
+	}
+	size := float64(stat.Size())
+
+	if size < 0xF000 {
+		return "", common.VideoFileIsTooSmall
+	}
+
+	samplePositions := [samplingPoints]int64{
+		4 * 1024,
+		int64(math.Floor(size / 4)),
+		int64(math.Floor(size / 4 * 2)),
+		int64(math.Floor(size / 4 * 3)),
+		int64(size - 8*1024)}
+
+	fullBlock := make([]byte, samplingPoints*onePointLen)
+	index := 0
+
+	for _, position := range samplePositions {
+
+		//f, err := os.Create(filepath.Join("c:\\Tmp", fmt.Sprintf("%d", position)+".videoPart"))
+		//if err != nil {
+		//	return "", err
+		//}
+
+		oneBlock := make([]byte, onePointLen)
+		_, err = fp.ReadAt(oneBlock, position)
+		if err != nil {
+			//_ = f.Close()
+			return "", err
+		}
+
+		for _, b := range oneBlock {
+			fullBlock[index] = b
+			index++
+		}
+
+		//_, err = f.Write(oneBlock)
+		//if err != nil {
+		//	return "", err
+		//}
+		//_ = f.Close()
+	}
+
+	h.Write(fullBlock)
+	hashBytes := h.Sum(nil)
+
+	return fmt.Sprintf("%x", md5.Sum(hashBytes)), nil
+}
+
+const (
+	samplingPoints = 5
+	onePointLen    = 4 * 1024
+)
+
+const checkHash = "f08d48a3e2cd6a02f9fd8ac92743dd3e"

internal/pkg/sub_file_hash/sub_file_hash_test.go

@@ -0,0 +1,27 @@
+package sub_file_hash
+
+import (
+	"github.com/allanpk716/ChineseSubFinder/internal/pkg/unit_test_helper"
+	"path/filepath"
+	"testing"
+)
+
+func TestCalculate(t *testing.T) {
+
+	rootDir := unit_test_helper.GetTestDataResourceRootPath([]string{"sub_spplier"}, 4, true)
+	rootDir = filepath.Join(rootDir, "csf")
+
+	gVideoFPath, err := unit_test_helper.GenerateCSFVideoFile(rootDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	calculate, err := Calculate(gVideoFPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if calculate != checkHash {
+		t.Fatal("Hash not the same")
+	}
+}

internal/pkg/sub_share_center/share_sub_cache_helper.go

@@ -3,11 +3,12 @@ package sub_share_center
 import (
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/log_helper"
 	"github.com/allanpk716/ChineseSubFinder/internal/pkg/my_util"
+	"os"
 	"path/filepath"
 )
 
 // CopySub2Cache 检测原有字幕是否存在，然后放到缓存目录中
-func CopySub2Cache(orgSubFileFPath string, year int) (bool, string) {
+func CopySub2Cache(orgSubFileFPath, imdbID string, year int) (bool, string) {
 
 	nowFolderDir, err := my_util.GetShareFolderByYear(year)
 	if err != nil {
@@ -15,7 +16,13 @@ func CopySub2Cache(orgSubFileFPath string, year int) (bool, string) {
 		return false, ""
 	}
 
-	desSubFileFPath := filepath.Join(nowFolderDir, filepath.Base(orgSubFileFPath))
+	err = os.MkdirAll(filepath.Join(nowFolderDir, imdbID), os.ModePerm)
+	if err != nil {
+		log_helper.GetLogger().Errorln("CheckOrgSubFileExistAndCopy2Cache.MkdirAll", err)
+		return false, ""
+	}
+
+	desSubFileFPath := filepath.Join(nowFolderDir, imdbID, filepath.Base(orgSubFileFPath))
 	err = my_util.CopyFile(orgSubFileFPath, desSubFileFPath)
 	if err != nil {
 		log_helper.GetLogger().Errorln("CheckOrgSubFileExistAndCopy2Cache.CopyFile", err)

internal/pkg/unit_test_helper/unit_test_helper.go

@@ -139,6 +139,54 @@ func GenerateXunleiVideoFile(videoPartsRootPath string) (string, error) {
 	return outVideoFPath, nil
 }
 
+// GenerateCSFVideoFile 这里为 CSF 的接口专门生成一个视频文件，瑞克和莫蒂 (2013)\Season 5\S05E09 .mkv
+func GenerateCSFVideoFile(videoPartsRootPath string) (string, error) {
+
+	const videoSize int64 = 640302895
+	const videoName = "S05E09.mkv"
+	const ext = ".videoPart"
+	partNames := []string{"4096", "160075723", "320151447", "480227171", "640294703"}
+
+	outVideoFPath := filepath.Join(videoPartsRootPath, videoName)
+
+	f, err := os.Create(outVideoFPath)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	if err := f.Truncate(videoSize); err != nil {
+		return "", err
+	}
+
+	/*
+		一共有 5 个检测点
+	*/
+	for _, name := range partNames {
+
+		partF, err := os.Open(filepath.Join(videoPartsRootPath, name+ext))
+		if err != nil {
+			return "", err
+		}
+		partAll, err := io.ReadAll(partF)
+		if err != nil {
+			return "", err
+		}
+		int64Numb, err := strconv.ParseInt(name, 10, 64)
+		if err != nil {
+			return "", err
+		}
+		_, err = f.WriteAt(partAll, int64Numb)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return outVideoFPath, nil
+}
+
 // copyTestData 单元测试前把测试的数据 copy 一份出来操作，src 目录中默认应该有一个 org 原始数据文件夹，然后需要复制一份 test 文件夹出来
 func copyTestData(srcDir string) (string, error) {