保存进度

Signed-off-by: allan716 <[email protected]>

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/baabaaox/go-webrtcvad v1.0.1
 	github.com/beevik/etree v1.1.0
 	github.com/bodgit/sevenzip v1.1.0
-	github.com/emirpasic/gods v1.12.0
+	github.com/emirpasic/gods v1.18.1
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-contrib/cors v1.3.1
 	github.com/gin-gonic/gin v1.7.7

go.sum

@@ -150,6 +150,8 @@ github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy0
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=

pkg/downloader/downloader.go

@@ -5,6 +5,10 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/allanpk716/ChineseSubFinder/pkg/manual_upload_sub_2_local"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/save_sub_helper"
+
 	"github.com/allanpk716/ChineseSubFinder/pkg/scan_logic"
 
 	"github.com/allanpk716/ChineseSubFinder/pkg"
@@ -34,15 +38,17 @@ type Downloader struct {
 	fileDownloader           *file_downloader.FileDownloader
 	ctx                      context.Context
 	cancel                   context.CancelFunc
-	subSupplierHub           *subSupplier.SubSupplierHub                  // 字幕提供源的集合，这个需要定时进行扫描，这些字幕源是否有效，以及下载验证码信息
-	mk                       *markSystem.MarkingSystem                    // MarkingSystem，字幕的评价系统
-	subFormatter             ifaces.ISubFormatter                         // 字幕格式化命名的实现
-	subNameFormatter         subCommon.FormatterName                      // 从 inSubFormatter 推断出来
-	subTimelineFixerHelperEx *sub_timeline_fixer.SubTimelineFixerHelperEx // 字幕时间轴校正
-	downloaderLock           sync.Mutex                                   // 取消执行 task control 的 Lock
-	downloadQueue            *task_queue.TaskQueue                        // 需要下载的视频的队列
-	embyHelper               *embyHelper.EmbyHelper                       // Emby 的实例
-	ScanLogic                *scan_logic.ScanLogic                        // 是否扫描逻辑
+	subSupplierHub           *subSupplier.SubSupplierHub                      // 字幕提供源的集合，这个需要定时进行扫描，这些字幕源是否有效，以及下载验证码信息
+	mk                       *markSystem.MarkingSystem                        // MarkingSystem，字幕的评价系统
+	subFormatter             ifaces.ISubFormatter                             // 字幕格式化命名的实现
+	subNameFormatter         subCommon.FormatterName                          // 从 inSubFormatter 推断出来
+	subTimelineFixerHelperEx *sub_timeline_fixer.SubTimelineFixerHelperEx     // 字幕时间轴校正
+	downloaderLock           sync.Mutex                                       // 取消执行 task control 的 Lock
+	downloadQueue            *task_queue.TaskQueue                            // 需要下载的视频的队列
+	embyHelper               *embyHelper.EmbyHelper                           // Emby 的实例
+	ScanLogic                *scan_logic.ScanLogic                            // 是否扫描逻辑
+	SaveSubHelper            *save_sub_helper.SaveSubHelper                   // 保存字幕的逻辑
+	ManualUploadSub2Local    *manual_upload_sub_2_local.ManualUploadSub2Local // 手动上传字幕到本地
 
 	cacheLocker   sync.Mutex
 	movieInfoMap  map[string]MovieInfo  // 给 Web 界面使用的，Key: VideoFPath
@@ -93,6 +99,13 @@ func NewDownloader(inSubFormatter ifaces.ISubFormatter, fileDownloader *file_dow
 
 	downloader.ScanLogic = scan_logic.NewScanLogic(downloader.log)
 
+	downloader.SaveSubHelper = save_sub_helper.NewSaveSubHelper(
+		downloader.log,
+		downloader.settings, downloader.subFormatter,
+		downloader.subTimelineFixerHelperEx)
+
+	downloader.ManualUploadSub2Local = manual_upload_sub_2_local.NewManualUploadSub2Local(downloader.log, downloader.SaveSubHelper)
+
 	downloader.movieInfoMap = make(map[string]MovieInfo)
 	downloader.seasonInfoMap = make(map[string]SeasonInfo)
 

pkg/downloader/downloader_things.go

@@ -3,7 +3,6 @@ package downloader
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 
 	"github.com/allanpk716/ChineseSubFinder/pkg"
@@ -12,8 +11,6 @@ import (
 	"github.com/allanpk716/ChineseSubFinder/pkg/types/series"
 	"github.com/allanpk716/ChineseSubFinder/pkg/types/subparser"
 
-	"github.com/allanpk716/ChineseSubFinder/pkg/change_file_encode"
-	"github.com/allanpk716/ChineseSubFinder/pkg/chs_cht_changer"
 	"github.com/allanpk716/ChineseSubFinder/pkg/decode"
 	subcommon "github.com/allanpk716/ChineseSubFinder/pkg/sub_formatter/common"
 	"github.com/allanpk716/ChineseSubFinder/pkg/sub_helper"
@@ -73,7 +70,7 @@ func (d *Downloader) oneVideoSelectBestSub(oneVideoFullPath string, organizeSubF
 			bSetDefault = false
 		}
 		// 找到了，写入文件
-		err = d.writeSubFile2VideoPath(oneVideoFullPath, *finalSubFile, "", bSetDefault, false)
+		err = d.SaveSubHelper.WriteSubFile2VideoPath(oneVideoFullPath, *finalSubFile, "", bSetDefault, false)
 		if err != nil {
 			return errors.New(fmt.Sprintf("SaveMultiSub: %v, writeSubFile2VideoPath, Error: %v ", d.settings.AdvancedSettings.SaveMultiSub, err))
 		}
@@ -98,7 +95,7 @@ func (d *Downloader) oneVideoSelectBestSub(oneVideoFullPath string, organizeSubF
 				if i == 0 {
 					setDefault = true
 				}
-				err = d.writeSubFile2VideoPath(oneVideoFullPath, file, siteNames[i], setDefault, false)
+				err = d.SaveSubHelper.WriteSubFile2VideoPath(oneVideoFullPath, file, siteNames[i], setDefault, false)
 				if err != nil {
 					return errors.New(fmt.Sprintf("SaveMultiSub: %v, writeSubFile2VideoPath, Error: %v ", d.settings.AdvancedSettings.SaveMultiSub, err))
 				}
@@ -112,7 +109,7 @@ func (d *Downloader) oneVideoSelectBestSub(oneVideoFullPath string, organizeSubF
 				那么就比较麻烦，干脆，normal 的命名格式化实例，就不设置 default 了，forced 不想用，因为可能会跟你手动选择的字幕冲突（下次观看的时候，理论上也可能不会）
 			*/
 			for i := len(finalSubFiles) - 1; i > -1; i-- {
-				err = d.writeSubFile2VideoPath(oneVideoFullPath, finalSubFiles[i], siteNames[i], false, false)
+				err = d.SaveSubHelper.WriteSubFile2VideoPath(oneVideoFullPath, finalSubFiles[i], siteNames[i], false, false)
 				if err != nil {
 					return errors.New(fmt.Sprintf("SaveMultiSub: %v, writeSubFile2VideoPath, Error: %v ", d.settings.AdvancedSettings.SaveMultiSub, err))
 				}
@@ -170,69 +167,3 @@ func (d *Downloader) saveFullSeasonSub(seriesInfo *series.SeriesInfo, organizeSu
 
 	return fullSeasonSubDict
 }
-
-// 在前面需要进行语言的筛选、排序，这里仅仅是存储， extraSubPreName 这里传递是字幕的网站，有就认为是多字幕的存储。空就是单字幕，单字幕就可以setDefault
-func (d *Downloader) writeSubFile2VideoPath(videoFileFullPath string, finalSubFile subparser.FileInfo, extraSubPreName string, setDefault bool, skipExistFile bool) error {
-	defer d.log.Infoln("----------------------------------")
-	videoRootPath := filepath.Dir(videoFileFullPath)
-	subNewName, subNewNameWithDefault, _ := d.subFormatter.GenerateMixSubName(videoFileFullPath, finalSubFile.Ext, finalSubFile.Lang, extraSubPreName)
-
-	desSubFullPath := filepath.Join(videoRootPath, subNewName)
-	if setDefault == true {
-		// 先判断没有 default 的字幕是否存在了，在的话，先删除，然后再写入
-		if pkg.IsFile(desSubFullPath) == true {
-			_ = os.Remove(desSubFullPath)
-		}
-		desSubFullPath = filepath.Join(videoRootPath, subNewNameWithDefault)
-	}
-
-	if skipExistFile == true {
-		// 需要判断文件是否存在在，有则跳过
-		if pkg.IsFile(desSubFullPath) == true {
-			d.log.Infoln("OrgSubName:", finalSubFile.Name)
-			d.log.Infoln("Sub Skip DownAt:", desSubFullPath)
-			return nil
-		}
-	}
-	// 最后写入字幕
-	err := pkg.WriteFile(desSubFullPath, finalSubFile.Data)
-	if err != nil {
-		return err
-	}
-	d.log.Infoln("----------------------------------")
-	d.log.Infoln("OrgSubName:", finalSubFile.Name)
-	d.log.Infoln("SubDownAt:", desSubFullPath)
-
-	// 然后还需要判断是否需要校正字幕的时间轴
-	if d.settings.AdvancedSettings.FixTimeLine == true {
-		err = d.subTimelineFixerHelperEx.Process(videoFileFullPath, desSubFullPath)
-		if err != nil {
-			return err
-		}
-	}
-	// 判断是否需要转换字幕的编码
-	if d.settings.ExperimentalFunction.AutoChangeSubEncode.Enable == true {
-		d.log.Infoln("----------------------------------")
-		d.log.Infoln("change_file_encode to", d.settings.ExperimentalFunction.AutoChangeSubEncode.GetDesEncodeType())
-		err = change_file_encode.Process(desSubFullPath, d.settings.ExperimentalFunction.AutoChangeSubEncode.DesEncodeType)
-		if err != nil {
-			return err
-		}
-	}
-
-	// 判断是否需要进行简繁互转
-	// 一定得是 UTF-8 才能够执行简繁转换
-	// 测试了先转 UTF-8 进行简繁转换然后再转 GBK，有些时候会出错，所以还是不支持这样先
-	if d.settings.ExperimentalFunction.AutoChangeSubEncode.Enable == true &&
-		d.settings.ExperimentalFunction.AutoChangeSubEncode.DesEncodeType == 0 &&
-		d.settings.ExperimentalFunction.ChsChtChanger.Enable == true {
-		d.log.Infoln("----------------------------------")
-		d.log.Infoln("chs_cht_changer to", d.settings.ExperimentalFunction.ChsChtChanger.GetDesChineseLanguageTypeString())
-		err = chs_cht_changer.Process(desSubFullPath, d.settings.ExperimentalFunction.ChsChtChanger.DesChineseLanguageType)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/manual_upload_sub_2_local/processor.go

@@ -0,0 +1,179 @@
+package manual_upload_sub_2_local
+
+import (
+	"sync"
+
+	cb "github.com/emirpasic/gods/queues/circularbuffer"
+
+	"github.com/pkg/errors"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/save_sub_helper"
+	subCommon "github.com/allanpk716/ChineseSubFinder/pkg/sub_formatter/common"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_parser/ass"
+	"github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_parser/srt"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/sub_parser_hub"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/sub_helper"
+
+	"github.com/sirupsen/logrus"
+
+	llq "github.com/emirpasic/gods/queues/linkedlistqueue"
+	"github.com/emirpasic/gods/sets/hashset"
+)
+
+type ManualUploadSub2Local struct {
+	log              *logrus.Logger
+	saveSubHelper    *save_sub_helper.SaveSubHelper // 保存字幕的逻辑
+	subNameFormatter subCommon.FormatterName        // 从 inSubFormatter 推断出来
+	processQueue     *llq.Queue
+	jobSet           *hashset.Set
+	addOneSignal     chan interface{}
+	addLocker        sync.Mutex
+	subParserHub     *sub_parser_hub.SubParserHub
+	workingJob       string // 正在操作的任务的路径
+	jobStatusQueue   *cb.Queue
+}
+
+func NewManualUploadSub2Local(log *logrus.Logger, saveSubHelper *save_sub_helper.SaveSubHelper) *ManualUploadSub2Local {
+
+	m := &ManualUploadSub2Local{
+		log:            log,
+		saveSubHelper:  saveSubHelper,
+		processQueue:   llq.New(),
+		addOneSignal:   make(chan interface{}, 1),
+		subParserHub:   sub_parser_hub.NewSubParserHub(log, ass.NewParser(log), srt.NewParser(log)),
+		workingJob:     "",
+		jobStatusQueue: cb.New(100),
+	}
+
+	// 这里就不单独弄一个 settings.SubNameFormatter 字段来传递值了，因为 inSubFormatter 就已经知道是什么 formatter 了
+	m.subNameFormatter = subCommon.FormatterName(saveSubHelper.SubFormatter.GetFormatterFormatterName())
+
+	go func(mu *ManualUploadSub2Local) {
+		for {
+			select {
+			case _ = <-mu.addOneSignal:
+				// 有新任务了
+				m.dealers()
+			}
+		}
+	}(m)
+
+	return m
+}
+
+// IsJobInQueue 是否正在队列中排队，或者正在被处理
+func (m *ManualUploadSub2Local) IsJobInQueue(job *Job) bool {
+	m.addLocker.Lock()
+	defer func() {
+		m.addLocker.Unlock()
+	}()
+	if m.jobSet.Contains(job.VideoFPath) == true {
+		// 已经在队列中了
+		return true
+	} else {
+		// 还有一种可能，任务从队列拿出来了，正在处理，那么在外部开来也还是在队列中的
+		if m.workingJob == job.VideoFPath {
+			return true
+		}
+	}
+	return false
+}
+
+func (m *ManualUploadSub2Local) Add(job *Job) string {
+
+	m.addLocker.Lock()
+	defer func() {
+		m.addLocker.Unlock()
+	}()
+
+	if m.jobSet.Contains(job.VideoFPath) == true {
+		// 已经在队列中了
+		return ""
+	}
+
+	//randomKey := pkg.RandStringBytesMaskImprSrcSB(10)
+
+	m.processQueue.Enqueue(job)
+	m.jobSet.Add(job.VideoFPath)
+	// 通知有新任务了
+	m.addOneSignal <- struct{}{}
+
+	return ""
+}
+
+func (m *ManualUploadSub2Local) dealers() {
+
+	m.addLocker.Lock()
+	if m.processQueue.Empty() == true {
+		// 没有任务了
+		m.addLocker.Unlock()
+		return
+	}
+	job, ok := m.processQueue.Dequeue()
+	if ok == false {
+		// 没有任务了
+		m.addLocker.Unlock()
+		return
+	}
+	// 移除这个任务
+	m.jobSet.Remove(job.(*Job).VideoFPath)
+	// 标记这个正在处理
+	m.workingJob = job.(*Job).VideoFPath
+	m.addLocker.Unlock()
+	// 具体处理这个任务
+	err := m.processSub(job.(*Job))
+	if err != nil {
+		m.log.Error(err)
+	}
+}
+
+func (m *ManualUploadSub2Local) processSub(job *Job) error {
+
+	defer func() {
+		// 任务处理完了
+		m.addLocker.Lock()
+		m.workingJob = ""
+		m.addLocker.Unlock()
+	}()
+
+	// 不管是不是保存多个字幕，都要先扫描本地的字幕，进行 .Default .Forced 去除
+	// 这个视频的所有字幕，去除 .default .Forced 标记
+	err := sub_helper.SearchVideoMatchSubFileAndRemoveExtMark(m.log, job.VideoFPath)
+	if err != nil {
+		// 找个错误可以忍
+		m.log.Errorln("SearchVideoMatchSubFileAndRemoveExtMark,", job.VideoFPath, err)
+	}
+
+	bFind, subFileInfo, err := m.subParserHub.DetermineFileTypeFromFile(job.SubFPath)
+	if err != nil {
+		err = errors.New("DetermineFileTypeFromFile," + job.SubFPath + "," + err.Error())
+		return err
+	}
+	if bFind == false {
+		err = errors.New("DetermineFileTypeFromFile," + job.SubFPath + ",not support SubType")
+		return err
+	}
+	if m.subNameFormatter == subCommon.Emby {
+		err = m.saveSubHelper.WriteSubFile2VideoPath(job.VideoFPath, *subFileInfo, "manual", true, false)
+		if err != nil {
+			err = errors.New("WriteSubFile2VideoPath," + job.VideoFPath + "," + err.Error())
+			return err
+		}
+	} else {
+		err = m.saveSubHelper.WriteSubFile2VideoPath(job.VideoFPath, *subFileInfo, "manual", false, false)
+		if err != nil {
+			err = errors.New("WriteSubFile2VideoPath," + job.VideoFPath + "," + err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+type Job struct {
+	VideoFPath string
+	SubFPath   string
+}

pkg/manual_upload_sub_2_local/processor_test.go

@@ -0,0 +1,10 @@
+package manual_upload_sub_2_local
+
+import (
+	"testing"
+)
+
+func TestNewManualUploadSub2Local(t *testing.T) {
+
+	NewManualUploadSub2Local(nil, nil)
+}

pkg/save_sub_helper/save_sub_helper.go

@@ -0,0 +1,92 @@
+package save_sub_helper
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg"
+	"github.com/allanpk716/ChineseSubFinder/pkg/change_file_encode"
+	"github.com/allanpk716/ChineseSubFinder/pkg/chs_cht_changer"
+	"github.com/allanpk716/ChineseSubFinder/pkg/ifaces"
+	"github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_timeline_fixer"
+	"github.com/allanpk716/ChineseSubFinder/pkg/settings"
+	"github.com/allanpk716/ChineseSubFinder/pkg/types/subparser"
+	"github.com/sirupsen/logrus"
+)
+
+type SaveSubHelper struct {
+	log                      *logrus.Logger
+	settings                 *settings.Settings
+	SubFormatter             ifaces.ISubFormatter                         // 字幕格式化命名的实现
+	subTimelineFixerHelperEx *sub_timeline_fixer.SubTimelineFixerHelperEx // 字幕时间轴校正
+}
+
+func NewSaveSubHelper(log *logrus.Logger, settings *settings.Settings, subFormatter ifaces.ISubFormatter, subTimelineFixerHelperEx *sub_timeline_fixer.SubTimelineFixerHelperEx) *SaveSubHelper {
+	return &SaveSubHelper{settings: settings, log: log, SubFormatter: subFormatter, subTimelineFixerHelperEx: subTimelineFixerHelperEx}
+}
+
+// WriteSubFile2VideoPath 在前面需要进行语言的筛选、排序，这里仅仅是存储， extraSubPreName 这里传递是字幕的网站，有就认为是多字幕的存储。空就是单字幕，单字幕就可以setDefault
+func (s *SaveSubHelper) WriteSubFile2VideoPath(videoFileFullPath string, finalSubFile subparser.FileInfo, extraSubPreName string, setDefault bool, skipExistFile bool) error {
+	defer s.log.Infoln("----------------------------------")
+	videoRootPath := filepath.Dir(videoFileFullPath)
+	subNewName, subNewNameWithDefault, _ := s.SubFormatter.GenerateMixSubName(videoFileFullPath, finalSubFile.Ext, finalSubFile.Lang, extraSubPreName)
+
+	desSubFullPath := filepath.Join(videoRootPath, subNewName)
+	if setDefault == true {
+		// 先判断没有 default 的字幕是否存在了，在的话，先删除，然后再写入
+		if pkg.IsFile(desSubFullPath) == true {
+			_ = os.Remove(desSubFullPath)
+		}
+		desSubFullPath = filepath.Join(videoRootPath, subNewNameWithDefault)
+	}
+
+	if skipExistFile == true {
+		// 需要判断文件是否存在在，有则跳过
+		if pkg.IsFile(desSubFullPath) == true {
+			s.log.Infoln("OrgSubName:", finalSubFile.Name)
+			s.log.Infoln("Sub Skip DownAt:", desSubFullPath)
+			return nil
+		}
+	}
+	// 最后写入字幕
+	err := pkg.WriteFile(desSubFullPath, finalSubFile.Data)
+	if err != nil {
+		return err
+	}
+	s.log.Infoln("----------------------------------")
+	s.log.Infoln("OrgSubName:", finalSubFile.Name)
+	s.log.Infoln("SubDownAt:", desSubFullPath)
+
+	// 然后还需要判断是否需要校正字幕的时间轴
+	if s.settings.AdvancedSettings.FixTimeLine == true {
+		err = s.subTimelineFixerHelperEx.Process(videoFileFullPath, desSubFullPath)
+		if err != nil {
+			return err
+		}
+	}
+	// 判断是否需要转换字幕的编码
+	if s.settings.ExperimentalFunction.AutoChangeSubEncode.Enable == true {
+		s.log.Infoln("----------------------------------")
+		s.log.Infoln("change_file_encode to", s.settings.ExperimentalFunction.AutoChangeSubEncode.GetDesEncodeType())
+		err = change_file_encode.Process(desSubFullPath, s.settings.ExperimentalFunction.AutoChangeSubEncode.DesEncodeType)
+		if err != nil {
+			return err
+		}
+	}
+
+	// 判断是否需要进行简繁互转
+	// 一定得是 UTF-8 才能够执行简繁转换
+	// 测试了先转 UTF-8 进行简繁转换然后再转 GBK，有些时候会出错，所以还是不支持这样先
+	if s.settings.ExperimentalFunction.AutoChangeSubEncode.Enable == true &&
+		s.settings.ExperimentalFunction.AutoChangeSubEncode.DesEncodeType == 0 &&
+		s.settings.ExperimentalFunction.ChsChtChanger.Enable == true {
+		s.log.Infoln("----------------------------------")
+		s.log.Infoln("chs_cht_changer to", s.settings.ExperimentalFunction.ChsChtChanger.GetDesChineseLanguageTypeString())
+		err = chs_cht_changer.Process(desSubFullPath, s.settings.ExperimentalFunction.ChsChtChanger.DesChineseLanguageType)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}