package downloader import ( "errors" "fmt" "sync" "github.com/allanpk716/ChineseSubFinder/pkg/ifaces" common2 "github.com/allanpk716/ChineseSubFinder/pkg/types/common" taskQueue2 "github.com/allanpk716/ChineseSubFinder/pkg/types/task_queue" embyHelper "github.com/allanpk716/ChineseSubFinder/pkg/logic/emby_helper" "github.com/allanpk716/ChineseSubFinder/pkg/logic/file_downloader" markSystem "github.com/allanpk716/ChineseSubFinder/pkg/logic/mark_system" "github.com/allanpk716/ChineseSubFinder/pkg/logic/pre_download_process" subSupplier "github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_supplier" "github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_supplier/csf" "github.com/allanpk716/ChineseSubFinder/pkg/logic/sub_timeline_fixer" "github.com/allanpk716/ChineseSubFinder/pkg/decode" "github.com/allanpk716/ChineseSubFinder/pkg/global_value" "github.com/allanpk716/ChineseSubFinder/pkg/log_helper" "github.com/allanpk716/ChineseSubFinder/pkg/my_folder" "github.com/allanpk716/ChineseSubFinder/pkg/my_util" "github.com/allanpk716/ChineseSubFinder/pkg/settings" subCommon "github.com/allanpk716/ChineseSubFinder/pkg/sub_formatter/common" "github.com/allanpk716/ChineseSubFinder/pkg/task_queue" "github.com/sirupsen/logrus" "golang.org/x/net/context" ) // Downloader 实例化一次用一次，不要反复的使用，很多临时标志位需要清理。 type Downloader struct { settings *settings.Settings log *logrus.Logger fileDownloader *file_downloader.FileDownloader ctx context.Context cancel context.CancelFunc subSupplierHub *subSupplier.SubSupplierHub // 字幕提供源的集合，这个需要定时进行扫描，这些字幕源是否有效，以及下载验证码信息 mk *markSystem.MarkingSystem // MarkingSystem，字幕的评价系统 subFormatter ifaces.ISubFormatter // 字幕格式化命名的实现 subNameFormatter subCommon.FormatterName // 从 inSubFormatter 推断出来 subTimelineFixerHelperEx *sub_timeline_fixer.SubTimelineFixerHelperEx // 字幕时间轴校正 downloaderLock sync.Mutex // 取消执行 task control 的 Lock downloadQueue *task_queue.TaskQueue // 需要下载的视频的队列 embyHelper *embyHelper.EmbyHelper // Emby 的实例 cacheLocker sync.Mutex movieInfoMap map[string]MovieInfo // 给 Web 界面使用的，Key: VideoFPath seasonInfoMap map[string]SeasonInfo // 给 Web 界面使用的,Key: RootDirPath } func NewDownloader(inSubFormatter ifaces.ISubFormatter, fileDownloader *file_downloader.FileDownloader, downloadQueue *task_queue.TaskQueue) *Downloader { var downloader Downloader downloader.fileDownloader = fileDownloader downloader.subFormatter = inSubFormatter downloader.fileDownloader = fileDownloader downloader.log = fileDownloader.Log // 参入设置信息 downloader.settings = fileDownloader.Settings // 检测是否某些参数超出范围 downloader.settings.Check() // 这里就不单独弄一个 settings.SubNameFormatter 字段来传递值了，因为 inSubFormatter 就已经知道是什么 formatter 了 downloader.subNameFormatter = subCommon.FormatterName(downloader.subFormatter.GetFormatterFormatterName()) var sitesSequence = make([]string, 0) // TODO 这里写固定了抉择字幕的顺序 sitesSequence = append(sitesSequence, common2.SubSiteZiMuKu) sitesSequence = append(sitesSequence, common2.SubSiteSubHd) sitesSequence = append(sitesSequence, common2.SubSiteChineseSubFinder) sitesSequence = append(sitesSequence, common2.SubSiteAssrt) sitesSequence = append(sitesSequence, common2.SubSiteA4K) sitesSequence = append(sitesSequence, common2.SubSiteShooter) sitesSequence = append(sitesSequence, common2.SubSiteXunLei) downloader.mk = markSystem.NewMarkingSystem(downloader.log, sitesSequence, downloader.settings.AdvancedSettings.SubTypePriority) // 初始化，字幕校正的实例 downloader.subTimelineFixerHelperEx = sub_timeline_fixer.NewSubTimelineFixerHelperEx(downloader.log, *downloader.settings.TimelineFixerSettings) if downloader.settings.AdvancedSettings.FixTimeLine == true { downloader.subTimelineFixerHelperEx.Check() } // 任务队列 downloader.downloadQueue = downloadQueue // 单个任务的超时设置 downloader.ctx, downloader.cancel = context.WithCancel(context.Background()) // 用于字幕下载后的刷新 if downloader.settings.EmbySettings.Enable == true { downloader.embyHelper = embyHelper.NewEmbyHelper(downloader.log, downloader.settings) } downloader.movieInfoMap = make(map[string]MovieInfo) downloader.seasonInfoMap = make(map[string]SeasonInfo) err := downloader.loadVideoListCache() if err != nil { downloader.log.Errorln("loadVideoListCache error:", err) } return &downloader } // SupplierCheck 检查字幕源是否有效，会影响后续的字幕源是否参与下载 func (d *Downloader) SupplierCheck() { defer func() { if p := recover(); p != nil { d.log.Errorln("Downloader.SupplierCheck() panic") my_util.PrintPanicStack(d.log) } d.downloaderLock.Unlock() d.log.Infoln("Download.SupplierCheck() End") }() d.downloaderLock.Lock() d.log.Infoln("Download.SupplierCheck() Start ...") //// 创建一个 chan 用于任务的中断和超时 //done := make(chan interface{}, 1) //// 接收内部任务的 panic //panicChan := make(chan interface{}, 1) // //go func() { // defer func() { // if p := recover(); p != nil { // panicChan <- p // } // // close(done) // close(panicChan) // }() // 下载前的初始化 d.log.Infoln("PreDownloadProcess.Init().Check().Wait()...") if d.settings.SpeedDevMode == true { // 这里是调试使用的，指定了只用一个字幕源 subSupplierHub := subSupplier.NewSubSupplierHub(csf.NewSupplier(d.fileDownloader)) d.subSupplierHub = subSupplierHub } else { preDownloadProcess := pre_download_process.NewPreDownloadProcess(d.fileDownloader) err := preDownloadProcess.Init().Check().Wait() if err != nil { //done <- errors.New(fmt.Sprintf("NewPreDownloadProcess Error: %v", err)) d.log.Errorln(errors.New(fmt.Sprintf("NewPreDownloadProcess Error: %v", err))) } else { // 更新 SubSupplierHub 实例 d.subSupplierHub = preDownloadProcess.SubSupplierHub //done <- nil } } // done <- nil //}() // //select { //case err := <-done: // if err != nil { // d.log.Errorln(err) // } // break //case p := <-panicChan: // // 遇到内部的 panic，向外抛出 // panic(p) //case <-d.ctx.Done(): // { // d.log.Errorln("cancel SupplierCheck") // return // } //} } // QueueDownloader 从字幕队列中取一个视频的字幕下载任务出来，并且开始下载 func (d *Downloader) QueueDownloader() { d.log.Debugln("Download.QueueDownloader() Try Start ...") d.downloaderLock.Lock() d.log.Debugln("Download.QueueDownloader() Start ...") defer func() { if p := recover(); p != nil { d.log.Errorln("Downloader.QueueDownloader() panic") my_util.PrintPanicStack(d.log) } d.downloaderLock.Unlock() d.log.Debugln("Download.QueueDownloader() End") }() var downloadCounter int64 downloadCounter = 0 // 移除查过三个月的 Done 任务 d.downloadQueue.BeforeGetOneJob() // 从队列取数据出来，见《任务生命周期》 bok, oneJob, err := d.downloadQueue.GetOneJob() if err != nil { d.log.Errorln("d.downloadQueue.GetOneWaitingJob()", err) return } if bok == false { d.log.Debugln("Download Queue Is Empty, Skip This Time") return } // -------------------------------------------------- // 这个任务如果是 series 那么需要考虑是否原始存入的信息是缺失的，需要补全 { if oneJob.VideoType == common2.Series && (oneJob.SeriesRootDirPath == "" || oneJob.Season <= 0 || oneJob.Episode <= 0) { // 连续剧的时候需要额外提交信息 epsVideoNfoInfo, err := decode.GetVideoNfoInfo4OneSeriesEpisode(oneJob.VideoFPath) if err != nil { d.log.Errorln("decode.GetVideoNfoInfo4OneSeriesEpisode()", err) d.log.Infoln("maybe your moved video file to another place or delete it, so will delete this job") bok, err = d.downloadQueue.Del(oneJob.Id) if err != nil { d.log.Errorln("d.downloadQueue.Del()", err) return } if bok == false { d.log.Errorln(fmt.Sprintf("d.downloadQueue.Del(%d) == false", oneJob.Id)) return } return } seriesInfoDirPath := decode.GetSeriesDirRootFPath(oneJob.VideoFPath) if seriesInfoDirPath == "" { d.log.Errorln(fmt.Sprintf("decode.GetSeriesDirRootFPath == Empty, %s", oneJob.VideoFPath)) d.log.Infoln("you need check the directory structure of a series, so will delete this job") bok, err = d.downloadQueue.Del(oneJob.Id) if err != nil { d.log.Errorln("d.downloadQueue.Del()", err) return } if bok == false { d.log.Errorln(fmt.Sprintf("d.downloadQueue.Del(%d) == false", oneJob.Id)) return } return } oneJob.Season = epsVideoNfoInfo.Season oneJob.Episode = epsVideoNfoInfo.Episode oneJob.SeriesRootDirPath = seriesInfoDirPath } } // -------------------------------------------------- // 这个视频文件不存在了 { isBlue, _, _ := decode.IsFakeBDMVWorked(oneJob.VideoFPath) if isBlue == false && my_util.IsFile(oneJob.VideoFPath) == false { // 不是蓝光，那么就判断文件是否存在，不存在，那么就标记 ignore bok, err = d.downloadQueue.Del(oneJob.Id) if err != nil { d.log.Errorln("d.downloadQueue.Del()", err) return } if bok == false { d.log.Errorln(fmt.Sprintf("d.downloadQueue.Del(%d) == false", oneJob.Id)) return } d.log.Infoln(oneJob.VideoFPath, "is missing, Delete This Job") return } } // -------------------------------------------------- // 判断是否看过，这个只有 Emby 情况下才会生效 { isPlayed := false if d.embyHelper != nil { // 在拿出来后，如果是有内部媒体服务器媒体 ID 的，那么就去查询是否已经观看过了 isPlayed, err = d.embyHelper.IsVideoPlayed(oneJob.MediaServerInsideVideoID) if err != nil { d.log.Errorln("d.embyHelper.IsVideoPlayed()", oneJob.VideoFPath, err) return } } // TODO 暂时屏蔽掉 http api 提交的已看字幕的接口上传 // 不管如何，只要是发现数据库中有 HTTP API 提交的信息，就认为是看过 //var videoPlayedInfos []models.ThirdPartSetVideoPlayedInfo //dao.GetDb().Where("physical_video_file_full_path = ?", oneJob.VideoFPath).Find(&videoPlayedInfos) //if len(videoPlayedInfos) > 0 { // isPlayed = true //} // -------------------------------------------------- // 如果已经播放过 且 这个任务的优先级 > 3 ，不是很急的那种，说明是可以设置忽略继续下载的 if isPlayed == true && oneJob.TaskPriority > task_queue.HighTaskPriorityLevel { // 播放过了，那么就标记 ignore oneJob.JobStatus = taskQueue2.Ignore bok, err = d.downloadQueue.Update(oneJob) if err != nil { d.log.Errorln("d.downloadQueue.Update()", err) return } if bok == false { d.log.Errorln("d.downloadQueue.Update() Failed") return } d.log.Infoln("Is Played, Ignore This Job") return } } // -------------------------------------------------- // 判断是否需要跳过，因为如果是 Normal 扫描出来的，那么可能因为视频时间久远，下载一次即可 { if oneJob.TaskPriority > task_queue.HighTaskPriorityLevel { // 优先级大于 3，那么就不是很急的任务，才需要判断 if oneJob.VideoType == common2.Movie { if d.subSupplierHub.MovieNeedDlSub(oneJob.VideoFPath, false) == false { // 需要标记忽略 oneJob.JobStatus = taskQueue2.Ignore bok, err = d.downloadQueue.Update(oneJob) if err != nil { d.log.Errorln("d.downloadQueue.Update()", err) return } if bok == false { d.log.Errorln("d.downloadQueue.Update() Failed") return } d.log.Infoln("MovieNeedDlSub == false, Ignore This Job") return } } else if oneJob.VideoType == common2.Series { bNeedDlSub, seriesInfo, err := d.subSupplierHub.SeriesNeedDlSub(oneJob.SeriesRootDirPath, false, false) if err != nil { d.log.Errorln("SeriesNeedDlSub", err) return } needMarkSkip := false if bNeedDlSub == false { // 需要跳过 needMarkSkip = true } else { // 需要下载的 Eps 是否与 Normal 判断这个连续剧中有那些剧集需要下载的，情况符合。通过下载的时间来判断 epsKey := my_util.GetEpisodeKeyName(oneJob.Season, oneJob.Episode) _, found := seriesInfo.NeedDlEpsKeyList[epsKey] if found == false { // 需要跳过 needMarkSkip = true } } if needMarkSkip == true { // 需要标记忽略 oneJob.JobStatus = taskQueue2.Ignore bok, err = d.downloadQueue.Update(oneJob) if err != nil { d.log.Errorln("d.downloadQueue.Update()", err) return } if bok == false { d.log.Errorln("d.downloadQueue.Update() Failed") return } d.log.Infoln("SeriesNeedDlSub == false, Ignore This Job") return } } } } // 取出来后，需要标记为正在下载 oneJob.JobStatus = taskQueue2.Downloading bok, err = d.downloadQueue.Update(oneJob) if err != nil { d.log.Errorln("d.downloadQueue.Update()", err) return } if bok == false { d.log.Errorln("d.downloadQueue.Update() Failed") return } // ------------------------------------------------------------------------ // 开始标记，这个是单次扫描的开始，要注意格式，在日志的内部解析识别单个日志开头的时候需要特殊的格式 d.log.Infoln("------------------------------------------") d.log.Infoln(log_helper.OnceSubsScanStart + "#" + oneJob.Id) // ------------------------------------------------------------------------ defer func() { d.log.Infoln(log_helper.OnceSubsScanEnd) d.log.Infoln("------------------------------------------") }() downloadCounter++ // 创建一个 chan 用于任务的中断和超时 done := make(chan interface{}, 1) // 接收内部任务的 panic panicChan := make(chan interface{}, 1) go func() { defer func() { if p := recover(); p != nil { panicChan <- p } close(done) close(panicChan) // 没下载完毕一次，进行一次缓存和 Chrome 的清理 err = my_folder.ClearRootTmpFolder() if err != nil { d.log.Error("ClearRootTmpFolder", err) } if global_value.LiteMode() == false { my_util.CloseChrome(d.log) } }() if oneJob.VideoType == common2.Movie { // 电影 // 具体的下载逻辑 func() done <- d.movieDlFunc(d.ctx, oneJob, downloadCounter) } else if oneJob.VideoType == common2.Series { // 连续剧 // 具体的下载逻辑 func() done <- d.seriesDlFunc(d.ctx, oneJob, downloadCounter) } else { d.log.Errorln("oneJob.VideoType not support, oneJob.VideoType = ", oneJob.VideoType) done <- nil } }() select { case err := <-done: // 跳出 select，可以外层继续，不会阻塞在这里 if err != nil { d.log.Errorln(err) } // 刷新视频的缓存结构 d.UpdateInfo(oneJob) break case p := <-panicChan: // 遇到内部的 panic，向外抛出 panic(p) case <-d.ctx.Done(): { // 取消这个 context d.log.Warningln("cancel Downloader.QueueDownloader()") return } } } func (d *Downloader) Cancel() { if d == nil { return } d.cancel() d.log.Infoln("Downloader.Cancel()") }