ChineseSubFinder/pkg/logic/sub_timeline_fixer/SubTimelineFixerHelperEx.go

package sub_timeline_fixer

import (
	"errors"
	"math"
	"os"

	"github.com/ChineseSubFinder/ChineseSubFinder/pkg"

	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/types/subparser"

	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/ffmpeg_helper"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/logic/sub_parser/ass"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/logic/sub_parser/srt"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/settings"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/sub_parser_hub"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/sub_timeline_fixer"
	"github.com/ChineseSubFinder/ChineseSubFinder/pkg/vad"
	"github.com/emirpasic/gods/maps/treemap"
	"github.com/emirpasic/gods/utils"
	"github.com/sirupsen/logrus"
)

type SubTimelineFixerHelperEx struct {
	log                 *logrus.Logger
	ffmpegHelper        *ffmpeg_helper.FFMPEGHelper
	subParserHub        *sub_parser_hub.SubParserHub
	timelineFixPipeLine *sub_timeline_fixer.Pipeline
	fixerConfig         settings.TimelineFixerSettings
	needDownloadFFMPeg  bool
}

func NewSubTimelineFixerHelperEx(log *logrus.Logger, fixerConfig settings.TimelineFixerSettings) *SubTimelineFixerHelperEx {

	fixerConfig.Check()

	return &SubTimelineFixerHelperEx{
		log:                 log,
		ffmpegHelper:        ffmpeg_helper.NewFFMPEGHelper(log),
		subParserHub:        sub_parser_hub.NewSubParserHub(log, ass.NewParser(log), srt.NewParser(log)),
		timelineFixPipeLine: sub_timeline_fixer.NewPipeline(fixerConfig.MaxOffsetTime),
		fixerConfig:         fixerConfig,
		needDownloadFFMPeg:  false,
	}
}

// Check 是否安装了 ffmpeg 和 ffprobe
func (s *SubTimelineFixerHelperEx) Check() bool {
	version, err := s.ffmpegHelper.Version()
	if err != nil {
		s.needDownloadFFMPeg = false
		s.log.Errorln("Need Install ffmpeg and ffprobe !")
		return false
	}
	s.needDownloadFFMPeg = true
	s.log.Infoln(version)
	return true
}

func (s *SubTimelineFixerHelperEx) Process(videoFileFullPath, srcSubFPath string) error {

	if s.needDownloadFFMPeg == false {
		s.log.Errorln("Need Install ffmpeg and ffprobe, Can't Do TimeLine Fix")
		return nil
	}

	var infoSrc *subparser.FileInfo
	var pipeResultMax sub_timeline_fixer.PipeResult
	bProcess := false
	bok := false
	var ffmpegInfo *ffmpeg_helper.FFMPEGInfo
	var err error
	// 先尝试获取内置字幕的信息
	bok, ffmpegInfo, err = s.ffmpegHelper.ExportFFMPEGInfo(videoFileFullPath, ffmpeg_helper.Subtitle)
	if err != nil {
		return err
	}
	if bok == false {
		return errors.New("SubTimelineFixerHelperEx.Process.ExportFFMPEGInfo = false Subtitle -- " + videoFileFullPath)
	}

	// 这个需要提前考虑，如果只有一个内置的字幕，且这个字幕的大小小于 2kb，那么认为这个字幕是有问题的，就直接切换到 audio 校正
	oneSubAndIsError := false
	if len(ffmpegInfo.SubtitleInfoList) == 1 {
		fi, err := os.Stat(ffmpegInfo.SubtitleInfoList[0].FullPath)
		if err != nil {
			oneSubAndIsError = true
		} else {
			if fi.Size() <= 2048 {
				oneSubAndIsError = true
			}
		}
	}
	// 内置的字幕，这里只列举一种格式出来，其实会有一个字幕的 srt 和 ass 两种格式都导出存在
	if ffmpegInfo.SubtitleInfoList == nil || len(ffmpegInfo.SubtitleInfoList) <= 0 || oneSubAndIsError == true {

		if ffmpegInfo.AudioInfoList == nil || len(ffmpegInfo.AudioInfoList) == 0 {
			return errors.New("SubTimelineFixerHelperEx.Process.ExportFFMPEGInfo Can`t Find SubTitle And Audio To Export -- " + videoFileFullPath)
		}

		// 如果内置字幕没有，那么就需要尝试获取音频信息
		bok, ffmpegInfo, err = s.ffmpegHelper.ExportFFMPEGInfo(videoFileFullPath, ffmpeg_helper.Audio)
		if err != nil {
			return err
		}
		if bok == false {
			return errors.New("SubTimelineFixerHelperEx.Process.ExportFFMPEGInfo = false Audio -- " + videoFileFullPath)
		}

		// 使用音频进行时间轴的校正
		if len(ffmpegInfo.AudioInfoList) <= 0 {
			s.log.Warnln("Can`t find audio info, skip time fix --", videoFileFullPath)
			return nil
		}
		bProcess, infoSrc, pipeResultMax, err = s.ProcessByAudioFile(ffmpegInfo.AudioInfoList[0].FullPath, srcSubFPath)
		if err != nil {
			return err
		}
	} else {
		// 使用内置的字幕进行时间轴的校正，这里需要考虑一个问题，内置的字幕可能是有问题的（先考虑一种，就是字幕的长度不对，是一小段的）
		// 那么就可以比较多个内置字幕的大小选择大的去使用
		// 如果有多个内置的字幕，还是要判断下的，选体积最大的那个吧
		fileSizes := treemap.NewWith(utils.Int64Comparator)
		for index, info := range ffmpegInfo.SubtitleInfoList {
			fi, err := os.Stat(info.FullPath)
			if err != nil {
				fileSizes.Put(0, index)
			} else {
				fileSizes.Put(fi.Size(), index)
			}
		}
		_, index := fileSizes.Max()
		baseSubFPath := ffmpegInfo.SubtitleInfoList[index.(int)].FullPath
		bProcess, infoSrc, pipeResultMax, err = s.ProcessBySubFile(baseSubFPath, srcSubFPath)
		if err != nil {
			return err
		}
	}

	// 开始调整字幕时间轴
	if bProcess == false || math.Abs(pipeResultMax.GetOffsetTime()) < s.fixerConfig.MinOffset {
		s.log.Infoln("Skip TimeLine Fix -- OffsetTime:", pipeResultMax.GetOffsetTime(), srcSubFPath)
		return nil
	}
	err = s.changeTimeLineAndSave(infoSrc, pipeResultMax, srcSubFPath)
	if err != nil {
		return err
	}
	s.log.Infoln("TimeLine Fix -- Score:", pipeResultMax.Score, srcSubFPath)
	s.log.Infoln("Fix Offset:", pipeResultMax.GetOffsetTime(), srcSubFPath)
	s.log.Infoln("BackUp Org SubFile:", pipeResultMax.GetOffsetTime(), srcSubFPath+sub_timeline_fixer.BackUpExt)

	return nil
}

func (s *SubTimelineFixerHelperEx) ProcessBySubFileInfo(infoBase *subparser.FileInfo, infoSrc *subparser.FileInfo) (bool, *subparser.FileInfo, sub_timeline_fixer.PipeResult, error) {

	// ---------------------------------------------------------------------------------------
	pipeResult, err := s.timelineFixPipeLine.CalcOffsetTime(infoBase, infoSrc, nil, false)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}

	return true, infoSrc, pipeResult, nil
}

func (s *SubTimelineFixerHelperEx) ProcessBySubFile(baseSubFileFPath, srcSubFileFPath string) (bool, *subparser.FileInfo, sub_timeline_fixer.PipeResult, error) {

	bFind, infoBase, err := s.subParserHub.DetermineFileTypeFromFile(baseSubFileFPath)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}
	if bFind == false {
		s.log.Warnln("ProcessBySubFile.DetermineFileTypeFromFile sub not match --", baseSubFileFPath)
		return false, nil, sub_timeline_fixer.PipeResult{}, nil
	}

	bFind, infoSrc, err := s.subParserHub.DetermineFileTypeFromFile(srcSubFileFPath)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}
	if bFind == false {
		s.log.Warnln("ProcessBySubFile.DetermineFileTypeFromFile sub not match --", srcSubFileFPath)
		return false, nil, sub_timeline_fixer.PipeResult{}, nil
	}

	return s.ProcessBySubFileInfo(infoBase, infoSrc)
}

func (s *SubTimelineFixerHelperEx) ProcessByAudioVAD(audioVADInfos []vad.VADInfo, infoSrc *subparser.FileInfo) (bool, *subparser.FileInfo, sub_timeline_fixer.PipeResult, error) {

	// ---------------------------------------------------------------------------------------
	pipeResult, err := s.timelineFixPipeLine.CalcOffsetTime(nil, infoSrc, audioVADInfos, false)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}

	return true, infoSrc, pipeResult, nil
}

func (s *SubTimelineFixerHelperEx) ProcessByAudioFile(baseAudioFileFPath, srcSubFileFPath string) (bool, *subparser.FileInfo, sub_timeline_fixer.PipeResult, error) {

	audioVADInfos, err := vad.GetVADInfoFromAudio(vad.AudioInfo{
		FileFullPath: baseAudioFileFPath,
		SampleRate:   16000,
		BitDepth:     16,
	}, true)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}

	bFind, infoSrc, err := s.subParserHub.DetermineFileTypeFromFile(srcSubFileFPath)
	if err != nil {
		return false, nil, sub_timeline_fixer.PipeResult{}, err
	}
	if bFind == false {
		s.log.Warnln("ProcessByAudioFile.DetermineFileTypeFromFile sub not match --", srcSubFileFPath)
		return false, nil, sub_timeline_fixer.PipeResult{}, nil
	}

	return s.ProcessByAudioVAD(audioVADInfos, infoSrc)
}

func (s *SubTimelineFixerHelperEx) IsVideoCanExportSubtitleAndAudio(videoFileFullPath string) (bool, *ffmpeg_helper.FFMPEGInfo, []vad.VADInfo, *subparser.FileInfo, error) {

	// 先尝试获取内置字幕的信息
	bok, ffmpegInfo, err := s.ffmpegHelper.ExportFFMPEGInfo(videoFileFullPath, ffmpeg_helper.SubtitleAndAudio)
	if err != nil {
		return false, nil, nil, nil, err
	}
	if bok == false {
		return false, nil, nil, nil, nil
	}
	// ---------------------------------------------------------------------------------------
	// 音频
	if len(ffmpegInfo.AudioInfoList) <= 0 {
		return false, nil, nil, nil, nil
	}
	audioVADInfos, err := vad.GetVADInfoFromAudio(vad.AudioInfo{
		FileFullPath: ffmpegInfo.AudioInfoList[0].FullPath,
		SampleRate:   16000,
		BitDepth:     16,
	}, true)
	if err != nil {
		return false, nil, nil, nil, err
	}
	// ---------------------------------------------------------------------------------------
	// 字幕
	if len(ffmpegInfo.SubtitleInfoList) <= 0 {
		return false, nil, nil, nil, nil
	}
	// 使用内置的字幕进行时间轴的校正，这里需要考虑一个问题，内置的字幕可能是有问题的（先考虑一种，就是字幕的长度不对，是一小段的）
	// 那么就可以比较多个内置字幕的大小选择大的去使用
	// 如果有多个内置的字幕，还是要判断下的，选体积最大的那个吧
	fileSizes := treemap.NewWith(utils.Int64Comparator)
	for index, info := range ffmpegInfo.SubtitleInfoList {
		fi, err := os.Stat(info.FullPath)
		if err != nil {
			fileSizes.Put(0, index)
		} else {
			fileSizes.Put(fi.Size(), index)
		}
	}
	_, index := fileSizes.Max()
	baseSubFPath := ffmpegInfo.SubtitleInfoList[index.(int)].FullPath
	bFind, infoBase, err := s.subParserHub.DetermineFileTypeFromFile(baseSubFPath)
	if err != nil {
		return false, nil, nil, nil, err
	}
	if bFind == false {
		return false, nil, nil, nil, nil
	}
	// ---------------------------------------------------------------------------------------

	return true, ffmpegInfo, audioVADInfos, infoBase, nil
}

func (s *SubTimelineFixerHelperEx) IsMatchBySubFile(ffmpegInfo *ffmpeg_helper.FFMPEGInfo, audioVADInfos []vad.VADInfo, infoBase *subparser.FileInfo, srcSubFileFPath string, config CompareConfig) (bool, *MatchResult, error) {

	bFind, srcBase, err := s.subParserHub.DetermineFileTypeFromFile(srcSubFileFPath)
	if err != nil {
		return false, nil, err
	}
	if bFind == false {
		return false, nil, nil
	}
	// ---------------------------------------------------------------------------------------
	// 音频
	s.log.Infoln("IsMatchBySubFile:", srcSubFileFPath)
	bProcess, _, pipeResultMaxAudio, err := s.ProcessByAudioVAD(audioVADInfos, srcBase)
	if err != nil {
		return false, nil, err
	}
	if bProcess == false {
		return false, nil, nil
	}
	// ---------------------------------------------------------------------------------------
	// 字幕
	bProcess, _, pipeResultMaxSub, err := s.ProcessBySubFileInfo(infoBase, srcBase)
	if err != nil {
		return false, nil, err
	}
	if bProcess == false {
		return false, nil, nil
	}

	targetSubEndTime := pkg.Time2SecondNumber(srcBase.GetEndTime())

	matchResult := &MatchResult{
		VideoDuration:          ffmpegInfo.Duration,
		TargetSubEndTime:       targetSubEndTime,
		AudioCompareScore:      pipeResultMaxAudio.Score,
		AudioCompareOffsetTime: pipeResultMaxAudio.GetOffsetTime(),
		SubCompareScore:        pipeResultMaxSub.Score,
		SubCompareOffsetTime:   pipeResultMaxSub.GetOffsetTime(),
	}
	// ---------------------------------------------------------------------------------------
	// 分数需要大于某个值
	if pipeResultMaxAudio.Score < config.MinScore || pipeResultMaxSub.Score < config.MinScore {
		return false, matchResult, nil
	}
	// 两种方式获取到的时间轴的偏移量，差值需要在一定范围内
	if math.Abs(pipeResultMaxAudio.GetOffsetTime()-pipeResultMaxSub.GetOffsetTime()) > config.OffsetRange {
		return false, matchResult, nil
	}
	// ---------------------------------------------------------------------------------------
	// 待判断的字幕的时间长度要小于等于视频的总长度
	if targetSubEndTime > ffmpegInfo.Duration {
		return false, matchResult, nil
	}
	// ---------------------------------------------------------------------------------------
	// 两个对比字幕的对白数量不能超过 10%
	minRage := float64(len(infoBase.Dialogues)) * config.DialoguesDifferencePercentage
	if math.Abs(float64(len(srcBase.Dialogues)-len(infoBase.Dialogues))) > minRage {
		return false, matchResult, nil
	}
	return true, matchResult, nil
}

func (s *SubTimelineFixerHelperEx) changeTimeLineAndSave(infoSrc *subparser.FileInfo, pipeResult sub_timeline_fixer.PipeResult, desSubSaveFPath string) error {
	/*
		修复的字幕先存放到缓存目录，然后需要把原有的字幕进行“备份”，改名，然后再替换过来
	*/
	subFileName := desSubSaveFPath + sub_timeline_fixer.TmpExt
	if pkg.IsFile(subFileName) == true {
		err := os.Remove(subFileName)
		if err != nil {
			return err
		}
	}
	_, err := s.timelineFixPipeLine.FixSubFileTimeline(infoSrc, pipeResult.ScaledFileInfo, pipeResult.GetOffsetTime(), subFileName)
	if err != nil {
		return err
	}

	if pkg.IsFile(desSubSaveFPath+sub_timeline_fixer.BackUpExt) == true {
		err = os.Remove(desSubSaveFPath + sub_timeline_fixer.BackUpExt)
		if err != nil {
			return err
		}
	}

	err = os.Rename(desSubSaveFPath, desSubSaveFPath+sub_timeline_fixer.BackUpExt)
	if err != nil {
		return err
	}

	err = os.Rename(subFileName, desSubSaveFPath)
	if err != nil {
		return err
	}

	return nil
}

type CompareConfig struct {
	MinScore                      float64 // 最低的分数
	OffsetRange                   float64 // 偏移量的范围
	DialoguesDifferencePercentage float64 // 两个字幕的对白字幕差异百分比
}

type MatchResult struct {
	VideoDuration          float64 // 视频的时长
	TargetSubEndTime       float64 // 目标字幕的结束时间
	AudioCompareScore      float64 // 音频的对比分数
	AudioCompareOffsetTime float64 // 音频的对比偏移量
	SubCompareScore        float64 // 字幕的对比分数
	SubCompareOffsetTime   float64 // 字幕的对比偏移量

}