提高并发测试的能力

Signed-off-by: allan716 <[email protected]>

TestCode/test_statistics_subs_score.go

@@ -1,9 +1,15 @@
 package TestCode
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
+
+	common2 "github.com/allanpk716/ChineseSubFinder/pkg/types/common"
+
+	"github.com/allanpk716/ChineseSubFinder/pkg/task_control"
 
 	"github.com/sirupsen/logrus"
 
@@ -261,8 +267,17 @@ func statistics_subs_score_is_match(
 		logger.Errorln("SetCellValue H Header", err)
 		return
 	}
-
-	subCounter := 1
+	// --------------------------------------------------
+	// 并发控制
+	var taskControl *task_control.TaskControl
+	taskControl, err = task_control.NewTaskControl(6, logger)
+	if err != nil {
+		logger.Errorln("NewTaskControl", err)
+		return
+	}
+	taskControl.SetCtxProcessFunc("ScanSubPlayedPool", dealOne, common2.ScanPlayedSubTimeOut)
+	// --------------------------------------------------
+	subCounter = 1
 	err = filepath.Walk(subSearchRootPath,
 		func(path string, info os.FileInfo, err error) error {
 			if err != nil {
@@ -276,68 +291,26 @@ func statistics_subs_score_is_match(
 				return nil
 			}
 
-			bok, matchResult, err := s.IsMatchBySubFile(ffmpegInfo, audioVADInfos, infoBase, path,
-				sub_timeline_fixer.CompareConfig{
-					MinScore:                      40000,
-					OffsetRange:                   2,
-					DialoguesDifferencePercentage: 0.1,
-				})
+			// 并发控制
+			err = taskControl.Invoke(&task_control.TaskData{
+				Index: 0,
+				Count: 0,
+				DataEx: ExcelInputData{
+					Logger:            logger,
+					F:                 f,
+					S:                 s,
+					FfmpegInfo:        ffmpegInfo,
+					AudioVADInfos:     audioVADInfos,
+					InfoBase:          infoBase,
+					NowTargetSubFPath: path,
+					SubFileName:       info.Name(),
+					SheetName:         sheetName,
+				},
+			})
 			if err != nil {
-				return nil
+				logger.Errorln("Invoke", err)
 			}
 
-			if bok == false && matchResult == nil {
-				return nil
-			}
-
-			subCounter++
-			err = f.SetCellValue(sheetName, fmt.Sprintf("A%d", subCounter+1), info.Name())
-			if err != nil {
-				logger.Errorln("SetCellValue A", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("B%d", subCounter+1), matchResult.AudioCompareScore)
-			if err != nil {
-				logger.Errorln("SetCellValue B", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("C%d", subCounter+1), matchResult.AudioCompareOffsetTime)
-			if err != nil {
-				logger.Errorln("SetCellValue C", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("D%d", subCounter+1), matchResult.SubCompareScore)
-			if err != nil {
-				logger.Errorln("SetCellValue D", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("E%d", subCounter+1), matchResult.SubCompareOffsetTime)
-			if err != nil {
-				logger.Errorln("SetCellValue E", info.Name(), subCounter+1, err)
-				return nil
-			}
-			iTrue := 0
-			if bok == true {
-				iTrue = 1
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("F%d", subCounter+1), iTrue)
-			if err != nil {
-				logger.Errorln("SetCellValue F", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("G%d", subCounter+1), matchResult.VideoDuration)
-			if err != nil {
-				logger.Errorln("SetCellValue G", info.Name(), subCounter+1, err)
-				return nil
-			}
-			err = f.SetCellValue(sheetName, fmt.Sprintf("H%d", subCounter+1), matchResult.TargetSubEndTime)
-			if err != nil {
-				logger.Errorln("SetCellValue H", info.Name(), subCounter+1, err)
-				return nil
-			}
-
-			logger.Infoln(subCounter, path, info.Size())
-
 			return nil
 		})
 	if err != nil {
@@ -345,6 +318,8 @@ func statistics_subs_score_is_match(
 		return
 	}
 
+	taskControl.Hold()
+
 	f.SetActiveSheet(newSheet)
 	err = f.SaveAs(fmt.Sprintf("%s.xlsx", excelFileName))
 	if err != nil {
@@ -354,3 +329,101 @@ func statistics_subs_score_is_match(
 
 	logger.Infoln("Done")
 }
+
+func dealOne(ctx context.Context, inData interface{}) error {
+
+	taskData := inData.(*task_control.TaskData)
+	excelInputData := taskData.DataEx.(ExcelInputData)
+
+	bok, matchResult, err := excelInputData.S.IsMatchBySubFile(
+		excelInputData.FfmpegInfo,
+		excelInputData.AudioVADInfos,
+		excelInputData.InfoBase,
+		excelInputData.NowTargetSubFPath,
+		sub_timeline_fixer.CompareConfig{
+			MinScore:                      40000,
+			OffsetRange:                   2,
+			DialoguesDifferencePercentage: 0.1,
+		})
+	if err != nil {
+		return nil
+	}
+
+	if bok == false && matchResult == nil {
+		return nil
+	}
+	counterLock.Lock()
+	defer counterLock.Unlock()
+
+	subCounter++
+
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("A%d", subCounter+1), excelInputData.SubFileName)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue A", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("B%d", subCounter+1), matchResult.AudioCompareScore)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue B", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("C%d", subCounter+1), matchResult.AudioCompareOffsetTime)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue C", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("D%d", subCounter+1), matchResult.SubCompareScore)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue D", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("E%d", subCounter+1), matchResult.SubCompareOffsetTime)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue E", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	iTrue := 0
+	if bok == true {
+		iTrue = 1
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("F%d", subCounter+1), iTrue)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue F", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("G%d", subCounter+1), matchResult.VideoDuration)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue G", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+	err = excelInputData.F.SetCellValue(excelInputData.SheetName, fmt.Sprintf("H%d", subCounter+1), matchResult.TargetSubEndTime)
+	if err != nil {
+		excelInputData.Logger.Errorln("SetCellValue H", excelInputData.SubFileName, subCounter+1, err)
+		return nil
+	}
+
+	excelInputData.Logger.Infoln(subCounter, excelInputData.NowTargetSubFPath)
+
+	return nil
+}
+
+var counterLock sync.Mutex
+var subCounter int
+
+type ExcelInputData struct {
+	Logger            *logrus.Logger
+	F                 *excelize.File
+	S                 *sub_timeline_fixer.SubTimelineFixerHelperEx
+	FfmpegInfo        *ffmpeg_helper.FFMPEGInfo
+	AudioVADInfos     []vad.VADInfo
+	InfoBase          *subparser.FileInfo
+	NowTargetSubFPath string
+	SubFileName       string
+	SheetName         string
+}
+
+type ExcelMathResult struct {
+	Index       int
+	Name        string
+	MatchResult *sub_timeline_fixer.MatchResult
+}

pkg/logic/sub_timeline_fixer/SubTimelineFixerHelperEx.go

@@ -274,7 +274,7 @@ func (s SubTimelineFixerHelperEx) IsVideoCanExportSubtitleAndAudio(videoFileFull
 	return true, ffmpegInfo, audioVADInfos, infoBase, nil
 }
 
-func (s SubTimelineFixerHelperEx) IsMatchBySubFile(ffmpegInfo *ffmpeg_helper.FFMPEGInfo, audioVADInfos []vad.VADInfo, infoBase *subparser.FileInfo, srcSubFileFPath string, config CompareConfig) (bool, *MathResult, error) {
+func (s SubTimelineFixerHelperEx) IsMatchBySubFile(ffmpegInfo *ffmpeg_helper.FFMPEGInfo, audioVADInfos []vad.VADInfo, infoBase *subparser.FileInfo, srcSubFileFPath string, config CompareConfig) (bool, *MatchResult, error) {
 
 	bFind, srcBase, err := s.subParserHub.DetermineFileTypeFromFile(srcSubFileFPath)
 	if err != nil {
@@ -305,7 +305,7 @@ func (s SubTimelineFixerHelperEx) IsMatchBySubFile(ffmpegInfo *ffmpeg_helper.FFM
 
 	targetSubEndTime := my_util.Time2SecondNumber(srcBase.GetEndTime())
 
-	matchResult := &MathResult{
+	matchResult := &MatchResult{
 		VideoDuration:          ffmpegInfo.Duration,
 		TargetSubEndTime:       targetSubEndTime,
 		AudioCompareScore:      pipeResultMaxAudio.Score,
@@ -378,7 +378,7 @@ type CompareConfig struct {
 	DialoguesDifferencePercentage float64 // 两个字幕的对白字幕差异百分比
 }
 
-type MathResult struct {
+type MatchResult struct {
 	VideoDuration          float64 // 视频的时长
 	TargetSubEndTime       float64 // 目标字幕的结束时间
 	AudioCompareScore      float64 // 音频的对比分数