lib/model: Incremental block stats usage reporting

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4498
LGTM: calmh

cmd/syncthing/gui.go

@@ -104,7 +104,7 @@ type modelIntf interface {
 	CurrentSequence(folder string) (int64, bool)
 	RemoteSequence(folder string) (int64, bool)
 	State(folder string) (string, time.Time, error)
-	UsageReportingStats(version int) map[string]interface{}
+	UsageReportingStats(version int, preview bool) map[string]interface{}
 }
 
 type configIntf interface {
@@ -980,7 +980,7 @@ func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) {
 	if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
 		version = val
 	}
-	sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version))
+	sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version, true))
 }
 
 func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {

cmd/syncthing/mocked_model_test.go

@@ -115,6 +115,6 @@ func (m *mockedModel) State(folder string) (string, time.Time, error) {
 	return "", time.Time{}, nil
 }
 
-func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} {
+func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
 	return nil
 }

cmd/syncthing/usage_report.go

@@ -36,7 +36,7 @@ const usageReportVersion = 3
 
 // reportData returns the data to be sent in a usage report. It's used in
 // various places, so not part of the usageReportingManager object.
-func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} {
+func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int, preview bool) map[string]interface{} {
 	opts := cfg.Options()
 	res := make(map[string]interface{})
 	res["urVersion"] = version
@@ -310,7 +310,7 @@ func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf,
 		res["guiStats"] = guiStatsInterface
 	}
 
-	for key, value := range m.UsageReportingStats(version) {
+	for key, value := range m.UsageReportingStats(version, preview) {
 		res[key] = value
 	}
 
@@ -338,7 +338,7 @@ func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectio
 }
 
 func (s *usageReportingService) sendUsageReport() error {
-	d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted)
+	d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted, false)
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(d)
 

lib/model/folder.go

@@ -73,10 +73,6 @@ func (f *folder) Stop() {
 	f.cancel()
 }
 
-func (f *folder) BlockStats() map[string]int {
-	return nil
-}
-
 // CheckHealth checks the folder for common errors, updates the folder state
 // and returns the current folder error, or nil if the folder is healthy.
 func (f *folder) CheckHealth() error {

lib/model/model.go

@@ -54,7 +54,6 @@ type service interface {
 	Scan(subs []string) error
 	Serve()
 	Stop()
-	BlockStats() map[string]int
 	CheckHealth() error
 
 	getState() (folderState, time.Time, error)
@@ -397,19 +396,20 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) {
 	m.fmut.Unlock()
 }
 
-func (m *Model) UsageReportingStats(version int) map[string]interface{} {
+func (m *Model) UsageReportingStats(version int, preview bool) map[string]interface{} {
 	stats := make(map[string]interface{})
 	if version >= 3 {
 		// Block stats
-		m.fmut.Lock()
-		blockStats := make(map[string]int)
-		for _, folder := range m.folderRunners {
-			for k, v := range folder.BlockStats() {
-				blockStats[k] += v
+		blockStatsMut.Lock()
+		copyBlockStats := make(map[string]int)
+		for k, v := range blockStats {
+			copyBlockStats[k] = v
+			if !preview {
+				blockStats[k] = 0
 			}
 		}
-		m.fmut.Unlock()
-		stats["blockStats"] = blockStats
+		blockStatsMut.Unlock()
+		stats["blockStats"] = copyBlockStats
 
 		// Transport stats
 		m.pmut.Lock()

lib/model/rwfolder.go

@@ -29,6 +29,11 @@ import (
 	"github.com/syncthing/syncthing/lib/weakhash"
 )
 
+var (
+	blockStats    = make(map[string]int)
+	blockStatsMut = sync.NewMutex()
+)
+
 func init() {
 	folderFactories[config.FolderTypeSendReceive] = newSendReceiveFolder
 }
@@ -91,9 +96,6 @@ type sendReceiveFolder struct {
 
 	errors    map[string]string // path -> error string
 	errorsMut sync.Mutex
-
-	blockStats    map[string]int
-	blockStatsMut sync.Mutex
 }
 
 func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
@@ -107,9 +109,6 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
 		pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.
 
 		errorsMut: sync.NewMutex(),
-
-		blockStats:    make(map[string]int),
-		blockStatsMut: sync.NewMutex(),
 	}
 
 	f.configureCopiersAndPullers()
@@ -886,10 +885,10 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
 	}
 
 	if err == nil {
-		f.blockStatsMut.Lock()
-		f.blockStats["total"] += len(target.Blocks)
-		f.blockStats["renamed"] += len(target.Blocks)
-		f.blockStatsMut.Unlock()
+		blockStatsMut.Lock()
+		blockStats["total"] += len(target.Blocks)
+		blockStats["renamed"] += len(target.Blocks)
+		blockStatsMut.Unlock()
 
 		// The file was renamed, so we have handled both the necessary delete
 		// of the source and the creation of the target. Fix-up the metadata,
@@ -1457,14 +1456,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 			if err != nil {
 				f.newError("finisher", state.file.Name, err)
 			} else {
-				f.blockStatsMut.Lock()
-				f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
-				f.blockStats["reused"] += state.reused
-				f.blockStats["pulled"] += state.pullTotal
-				f.blockStats["copyOrigin"] += state.copyOrigin
-				f.blockStats["copyOriginShifted"] += state.copyOriginShifted
-				f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
-				f.blockStatsMut.Unlock()
+				blockStatsMut.Lock()
+				blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
+				blockStats["reused"] += state.reused
+				blockStats["pulled"] += state.pullTotal
+				// copyOriginShifted is counted towards copyOrigin due to progress bar reasons
+				// for reporting reasons we want to separate these.
+				blockStats["copyOrigin"] += state.copyOrigin - state.copyOriginShifted
+				blockStats["copyOriginShifted"] += state.copyOriginShifted
+				blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
+				blockStatsMut.Unlock()
 			}
 
 			events.Default.Log(events.ItemFinished, map[string]interface{}{
@@ -1482,16 +1483,6 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 	}
 }
 
-func (f *sendReceiveFolder) BlockStats() map[string]int {
-	f.blockStatsMut.Lock()
-	stats := make(map[string]int)
-	for k, v := range f.blockStats {
-		stats[k] = v
-	}
-	f.blockStatsMut.Unlock()
-	return stats
-}
-
 // Moves the given filename to the front of the job queue
 func (f *sendReceiveFolder) BringToFront(filename string) {
 	f.queue.BringToFront(filename)