lib/model: Optimize puller for meta only changes (#5622)

lib/model/folder_sendrecv.go

@@ -292,7 +292,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 
 func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
 	changed := 0
-	var processDirectly []protocol.FileInfo
 	var dirDeletions []protocol.FileInfo
 	fileDeletions := map[string]protocol.FileInfo{}
 	buckets := map[string][]protocol.FileInfo{}
@@ -346,8 +345,16 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 			changed++
 
 		case file.Type == protocol.FileInfoTypeFile:
-			// Queue files for processing after directories and symlinks.
-			f.queue.Push(file.Name, file.Size, file.ModTime())
+			curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
+			if _, need := blockDiff(curFile.Blocks, file.Blocks); hasCurFile && len(need) == 0 {
+				// We are supposed to copy the entire file, and then fetch nothing. We
+				// are only updating metadata, so we don't actually *need* to make the
+				// copy.
+				f.shortcutFile(file, curFile, dbUpdateChan)
+			} else {
+				// Queue files for processing after directories and symlinks.
+				f.queue.Push(file.Name, file.Size, file.ModTime())
+			}
 
 		case runtime.GOOS == "windows" && file.IsSymlink():
 			file.SetUnsupported(f.shortID)
@@ -355,11 +362,23 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 			dbUpdateChan <- dbUpdateJob{file, dbUpdateInvalidate}
 			changed++
 
-		default:
-			// Directories, symlinks
-			l.Debugln(f, "to be processed directly", file)
-			processDirectly = append(processDirectly, file)
+		case file.IsDirectory() && !file.IsSymlink():
+			changed++
+			l.Debugln(f, "Handling directory", file.Name)
+			if f.checkParent(file.Name, scanChan) {
+				f.handleDir(file, dbUpdateChan, scanChan)
+			}
+
+		case file.IsSymlink():
 			changed++
+			l.Debugln(f, "Handling symlink", file.Name)
+			if f.checkParent(file.Name, scanChan) {
+				f.handleSymlink(file, dbUpdateChan, scanChan)
+			}
+
+		default:
+			l.Warnln(file)
+			panic("unhandleable item type, can't happen")
 		}
 
 		return true
@@ -371,39 +390,6 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 	default:
 	}
 
-	// Sort the "process directly" pile by number of path components. This
-	// ensures that we handle parents before children.
-
-	sort.Sort(byComponentCount(processDirectly))
-
-	// Process the list.
-
-	for _, fi := range processDirectly {
-		select {
-		case <-f.ctx.Done():
-			return changed, fileDeletions, dirDeletions, f.ctx.Err()
-		default:
-		}
-
-		if !f.checkParent(fi.Name, scanChan) {
-			continue
-		}
-
-		switch {
-		case fi.IsDirectory() && !fi.IsSymlink():
-			l.Debugln(f, "Handling directory", fi.Name)
-			f.handleDir(fi, dbUpdateChan, scanChan)
-
-		case fi.IsSymlink():
-			l.Debugln(f, "Handling symlink", fi.Name)
-			f.handleSymlink(fi, dbUpdateChan, scanChan)
-
-		default:
-			l.Warnln(fi)
-			panic("unhandleable item type, can't happen")
-		}
-	}
-
 	// Now do the file queue. Reorder it according to configuration.
 
 	switch f.Order {
@@ -1020,15 +1006,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, dbUpdateChan chan<- dbUpdateJob) {
 	curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
 
-	have, need := blockDiff(curFile.Blocks, file.Blocks)
-
-	if hasCurFile && len(need) == 0 {
-		// We are supposed to copy the entire file, and then fetch nothing. We
-		// are only updating metadata, so we don't actually *need* to make the
-		// copy.
-		f.shortcutFile(file, curFile, dbUpdateChan)
-		return
-	}
+	have, _ := blockDiff(curFile.Blocks, file.Blocks)
 
 	tempName := fs.TempName(file.Name)
 
@@ -1974,32 +1952,6 @@ func (l fileErrorList) Swap(a, b int) {
 	l[a], l[b] = l[b], l[a]
 }
 
-// byComponentCount sorts by the number of path components in Name, that is
-// "x/y" sorts before "foo/bar/baz".
-type byComponentCount []protocol.FileInfo
-
-func (l byComponentCount) Len() int {
-	return len(l)
-}
-
-func (l byComponentCount) Less(a, b int) bool {
-	return componentCount(l[a].Name) < componentCount(l[b].Name)
-}
-
-func (l byComponentCount) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-
-func componentCount(name string) int {
-	count := 0
-	for _, codepoint := range name {
-		if codepoint == fs.PathSeparator {
-			count++
-		}
-	}
-	return count
-}
-
 func conflictName(name, lastModBy string) string {
 	ext := filepath.Ext(name)
 	return name[:len(name)-len(ext)] + time.Now().Format(".sync-conflict-20060102-150405-") + lastModBy + ext