Also keep GlobalSize in RAM

lib/db/leveldb.go

@@ -167,13 +167,9 @@ func globalKeyFolder(key []byte) []byte {
 	return folder[:izero]
 }
 
-func isLocalDevice(dev []byte) bool {
-	return bytes.Equal(dev, protocol.LocalDeviceID[:])
-}
-
 type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64
 
-func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) int64 {
+func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
 	runtime.GC()
 
 	sort.Sort(fileList(fs)) // sort list on name, same as in the database
@@ -200,6 +196,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 	fsi := 0
 	var maxLocalVer int64
 
+	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for {
 		var newName, oldName []byte
 		moreFs := fsi < len(fs)
@@ -227,10 +224,13 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
+			if isLocalDevice {
+				localSize.addFile(fs[fsi])
+			}
 			if fs[fsi].IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, newName)
+				ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
+				ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
 			}
 			fsi++
 
@@ -246,10 +246,14 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 				if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
 				}
+				if isLocalDevice {
+					localSize.removeFile(ef)
+					localSize.addFile(fs[fsi])
+				}
 				if fs[fsi].IsInvalid() {
-					ldbRemoveFromGlobal(snap, batch, folder, device, newName)
+					ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
 				} else {
-					ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
+					ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
 				}
 			} else {
 				l.Debugln("generic replace; equal - ignore")
@@ -289,19 +293,19 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 	return maxLocalVer
 }
 
-func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 {
+func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	// TODO: Return the remaining maxLocalVer?
-	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
+	return ldbGenericReplace(db, folder, device, fs, localSize, globalSize, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		// Database has a file that we are missing. Remove it.
 		l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
-		ldbRemoveFromGlobal(db, batch, folder, device, name)
+		ldbRemoveFromGlobal(db, batch, folder, device, name, globalSize)
 		l.Debugf("batch.Delete %p %x", batch, dbi.Key())
 		batch.Delete(dbi.Key())
 		return 0
 	})
 }
 
-func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, size *sizeTracker) int64 {
+func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	runtime.GC()
 
 	batch := new(leveldb.Batch)
@@ -318,24 +322,24 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, si
 
 	var maxLocalVer int64
 	var fk []byte
-	localDev := isLocalDevice(device)
+	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for _, f := range fs {
 		name := []byte(f.Name)
 		fk = deviceKeyInto(fk[:cap(fk)], folder, device, name)
 		l.Debugf("snap.Get %p %x", snap, fk)
 		bs, err := snap.Get(fk, nil)
 		if err == leveldb.ErrNotFound {
-			if localDev {
-				size.addFile(f)
+			if isLocalDevice {
+				localSize.addFile(f)
 			}
 
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
 			if f.IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, name)
+				ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, f)
+				ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
 			}
 			continue
 		}
@@ -348,18 +352,18 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, si
 		// Flags might change without the version being bumped when we set the
 		// invalid flag on an existing file.
 		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
-			if localDev {
-				size.removeFile(ef)
-				size.addFile(f)
+			if isLocalDevice {
+				localSize.removeFile(ef)
+				localSize.addFile(f)
 			}
 
 			if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
 			if f.IsInvalid() {
-				ldbRemoveFromGlobal(snap, batch, folder, device, name)
+				ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
 			} else {
-				ldbUpdateGlobal(snap, batch, folder, device, f)
+				ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
 			}
 		}
 
@@ -404,7 +408,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
 // ldbUpdateGlobal adds this device+version to the version list for the given
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.
-func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo) bool {
+func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo, globalSize *sizeTracker) bool {
 	l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version)
 	name := []byte(file.Name)
 	gk := globalKey(folder, name)
@@ -414,7 +418,8 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 	}
 
 	var fl versionList
-
+	var oldFile protocol.FileInfo
+	var hasOldFile bool
 	// Remove the device from the current version list
 	if svl != nil {
 		err = fl.UnmarshalXDR(svl)
@@ -422,6 +427,12 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 			panic(err)
 		}
 
+		if len(fl.versions) > 0 {
+			// Keep the current neweset file around so we can subtract it from
+			// the globalSize if we replace it.
+			oldFile, hasOldFile = ldbGet(db, folder, fl.versions[0].device, name)
+		}
+
 		for i := range fl.versions {
 			if bytes.Compare(fl.versions[i].device, device) == 0 {
 				if fl.versions[i].version.Equal(file.Version) {
@@ -439,6 +450,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 		version: file.Version,
 	}
 
+	insertedAt := -1
 	// Find a position in the list to insert this file. The file at the front
 	// of the list is the newer, the "global".
 	for i := range fl.versions {
@@ -447,6 +459,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 			// The version at this point in the list is equal to or lesser
 			// ("older") than us. We insert ourselves in front of it.
 			fl.versions = insertVersion(fl.versions, i, nv)
+			insertedAt = i
 			goto done
 
 		case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
@@ -462,6 +475,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 			}
 			if file.WinsConflict(of) {
 				fl.versions = insertVersion(fl.versions, i, nv)
+				insertedAt = i
 				goto done
 			}
 		}
@@ -469,8 +483,20 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
 
 	// We didn't find a position for an insert above, so append to the end.
 	fl.versions = append(fl.versions, nv)
+	insertedAt = len(fl.versions) - 1
 
 done:
+	if insertedAt == 0 {
+		// We just inserted a new newest version. Fixup the global size
+		// calculation.
+		if !file.Version.Equal(oldFile.Version) {
+			globalSize.addFile(file)
+			if hasOldFile {
+				globalSize.removeFile(oldFile)
+			}
+		}
+	}
+
 	l.Debugf("batch.Put %p %x", batch, gk)
 	l.Debugf("new global after update: %v", fl)
 	batch.Put(gk, fl.MustMarshalXDR())
@@ -488,7 +514,7 @@ func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
 // ldbRemoveFromGlobal removes the device from the global version list for the
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
-func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte) {
+func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte, globalSize *sizeTracker) {
 	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
 
 	gk := globalKey(folder, file)
@@ -505,8 +531,17 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 		panic(err)
 	}
 
+	removed := false
 	for i := range fl.versions {
 		if bytes.Compare(fl.versions[i].device, device) == 0 {
+			if i == 0 && globalSize != nil {
+				f, ok := ldbGet(db, folder, device, file)
+				if !ok {
+					panic("removing nonexistent file")
+				}
+				globalSize.removeFile(f)
+				removed = true
+			}
 			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
 			break
 		}
@@ -519,6 +554,13 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 		l.Debugf("batch.Put %p %x", batch, gk)
 		l.Debugf("new global after remove: %v", fl)
 		batch.Put(gk, fl.MustMarshalXDR())
+		if removed {
+			f, ok := ldbGet(db, folder, fl.versions[0].device, file)
+			if !ok {
+				panic("new global is nonexistent file")
+			}
+			globalSize.addFile(f)
+		}
 	}
 }
 
@@ -579,7 +621,7 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
 		case "", ".", "..", "/": // A few obviously invalid filenames
 			l.Infof("Dropping invalid filename %q from database", f.Name)
 			batch := new(leveldb.Batch)
-			ldbRemoveFromGlobal(db, batch, folder, device, nil)
+			ldbRemoveFromGlobal(db, batch, folder, device, nil, nil)
 			batch.Delete(dbi.Key())
 			db.Write(batch, nil)
 			continue
@@ -914,7 +956,7 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	return tf, err
 }
 
-func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
+func ldbCheckGlobals(db *leveldb.DB, folder []byte, globalSize *sizeTracker) {
 	defer runtime.GC()
 
 	snap, err := db.GetSnapshot()
@@ -951,8 +993,9 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 
 		name := globalKeyName(gk)
 		var newVL versionList
-		for _, version := range vl.versions {
+		for i, version := range vl.versions {
 			fk = deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
+
 			l.Debugf("snap.Get %p %x", snap, fk)
 			_, err := snap.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
@@ -962,6 +1005,14 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
 				panic(err)
 			}
 			newVL.versions = append(newVL.versions, version)
+
+			if i == 0 {
+				fi, ok := ldbGet(snap, folder, version.device, name)
+				if !ok {
+					panic("nonexistent global master file")
+				}
+				globalSize.addFile(fi)
+			}
 		}
 
 		if len(newVL.versions) != len(vl.versions) {

lib/db/set.go

@@ -27,7 +27,8 @@ type FileSet struct {
 	folder       string
 	db           *leveldb.DB
 	blockmap     *BlockMap
-	sizeTracker
+	localSize    sizeTracker
+	globalSize   sizeTracker
 }
 
 // FileIntf is the set of methods implemented by both protocol.FileInfo and
@@ -57,6 +58,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
 	if f.IsInvalid() {
 		return
 	}
+
 	s.mut.Lock()
 	if f.IsDeleted() {
 		s.deleted++
@@ -71,6 +73,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
 	if f.IsInvalid() {
 		return
 	}
+
 	s.mut.Lock()
 	if f.IsDeleted() {
 		s.deleted--
@@ -78,6 +81,9 @@ func (s *sizeTracker) removeFile(f FileIntf) {
 		s.files--
 	}
 	s.bytes -= f.Size()
+	if s.deleted < 0 || s.files < 0 {
+		panic("bug: removed more than added")
+	}
 	s.mut.Unlock()
 }
 
@@ -96,7 +102,7 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 		mutex:        sync.NewMutex(),
 	}
 
-	ldbCheckGlobals(db, []byte(folder))
+	ldbCheckGlobals(db, []byte(folder), &s.globalSize)
 
 	var deviceID protocol.DeviceID
 	ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
@@ -105,7 +111,7 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
 			s.localVersion[deviceID] = f.LocalVersion
 		}
 		if deviceID == protocol.LocalDeviceID {
-			s.addFile(f)
+			s.localSize.addFile(f)
 		}
 		return true
 	})
@@ -120,7 +126,7 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 	normalizeFilenames(fs)
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
-	s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs)
+	s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
 	if len(fs) == 0 {
 		// Reset the local version if all files were removed.
 		s.localVersion[device] = 0
@@ -149,7 +155,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 		s.blockmap.Discard(discards)
 		s.blockmap.Update(updates)
 	}
-	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs, &s.sizeTracker); lv > s.localVersion[device] {
+	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize); lv > s.localVersion[device] {
 		s.localVersion[device] = lv
 	}
 }
@@ -225,6 +231,14 @@ func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
 	return s.localVersion[device]
 }
 
+func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
+	return s.localSize.Size()
+}
+
+func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
+	return s.globalSize.Size()
+}
+
 // ListFolders returns the folder IDs seen in the database.
 func ListFolders(db *leveldb.DB) []string {
 	return ldbListFolders(db)

lib/model/model.go

@@ -329,8 +329,6 @@ func (m *Model) FolderStatistics() map[string]stats.FolderStatistics {
 // Completion returns the completion status, in percent, for the given device
 // and folder.
 func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
-	var tot int64
-
 	m.fmut.RLock()
 	rf, ok := m.folderFiles[folder]
 	m.fmut.RUnlock()
@@ -338,29 +336,22 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		return 0 // Folder doesn't exist, so we hardly have any of it
 	}
 
-	rf.WithGlobalTruncated(func(f db.FileIntf) bool {
-		if !f.IsDeleted() {
-			tot += f.Size()
-		}
-		return true
-	})
-
+	_, _, tot := rf.GlobalSize()
 	if tot == 0 {
 		return 100 // Folder is empty, so we have all of it
 	}
 
 	var need int64
 	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
-		if !f.IsDeleted() {
-			need += f.Size()
-		}
+		need += f.Size()
 		return true
 	})
 
-	res := 100 * (1 - float64(need)/float64(tot))
-	l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, device, folder, res, need, tot)
+	needRatio := float64(need) / float64(tot)
+	completionPct := 100 * (1 - needRatio)
+	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)
 
-	return res
+	return completionPct
 }
 
 func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
@@ -389,13 +380,7 @@ func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithGlobalTruncated(func(f db.FileIntf) bool {
-			fs, de, by := sizeOfFile(f)
-			nfiles += fs
-			deleted += de
-			bytes += by
-			return true
-		})
+		nfiles, deleted, bytes = rf.GlobalSize()
 	}
 	return
 }
@@ -406,7 +391,7 @@ func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		nfiles, deleted, bytes = rf.Size()
+		nfiles, deleted, bytes = rf.LocalSize()
 	}
 	return
 }