lib/db: Add index to track locally needed files (#4958)

To optimize WithNeed, which is called for the local device whenever an index
update is received. No tracking for remote devices to conserve db space, as
WithNeed is only queried for completion.

lib/db/leveldb.go

@@ -13,7 +13,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const dbVersion = 2
+const dbVersion = 3
 
 const (
 	KeyTypeDevice = iota
@@ -28,13 +28,14 @@ const (
 	KeyTypeFolderMeta
 	KeyTypeMiscData
 	KeyTypeSequence
+	KeyTypeNeed
 )
 
-func (l VersionList) String() string {
+func (vl VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
 	b.WriteString("{")
-	for i, v := range l.Versions {
+	for i, v := range vl.Versions {
 		if i > 0 {
 			b.WriteString(", ")
 		}
@@ -45,18 +46,90 @@ func (l VersionList) String() string {
 	return b.String()
 }
 
+// update brings the VersionList up to date with file. It returns the updated
+// VersionList, a potentially removed old FileVersion and its index, as well as
+// the index where the new FileVersion was inserted.
+func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
+	removedAt, insertedAt = -1, -1
+	for i, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			removedAt = i
+			removedFV = v
+			vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
+			break
+		}
+	}
+
+	nv := FileVersion{
+		Device:  device,
+		Version: file.Version,
+		Invalid: file.Invalid,
+	}
+	for i, v := range vl.Versions {
+		switch v.Version.Compare(file.Version) {
+		case protocol.Equal:
+			if nv.Invalid {
+				continue
+			}
+			fallthrough
+
+		case protocol.Lesser:
+			// The version at this point in the list is equal to or lesser
+			// ("older") than us. We insert ourselves in front of it.
+			vl = vl.insertAt(i, nv)
+			return vl, removedFV, removedAt, i
+
+		case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
+			// The version at this point is in conflict with us. We must pull
+			// the actual file metadata to determine who wins. If we win, we
+			// insert ourselves in front of the loser here. (The "Lesser" and
+			// "Greater" in the condition above is just based on the device
+			// IDs in the version vector, which is not the only thing we use
+			// to determine the winner.)
+			//
+			// A surprise missing file entry here is counted as a win for us.
+			if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
+				vl = vl.insertAt(i, nv)
+				return vl, removedFV, removedAt, i
+			}
+		}
+	}
+
+	// We didn't find a position for an insert above, so append to the end.
+	vl.Versions = append(vl.Versions, nv)
+
+	return vl, removedFV, removedAt, len(vl.Versions) - 1
+}
+
+func (vl VersionList) insertAt(i int, v FileVersion) VersionList {
+	vl.Versions = append(vl.Versions, FileVersion{})
+	copy(vl.Versions[i+1:], vl.Versions[i:])
+	vl.Versions[i] = v
+	return vl
+}
+
+func (vl VersionList) Get(device []byte) (FileVersion, bool) {
+	for _, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			return v, true
+		}
+	}
+
+	return FileVersion{}, false
+}
+
 type fileList []protocol.FileInfo
 
-func (l fileList) Len() int {
-	return len(l)
+func (fl fileList) Len() int {
+	return len(fl)
 }
 
-func (l fileList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
+func (fl fileList) Swap(a, b int) {
+	fl[a], fl[b] = fl[b], fl[a]
 }
 
-func (l fileList) Less(a, b int) bool {
-	return l[a].Name < l[b].Name
+func (fl fileList) Less(a, b int) bool {
+	return fl[a].Name < fl[b].Name
 }
 
 // Flush batches to disk when they contain this many records.

lib/db/leveldb_dbinstance.go

@@ -103,6 +103,9 @@ func (db *Instance) UpdateSchema() {
 	if prevVersion <= 1 {
 		db.updateSchema1to2()
 	}
+	if prevVersion <= 2 {
+		db.updateSchema2to3()
+	}
 
 	l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion)
 	miscDB.PutInt64("dbVersion", dbVersion)
@@ -310,26 +313,32 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
 }
 
 func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
-	k := db.globalKey(folder, file)
-
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	bs, err := t.Get(k, nil)
+	_, _, f, ok := db.getGlobalInto(t, nil, nil, folder, file, truncate)
+	return f, ok
+}
+
+func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
+	gk = db.globalKeyInto(gk, folder, file)
+
+	bs, err := t.Get(gk, nil)
 	if err != nil {
-		return nil, false
+		return gk, dk, nil, false
 	}
 
 	vl, ok := unmarshalVersionList(bs)
 	if !ok {
-		return nil, false
+		return gk, dk, nil, false
 	}
 
-	if fi, ok := db.getFileTrunc(db.deviceKey(folder, vl.Versions[0].Device, file), truncate); ok {
-		return fi, true
+	dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
+	if fi, ok := db.getFileTrunc(dk, truncate); ok {
+		return gk, dk, fi, true
 	}
 
-	return nil, false
+	return gk, dk, nil, false
 }
 
 func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
@@ -409,6 +418,11 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 }
 
 func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
+	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
+		db.withNeedLocal(folder, truncate, fn)
+		return
+	}
+
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -422,22 +436,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 			continue
 		}
 
-		have := false // If we have the file, any version
-		need := false // If we have a lower version of the file
-		var haveFileVersion FileVersion
-		for _, v := range vl.Versions {
-			if bytes.Equal(v.Device, device) {
-				have = true
-				haveFileVersion = v
-				// XXX: This marks Concurrent (i.e. conflicting) changes as
-				// needs. Maybe we should do that, but it needs special
-				// handling in the puller.
-				need = !v.Version.GreaterEqual(vl.Versions[0].Version)
-				break
-			}
-		}
-
-		if have && !need {
+		haveFV, have := vl.Get(device)
+		// XXX: This marks Concurrent (i.e. conflicting) changes as
+		// needs. Maybe we should do that, but it needs special
+		// handling in the puller.
+		if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
 			continue
 		}
 
@@ -474,7 +477,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 				break
 			}
 
-			l.Debugf("need folder=%q device=%v name=%q need=%v have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveFileVersion.Invalid, haveFileVersion.Version, needVersion, needDevice)
+			l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
 
 			if !fn(gf) {
 				return
@@ -486,6 +489,28 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 	}
 }
 
+func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
+	t := db.newReadOnlyTransaction()
+	defer t.close()
+
+	dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	defer dbi.Release()
+
+	var dk []byte
+	var gk []byte
+	var f FileIntf
+	var ok bool
+	for dbi.Next() {
+		gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
+		if !ok {
+			continue
+		}
+		if !fn(f) {
+			return
+		}
+	}
+}
+
 func (db *Instance) ListFolders() []string {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
@@ -511,36 +536,21 @@ func (db *Instance) ListFolders() []string {
 }
 
 func (db *Instance) dropFolder(folder []byte) {
-	t := db.newReadOnlyTransaction()
+	t := db.newReadWriteTransaction()
 	defer t.close()
 
-	// Remove all items related to the given folder from the device->file bucket
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
-	for dbi.Next() {
-		itemFolder := db.deviceKeyFolder(dbi.Key())
-		if bytes.Equal(folder, itemFolder) {
-			db.Delete(dbi.Key(), nil)
-		}
+	for _, key := range [][]byte{
+		// Remove all items related to the given folder from the device->file bucket
+		db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
+		// Remove all sequences related to the folder
+		db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
+		// Remove all items related to the given folder from the global bucket
+		db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+		// Remove all needs related to the folder
+		db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+	} {
+		t.deleteKeyPrefix(key)
 	}
-	dbi.Release()
-
-	// Remove all sequences related to the folder
-	sequenceKey := db.sequenceKey([]byte(folder), 0)
-	dbi = t.NewIterator(util.BytesPrefix(sequenceKey[:keyPrefixLen+keyFolderLen]), nil)
-	for dbi.Next() {
-		db.Delete(dbi.Key(), nil)
-	}
-	dbi.Release()
-
-	// Remove all items related to the given folder from the global bucket
-	dbi = t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
-	for dbi.Next() {
-		itemFolder, ok := db.globalKeyFolder(dbi.Key())
-		if ok && bytes.Equal(folder, itemFolder) {
-			db.Delete(dbi.Key(), nil)
-		}
-	}
-	dbi.Release()
 }
 
 func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
@@ -680,13 +690,14 @@ func (db *Instance) updateSchema0to1() {
 	l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded)
 }
 
+// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
+// to allow iteration in sequence order (simplifies sending indexes).
 func (db *Instance) updateSchema1to2() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
 	var sk []byte
 	var dk []byte
-
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
@@ -699,6 +710,34 @@ func (db *Instance) updateSchema1to2() {
 	}
 }
 
+// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
+func (db *Instance) updateSchema2to3() {
+	t := db.newReadWriteTransaction()
+	defer t.close()
+
+	var nk []byte
+	var dk []byte
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+		db.withGlobal(folder, nil, true, func(f FileIntf) bool {
+			name := []byte(f.FileName())
+			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
+			var v protocol.Vector
+			haveFile, ok := db.getFileTrunc(dk, true)
+			if ok {
+				v = haveFile.FileVersion()
+			}
+			if !need(f, ok, v) {
+				return true
+			}
+			nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
+			t.Put(nk, nil)
+			t.checkFlush()
+			return true
+		})
+	}
+}
+
 // deviceKey returns a byte slice encoding the following information:
 //	   keyTypeDevice (1 byte)
 //	   folder (4 bytes)
@@ -755,7 +794,7 @@ func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
 	gk[0] = KeyTypeGlobal
 	binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
 	copy(gk[keyPrefixLen+keyFolderLen:], file)
-	return gk[:reqLen]
+	return gk
 }
 
 // globalKeyName returns the filename from the key
@@ -768,6 +807,17 @@ func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
 	return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
 }
 
+// needKey is a globalKey with a different prefix
+func (db *Instance) needKey(folder, file []byte) []byte {
+	return db.needKeyInto(nil, folder, file)
+}
+
+func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
+	k = db.globalKeyInto(k, folder, file)
+	k[0] = KeyTypeNeed
+	return k
+}
+
 // sequenceKey returns a byte slice encoding the following information:
 //	   KeyTypeSequence (1 byte)
 //	   folder (4 bytes)
@@ -782,7 +832,7 @@ func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
 	k[0] = KeyTypeSequence
 	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
 	binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
-	return k[:reqLen]
+	return k
 }
 
 // sequenceKeySequence returns the sequence number from the key

lib/db/leveldb_test.go

@@ -224,3 +224,99 @@ func TestInvalidFiles(t *testing.T) {
 		t.Error("quux should not be invalid")
 	}
 }
+
+const myID = 1
+
+var (
+	remoteDevice0, remoteDevice1 protocol.DeviceID
+	update0to3Folder             = "UpdateSchema0to3"
+	invalid                      = "invalid"
+	slashPrefixed                = "/notgood"
+	haveUpdate0to3               map[protocol.DeviceID]fileList
+)
+
+func init() {
+	remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
+	remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
+	haveUpdate0to3 = map[protocol.DeviceID]fileList{
+		protocol.LocalDeviceID: {
+			protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+			protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		},
+		remoteDevice0: {
+			protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+			protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+			protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
+		},
+		remoteDevice1: {
+			protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+			protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+			protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
+		},
+	}
+}
+
+func TestUpdate0to3(t *testing.T) {
+	ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	db := newDBInstance(ldb, "<memory>")
+
+	folder := []byte(update0to3Folder)
+
+	db.updateSchema0to1()
+
+	if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
+		t.Error("File prefixed by '/' was not removed during transition to schema 1")
+	}
+
+	if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil {
+		t.Error("Invalid file wasn't added to global list")
+	}
+
+	db.updateSchema1to2()
+
+	found := false
+	db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		l.Infoln(f)
+		if found {
+			t.Error("Unexpected additional file via sequence", f.FileName())
+			return true
+		}
+		if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) {
+			found = true
+		} else {
+			t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
+		}
+		return true
+	})
+	if !found {
+		t.Error("Local file wasn't added to sequence bucket", err)
+	}
+
+	db.updateSchema2to3()
+
+	need := map[string]protocol.FileInfo{
+		haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
+		haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
+		haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
+	}
+	db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
+		e, ok := need[fi.FileName()]
+		if !ok {
+			t.Error("Got unexpected needed file:", fi.FileName())
+		}
+		f := fi.(protocol.FileInfo)
+		delete(need, f.Name)
+		if !f.IsEquivalent(e, true, true) {
+			t.Errorf("Wrong needed file, got %v, expected %v", f, e)
+		}
+		return true
+	})
+	for n := range need {
+		t.Errorf(`Missing needed file "%v"`, n)
+	}
+}

lib/db/leveldb_transactions.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // A readOnlyTransaction represents a database snapshot.
@@ -85,115 +86,87 @@ func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protoco
 // If the file does not have an entry in the global list, it is created.
 func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) bool {
 	l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.Invalid)
-	name := []byte(file.Name)
-	svl, _ := t.Get(gk, nil) // skip error, we check len(svl) != 0 later
 
 	var fl VersionList
-	var oldFile protocol.FileInfo
-	var hasOldFile bool
-	// Remove the device from the current version list
-	if len(svl) != 0 {
-		fl.Unmarshal(svl) // skip error, range handles success case
-		for i := range fl.Versions {
-			if bytes.Equal(fl.Versions[i].Device, device) {
-				if fl.Versions[i].Version.Equal(file.Version) && fl.Versions[i].Invalid == file.Invalid {
-					// No need to do anything
-					return false
-				}
-
-				if i == 0 {
-					// Keep the current newest file around so we can subtract it from
-					// the metadata if we replace it.
-					oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
-				}
-
-				fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
-				break
-			}
-		}
+	if svl, err := t.Get(gk, nil); err == nil {
+		fl.Unmarshal(svl) // Ignore error, continue with empty fl
 	}
-
-	nv := FileVersion{
-		Device:  device,
-		Version: file.Version,
-		Invalid: file.Invalid,
+	fl, removedFV, removedAt, insertedAt := fl.update(folder, device, file, t.db)
+	if insertedAt == -1 {
+		l.Debugln("update global; same version, global unchanged")
+		return false
 	}
 
-	insertedAt := -1
-	// Find a position in the list to insert this file. The file at the front
-	// of the list is the newer, the "global".
-insert:
-	for i := range fl.Versions {
-		switch fl.Versions[i].Version.Compare(file.Version) {
-		case protocol.Equal:
-			if nv.Invalid {
-				continue insert
-			}
-			fallthrough
-
-		case protocol.Lesser:
-			// The version at this point in the list is equal to or lesser
-			// ("older") than us. We insert ourselves in front of it.
-			fl.Versions = insertVersion(fl.Versions, i, nv)
-			insertedAt = i
-			break insert
-
-		case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
-			// The version at this point is in conflict with us. We must pull
-			// the actual file metadata to determine who wins. If we win, we
-			// insert ourselves in front of the loser here. (The "Lesser" and
-			// "Greater" in the condition above is just based on the device
-			// IDs in the version vector, which is not the only thing we use
-			// to determine the winner.)
-			//
-			// A surprise missing file entry here is counted as a win for us.
-			if of, ok := t.getFile(folder, fl.Versions[i].Device, name); !ok || file.WinsConflict(of) {
-				fl.Versions = insertVersion(fl.Versions, i, nv)
-				insertedAt = i
-				break insert
-			}
-		}
+	if removedAt != 0 && insertedAt != 0 {
+		l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
+		t.Put(gk, mustMarshal(&fl))
+		return true
 	}
 
-	if insertedAt == -1 {
-		// We didn't find a position for an insert above, so append to the end.
-		fl.Versions = append(fl.Versions, nv)
-		insertedAt = len(fl.Versions) - 1
-	}
-	// Fixup the global size calculation.
-	if hasOldFile {
-		// We removed the previous newest version
+	name := []byte(file.Name)
+
+	// Remove the old global from the global size counter
+	var oldGlobalFV FileVersion
+	if removedAt == 0 {
+		oldGlobalFV = removedFV
+	} else if len(fl.Versions) > 1 {
+		// The previous newest version is now at index 1
+		oldGlobalFV = fl.Versions[1]
+	}
+	if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
+		// A failure to get the file here is surprising and our
+		// global size data will be incorrect until a restart...
 		meta.removeFile(globalDeviceID, oldFile)
-		if insertedAt == 0 {
-			// inserted a new newest version
-			meta.addFile(globalDeviceID, file)
-		} else {
-			// The previous second version is now the first
-			if newGlobal, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
-				// A failure to get the file here is surprising and our
-				// global size data will be incorrect until a restart...
-				meta.addFile(globalDeviceID, newGlobal)
-			}
-		}
-	} else if insertedAt == 0 {
-		// We just inserted a new newest version.
-		meta.addFile(globalDeviceID, file)
-		if len(fl.Versions) > 1 {
-			// The previous newest version is now at index 1, grab it from there.
-			if oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name); ok {
-				// A failure to get the file here is surprising and our
-				// global size data will be incorrect until a restart...
-				meta.removeFile(globalDeviceID, oldFile)
-			}
+	}
+
+	// Add the new global to the global size counter
+	var newGlobal protocol.FileInfo
+	if insertedAt == 0 {
+		// Inserted a new newest version
+		newGlobal = file
+	} else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
+		// The previous second version is now the first
+		newGlobal = new
+	} else {
+		panic("This file must exist in the db")
+	}
+	meta.addFile(globalDeviceID, newGlobal)
+
+	// Fixup the list of files we need.
+	nk := t.db.needKey(folder, name)
+	hasNeeded, _ := t.db.Has(nk, nil)
+	if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
+		if !hasNeeded {
+			l.Debugf("local need insert; folder=%q, name=%q", folder, name)
+			t.Put(nk, nil)
 		}
+	} else if hasNeeded {
+		l.Debugf("local need delete; folder=%q, name=%q", folder, name)
+		t.Delete(nk)
 	}
 
-	l.Debugf("new global after update: %v", fl)
+	l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
 	t.Put(gk, mustMarshal(&fl))
 
 	return true
 }
 
+func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool {
+	// We never need an invalid file.
+	if global.IsInvalid() {
+		return false
+	}
+	// We don't need a deleted file if we don't have it.
+	if global.IsDeleted() && !haveLocal {
+		return false
+	}
+	// We don't need the global file if we already have the same version.
+	if haveLocal && localVersion.Equal(global.FileVersion()) {
+		return false
+	}
+	return true
+}
+
 // removeFromGlobal removes the device from the global version list for the
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
@@ -246,11 +219,13 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
 	}
 }
 
-func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
-	t := append(vl, FileVersion{})
-	copy(t[i+1:], t[i:])
-	t[i] = v
-	return t
+func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) {
+	dbi := t.NewIterator(util.BytesPrefix(prefix), nil)
+	for dbi.Next() {
+		t.Delete(dbi.Key())
+		t.checkFlush()
+	}
+	dbi.Release()
 }
 
 type marshaller interface {

lib/db/set.go

@@ -44,6 +44,7 @@ type FileIntf interface {
 	HasPermissionBits() bool
 	SequenceNo() int64
 	BlockSize() int
+	FileVersion() protocol.Vector
 }
 
 // The Iterator is called with either a protocol.FileInfo or a

lib/db/structs.go

@@ -75,6 +75,10 @@ func (f FileInfoTruncated) SequenceNo() int64 {
 	return f.Sequence
 }
 
+func (f FileInfoTruncated) FileVersion() protocol.Vector {
+	return f.Version
+}
+
 func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo {
 	return protocol.FileInfo{
 		Name:         f.Name,

lib/db/testdata/v0.14.45-update0to3.db.jsons

@@ -0,0 +1,22 @@
+{"k":"AAAAAAAAAAABL25vdGdvb2Q=","v":"Cggvbm90Z29vZEoHCgUIARDoB1ACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHw=="}
+{"k":"AAAAAAAAAAABYQ==","v":"CgFhSgcKBQgBEOgHUAGCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f"}
+{"k":"AAAAAAAAAAACYg==","v":"CgFiSgcKBQgBEOkHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIA=="}
+{"k":"AAAAAAAAAAACYw==","v":"CgFjOAFKBwoFCAEQ6geCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
+{"k":"AAAAAAAAAAACZA==","v":"CgFkSgcKBQgBEOsHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
+{"k":"AAAAAAAAAAADYw==","v":"CgFjSgcKBQgBEOoHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
+{"k":"AAAAAAAAAAADZA==","v":"CgFkOAFKBwoFCAEQ6weCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
+{"k":"AAAAAAAAAAADaW52YWxpZA==","v":"CgdpbnZhbGlkOAFKBwoFCAEQ7AeCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
+{"k":"AQAAAAAvbm90Z29vZA==","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
+{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
+{"k":"AQAAAABi","v":"CisKBwoFCAEQ6QcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/At"}
+{"k":"AQAAAABj","v":"CisKBwoFCAEQ6gcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcCi0KBwoFCAEQ6gcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtGAE="}
+{"k":"AQAAAABk","v":"CisKBwoFCAEQ6wcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCi0KBwoFCAEQ6wcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
+{"k":"AQAAAABpbnZhbGlk","v":"Ci0KBwoFCAEQ7AcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
+{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHy9ub3Rnb29k","v":"AAAAAA=="}
+{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2E=","v":"AAAAAA=="}
+{"k":"BgAAAAAAAAAA","v":"VXBkYXRlU2NoZW1hMHRvMw=="}
+{"k":"BwAAAAAAAAAA","v":""}
+{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
+{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="}
+{"k":"BwAAAAMAAAAA","v":"R5QD/970lrX15bycChUiHnAHMWRQn6MHYa9jCU9vlFw="}
+{"k":"CQAAAAA=","v":"CicIAjACigEg//////////////////////////////////////////8KJwgFMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAKKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLQolCAGKASBHlAP/3vSWtfXlvJwKFSIecAcxZFCfowdhr2MJT2+UXBCyn6iaw4HGlxU="}

lib/db/util_test.go

@@ -11,8 +11,6 @@ import (
 	"io"
 	"os"
 
-	"github.com/syncthing/syncthing/lib/fs"
-	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -58,37 +56,51 @@ func openJSONS(file string) (*leveldb.DB, error) {
 	return db, nil
 }
 
-func generateIgnoredFilesDB() {
-	// This generates a database with files with invalid flags, local and
-	// remote, in the format used in 0.14.48.
+// The following commented tests were used to generate jsons files to stdout for
+// future tests and are kept here for reference (reuse).
 
-	db := OpenMemory()
-	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
-	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
-		{ // invalid (ignored) file
-			Name:    "foo",
-			Type:    protocol.FileInfoTypeFile,
-			Invalid: true,
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
-		},
-		{ // regular file
-			Name:    "bar",
-			Type:    protocol.FileInfoTypeFile,
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
-		},
-	})
-	fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
-		{ // invalid file
-			Name:    "baz",
-			Type:    protocol.FileInfoTypeFile,
-			Invalid: true,
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
-		},
-		{ // regular file
-			Name:    "quux",
-			Type:    protocol.FileInfoTypeFile,
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
-		},
-	})
-	writeJSONS(os.Stdout, db.DB)
-}
+// TestGenerateIgnoredFilesDB generates a database with files with invalid flags,
+// local and remote, in the format used in 0.14.48.
+// func TestGenerateIgnoredFilesDB(t *testing.T) {
+// 	db := OpenMemory()
+// 	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
+// 	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+// 		{ // invalid (ignored) file
+// 			Name:    "foo",
+// 			Type:    protocol.FileInfoTypeFile,
+// 			Invalid: true,
+// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
+// 		},
+// 		{ // regular file
+// 			Name:    "bar",
+// 			Type:    protocol.FileInfoTypeFile,
+// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
+// 		},
+// 	})
+// 	fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
+// 		{ // invalid file
+// 			Name:    "baz",
+// 			Type:    protocol.FileInfoTypeFile,
+// 			Invalid: true,
+// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
+// 		},
+// 		{ // regular file
+// 			Name:    "quux",
+// 			Type:    protocol.FileInfoTypeFile,
+// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
+// 		},
+// 	})
+// 	writeJSONS(os.Stdout, db.DB)
+// }
+
+// TestGenerateUpdate0to3DB generates a database with files with invalid flags, prefixed
+// by a slash and other files to test database migration from version 0 to 3, in the
+// format used in 0.14.45.
+// func TestGenerateUpdate0to3DB(t *testing.T) {
+// 	db := OpenMemory()
+// 	fs := NewFileSet(update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
+// 	for devID, files := range haveUpdate0to3 {
+// 		fs.Update(devID, files)
+// 	}
+// 	writeJSONS(os.Stdout, db.DB)
+// }

lib/model/requests_test.go

@@ -346,7 +346,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
 		expected := map[string]struct{}{invIgn: {}, ign: {}, ignExisting: {}}
 		for _, f := range fs {
 			if _, ok := expected[f.Name]; !ok {
-				t.Fatalf("Unexpected file %v was added to index", f.Name)
+				t.Errorf("Unexpected file %v was added to index", f.Name)
 			}
 			if !f.Invalid {
 				t.Errorf("File %v wasn't marked as invalid", f.Name)

lib/protocol/bep_extensions.go

@@ -95,6 +95,10 @@ func (f FileInfo) SequenceNo() int64 {
 	return f.Sequence
 }
 
+func (f FileInfo) FileVersion() Vector {
+	return f.Version
+}
+
 // WinsConflict returns true if "f" is the one to choose when it is in
 // conflict with "other".
 func (f FileInfo) WinsConflict(other FileInfo) bool {