lib/db: Catch unignored/conflicting files as needed (fixes #5053) (#5054)

lib/db/leveldb_dbinstance_updateschema.go

@@ -20,10 +20,11 @@ import (
 //   2: v0.14.48
 //   3: v0.14.49
 //   4: v0.14.49
-//   5: v0.14.50
+//   5: v0.14.49
+//   6: v0.14.50
 const (
-	dbVersion             = 5
-	dbMinSyncthingVersion = "v0.14.49"
+	dbVersion             = 6
+	dbMinSyncthingVersion = "v0.14.50"
 )
 
 type databaseDowngradeError struct {
@@ -62,12 +63,12 @@ func (db *Instance) updateSchema() error {
 	if prevVersion < 3 {
 		db.updateSchema2to3()
 	}
-	// This update fixes a problem that only exists in dbVersion 3.
-	if prevVersion == 3 {
-		db.updateSchema3to4()
+	// This update fixes problems existing in versions 3 and 4
+	if prevVersion == 3 || prevVersion == 4 {
+		db.updateSchemaTo5()
 	}
-	if prevVersion < 5 {
-		db.updateSchema4to5()
+	if prevVersion < 6 {
+		db.updateSchema5to6()
 	}
 
 	miscDB.PutInt64("dbVersion", dbVersion)
@@ -192,10 +193,11 @@ func (db *Instance) updateSchema2to3() {
 	}
 }
 
-// updateSchema3to4 resets the need bucket due a bug existing in dbVersion 3 /
-// v0.14.49-rc.1
+// updateSchemaTo5 resets the need bucket due to bugs existing in the v0.14.49
+// release candidates (dbVersion 3 and 4)
 // https://github.com/syncthing/syncthing/issues/5007
-func (db *Instance) updateSchema3to4() {
+// https://github.com/syncthing/syncthing/issues/5053
+func (db *Instance) updateSchemaTo5() {
 	t := db.newReadWriteTransaction()
 	var nk []byte
 	for _, folderStr := range db.ListFolders() {
@@ -207,7 +209,7 @@ func (db *Instance) updateSchema3to4() {
 	db.updateSchema2to3()
 }
 
-func (db *Instance) updateSchema4to5() {
+func (db *Instance) updateSchema5to6() {
 	// For every local file with the Invalid bit set, clear the Invalid bit and
 	// set LocalFlags = FlagLocalIgnored.
 

lib/db/leveldb_transactions.go

@@ -99,31 +99,6 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 
 	name := []byte(file.Name)
 
-	if removedAt != 0 && insertedAt != 0 {
-		if bytes.Equal(device, protocol.LocalDeviceID[:]) && file.Version.Equal(fl.Versions[0].Version) {
-			l.Debugf("local need delete; folder=%q, name=%q", folder, name)
-			t.Delete(t.db.needKey(folder, name))
-		}
-		l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
-		t.Put(gk, mustMarshal(&fl))
-		return true
-	}
-
-	// Remove the old global from the global size counter
-	var oldGlobalFV FileVersion
-	if removedAt == 0 {
-		oldGlobalFV = removedFV
-	} else if len(fl.Versions) > 1 {
-		// The previous newest version is now at index 1
-		oldGlobalFV = fl.Versions[1]
-	}
-	if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
-		// A failure to get the file here is surprising and our
-		// global size data will be incorrect until a restart...
-		meta.removeFile(globalDeviceID, oldFile)
-	}
-
-	// Add the new global to the global size counter
 	var newGlobal protocol.FileInfo
 	if insertedAt == 0 {
 		// Inserted a new newest version
@@ -134,7 +109,6 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 	} else {
 		panic("This file must exist in the db")
 	}
-	meta.addFile(globalDeviceID, newGlobal)
 
 	// Fixup the list of files we need.
 	nk := t.db.needKey(folder, name)
@@ -149,6 +123,29 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 		t.Delete(nk)
 	}
 
+	if removedAt != 0 && insertedAt != 0 {
+		l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
+		t.Put(gk, mustMarshal(&fl))
+		return true
+	}
+
+	// Remove the old global from the global size counter
+	var oldGlobalFV FileVersion
+	if removedAt == 0 {
+		oldGlobalFV = removedFV
+	} else if len(fl.Versions) > 1 {
+		// The previous newest version is now at index 1
+		oldGlobalFV = fl.Versions[1]
+	}
+	if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
+		// A failure to get the file here is surprising and our
+		// global size data will be incorrect until a restart...
+		meta.removeFile(globalDeviceID, oldFile)
+	}
+
+	// Add the new global to the global size counter
+	meta.addFile(globalDeviceID, newGlobal)
+
 	l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
 	t.Put(gk, mustMarshal(&fl))
 

lib/db/set_test.go

@@ -1065,6 +1065,36 @@ func TestNeedDeleted(t *testing.T) {
 	}
 }
 
+func TestNeedAfterUnignore(t *testing.T) {
+	ldb := db.OpenMemory()
+
+	folder := "test"
+	file := "foo"
+	s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
+
+	remID := remoteDevice0.Short()
+
+	// Initial state: Devices in sync, locally ignored
+	local := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
+	local.SetIgnored(myID)
+	remote := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
+	s.Update(protocol.LocalDeviceID, fileList{local})
+	s.Update(remoteDevice0, fileList{remote})
+
+	// Unignore locally -> conflicting changes. Remote is newer, thus winning.
+	local.Version = local.Version.Update(myID)
+	local.Version = local.Version.DropOthers(myID)
+	local.LocalFlags = 0
+	local.ModifiedS = 0
+	s.Update(protocol.LocalDeviceID, fileList{local})
+
+	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
+		t.Fatal("Expected one local need, got", need)
+	} else if !need[0].IsEquivalent(remote, false, false) {
+		t.Fatalf("Got %v, expected %v", need[0], remote)
+	}
+}
+
 func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo) {
 	fs.Drop(device)
 	fs.Update(device, files)