cmd/syncthing, lib/db: Be nicer about dropping deltas on upgrade (#4798)

When dropping delta index IDs due to upgrade, only drop our local one.
Previously, when dropping all of them, we would trigger a full send in
both directions on first connect after upgrade. Then the other side
would upgrade, doing the same thing. Net effect is full index data gets
sent twice in both directions.

With this change we just drop our local ID, meaning we will send our
full index on first connect after upgrade. When the other side upgrades,
they will do the same. This is a bit less cruel.

cmd/syncthing/main.go

@@ -726,7 +726,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	if runtimeOptions.resetDeltaIdxs {
 		l.Infoln("Reinitializing delta index IDs")
-		ldb.DropDeltaIndexIDs()
+		ldb.DropLocalDeltaIndexIDs()
+		ldb.DropRemoteDeltaIndexIDs()
 	}
 
 	protectedFiles := []string{
@@ -762,8 +763,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		}
 
 		// Drop delta indexes in case we've changed random stuff we
-		// shouldn't have.
-		ldb.DropDeltaIndexIDs()
+		// shouldn't have. We will resend our index on next connect.
+		ldb.DropLocalDeltaIndexIDs()
 
 		// Remember the new version.
 		miscDB.PutString("prevVersion", Version)

lib/db/leveldb_dbinstance.go

@@ -745,6 +745,15 @@ func (db *Instance) indexIDKey(device, folder []byte) []byte {
 	return k
 }
 
+func (db *Instance) indexIDDevice(key []byte) []byte {
+	device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+	if !ok {
+		// uuh ...
+		return nil
+	}
+	return device
+}
+
 func (db *Instance) mtimesKey(folder []byte) []byte {
 	prefix := make([]byte, 5) // key type + 4 bytes folder idx number
 	prefix[0] = KeyTypeVirtualMtime
@@ -759,10 +768,33 @@ func (db *Instance) folderMetaKey(folder []byte) []byte {
 	return prefix
 }
 
-// DropDeltaIndexIDs removes all index IDs from the database. This will
-// cause a full index transmission on the next connection.
-func (db *Instance) DropDeltaIndexIDs() {
-	db.dropPrefix([]byte{KeyTypeIndexID})
+// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
+// the database. This will cause a full index transmission on the next
+// connection.
+func (db *Instance) DropLocalDeltaIndexIDs() {
+	db.dropDeltaIndexIDs(true)
+}
+
+// DropRemoteDeltaIndexIDs removes all index IDs for the other devices than
+// the local one from the database. This will cause them to send us a full
+// index on the next connection.
+func (db *Instance) DropRemoteDeltaIndexIDs() {
+	db.dropDeltaIndexIDs(false)
+}
+
+func (db *Instance) dropDeltaIndexIDs(local bool) {
+	t := db.newReadWriteTransaction()
+	defer t.close()
+
+	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		device := db.indexIDDevice(dbi.Key())
+		if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
+			t.Delete(dbi.Key())
+		}
+	}
 }
 
 func (db *Instance) dropMtimes(folder []byte) {

lib/db/leveldb_test.go

@@ -9,6 +9,8 @@ package db
 import (
 	"bytes"
 	"testing"
+
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func TestDeviceKey(t *testing.T) {
@@ -62,3 +64,91 @@ func TestGlobalKey(t *testing.T) {
 		t.Error("should not have been found")
 	}
 }
+
+func TestDropIndexIDs(t *testing.T) {
+	db := OpenMemory()
+
+	d1 := []byte("device67890123456789012345678901")
+	d2 := []byte("device12345678901234567890123456")
+
+	// Set some index IDs
+
+	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
+	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
+	db.setIndexID(d1, []byte("foo"), 3)
+	db.setIndexID(d1, []byte("bar"), 4)
+	db.setIndexID(d2, []byte("foo"), 5)
+	db.setIndexID(d2, []byte("bar"), 6)
+
+	// Verify them
+
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
+		t.Fatal("fail local 1")
+	}
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
+		t.Fatal("fail local 2")
+	}
+	if db.getIndexID(d1, []byte("foo")) != 3 {
+		t.Fatal("fail remote 1")
+	}
+	if db.getIndexID(d1, []byte("bar")) != 4 {
+		t.Fatal("fail remote 2")
+	}
+	if db.getIndexID(d2, []byte("foo")) != 5 {
+		t.Fatal("fail remote 3")
+	}
+	if db.getIndexID(d2, []byte("bar")) != 6 {
+		t.Fatal("fail remote 4")
+	}
+
+	// Drop the local ones, verify only they got dropped
+
+	db.DropLocalDeltaIndexIDs()
+
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 0 {
+		t.Fatal("fail local 1")
+	}
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 0 {
+		t.Fatal("fail local 2")
+	}
+	if db.getIndexID(d1, []byte("foo")) != 3 {
+		t.Fatal("fail remote 1")
+	}
+	if db.getIndexID(d1, []byte("bar")) != 4 {
+		t.Fatal("fail remote 2")
+	}
+	if db.getIndexID(d2, []byte("foo")) != 5 {
+		t.Fatal("fail remote 3")
+	}
+	if db.getIndexID(d2, []byte("bar")) != 6 {
+		t.Fatal("fail remote 4")
+	}
+
+	// Set local ones again
+
+	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
+	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
+
+	// Drop the remote ones, verify only they got dropped
+
+	db.DropRemoteDeltaIndexIDs()
+
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
+		t.Fatal("fail local 1")
+	}
+	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
+		t.Fatal("fail local 2")
+	}
+	if db.getIndexID(d1, []byte("foo")) != 0 {
+		t.Fatal("fail remote 1")
+	}
+	if db.getIndexID(d1, []byte("bar")) != 0 {
+		t.Fatal("fail remote 2")
+	}
+	if db.getIndexID(d2, []byte("foo")) != 0 {
+		t.Fatal("fail remote 3")
+	}
+	if db.getIndexID(d2, []byte("bar")) != 0 {
+		t.Fatal("fail remote 4")
+	}
+}