lib/protocol: Avoid data loss on database wipe by higher version numbers (fixes #3876) (#6605)

This makes version vector values clock based instead of just incremented
from zero. The effect is that a vector that is created from scratch
(after database reset) will have a higher value for the local device
than what it could have been previously, causing a conflict. That is, if
we are A and we had

    {A: 42, B: 12}

in the old scheme, a reset and rescan would give us

    {A: 1}

which is a strict ancestor of the older file (this might be wrong). With
the new scheme we would instead have

    {A: someClockTime, b: otherClockTime}

and the new version after reset would become

    {A: someClockTime+delta}

which is in conflict with the previous entry (better).
In case the clocks are wrong (current time is less than the value in the
vector) we fall back to just simple increment like today.

This scheme is ineffective if we suffer a database reset while at the
same time setting the clock back far into the past. It's however no
worse than what we already do.

This loses the ability to emit the "added" event, as we can't look for
the magic 1 entry any more. That event was however already broken
(#5541).

Another place where we infer meaning from the vector itself is in
receive only folders, but there the only criteria is that the vector is
one item long and includes just ourselves, which remains the case with
this change.

* wip

lib/model/folder.go

@@ -904,19 +904,8 @@ func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events
 		objType := "file"
 		action := "modified"
 
-		switch {
-		case file.IsDeleted():
+		if file.IsDeleted() {
 			action = "deleted"
-
-		// If our local vector is version 1 AND it is the only version
-		// vector so far seen for this file then it is a new file.  Else if
-		// it is > 1 it's not new, and if it is 1 but another shortId
-		// version vector exists then it is new for us but created elsewhere
-		// so the file is still not new but modified by us. Only if it is
-		// truly new do we change this to 'added', else we leave it as
-		// 'modified'.
-		case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1:
-			action = "added"
 		}
 
 		if file.IsSymlink() {

lib/protocol/vector.go

@@ -2,6 +2,8 @@
 
 package protocol
 
+import "time"
+
 // The Vector type represents a version vector. The zero value is a usable
 // version vector. The vector has slice semantics and some operations on it
 // are "append-like" in that they may return the same vector modified, or v
@@ -13,23 +15,39 @@ package protocol
 // one. If it is possible, the vector v is updated and returned. If it is not,
 // a copy will be created, updated and returned.
 func (v Vector) Update(id ShortID) Vector {
+	now := uint64(time.Now().Unix())
+	return v.updateWithNow(id, now)
+}
+
+func (v Vector) updateWithNow(id ShortID, now uint64) Vector {
 	for i := range v.Counters {
 		if v.Counters[i].ID == id {
 			// Update an existing index
-			v.Counters[i].Value++
+			v.Counters[i].Value = max(v.Counters[i].Value+1, now)
 			return v
 		} else if v.Counters[i].ID > id {
 			// Insert a new index
 			nv := make([]Counter, len(v.Counters)+1)
 			copy(nv, v.Counters[:i])
 			nv[i].ID = id
-			nv[i].Value = 1
+			nv[i].Value = max(1, now)
 			copy(nv[i+1:], v.Counters[i:])
 			return Vector{Counters: nv}
 		}
 	}
+
 	// Append a new index
-	return Vector{Counters: append(v.Counters, Counter{ID: id, Value: 1})}
+	return Vector{Counters: append(v.Counters, Counter{
+		ID:    id,
+		Value: max(1, now),
+	})}
+}
+
+func max(a, b uint64) uint64 {
+	if a > b {
+		return a
+	}
+	return b
 }
 
 // Merge returns the vector containing the maximum indexes from v and b. If it

lib/protocol/vector_test.go

@@ -12,8 +12,8 @@ func TestUpdate(t *testing.T) {
 
 	// Append
 
-	v = v.Update(42)
-	expected := Vector{Counters: []Counter{{ID: 42, Value: 1}}}
+	v = v.updateWithNow(42, 5)
+	expected := Vector{Counters: []Counter{{ID: 42, Value: 5}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -21,17 +21,17 @@ func TestUpdate(t *testing.T) {
 
 	// Insert at front
 
-	v = v.Update(36)
-	expected = Vector{Counters: []Counter{{ID: 36, Value: 1}, {ID: 42, Value: 1}}}
+	v = v.updateWithNow(36, 6)
+	expected = Vector{Counters: []Counter{{ID: 36, Value: 6}, {ID: 42, Value: 5}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
 	}
 
-	// Insert in moddle
+	// Insert in middle
 
-	v = v.Update(37)
-	expected = Vector{Counters: []Counter{{ID: 36, Value: 1}, {ID: 37, Value: 1}, {ID: 42, Value: 1}}}
+	v = v.updateWithNow(37, 7)
+	expected = Vector{Counters: []Counter{{ID: 36, Value: 6}, {ID: 37, Value: 7}, {ID: 42, Value: 5}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)
@@ -39,8 +39,26 @@ func TestUpdate(t *testing.T) {
 
 	// Update existing
 
-	v = v.Update(37)
-	expected = Vector{Counters: []Counter{{ID: 36, Value: 1}, {ID: 37, Value: 2}, {ID: 42, Value: 1}}}
+	v = v.updateWithNow(37, 1)
+	expected = Vector{Counters: []Counter{{ID: 36, Value: 6}, {ID: 37, Value: 8}, {ID: 42, Value: 5}}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Update existing with higher current time
+
+	v = v.updateWithNow(37, 100)
+	expected = Vector{Counters: []Counter{{ID: 36, Value: 6}, {ID: 37, Value: 100}, {ID: 42, Value: 5}}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Update existing with lower current time
+
+	v = v.updateWithNow(37, 50)
+	expected = Vector{Counters: []Counter{{ID: 36, Value: 6}, {ID: 37, Value: 101}, {ID: 42, Value: 5}}}
 
 	if v.Compare(expected) != Equal {
 		t.Errorf("Update error, %+v != %+v", v, expected)