lib/db: Recover sequence number and metadata on startup (fixes #6335) (#6336)

lib/db: Recover sequence number and metadata on startup (fixes #6335)

If we crashed after writing new file entries but before updating
metadata in the database the sequence number and metadata will be wrong.
This fixes that.

lib/db/set.go

@@ -71,7 +71,7 @@ func init() {
 }
 
 func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
-	var s = FileSet{
+	var s = &FileSet{
 		folder:      folder,
 		fs:          fs,
 		db:          db,
@@ -79,26 +79,34 @@ func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
 		updateMutex: sync.NewMutex(),
 	}
 
-	if err := s.meta.fromDB(db, []byte(folder)); err != nil {
-		l.Infof("No stored folder metadata for %q: recalculating", folder)
-		if err := s.recalcCounts(); backend.IsClosed(err) {
+	recalc := func() *FileSet {
+		if err := s.recalcMeta(); backend.IsClosed(err) {
 			return nil
 		} else if err != nil {
 			panic(err)
 		}
-	} else if age := time.Since(s.meta.Created()); age > databaseRecheckInterval {
+		return s
+	}
+
+	if err := s.meta.fromDB(db, []byte(folder)); err != nil {
+		l.Infof("No stored folder metadata for %q; recalculating", folder)
+		return recalc()
+	}
+
+	if metaOK := s.verifyLocalSequence(); !metaOK {
+		l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder)
+		return recalc()
+	}
+
+	if age := time.Since(s.meta.Created()); age > databaseRecheckInterval {
 		l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, age)
-		if err := s.recalcCounts(); backend.IsClosed(err) {
-			return nil
-		} else if err != nil {
-			panic(err)
-		}
+		return recalc()
 	}
 
-	return &s
+	return s
 }
 
-func (s *FileSet) recalcCounts() error {
+func (s *FileSet) recalcMeta() error {
 	s.meta = newMetadataTracker()
 
 	if err := s.db.checkGlobals([]byte(s.folder), s.meta); err != nil {
@@ -123,6 +131,32 @@ func (s *FileSet) recalcCounts() error {
 	return s.meta.toDB(s.db, []byte(s.folder))
 }
 
+// Verify the local sequence number from actual sequence entries. Returns
+// true if it was all good, or false if a fixup was necessary.
+func (s *FileSet) verifyLocalSequence() bool {
+	// Walk the sequence index from the current (supposedly) highest
+	// sequence number and raise the alarm if we get anything. This recovers
+	// from the occasion where we have written sequence entries to disk but
+	// not yet written new metadata to disk.
+	//
+	// Note that we can have the same thing happen for remote devices but
+	// there it's not a problem -- we'll simply advertise a lower sequence
+	// number than we've actually seen and receive some duplicate updates
+	// and then be in sync again.
+
+	curSeq := s.meta.Sequence(protocol.LocalDeviceID)
+
+	snap := s.Snapshot()
+	ok := true
+	snap.WithHaveSequence(curSeq+1, func(fi FileIntf) bool {
+		ok = false // we got something, which we should not have
+		return false
+	})
+	snap.Release()
+
+	return ok
+}
+
 func (s *FileSet) Drop(device protocol.DeviceID) {
 	l.Debugf("%s Drop(%v)", s.folder, device)