chore: forget deleted files older than six months (fixes #6284) (#10023)

This reduces the number of file entries we carry in the database,
sometimes significantly. The downside is that if a file is deleted while
a device is offline, and that device comes back more than the cutoff
interval (six months) later, those files will get resurrected at some
point.

internal/db/sqlite/db_service.go

@@ -8,6 +8,7 @@ package sqlite
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/syncthing/syncthing/internal/db"
@@ -16,6 +17,7 @@ import (
 const (
 	internalMetaPrefix = "dbsvc"
 	lastMaintKey       = "lastMaint"
+	MaxDeletedFileAge  = 180 * 24 * time.Hour
 )
 
 type Service struct {
@@ -24,6 +26,10 @@ type Service struct {
 	internalMeta        *db.Typed
 }
 
+func (s *Service) String() string {
+	return fmt.Sprintf("sqlite.service@%p", s)
+}
+
 func newService(sdb *DB, maintenanceInterval time.Duration) *Service {
 	return &Service{
 		sdb:                 sdb,
@@ -73,6 +79,9 @@ func (s *Service) periodic(ctx context.Context) error {
 	t1 := time.Now()
 	defer func() { l.Debugln("Periodic done in", time.Since(t1), "+", t1.Sub(t0)) }()
 
+	if err := s.garbageCollectOldDeletedLocked(); err != nil {
+		return wrap(err)
+	}
 	if err := s.garbageCollectBlocklistsAndBlocksLocked(ctx); err != nil {
 		return wrap(err)
 	}
@@ -85,6 +94,22 @@ func (s *Service) periodic(ctx context.Context) error {
 	return nil
 }
 
+func (s *Service) garbageCollectOldDeletedLocked() error {
+	// Remove deleted files that are marked as not needed (we have processed
+	// them) and they were deleted more than MaxDeletedFileAge ago.
+	res, err := s.sdb.stmt(`
+		DELETE FROM files
+		WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
+	`).Exec(time.Now().Add(-MaxDeletedFileAge).UnixNano())
+	if err != nil {
+		return wrap(err)
+	}
+	if aff, err := res.RowsAffected(); err == nil {
+		l.Debugln("Removed old deleted file records:", aff)
+	}
+	return nil
+}
+
 func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) error {
 	// Remove all blocklists not referred to by any files and, by extension,
 	// any blocks not referred to by a blocklist. This is an expensive

lib/syncthing/utils.go

@@ -251,6 +251,11 @@ func TryMigrateDatabase() error {
 			return err
 		}
 		_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
+			if fi.Deleted && time.Since(fi.ModTime()) > sqlite.MaxDeletedFileAge {
+				// Skip deleted files that match the garbage collection
+				// criteria in the database
+				return true
+			}
 			fis <- fi
 			return true
 		})