chore: configurable delete retention interval (#10030)

Command line flag, as it also needs to be able to take effect during
migration.

cmd/syncthing/main.go

@@ -161,24 +161,25 @@ func (c *CLI) AfterApply() error {
 type serveCmd struct {
 	buildSpecificOptions
 
-	AllowNewerConfig      bool          `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
-	Audit                 bool          `help:"Write events to audit file" env:"STAUDIT"`
-	AuditFile             string        `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
-	DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTINTERVAL"`
-	GUIAddress            string        `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
-	GUIAPIKey             string        `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
-	LogFile               string        `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
-	LogFlags              int           `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
-	LogMaxFiles           int           `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STNUMLOGFILES"`
-	LogMaxSize            int           `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
-	NoBrowser             bool          `help:"Do not start browser" env:"STNOBROWSER"`
-	NoDefaultFolder       bool          `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
-	NoPortProbing         bool          `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
-	NoRestart             bool          `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
-	NoUpgrade             bool          `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
-	Paused                bool          `help:"Start with all devices and folders paused" env:"STPAUSED"`
-	Unpaused              bool          `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
-	Verbose               bool          `help:"Print verbose log output" env:"STVERBOSE"`
+	AllowNewerConfig          bool          `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
+	Audit                     bool          `help:"Write events to audit file" env:"STAUDIT"`
+	AuditFile                 string        `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
+	DBMaintenanceInterval     time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTENANCEINTERVAL"`
+	DBDeleteRetentionInterval time.Duration `help:"Database deleted item retention interval" default:"4320h" env:"STDBDELETERETENTIONINTERVAL"`
+	GUIAddress                string        `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
+	GUIAPIKey                 string        `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
+	LogFile                   string        `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
+	LogFlags                  int           `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
+	LogMaxFiles               int           `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STLOGMAXOLDFILES"`
+	LogMaxSize                int           `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
+	NoBrowser                 bool          `help:"Do not start browser" env:"STNOBROWSER"`
+	NoDefaultFolder           bool          `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
+	NoPortProbing             bool          `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
+	NoRestart                 bool          `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
+	NoUpgrade                 bool          `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
+	Paused                    bool          `help:"Start with all devices and folders paused" env:"STPAUSED"`
+	Unpaused                  bool          `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
+	Verbose                   bool          `help:"Print verbose log output" env:"STVERBOSE"`
 
 	// Debug options below
 	DebugGUIAssetsDir   string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"`
@@ -487,12 +488,12 @@ func (c *serveCmd) syncthingMain() {
 		})
 	}
 
-	if err := syncthing.TryMigrateDatabase(); err != nil {
+	if err := syncthing.TryMigrateDatabase(c.DBDeleteRetentionInterval); err != nil {
 		l.Warnln("Failed to migrate old-style database:", err)
 		os.Exit(1)
 	}
 
-	sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database))
+	sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database), c.DBDeleteRetentionInterval)
 	if err != nil {
 		l.Warnln("Error opening database:", err)
 		os.Exit(1)

internal/db/sqlite/db.go

@@ -17,10 +17,12 @@ import (
 )
 
 type DB struct {
-	sql            *sqlx.DB
-	localDeviceIdx int64
-	updateLock     sync.Mutex
-	updatePoints   int
+	sql             *sqlx.DB
+	localDeviceIdx  int64
+	deleteRetention time.Duration
+
+	updateLock   sync.Mutex
+	updatePoints int
 
 	statementsMut sync.RWMutex
 	statements    map[string]*sqlx.Stmt
@@ -29,6 +31,14 @@ type DB struct {
 
 var _ db.DB = (*DB)(nil)
 
+type Option func(*DB)
+
+func WithDeleteRetention(d time.Duration) Option {
+	return func(s *DB) {
+		s.deleteRetention = d
+	}
+}
+
 func (s *DB) Close() error {
 	s.updateLock.Lock()
 	s.statementsMut.Lock()

internal/db/sqlite/db_open.go

@@ -21,7 +21,7 @@ import (
 
 const maxDBConns = 128
 
-func Open(path string) (*DB, error) {
+func Open(path string, opts ...Option) (*DB, error) {
 	// Open the database with options to enable foreign keys and recursive
 	// triggers (needed for the delete+insert triggers on row replace).
 	sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions)
@@ -36,7 +36,7 @@ func Open(path string) (*DB, error) {
 		// https://www.sqlite.org/pragma.html#pragma_optimize
 		return nil, wrap(err, "PRAGMA optimize")
 	}
-	return openCommon(sqlDB)
+	return openCommon(sqlDB, opts...)
 }
 
 // Open the database with options suitable for the migration inserts. This
@@ -73,7 +73,7 @@ func OpenTemp() (*DB, error) {
 	return Open(path)
 }
 
-func openCommon(sqlDB *sqlx.DB) (*DB, error) {
+func openCommon(sqlDB *sqlx.DB, opts ...Option) (*DB, error) {
 	if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil {
 		return nil, wrap(err, "PRAGMA auto_vacuum")
 	}
@@ -85,8 +85,15 @@ func openCommon(sqlDB *sqlx.DB) (*DB, error) {
 	}
 
 	db := &DB{
-		sql:        sqlDB,
-		statements: make(map[string]*sqlx.Stmt),
+		sql:             sqlDB,
+		deleteRetention: defaultDeleteRetention,
+		statements:      make(map[string]*sqlx.Stmt),
+	}
+	for _, opt := range opts {
+		opt(db)
+	}
+	if db.deleteRetention > 0 && db.deleteRetention < minDeleteRetention {
+		db.deleteRetention = minDeleteRetention
 	}
 
 	if err := db.runScripts("sql/schema/*"); err != nil {

internal/db/sqlite/db_service.go

@@ -15,9 +15,10 @@ import (
 )
 
 const (
-	internalMetaPrefix = "dbsvc"
-	lastMaintKey       = "lastMaint"
-	MaxDeletedFileAge  = 180 * 24 * time.Hour
+	internalMetaPrefix     = "dbsvc"
+	lastMaintKey           = "lastMaint"
+	defaultDeleteRetention = 180 * 24 * time.Hour
+	minDeleteRetention     = 24 * time.Hour
 )
 
 type Service struct {
@@ -101,12 +102,18 @@ func (s *Service) periodic(ctx context.Context) error {
 }
 
 func (s *Service) garbageCollectOldDeletedLocked() error {
+	if s.sdb.deleteRetention <= 0 {
+		l.Debugln("Delete retention is infinite, skipping cleanup")
+		return nil
+	}
+
 	// Remove deleted files that are marked as not needed (we have processed
 	// them) and they were deleted more than MaxDeletedFileAge ago.
+	l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention)
 	res, err := s.sdb.stmt(`
 		DELETE FROM files
 		WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
-	`).Exec(time.Now().Add(-MaxDeletedFileAge).UnixNano())
+	`).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano())
 	if err != nil {
 		return wrap(err)
 	}

lib/syncthing/utils.go

@@ -158,8 +158,8 @@ func copyFile(src, dst string) error {
 }
 
 // Opens a database
-func OpenDatabase(path string) (newdb.DB, error) {
-	sql, err := sqlite.Open(path)
+func OpenDatabase(path string, deleteRetention time.Duration) (newdb.DB, error) {
+	sql, err := sqlite.Open(path, sqlite.WithDeleteRetention(deleteRetention))
 	if err != nil {
 		return nil, err
 	}
@@ -170,7 +170,7 @@ func OpenDatabase(path string) (newdb.DB, error) {
 }
 
 // Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type
-func TryMigrateDatabase() error {
+func TryMigrateDatabase(deleteRetention time.Duration) error {
 	oldDBDir := locations.Get(locations.LegacyDatabase)
 	if _, err := os.Lstat(oldDBDir); err != nil {
 		// No old database
@@ -251,7 +251,7 @@ func TryMigrateDatabase() error {
 			return err
 		}
 		_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
-			if fi.Deleted && time.Since(fi.ModTime()) > sqlite.MaxDeletedFileAge {
+			if deleteRetention > 0 && fi.Deleted && time.Since(fi.ModTime()) > deleteRetention {
 				// Skip deleted files that match the garbage collection
 				// criteria in the database
 				return true