lib/db, lib/model: Refactor removing expired pending folders (#7537)

lib/db/observed.go

@@ -66,16 +66,11 @@ func (db *Lowlevel) PendingDevices() (map[protocol.DeviceID]ObservedDevice, erro
 	return res, nil
 }
 
-func (db *Lowlevel) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID, receiveEncrypted bool) error {
+func (db *Lowlevel) AddOrUpdatePendingFolder(id string, of ObservedFolder, device protocol.DeviceID) error {
 	key, err := db.keyer.GeneratePendingFolderKey(nil, device[:], []byte(id))
 	if err != nil {
 		return err
 	}
-	of := ObservedFolder{
-		Time:             time.Now().Truncate(time.Second),
-		Label:            label,
-		ReceiveEncrypted: receiveEncrypted,
-	}
 	bs, err := of.Marshal()
 	if err != nil {
 		return err
@@ -112,42 +107,6 @@ func (db *Lowlevel) RemovePendingFolder(id string) {
 	}
 }
 
-// RemovePendingFoldersBeforeTime removes entries for a specific device which are older
-// than a given timestamp or invalid.  It returns only the valid removed folder IDs.
-func (db *Lowlevel) RemovePendingFoldersBeforeTime(device protocol.DeviceID, oldest time.Time) ([]string, error) {
-	prefixKey, err := db.keyer.GeneratePendingFolderKey(nil, device[:], nil)
-	if err != nil {
-		return nil, err
-	}
-	iter, err := db.NewPrefixIterator(prefixKey)
-	if err != nil {
-		return nil, err
-	}
-	defer iter.Release()
-	oldest = oldest.Truncate(time.Second)
-	var res []string
-	for iter.Next() {
-		var of ObservedFolder
-		var folderID string
-		if err = of.Unmarshal(iter.Value()); err != nil {
-			l.Infof("Invalid pending folder entry, deleting from database: %x", iter.Key())
-		} else if of.Time.Before(oldest) {
-			folderID = string(db.keyer.FolderFromPendingFolderKey(iter.Key()))
-			l.Infof("Removing stale pending folder %s (%s) from device %s, last seen %v",
-				folderID, of.Label, device.Short(), of.Time)
-		} else {
-			// Keep entries younger or equal to the given timestamp
-			continue
-		}
-		if err := db.Delete(iter.Key()); err != nil {
-			l.Warnf("Failed to remove pending folder entry: %v", err)
-		} else if len(folderID) > 0 {
-			res = append(res, folderID)
-		}
-	}
-	return res, nil
-}
-
 // Consolidated information about a pending folder
 type PendingFolder struct {
 	OfferedBy map[protocol.DeviceID]ObservedFolder `json:"offeredBy"`

lib/model/model.go

@@ -1306,13 +1306,17 @@ func (m *model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 }
 
 func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.DeviceConfiguration, ccDeviceInfos map[string]*indexSenderStartInfo, indexSenders *indexSenderRegistry) ([]string, map[string]struct{}, error) {
-	handleTime := time.Now()
 	var folderDevice config.FolderDeviceConfiguration
 	tempIndexFolders := make([]string, 0, len(folders))
 	paused := make(map[string]struct{}, len(folders))
 	seenFolders := make(map[string]struct{}, len(folders))
 	updatedPending := make([]updatedPendingFolder, 0, len(folders))
 	deviceID := deviceCfg.DeviceID
+	expiredPending, err := m.db.PendingFoldersForDevice(deviceID)
+	if err != nil {
+		l.Infof("Could not get pending folders for cleanup: %v", err)
+	}
+	of := db.ObservedFolder{Time: time.Now().Truncate(time.Second)}
 	for _, folder := range folders {
 		seenFolders[folder.ID] = struct{}{}
 
@@ -1326,8 +1330,10 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
 				l.Infof("Ignoring folder %s from device %s since we are configured to", folder.Description(), deviceID)
 				continue
 			}
-			recvEnc := len(ccDeviceInfos[folder.ID].local.EncryptionPasswordToken) > 0
-			if err := m.db.AddOrUpdatePendingFolder(folder.ID, folder.Label, deviceID, recvEnc); err != nil {
+			delete(expiredPending, folder.ID)
+			of.Label = folder.Label
+			of.ReceiveEncrypted = len(ccDeviceInfos[folder.ID].local.EncryptionPasswordToken) > 0
+			if err := m.db.AddOrUpdatePendingFolder(folder.ID, of, deviceID); err != nil {
 				l.Warnf("Failed to persist pending folder entry to database: %v", err)
 			}
 			indexSenders.addPending(cfg, ccDeviceInfos[folder.ID])
@@ -1335,7 +1341,7 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
 				FolderID:         folder.ID,
 				FolderLabel:      folder.Label,
 				DeviceID:         deviceID,
-				ReceiveEncrypted: recvEnc,
+				ReceiveEncrypted: of.ReceiveEncrypted,
 			})
 			// DEPRECATED: Only for backwards compatibility, should be removed.
 			m.evLogger.Log(events.FolderRejected, map[string]string{
@@ -1412,18 +1418,16 @@ func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.Devi
 	}
 
 	indexSenders.removeAllExcept(seenFolders)
-	// All current pending folders were touched above, so discard any with older timestamps
-	expiredPending, err := m.db.RemovePendingFoldersBeforeTime(deviceID, handleTime)
-	if err != nil {
-		l.Infof("Could not clean up pending folder entries: %v", err)
+	for folder := range expiredPending {
+		m.db.RemovePendingFolderForDevice(folder, deviceID)
 	}
 	if len(updatedPending) > 0 || len(expiredPending) > 0 {
-		expiredPendingList := make([]map[string]string, len(expiredPending))
-		for i, folderID := range expiredPending {
-			expiredPendingList[i] = map[string]string{
+		expiredPendingList := make([]map[string]string, 0, len(expiredPending))
+		for folderID := range expiredPending {
+			expiredPendingList = append(expiredPendingList, map[string]string{
 				"folderID": folderID,
 				"deviceID": deviceID.String(),
-			}
+			})
 		}
 		m.evLogger.Log(events.PendingFoldersChanged, map[string]interface{}{
 			"added":   updatedPending,

lib/model/model_test.go

@@ -4173,7 +4173,11 @@ func TestPendingFolder(t *testing.T) {
 
 	setDevice(t, w, config.DeviceConfiguration{DeviceID: device2})
 	pfolder := "default"
-	if err := m.db.AddOrUpdatePendingFolder(pfolder, pfolder, device2, false); err != nil {
+	of := db.ObservedFolder{
+		Time:  time.Now().Truncate(time.Second),
+		Label: pfolder,
+	}
+	if err := m.db.AddOrUpdatePendingFolder(pfolder, of, device2); err != nil {
 		t.Fatal(err)
 	}
 	deviceFolders, err := m.PendingFolders(protocol.EmptyDeviceID)
@@ -4192,7 +4196,7 @@ func TestPendingFolder(t *testing.T) {
 		t.Fatal(err)
 	}
 	setDevice(t, w, config.DeviceConfiguration{DeviceID: device3})
-	if err := m.db.AddOrUpdatePendingFolder(pfolder, pfolder, device3, false); err != nil {
+	if err := m.db.AddOrUpdatePendingFolder(pfolder, of, device3); err != nil {
 		t.Fatal(err)
 	}
 	deviceFolders, err = m.PendingFolders(device2)