lib/versioner: Clean the versions dir of symlinks, not the full folder

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4289

lib/config/config.go

@@ -17,6 +17,8 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -29,7 +31,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 20
+	CurrentVersion       = 21
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -314,6 +316,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 19 {
 		convertV19V20(cfg)
 	}
+	if cfg.Version == 20 {
+		convertV20V21(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -363,6 +368,32 @@ func (cfg *Configuration) clean() error {
 	return nil
 }
 
+func convertV20V21(cfg *Configuration) {
+	for _, folder := range cfg.Folders {
+		switch folder.Versioning.Type {
+		case "simple", "trashcan":
+			// Clean out symlinks in the known place
+			cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
+		case "staggered":
+			versionDir := folder.Versioning.Params["versionsPath"]
+			if versionDir == "" {
+				// default place
+				cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
+			} else if filepath.IsAbs(versionDir) {
+				// absolute
+				cleanSymlinks(versionDir)
+			} else {
+				// relative to folder
+				cleanSymlinks(filepath.Join(folder.Path(), versionDir))
+			}
+		}
+	}
+
+	// there is also a symlink recovery step in Model.StartFolder()
+
+	cfg.Version = 21
+}
+
 func convertV19V20(cfg *Configuration) {
 	cfg.Options.MinHomeDiskFree = Size{Value: cfg.Options.DeprecatedMinHomeDiskFreePct, Unit: "%"}
 	cfg.Options.DeprecatedMinHomeDiskFreePct = 0
@@ -640,3 +671,23 @@ loop:
 	}
 	return devices[0:count]
 }
+
+func cleanSymlinks(dir string) {
+	if runtime.GOOS == "windows" {
+		// We don't do symlinks on Windows. Additionally, there may
+		// be things that look like symlinks that are not, which we
+		// should leave alone. Deduplicated files, for example.
+		return
+	}
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			l.Infoln("Removing incorrectly versioned symlink", path)
+			os.Remove(path)
+			return filepath.SkipDir
+		}
+		return nil
+	})
+}

lib/config/testdata/v21.xml

@@ -0,0 +1,15 @@
+<configuration version="21">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/model/model.go

@@ -178,8 +178,13 @@ func (m *Model) StartDeadlockDetector(timeout time.Duration) {
 func (m *Model) StartFolder(folder string) {
 	m.fmut.Lock()
 	m.pmut.Lock()
-	folderType := m.startFolderLocked(folder)
 	folderCfg := m.folderCfgs[folder]
+
+	if folderCfg.Versioning.Type != "" && m.cfg.RawCopy().OriginalVersion < 21 {
+		m.attemptSymlinkRecovery(folderCfg)
+	}
+
+	folderType := m.startFolderLocked(folder)
 	m.pmut.Unlock()
 	m.fmut.Unlock()
 
@@ -2721,3 +2726,76 @@ func rootedJoinedPath(root, rel string) (string, error) {
 
 	return joined, nil
 }
+
+func (m *Model) attemptSymlinkRecovery(fcfg config.FolderConfiguration) {
+	fs, ok := m.folderFiles[fcfg.ID]
+	if !ok {
+		return
+	}
+
+	// The window during which we had a broken release out, roughly.
+	startDate := time.Date(2017, 8, 8, 6, 0, 0, 0, time.UTC)
+	endDate := time.Date(2017, 8, 8, 12, 0, 0, 0, time.UTC)
+
+	// Look through all our files looking for deleted symlinks.
+	fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+		if !intf.IsSymlink() {
+			return true
+		}
+
+		symlinkPath, err := rootedJoinedPath(fcfg.Path(), intf.FileName())
+		if err != nil {
+			// odd
+			return true
+		}
+
+		if _, err := os.Lstat(symlinkPath); err == nil {
+			// The symlink exists. Our work here is done.
+			return true
+		}
+
+		fi := intf.(protocol.FileInfo)
+		if !fi.Deleted && fi.SymlinkTarget != "" {
+			// We haven't noticed the delete and put it into the
+			// index yet. Great! We can restore the symlink.
+			l.Infoln("Restoring incorrectly deleted symlink", symlinkPath)
+			os.Symlink(fi.SymlinkTarget, symlinkPath)
+			return true
+		}
+
+		// It's deleted. Check if it was deleted in the bad window.
+		if fi.ModTime().Before(startDate) || !fi.ModTime().Before(endDate) {
+			return true
+		}
+
+		// Try to find an older index entry.
+		for deviceID := range m.cfg.Devices() {
+			olderFI, ok := fs.Get(deviceID, fi.Name)
+			if !ok {
+				// This device doesn't have it.
+				continue
+			}
+			if olderFI.Deleted || !olderFI.IsSymlink() {
+				// The device has something deleted or not a
+				// symlink, doesn't help us.
+				continue
+			}
+			if olderFI.Version.GreaterEqual(fi.Version) {
+				// The device has something newer. We should
+				// chill and let the puller handle it. No
+				// need to look further for this specific
+				// symlink.
+				return true
+			}
+
+			if olderFI.SymlinkTarget != "" {
+				// It has symlink data. Restore the symlink.
+				l.Infoln("Restoring incorrectly deleted symlink", symlinkPath)
+				os.Symlink(olderFI.SymlinkTarget, symlinkPath)
+				return true
+			}
+		}
+
+		return true
+	})
+}

lib/model/model_test.go

@@ -141,6 +141,67 @@ func TestRequest(t *testing.T) {
 	}
 }
 
+func TestSymlinkRecovery(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("symlinks not supported on Windows")
+	}
+
+	ldb := db.OpenMemory()
+
+	fs := db.NewFileSet("default", ldb)
+
+	// device1 has an old entry
+	fs.Update(device1, []protocol.FileInfo{
+		{
+			Name:          "symlink-to-restore",
+			Type:          protocol.FileInfoTypeSymlink,
+			Version:       protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 42}}},
+			SymlinkTarget: "/tmp",
+		},
+	})
+
+	badTime := time.Date(2017, 8, 8, 9, 0, 0, 0, time.UTC).Unix()
+
+	// we have deleted it
+	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{
+			Name:      "symlink-to-restore",
+			Deleted:   true,
+			ModifiedS: badTime,
+			Type:      protocol.FileInfoTypeSymlink,
+			Version:   protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 42}, {ID: 2, Value: 1}}},
+		},
+	})
+
+	// Ensure the symlink does in fact not exist
+	symlinkPath := filepath.Join(defaultFolderConfig.Path(), "symlink-to-restore")
+	os.Remove(symlinkPath)
+	defer os.Remove(symlinkPath)
+	if _, err := os.Lstat(symlinkPath); err == nil {
+		t.Fatal("symlink should not exist")
+	}
+
+	// Start up
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
+
+	folderCfg := defaultFolderConfig
+	folderCfg.Versioning = config.VersioningConfiguration{
+		Type: "simple",
+	}
+
+	m.AddFolder(folderCfg)
+	m.StartFolder("default")
+	m.ServeBackground()
+	defer m.Stop()
+	m.ScanFolder("default")
+
+	// The symlink should have been restored as part of the StartFolder()
+
+	if _, err := os.Lstat(symlinkPath); err != nil {
+		t.Error("should have restored symlink")
+	}
+}
+
 func genFiles(n int) []protocol.FileInfo {
 	files := make([]protocol.FileInfo, n)
 	t := time.Now().Unix()

lib/versioner/external.go

@@ -27,8 +27,6 @@ type External struct {
 }
 
 func NewExternal(folderID, folderPath string, params map[string]string) Versioner {
-	cleanSymlinks(folderPath)
-
 	command := params["command"]
 
 	s := External{

lib/versioner/simple.go

@@ -26,8 +26,6 @@ type Simple struct {
 }
 
 func NewSimple(folderID, folderPath string, params map[string]string) Versioner {
-	cleanSymlinks(folderPath)
-
 	keep, err := strconv.Atoi(params["keep"])
 	if err != nil {
 		keep = 5 // A reasonable default

lib/versioner/staggered.go

@@ -39,8 +39,6 @@ type Staggered struct {
 }
 
 func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
-	cleanSymlinks(folderPath)
-
 	maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
 	if err != nil {
 		maxAge = 31536000 // Default: ~1 year

lib/versioner/trashcan.go

@@ -28,8 +28,6 @@ type Trashcan struct {
 }
 
 func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
-	cleanSymlinks(folderPath)
-
 	cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
 	// On error we default to 0, "do not clean out the trash can"
 

lib/versioner/versioner.go

@@ -8,12 +8,6 @@
 // simple default versioning scheme.
 package versioner
 
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-)
-
 type Versioner interface {
 	Archive(filePath string) error
 }
@@ -24,23 +18,3 @@ const (
 	TimeFormat = "20060102-150405"
 	TimeGlob   = "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]" // glob pattern matching TimeFormat
 )
-
-func cleanSymlinks(dir string) {
-	if runtime.GOOS == "windows" {
-		// We don't do symlinks on Windows. Additionally, there may
-		// be things that look like symlinks that are not, which we
-		// should leave alone. Deduplicated files, for example.
-		return
-	}
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.Mode()&os.ModeSymlink != 0 {
-			l.Infoln("Removing incorrectly versioned symlink", path)
-			os.Remove(path)
-			return filepath.SkipDir
-		}
-		return nil
-	})
-}