Merge pull request #1590 from calmh/long-filenames

Handle long filenames on Windows (fixes #1295)

cmd/syncthing/main.go

@@ -670,7 +670,7 @@ func defaultConfig(myName string) config.Configuration {
 	newCfg.Folders = []config.FolderConfiguration{
 		{
 			ID:              "default",
-			Path:            locations[locDefFolder],
+			RawPath:         locations[locDefFolder],
 			RescanIntervalS: 60,
 			Devices:         []config.FolderDeviceConfiguration{{DeviceID: myID}},
 		},

cmd/syncthing/main_test.go

@@ -21,8 +21,8 @@ import (
 
 func TestFolderErrors(t *testing.T) {
 	fcfg := config.FolderConfiguration{
-		ID:   "folder",
-		Path: "testdata/testfolder",
+		ID:      "folder",
+		RawPath: "testdata/testfolder",
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
@@ -62,7 +62,7 @@ func TestFolderErrors(t *testing.T) {
 
 	// Case 2 - new folder, marker created
 
-	fcfg.Path = "testdata/"
+	fcfg.RawPath = "testdata/"
 	cfg = config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})
@@ -110,7 +110,7 @@ func TestFolderErrors(t *testing.T) {
 	os.Remove("testdata/testfolder/.stfolder")
 	os.Remove("testdata/testfolder/")
 
-	fcfg.Path = "testdata/testfolder"
+	fcfg.RawPath = "testdata/testfolder"
 	cfg = config.Wrap("testdata/subfolder", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})

internal/config/config.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -70,7 +71,7 @@ func (orig Configuration) Copy() Configuration {
 
 type FolderConfiguration struct {
 	ID              string                      `xml:"id,attr" json:"id"`
-	Path            string                      `xml:"path,attr" json:"path"`
+	RawPath         string                      `xml:"path,attr" json:"path"`
 	Devices         []FolderDeviceConfiguration `xml:"device" json:"devices"`
 	ReadOnly        bool                        `xml:"ro,attr" json:"readOnly"`
 	RescanIntervalS int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
@@ -94,9 +95,37 @@ func (orig FolderConfiguration) Copy() FolderConfiguration {
 	return c
 }
 
+func (f FolderConfiguration) Path() string {
+	// This is intentionally not a pointer method, because things like
+	// cfg.Folders["default"].Path() should be valid.
+
+	// Attempt tilde expansion; leave unchanged in case of error
+	if path, err := osutil.ExpandTilde(f.RawPath); err == nil {
+		f.RawPath = path
+	}
+
+	// Attempt absolutification; leave unchanged in case of error
+	if !filepath.IsAbs(f.RawPath) {
+		// Abs() looks like a fairly expensive syscall on Windows, while
+		// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
+		// somewhat faster in the general case, hence the outer if...
+		if path, err := filepath.Abs(f.RawPath); err == nil {
+			f.RawPath = path
+		}
+	}
+
+	// Attempt to enable long filename support on Windows. We may still not
+	// have an absolute path here if the previous steps failed.
+	if runtime.GOOS == "windows" && filepath.IsAbs(f.RawPath) && !strings.HasPrefix(f.RawPath, `\\`) {
+		return `\\?\` + f.RawPath
+	}
+
+	return f.RawPath
+}
+
 func (f *FolderConfiguration) CreateMarker() error {
 	if !f.HasMarker() {
-		marker := filepath.Join(f.Path, ".stfolder")
+		marker := filepath.Join(f.Path(), ".stfolder")
 		fd, err := os.Create(marker)
 		if err != nil {
 			return err
@@ -109,7 +138,7 @@ func (f *FolderConfiguration) CreateMarker() error {
 }
 
 func (f *FolderConfiguration) HasMarker() bool {
-	_, err := os.Stat(filepath.Join(f.Path, ".stfolder"))
+	_, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
 	if err != nil {
 		return false
 	}
@@ -285,7 +314,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	for i := range cfg.Folders {
 		folder := &cfg.Folders[i]
 
-		if len(folder.Path) == 0 {
+		if len(folder.RawPath) == 0 {
 			folder.Invalid = "no directory configured"
 			continue
 		}
@@ -296,7 +325,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
 		// This way in the tests, we get away without OS specific separators
 		// in the test configs.
-		folder.Path = filepath.Dir(folder.Path + string(filepath.Separator))
+		folder.RawPath = filepath.Dir(folder.RawPath + string(filepath.Separator))
 
 		if folder.ID == "" {
 			folder.ID = "default"

internal/config/config_test.go

@@ -11,8 +11,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/syncthing/protocol"
@@ -78,7 +80,7 @@ func TestDeviceConfig(t *testing.T) {
 		expectedFolders := []FolderConfiguration{
 			{
 				ID:              "test",
-				Path:            "testdata",
+				RawPath:         "testdata",
 				Devices:         []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
 				ReadOnly:        true,
 				RescanIntervalS: 600,
@@ -297,10 +299,10 @@ func TestVersioningConfig(t *testing.T) {
 func TestIssue1262(t *testing.T) {
 	cfg, err := Load("testdata/issue-1262.xml", device4)
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 
-	actual := cfg.Folders()["test"].Path
+	actual := cfg.Folders()["test"].RawPath
 	expected := "e:"
 	if runtime.GOOS == "windows" {
 		expected = `e:\`
@@ -311,6 +313,51 @@ func TestIssue1262(t *testing.T) {
 	}
 }
 
+func TestWindowsPaths(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip("Not useful on non-Windows")
+		return
+	}
+
+	folder := FolderConfiguration{
+		RawPath: `e:\`,
+	}
+
+	expected := `\\?\e:\`
+	actual := folder.Path()
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+
+	folder.RawPath = `\\192.0.2.22\network\share`
+	expected = folder.RawPath
+	actual = folder.Path()
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+
+	folder.RawPath = `relative\path`
+	expected = folder.RawPath
+	actual = folder.Path()
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+}
+
+func TestFolderPath(t *testing.T) {
+	folder := FolderConfiguration{
+		RawPath: "~/tmp",
+	}
+
+	realPath := folder.Path()
+	if !filepath.IsAbs(realPath) {
+		t.Error(realPath, "should be absolute")
+	}
+	if strings.Contains(realPath, "~") {
+		t.Error(realPath, "should not contain ~")
+	}
+}
+
 func TestNewSaveLoad(t *testing.T) {
 	path := "testdata/temp.xml"
 	os.Remove(path)
@@ -391,8 +438,8 @@ func TestRequiresRestart(t *testing.T) {
 
 	newCfg = cfg
 	newCfg.Folders = append(newCfg.Folders, FolderConfiguration{
-		ID:   "t1",
-		Path: "t1",
+		ID:      "t1",
+		RawPath: "t1",
 	})
 	if !ChangeRequiresRestart(cfg, newCfg) {
 		t.Error("Adding a folder requires restart")
@@ -411,7 +458,7 @@ func TestRequiresRestart(t *testing.T) {
 	if ChangeRequiresRestart(cfg, newCfg) {
 		t.Error("No changes done yet")
 	}
-	newCfg.Folders[0].Path = "different"
+	newCfg.Folders[0].RawPath = "different"
 	if !ChangeRequiresRestart(cfg, newCfg) {
 		t.Error("Changing a folder requires restart")
 	}

internal/config/wrapper.go

@@ -159,12 +159,6 @@ func (w *Wrapper) Folders() map[string]FolderConfiguration {
 	if w.folderMap == nil {
 		w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders))
 		for _, fld := range w.cfg.Folders {
-			path, err := osutil.ExpandTilde(fld.Path)
-			if err != nil {
-				l.Warnln("home:", err)
-				continue
-			}
-			fld.Path = path
 			w.folderMap[fld.ID] = fld
 		}
 	}

internal/model/model.go

@@ -153,7 +153,7 @@ func (m *Model) StartFolderRW(folder string) {
 		if !ok {
 			l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
 		}
-		p.versioner = factory(folder, cfg.Path, cfg.Versioning.Params)
+		p.versioner = factory(folder, cfg.Path(), cfg.Versioning.Params)
 	}
 
 	if cfg.LenientMtimes {
@@ -730,7 +730,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
 	}
 	m.fmut.RLock()
-	fn := filepath.Join(m.folderCfgs[folder].Path, name)
+	fn := filepath.Join(m.folderCfgs[folder].Path(), name)
 	m.fmut.RUnlock()
 
 	var reader io.ReaderAt
@@ -813,7 +813,7 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
 		return lines, nil, fmt.Errorf("Folder %s does not exist", folder)
 	}
 
-	fd, err := os.Open(filepath.Join(cfg.Path, ".stignore"))
+	fd, err := os.Open(filepath.Join(cfg.Path(), ".stignore"))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return lines, nil, nil
@@ -844,7 +844,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		return fmt.Errorf("Folder %s does not exist", folder)
 	}
 
-	fd, err := ioutil.TempFile(cfg.Path, ".syncthing.stignore-"+folder)
+	fd, err := ioutil.TempFile(cfg.Path(), ".syncthing.stignore-"+folder)
 	if err != nil {
 		l.Warnln("Saving .stignore:", err)
 		return err
@@ -865,7 +865,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		return err
 	}
 
-	file := filepath.Join(cfg.Path, ".stignore")
+	file := filepath.Join(cfg.Path(), ".stignore")
 	err = osutil.Rename(fd.Name(), file)
 	if err != nil {
 		l.Warnln("Saving .stignore:", err)
@@ -1076,7 +1076,7 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
 	}
 
 	ignores := ignore.New(m.cfg.Options().CacheIgnoredFiles)
-	_ = ignores.Load(filepath.Join(cfg.Path, ".stignore")) // Ignore error, there might not be an .stignore
+	_ = ignores.Load(filepath.Join(cfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore
 	m.folderIgnores[cfg.ID] = ignores
 
 	m.addedFolder = true
@@ -1144,7 +1144,7 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
 		return errors.New("no such folder")
 	}
 
-	_ = ignores.Load(filepath.Join(folderCfg.Path, ".stignore")) // Ignore error, there might not be an .stignore
+	_ = ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore
 
 	// Required to make sure that we start indexing at a directory we're already
 	// aware off.
@@ -1170,7 +1170,7 @@ nextSub:
 	subs = unifySubs
 
 	w := &scanner.Walker{
-		Dir:           folderCfg.Path,
+		Dir:           folderCfg.Path(),
 		Subs:          subs,
 		Matcher:       ignores,
 		BlockSize:     protocol.BlockSize,
@@ -1268,7 +1268,7 @@ nextSub:
 					"size":     f.Size(),
 				})
 				batch = append(batch, nf)
-			} else if _, err := os.Lstat(filepath.Join(folderCfg.Path, f.Name)); err != nil {
+			} else if _, err := os.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
 				// File has been deleted.
 
 				// We don't specifically verify that the error is
@@ -1533,7 +1533,7 @@ func (m *Model) CheckFolderHealth(id string) error {
 		return errors.New("Folder does not exist")
 	}
 
-	fi, err := os.Stat(folder.Path)
+	fi, err := os.Stat(folder.Path())
 	if m.CurrentLocalVersion(id) > 0 {
 		// Safety check. If the cached index contains files but the
 		// folder doesn't exist, we have a problem. We would assume
@@ -1547,7 +1547,7 @@ func (m *Model) CheckFolderHealth(id string) error {
 	} else if os.IsNotExist(err) {
 		// If we don't have any files in the index, and the directory
 		// doesn't exist, try creating it.
-		err = os.MkdirAll(folder.Path, 0700)
+		err = os.MkdirAll(folder.Path(), 0700)
 		if err == nil {
 			err = folder.CreateMarker()
 		}

internal/model/model_test.go

@@ -35,8 +35,8 @@ func init() {
 	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
 
 	defaultFolderConfig = config.FolderConfiguration{
-		ID:   "default",
-		Path: "testdata",
+		ID:      "default",
+		RawPath: "testdata",
 		Devices: []config.FolderDeviceConfiguration{
 			{
 				DeviceID: device1,
@@ -540,7 +540,7 @@ func TestIgnores(t *testing.T) {
 		t.Error("No error")
 	}
 
-	m.AddFolder(config.FolderConfiguration{ID: "fresh", Path: "XXX"})
+	m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
 	ignores, _, err = m.GetIgnores("fresh")
 	if err != nil {
 		t.Error(err)
@@ -596,7 +596,7 @@ func TestROScanRecovery(t *testing.T) {
 
 	fcfg := config.FolderConfiguration{
 		ID:              "default",
-		Path:            "testdata/rotestfolder",
+		RawPath:         "testdata/rotestfolder",
 		RescanIntervalS: 1,
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
@@ -608,7 +608,7 @@ func TestROScanRecovery(t *testing.T) {
 		},
 	})
 
-	os.RemoveAll(fcfg.Path)
+	os.RemoveAll(fcfg.RawPath)
 
 	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
 
@@ -633,14 +633,14 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Mkdir(fcfg.Path, 0700)
+	os.Mkdir(fcfg.RawPath, 0700)
 
 	if err := waitFor("Folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
 	if err != nil {
 		t.Error(err)
 		return
@@ -652,14 +652,14 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
 
 	if err := waitFor("Folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(fcfg.Path)
+	os.Remove(fcfg.RawPath)
 
 	if err := waitFor("Folder path missing"); err != nil {
 		t.Error(err)
@@ -676,7 +676,7 @@ func TestRWScanRecovery(t *testing.T) {
 
 	fcfg := config.FolderConfiguration{
 		ID:              "default",
-		Path:            "testdata/rwtestfolder",
+		RawPath:         "testdata/rwtestfolder",
 		RescanIntervalS: 1,
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
@@ -688,7 +688,7 @@ func TestRWScanRecovery(t *testing.T) {
 		},
 	})
 
-	os.RemoveAll(fcfg.Path)
+	os.RemoveAll(fcfg.RawPath)
 
 	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
 
@@ -713,14 +713,14 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Mkdir(fcfg.Path, 0700)
+	os.Mkdir(fcfg.RawPath, 0700)
 
 	if err := waitFor("Folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
 	if err != nil {
 		t.Error(err)
 		return
@@ -732,14 +732,14 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
 
 	if err := waitFor("Folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(fcfg.Path)
+	os.Remove(fcfg.RawPath)
 
 	if err := waitFor("Folder path missing"); err != nil {
 		t.Error(err)

internal/model/rwfolder.go

@@ -82,7 +82,7 @@ func newRWFolder(m *Model, cfg config.FolderConfiguration) *rwFolder {
 		progressEmitter: m.progressEmitter,
 
 		folder:        cfg.ID,
-		dir:           cfg.Path,
+		dir:           cfg.Path(),
 		scanIntv:      time.Duration(cfg.RescanIntervalS) * time.Second,
 		ignorePerms:   cfg.IgnorePerms,
 		lenientMtimes: cfg.LenientMtimes,
@@ -852,7 +852,7 @@ func (p *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 		folderRoots := make(map[string]string)
 		p.model.fmut.RLock()
 		for folder, cfg := range p.model.folderCfgs {
-			folderRoots[folder] = cfg.Path
+			folderRoots[folder] = cfg.Path()
 		}
 		p.model.fmut.RUnlock()