Move index to index-v0.11.0.db (new format) and centralize location config

cmd/syncthing/gui.go

@@ -58,7 +58,7 @@ func init() {
 func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) error {
 	var err error
 
-	cert, err := loadCert(confDir, "https-")
+	cert, err := tls.LoadX509KeyPair(locations[locHttpsCertFile], locations[locHttpsKeyFile])
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
 		l.Infoln("Creating new HTTPS certificate")
@@ -71,8 +71,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 			name = tlsDefaultCommonName
 		}
 
-		newCertificate(confDir, "https-", name)
-		cert, err = loadCert(confDir, "https-")
+		cert, err = newCertificate(locations[locHttpsCertFile], locations[locHttpsKeyFile], name)
 	}
 	if err != nil {
 		return err

cmd/syncthing/gui_csrf.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -92,7 +91,7 @@ func newCsrfToken() string {
 }
 
 func saveCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
+	name := locations[locCsrfTokens]
 	tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
 
 	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
@@ -117,8 +116,7 @@ func saveCsrfTokens() {
 }
 
 func loadCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
-	f, err := os.Open(name)
+	f, err := os.Open(locations[locCsrfTokens])
 	if err != nil {
 		return
 	}

cmd/syncthing/locations.go

@@ -0,0 +1,109 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+)
+
+type locationEnum string
+
+// Use strings as keys to make printout and serialization of the locations map
+// more meaningful.
+const (
+	locConfigFile    locationEnum = "config"
+	locCertFile                   = "certFile"
+	locKeyFile                    = "keyFile"
+	locHttpsCertFile              = "httpsCertFile"
+	locHttpsKeyFile               = "httpsKeyFile"
+	locDatabase                   = "database"
+	locLogFile                    = "logFile"
+	locCsrfTokens                 = "csrfTokens"
+	locPanicLog                   = "panicLog"
+	locDefFolder                  = "defFolder"
+)
+
+// Platform dependent directories
+var baseDirs = map[string]string{
+	"config": defaultConfigDir(), // Overridden by -home flag
+	"home":   homeDir(),          // User's home directory, *not* -home flag
+}
+
+// Use the variables from baseDirs here
+var locations = map[locationEnum]string{
+	locConfigFile:    "${config}/config.xml",
+	locCertFile:      "${config}/cert.pem",
+	locKeyFile:       "${config}/key.pem",
+	locHttpsCertFile: "${config}/https-cert.pem",
+	locHttpsKeyFile:  "${config}/https-key.pem",
+	locDatabase:      "${config}/index-v0.11.0.db",
+	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
+	locCsrfTokens:    "${config}/csrftokens.txt",
+	locPanicLog:      "${config}/panic-20060102-150405.log", // passed through time.Format()
+	locDefFolder:     "${home}/Sync",
+}
+
+// expandLocations replaces the variables in the location map with actual
+// directory locations.
+func expandLocations() error {
+	for key, dir := range locations {
+		for varName, value := range baseDirs {
+			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
+		}
+		var err error
+		dir, err = osutil.ExpandTilde(dir)
+		if err != nil {
+			return err
+		}
+		locations[key] = dir
+	}
+	return nil
+}
+
+// defaultConfigDir returns the default configuration directory, as figured
+// out by various the environment variables present on each platform, or dies
+// trying.
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+	}
+}
+
+// homeDir returns the user's home directory, or dies trying.
+func homeDir() string {
+	home, err := osutil.ExpandTilde("~")
+	if err != nil {
+		l.Fatalln(err)
+	}
+	return home
+}

cmd/syncthing/main.go

@@ -196,17 +196,11 @@ var (
 )
 
 func main() {
-	defConfDir, err := getDefaultConfDir()
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
 	if runtime.GOOS == "windows" {
 		// On Windows, we use a log file by default. Setting the -logfile flag
-		// to the empty string disables this behavior.
+		// to "-" disables this behavior.
 
-		logFile = filepath.Join(defConfDir, "syncthing.log")
-		flag.StringVar(&logFile, "logfile", logFile, "Log file name (blank for stdout)")
+		flag.StringVar(&logFile, "logfile", "", "Log file name (use \"-\" for stdout)")
 
 		// We also add an option to hide the console window
 		flag.BoolVar(&noConsole, "no-console", false, "Hide console window")
@@ -226,23 +220,30 @@ func main() {
 	flag.BoolVar(&showVersion, "version", false, "Show version")
 	flag.StringVar(&upgradeTo, "upgrade-to", upgradeTo, "Force upgrade directly from specified URL")
 
-	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, defConfDir))
+	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, baseDirs["config"]))
 	flag.Parse()
 
 	if noConsole {
 		osutil.HideConsole()
 	}
 
-	if confDir == "" {
+	if confDir != "" {
 		// Not set as default above because the string can be really long.
-		confDir = defConfDir
+		baseDirs["config"] = confDir
+	}
+
+	if runtime.GOOS == "windows" {
+		if logFile == "" {
+			// Use the default log file location
+			logFile = locations[locLogFile]
+		} else if logFile == "-" {
+			// Don't use a logFile
+			logFile = ""
+		}
 	}
 
-	if confDir != defConfDir && filepath.Dir(logFile) == defConfDir {
-		// The user changed the config dir with -home, but not the log file
-		// location. In this case we assume they meant for the logfile to
-		// still live in it's default location *relative to the config dir*.
-		logFile = filepath.Join(confDir, "syncthing.log")
+	if err := expandLocations(); err != nil {
+		l.Fatalln(err)
 	}
 
 	if showVersion {
@@ -269,13 +270,13 @@ func main() {
 			}
 		}
 
-		cert, err := loadCert(dir, "")
+		certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 		if err == nil {
 			l.Warnln("Key exists; will not overwrite.")
 			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
 		} else {
-			newCertificate(dir, "", tlsDefaultCommonName)
-			cert, err = loadCert(dir, "")
+			cert, err = newCertificate(certFile, keyFile, tlsDefaultCommonName)
 			myID = protocol.NewDeviceID(cert.Certificate[0])
 			if err != nil {
 				l.Fatalln("load cert:", err)
@@ -301,17 +302,12 @@ func main() {
 		return
 	}
 
-	confDir, err := osutil.ExpandTilde(confDir)
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
-	if info, err := os.Stat(confDir); err == nil && !info.IsDir() {
-		l.Fatalln("Config directory", confDir, "is not a directory")
+	if info, err := os.Stat(baseDirs["config"]); err == nil && !info.IsDir() {
+		l.Fatalln("Config directory", baseDirs["config"], "is not a directory")
 	}
 
 	// Ensure that our home directory exists.
-	ensureDir(confDir, 0700)
+	ensureDir(baseDirs["config"], 0700)
 
 	if upgradeTo != "" {
 		err := upgrade.ToURL(upgradeTo)
@@ -337,7 +333,7 @@ func main() {
 
 		if doUpgrade {
 			// Use leveldb database locks to protect against concurrent upgrades
-			_, err = leveldb.OpenFile(filepath.Join(confDir, "index"), &opt.Options{OpenFilesCacheCapacity: 100})
+			_, err = leveldb.OpenFile(locations[locDatabase], &opt.Options{OpenFilesCacheCapacity: 100})
 			if err != nil {
 				l.Fatalln("Cannot upgrade, database seems to be locked. Is another copy of Syncthing already running?")
 			}
@@ -371,13 +367,12 @@ func syncthingMain() {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	events.Default.Log(events.Starting, map[string]string{"home": confDir})
+	events.Default.Log(events.Starting, map[string]string{"home": baseDirs["config"]})
 
 	// Ensure that that we have a certificate and key.
-	cert, err = loadCert(confDir, "")
+	cert, err = tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
 	if err != nil {
-		newCertificate(confDir, "", tlsDefaultCommonName)
-		cert, err = loadCert(confDir, "")
+		cert, err = newCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName)
 		if err != nil {
 			l.Fatalln("load cert:", err)
 		}
@@ -395,7 +390,7 @@ func syncthingMain() {
 
 	// Prepare to be able to save configuration
 
-	cfgFile := filepath.Join(confDir, "config.xml")
+	cfgFile := locations[locConfigFile]
 
 	var myName string
 
@@ -490,7 +485,7 @@ func syncthingMain() {
 		l.Infoln("Local networks:", strings.Join(networks, ", "))
 	}
 
-	dbFile := filepath.Join(confDir, "index")
+	dbFile := locations[locDatabase]
 	dbOpts := &opt.Options{OpenFilesCacheCapacity: 100}
 	ldb, err := leveldb.OpenFile(dbFile, dbOpts)
 	if err != nil && errors.IsCorrupted(err) {
@@ -665,16 +660,11 @@ func setupGUI(cfg *config.Wrapper, m *model.Model) {
 }
 
 func defaultConfig(myName string) config.Configuration {
-	defaultFolder, err := osutil.ExpandTilde("~/Sync")
-	if err != nil {
-		l.Fatalln("home:", err)
-	}
-
 	newCfg := config.New(myID)
 	newCfg.Folders = []config.FolderConfiguration{
 		{
 			ID:              "default",
-			Path:            defaultFolder,
+			Path:            locations[locDefFolder],
 			RescanIntervalS: 60,
 			Devices:         []config.FolderDeviceConfiguration{{DeviceID: myID}},
 		},
@@ -811,12 +801,7 @@ func renewUPnP(port int) {
 }
 
 func resetFolders() {
-	confDir, err := osutil.ExpandTilde(confDir)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	cfgFile := filepath.Join(confDir, "config.xml")
+	cfgFile := locations[locConfigFile]
 	cfg, err := config.Load(cfgFile, myID)
 	if err != nil {
 		log.Fatal(err)
@@ -832,8 +817,7 @@ func resetFolders() {
 		}
 	}
 
-	idx := filepath.Join(confDir, "index")
-	os.RemoveAll(idx)
+	os.RemoveAll(locations[locDatabase])
 }
 
 func restart() {
@@ -879,25 +863,6 @@ func ensureDir(dir string, mode int) {
 	}
 }
 
-func getDefaultConfDir() (string, error) {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing"), nil
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing"), nil
-
-	case "darwin":
-		return osutil.ExpandTilde("~/Library/Application Support/Syncthing")
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing"), nil
-		}
-		return osutil.ExpandTilde("~/.config/syncthing")
-	}
-}
-
 // getFreePort returns a free TCP port fort listening on. The ports given are
 // tried in succession and the first to succeed is returned. If none succeed,
 // a random high port is returned.

cmd/syncthing/monitor.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -164,7 +163,7 @@ func copyStderr(stderr io.ReadCloser, dst io.Writer) {
 			dst.Write([]byte(line))
 
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
-				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
+				panicFd, err = os.Create(time.Now().Format(locations[locPanicLog]))
 				if err != nil {
 					l.Warnln("Create panic log:", err)
 					continue

cmd/syncthing/tls.go

@@ -19,7 +19,6 @@ import (
 	mr "math/rand"
 	"net"
 	"os"
-	"path/filepath"
 	"time"
 )
 
@@ -28,13 +27,7 @@ const (
 	tlsDefaultCommonName = "syncthing"
 )
 
-func loadCert(dir string, prefix string) (tls.Certificate, error) {
-	cf := filepath.Join(dir, prefix+"cert.pem")
-	kf := filepath.Join(dir, prefix+"key.pem")
-	return tls.LoadX509KeyPair(cf, kf)
-}
-
-func newCertificate(dir, prefix, name string) {
+func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
 	l.Infof("Generating RSA key and certificate for %s...", name)
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
@@ -63,7 +56,7 @@ func newCertificate(dir, prefix, name string) {
 		l.Fatalln("create cert:", err)
 	}
 
-	certOut, err := os.Create(filepath.Join(dir, prefix+"cert.pem"))
+	certOut, err := os.Create(certFile)
 	if err != nil {
 		l.Fatalln("save cert:", err)
 	}
@@ -76,7 +69,7 @@ func newCertificate(dir, prefix, name string) {
 		l.Fatalln("save cert:", err)
 	}
 
-	keyOut, err := os.OpenFile(filepath.Join(dir, prefix+"key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		l.Fatalln("save key:", err)
 	}
@@ -88,6 +81,8 @@ func newCertificate(dir, prefix, name string) {
 	if err != nil {
 		l.Fatalln("save key:", err)
 	}
+
+	return tls.LoadX509KeyPair(certFile, keyFile)
 }
 
 type DowngradingListener struct {

test/.gitignore

@@ -15,5 +15,5 @@ dirs-*
 csrftokens.txt
 s4d
 http
-index
+h*/index*
 *.syncthing-reset*

test/cli_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func TestCLIReset(t *testing.T) {
-	dirs := []string{"s1", "s12-1", "h1/index"}
+	dirs := []string{"s1", "s12-1", "h1/index-v0.11.0.db"}
 
 	// Create directories that reset will remove
 

test/conflict_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestConflict(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -64,16 +64,11 @@ func TestConflict(t *testing.T) {
 	// startup, UPnP etc complete and make sure the sender has the full index
 	// before they connect.
 	for i := 0; i < 20; i++ {
-		resp, err := sender.post("/rest/scan?folder=default", nil)
+		err := sender.rescan("default")
 		if err != nil {
 			time.Sleep(time.Second)
 			continue
 		}
-		if resp.StatusCode != 200 {
-			resp.Body.Close()
-			time.Sleep(time.Second)
-			continue
-		}
 		break
 	}
 

test/filetype_test.go

@@ -61,7 +61,7 @@ func TestFileTypeChangeStaggeredVersioning(t *testing.T) {
 
 func testFileTypeChange(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/h2/config.xml

@@ -3,7 +3,9 @@
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
-        <versioning type="staggered"></versioning>
+        <versioning type="simple">
+            <param key="keep" val="5"></param>
+        </versioning>
         <lenientMtimes>false</lenientMtimes>
         <copiers>1</copiers>
         <pullers>16</pullers>

test/httpstress_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestStressHTTP(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s2", "h2/index")
+	err := removeAll("s2", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/ignore_test.go

@@ -22,7 +22,7 @@ func TestIgnores(t *testing.T) {
 	// Clean and start a syncthing instance
 
 	log.Println("Cleaning...")
-	err := removeAll("s1", "h1/index")
+	err := removeAll("s1", "h1/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/manypeers_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestManyPeers(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/parallell_scan_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestParallellScan(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "h1/index")
+	err := removeAll("s1", "h1/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/reconnect_test.go

@@ -32,7 +32,7 @@ func TestRestartSenderAndReceiverDuringTransfer(t *testing.T) {
 
 func testRestartDuringTransfer(t *testing.T, restartSender, restartReceiver bool, senderDelay, receiverDelay time.Duration) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/symlink_test.go

@@ -86,7 +86,7 @@ func TestSymlinksStaggeredVersioning(t *testing.T) {
 
 func testSymlinks(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/sync_test.go

@@ -79,7 +79,7 @@ func testSyncCluster(t *testing.T) {
 	err := removeAll("s1", "s12-1",
 		"s2", "s12-2", "s23-2",
 		"s3", "s23-3",
-		"h1/index", "h2/index", "h3/index")
+		"h1/index*", "h2/index*", "h3/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/transfer-bench_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestBenchmarkTransfer(t *testing.T) {
 	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
 	if err != nil {
 		t.Fatal(err)
 	}

test/util.go

@@ -241,17 +241,23 @@ func (i *inifiteReader) Read(bs []byte) (int, error) {
 // rm -rf
 func removeAll(dirs ...string) error {
 	for _, dir := range dirs {
-		// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
-		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.Mode()&0700 != 0700 {
-				os.Chmod(path, 0777)
-			}
-			return nil
-		})
-		os.RemoveAll(dir)
+		files, err := filepath.Glob(dir)
+		if err != nil {
+			return err
+		}
+		for _, file := range files {
+			// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
+			filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.Mode()&0700 != 0700 {
+					os.Chmod(path, 0777)
+				}
+				return nil
+			})
+			os.RemoveAll(file)
+		}
 	}
 	return nil
 }