Merge pull request #1337 from AudriusButkevicius/fileslevels

Add /rest/tree API call

cmd/syncthing/gui.go

@@ -130,6 +130,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	getRestMux.HandleFunc("/rest/system", restGetSystem)
 	getRestMux.HandleFunc("/rest/upgrade", restGetUpgrade)
 	getRestMux.HandleFunc("/rest/version", restGetVersion)
+	getRestMux.HandleFunc("/rest/tree", withModel(m, restGetTree))
 	getRestMux.HandleFunc("/rest/stats/device", withModel(m, restGetDeviceStats))
 	getRestMux.HandleFunc("/rest/stats/folder", withModel(m, restGetFolderStats))
 
@@ -262,6 +263,24 @@ func restGetVersion(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+func restGetTree(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	folder := qs.Get("folder")
+	prefix := qs.Get("prefix")
+	dirsonly := qs.Get("dirsonly") != ""
+
+	levels, err := strconv.Atoi(qs.Get("levels"))
+	if err != nil {
+		levels = -1
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	tree := m.GlobalDirectoryTree(folder, prefix, levels, dirsonly)
+
+	json.NewEncoder(w).Encode(tree)
+}
+
 func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	var folder = qs.Get("folder")

internal/db/leveldb.go

@@ -709,11 +709,9 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf,
 	return fi, true
 }
 
-func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
+func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iterator) {
 	runtime.GC()
 
-	start := globalKey(folder, nil)
-	limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff})
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -728,7 +726,7 @@ func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
 		snap.Release()
 	}()
 
-	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	dbi := snap.NewIterator(util.BytesPrefix(globalKey(folder, prefix)), nil)
 	defer dbi.Release()
 
 	for dbi.Next() {

internal/db/set.go

@@ -172,14 +172,21 @@ func (s *FileSet) WithGlobal(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobal()", s.folder)
 	}
-	ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
+	ldbWithGlobal(s.db, []byte(s.folder), nil, false, nativeFileIterator(fn))
 }
 
 func (s *FileSet) WithGlobalTruncated(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobalTruncated()", s.folder)
 	}
-	ldbWithGlobal(s.db, []byte(s.folder), true, nativeFileIterator(fn))
+	ldbWithGlobal(s.db, []byte(s.folder), nil, true, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithPrefixedGlobalTruncated()", s.folder, prefix)
+	}
+	ldbWithGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn))
 }
 
 func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {

internal/model/model.go

@@ -1417,6 +1417,69 @@ func (m *Model) RemoteLocalVersion(folder string) int64 {
 	return ver
 }
 
+func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} {
+	m.fmut.RLock()
+	files, ok := m.folderFiles[folder]
+	m.fmut.RUnlock()
+	if !ok {
+		return nil
+	}
+
+	output := make(map[string]interface{})
+	sep := string(filepath.Separator)
+	prefix = osutil.NativeFilename(prefix)
+
+	if prefix != "" && !strings.HasSuffix(prefix, sep) {
+		prefix = prefix + sep
+	}
+
+	files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
+		f := fi.(db.FileInfoTruncated)
+
+		if f.IsInvalid() || f.IsDeleted() || f.Name == prefix {
+			return true
+		}
+
+		f.Name = strings.Replace(f.Name, prefix, "", 1)
+
+		var dir, base string
+		if f.IsDirectory() && !f.IsSymlink() {
+			dir = f.Name
+		} else {
+			dir = filepath.Dir(f.Name)
+			base = filepath.Base(f.Name)
+		}
+
+		if levels > -1 && strings.Count(f.Name, sep) > levels {
+			return true
+		}
+
+		last := output
+		if dir != "." {
+			for _, path := range strings.Split(dir, sep) {
+				directory, ok := last[path]
+				if !ok {
+					newdir := make(map[string]interface{})
+					last[path] = newdir
+					last = newdir
+				} else {
+					last = directory.(map[string]interface{})
+				}
+			}
+		}
+
+		if !dirsonly && base != "" {
+			last[base] = []int64{
+				f.Modified, f.Size(),
+			}
+		}
+
+		return true
+	})
+
+	return output
+}
+
 func (m *Model) availability(folder, file string) []protocol.DeviceID {
 	// Acquire this lock first, as the value returned from foldersFiles can
 	// get heavily modified on Close()

internal/model/model_test.go

@@ -17,8 +17,13 @@ package model
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
 	"testing"
 	"time"
 
@@ -29,10 +34,31 @@ import (
 )
 
 var device1, device2 protocol.DeviceID
+var defaultConfig *config.Wrapper
+var defaultFolderConfig config.FolderConfiguration
 
 func init() {
 	device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
 	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
+
+	defaultFolderConfig = config.FolderConfiguration{
+		ID:   "default",
+		Path: "testdata",
+		Devices: []config.FolderDeviceConfiguration{
+			{
+				DeviceID: device1,
+			},
+		},
+	}
+	_defaultConfig := config.Configuration{
+		Folders: []config.FolderConfiguration{defaultFolderConfig},
+		Devices: []config.DeviceConfiguration{
+			{
+				DeviceID: device1,
+			},
+		},
+	}
+	defaultConfig = config.Wrap("/tmp/test", _defaultConfig)
 }
 
 var testDataExpected = map[string]protocol.FileInfo{
@@ -69,10 +95,10 @@ func init() {
 func TestRequest(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 
-	m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
 
 	// device1 shares default, but device2 doesn't
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata", Devices: []config.FolderDeviceConfiguration{{DeviceID: device1}}})
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 
 	// Existing, shared file
@@ -155,8 +181,8 @@ func genFiles(n int) []protocol.FileInfo {
 
 func BenchmarkIndex10000(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
 
@@ -168,8 +194,8 @@ func BenchmarkIndex10000(b *testing.B) {
 
 func BenchmarkIndex00100(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(100)
 
@@ -181,8 +207,8 @@ func BenchmarkIndex00100(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
 	m.Index(device1, "default", files)
@@ -195,8 +221,8 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
 	m.Index(device1, "default", files)
@@ -210,8 +236,8 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f00001(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 	files := genFiles(10000)
 	m.Index(device1, "default", files)
@@ -268,8 +294,8 @@ func (FakeConnection) Statistics() protocol.Statistics {
 
 func BenchmarkRequest(b *testing.B) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
 
 	const n = 1000
@@ -451,12 +477,8 @@ func TestIgnores(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
-	cfg := config.Wrap("/tmp", config.Configuration{
-		Folders: []config.FolderConfiguration{fcfg},
-	})
-	m := NewModel(cfg, "device", "syncthing", "dev", db)
-	m.AddFolder(fcfg)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 
 	expected := []string{
 		".*",
@@ -527,27 +549,9 @@ func TestIgnores(t *testing.T) {
 }
 
 func TestRefuseUnknownBits(t *testing.T) {
-	fcfg := config.FolderConfiguration{
-		ID:   "default",
-		Path: "testdata",
-		Devices: []config.FolderDeviceConfiguration{
-			{
-				DeviceID: device1,
-			},
-		},
-	}
-	cfg := config.Configuration{
-		Folders: []config.FolderConfiguration{fcfg},
-		Devices: []config.DeviceConfiguration{
-			{
-				DeviceID: device1,
-			},
-		},
-	}
-
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
-	m.AddFolder(fcfg)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 
 	m.ScanFolder("default")
 	m.Index(device1, "default", []protocol.FileInfo{
@@ -580,3 +584,474 @@ func TestRefuseUnknownBits(t *testing.T) {
 		t.Error("Valid file not found or name mismatch", ok, f)
 	}
 }
+
+func TestGlobalDirectoryTree(t *testing.T) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	b := func(isfile bool, path ...string) protocol.FileInfo {
+		var flags uint32 = protocol.FlagDirectory
+		blocks := []protocol.BlockInfo{}
+		if isfile {
+			flags = 0
+			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
+		}
+		return protocol.FileInfo{
+			Name:     filepath.Join(path...),
+			Flags:    flags,
+			Modified: 0x666,
+			Blocks:   blocks,
+		}
+	}
+
+	filedata := []int64{0x666, 0xa}
+
+	testdata := []protocol.FileInfo{
+		b(false, "another"),
+		b(false, "another", "directory"),
+		b(true, "another", "directory", "afile"),
+		b(false, "another", "directory", "with"),
+		b(false, "another", "directory", "with", "a"),
+		b(true, "another", "directory", "with", "a", "file"),
+		b(true, "another", "directory", "with", "file"),
+		b(true, "another", "file"),
+
+		b(false, "other"),
+		b(false, "other", "rand"),
+		b(false, "other", "random"),
+		b(false, "other", "random", "dir"),
+		b(false, "other", "random", "dirx"),
+		b(false, "other", "randomx"),
+
+		b(false, "some"),
+		b(false, "some", "directory"),
+		b(false, "some", "directory", "with"),
+		b(false, "some", "directory", "with", "a"),
+		b(true, "some", "directory", "with", "a", "file"),
+
+		b(true, "rootfile"),
+	}
+	expectedResult := map[string]interface{}{
+		"another": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"afile": filedata,
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{
+						"file": filedata,
+					},
+					"file": filedata,
+				},
+			},
+			"file": filedata,
+		},
+		"other": map[string]interface{}{
+			"rand": map[string]interface{}{},
+			"random": map[string]interface{}{
+				"dir":  map[string]interface{}{},
+				"dirx": map[string]interface{}{},
+			},
+			"randomx": map[string]interface{}{},
+		},
+		"some": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{
+						"file": filedata,
+					},
+				},
+			},
+		},
+		"rootfile": filedata,
+	}
+
+	mm := func(data interface{}) string {
+		bytes, err := json.Marshal(data)
+		if err != nil {
+			panic(err)
+		}
+		return string(bytes)
+	}
+
+	m.Index(device1, "default", testdata)
+
+	result := m.GlobalDirectoryTree("default", "", -1, false)
+
+	if !reflect.DeepEqual(result, expectedResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "another", -1, false)
+
+	if !reflect.DeepEqual(result, expectedResult["another"]) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult["another"]))
+	}
+
+	result = m.GlobalDirectoryTree("default", "", 0, false)
+	currentResult := map[string]interface{}{
+		"another":  map[string]interface{}{},
+		"other":    map[string]interface{}{},
+		"some":     map[string]interface{}{},
+		"rootfile": filedata,
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "", 1, false)
+	currentResult = map[string]interface{}{
+		"another": map[string]interface{}{
+			"directory": map[string]interface{}{},
+			"file":      filedata,
+		},
+		"other": map[string]interface{}{
+			"rand":    map[string]interface{}{},
+			"random":  map[string]interface{}{},
+			"randomx": map[string]interface{}{},
+		},
+		"some": map[string]interface{}{
+			"directory": map[string]interface{}{},
+		},
+		"rootfile": filedata,
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "", -1, true)
+	currentResult = map[string]interface{}{
+		"another": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{},
+				},
+			},
+		},
+		"other": map[string]interface{}{
+			"rand": map[string]interface{}{},
+			"random": map[string]interface{}{
+				"dir":  map[string]interface{}{},
+				"dirx": map[string]interface{}{},
+			},
+			"randomx": map[string]interface{}{},
+		},
+		"some": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{},
+				},
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "", 1, true)
+	currentResult = map[string]interface{}{
+		"another": map[string]interface{}{
+			"directory": map[string]interface{}{},
+		},
+		"other": map[string]interface{}{
+			"rand":    map[string]interface{}{},
+			"random":  map[string]interface{}{},
+			"randomx": map[string]interface{}{},
+		},
+		"some": map[string]interface{}{
+			"directory": map[string]interface{}{},
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "another", 0, false)
+	currentResult = map[string]interface{}{
+		"directory": map[string]interface{}{},
+		"file":      filedata,
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "some/directory", 0, false)
+	currentResult = map[string]interface{}{
+		"with": map[string]interface{}{},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "some/directory", 1, false)
+	currentResult = map[string]interface{}{
+		"with": map[string]interface{}{
+			"a": map[string]interface{}{},
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "some/directory", 2, false)
+	currentResult = map[string]interface{}{
+		"with": map[string]interface{}{
+			"a": map[string]interface{}{
+				"file": filedata,
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "another", -1, true)
+	currentResult = map[string]interface{}{
+		"directory": map[string]interface{}{
+			"with": map[string]interface{}{
+				"a": map[string]interface{}{},
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	// No prefix matching!
+	result = m.GlobalDirectoryTree("default", "som", -1, false)
+	currentResult = map[string]interface{}{}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+}
+
+func TestGlobalDirectorySelfFixing(t *testing.T) {
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	b := func(isfile bool, path ...string) protocol.FileInfo {
+		var flags uint32 = protocol.FlagDirectory
+		blocks := []protocol.BlockInfo{}
+		if isfile {
+			flags = 0
+			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
+		}
+		return protocol.FileInfo{
+			Name:     filepath.Join(path...),
+			Flags:    flags,
+			Modified: 0x666,
+			Blocks:   blocks,
+		}
+	}
+
+	filedata := []int64{0x666, 0xa}
+
+	testdata := []protocol.FileInfo{
+		b(true, "another", "directory", "afile"),
+		b(true, "another", "directory", "with", "a", "file"),
+		b(true, "another", "directory", "with", "file"),
+
+		b(false, "other", "random", "dirx"),
+		b(false, "other", "randomx"),
+
+		b(false, "some", "directory", "with", "x"),
+		b(true, "some", "directory", "with", "a", "file"),
+
+		b(false, "this", "is", "a", "deep", "invalid", "directory"),
+
+		b(true, "xthis", "is", "a", "deep", "invalid", "file"),
+	}
+	expectedResult := map[string]interface{}{
+		"another": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"afile": filedata,
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{
+						"file": filedata,
+					},
+					"file": filedata,
+				},
+			},
+		},
+		"other": map[string]interface{}{
+			"random": map[string]interface{}{
+				"dirx": map[string]interface{}{},
+			},
+			"randomx": map[string]interface{}{},
+		},
+		"some": map[string]interface{}{
+			"directory": map[string]interface{}{
+				"with": map[string]interface{}{
+					"a": map[string]interface{}{
+						"file": filedata,
+					},
+					"x": map[string]interface{}{},
+				},
+			},
+		},
+		"this": map[string]interface{}{
+			"is": map[string]interface{}{
+				"a": map[string]interface{}{
+					"deep": map[string]interface{}{
+						"invalid": map[string]interface{}{
+							"directory": map[string]interface{}{},
+						},
+					},
+				},
+			},
+		},
+		"xthis": map[string]interface{}{
+			"is": map[string]interface{}{
+				"a": map[string]interface{}{
+					"deep": map[string]interface{}{
+						"invalid": map[string]interface{}{
+							"file": filedata,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	mm := func(data interface{}) string {
+		bytes, err := json.Marshal(data)
+		if err != nil {
+			panic(err)
+		}
+		return string(bytes)
+	}
+
+	m.Index(device1, "default", testdata)
+
+	result := m.GlobalDirectoryTree("default", "", -1, false)
+
+	if !reflect.DeepEqual(result, expectedResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "xthis/is/a/deep", -1, false)
+	currentResult := map[string]interface{}{
+		"invalid": map[string]interface{}{
+			"file": filedata,
+		},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	result = m.GlobalDirectoryTree("default", "xthis/is/a/deep", -1, true)
+	currentResult = map[string]interface{}{
+		"invalid": map[string]interface{}{},
+	}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+
+	// !!! This is actually BAD, because we don't have enough level allowance
+	// to accept this file, hence the tree is left unbuilt !!!
+	result = m.GlobalDirectoryTree("default", "xthis", 1, false)
+	currentResult = map[string]interface{}{}
+
+	if !reflect.DeepEqual(result, currentResult) {
+		t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult))
+	}
+}
+
+func genDeepFiles(n, d int) []protocol.FileInfo {
+	rand.Seed(int64(n))
+	files := make([]protocol.FileInfo, n)
+	t := time.Now().Unix()
+	for i := 0; i < n; i++ {
+		path := ""
+		for i := 0; i <= d; i++ {
+			path = filepath.Join(path, strconv.Itoa(rand.Int()))
+		}
+
+		sofar := ""
+		for _, path := range filepath.SplitList(path) {
+			sofar = filepath.Join(sofar, path)
+			files[i] = protocol.FileInfo{
+				Name: sofar,
+			}
+			i++
+		}
+
+		files[i].Modified = t
+		files[i].Blocks = []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}}
+	}
+
+	return files
+}
+
+func BenchmarkTree_10000_50(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	m.ScanFolder("default")
+	files := genDeepFiles(10000, 50)
+
+	m.Index(device1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.GlobalDirectoryTree("default", "", -1, false)
+	}
+}
+
+func BenchmarkTree_10000_10(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	m.ScanFolder("default")
+	files := genDeepFiles(10000, 10)
+
+	m.Index(device1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.GlobalDirectoryTree("default", "", -1, false)
+	}
+}
+
+func BenchmarkTree_00100_50(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	m.ScanFolder("default")
+	files := genDeepFiles(100, 50)
+
+	m.Index(device1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.GlobalDirectoryTree("default", "", -1, false)
+	}
+}
+
+func BenchmarkTree_00100_10(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	m.ScanFolder("default")
+	files := genDeepFiles(100, 10)
+
+	m.Index(device1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.GlobalDirectoryTree("default", "", -1, false)
+	}
+}

internal/model/puller_test.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/scanner"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -77,8 +76,8 @@ func TestHandleFile(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
 
@@ -131,8 +130,8 @@ func TestHandleFileWithTemp(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
 
@@ -190,12 +189,9 @@ func TestCopierFinder(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 	requiredFile.Name = "file2"
 
-	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
-	cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
-
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
-	m.AddFolder(fcfg)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 	// Update index
 	m.updateLocal("default", existingFile)
 
@@ -268,12 +264,9 @@ func TestCopierCleanup(t *testing.T) {
 		return true
 	}
 
-	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
-	cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
-
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
-	m.AddFolder(fcfg)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 
 	// Create a file
 	file := protocol.FileInfo{
@@ -320,12 +313,9 @@ func TestCopierCleanup(t *testing.T) {
 // Make sure that the copier routine hashes the content when asked, and pulls
 // if it fails to find the block.
 func TestLastResortPulling(t *testing.T) {
-	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
-	cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
-
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
-	m.AddFolder(fcfg)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 
 	// Add a file to index (with the incorrect block representation, as content
 	// doesn't actually match the block list)
@@ -396,11 +386,11 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	cw := config.Wrap("/tmp/test", config.Configuration{})
-	m := NewModel(cw, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
 
-	emitter := NewProgressEmitter(cw)
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	emitter := NewProgressEmitter(defaultConfig)
 	go emitter.Serve()
 
 	p := Puller{
@@ -484,11 +474,10 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	cw := config.Wrap("/tmp/test", config.Configuration{})
-	m := NewModel(cw, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	m := NewModel(defaultConfig, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
 
-	emitter := NewProgressEmitter(cw)
+	emitter := NewProgressEmitter(defaultConfig)
 	go emitter.Serve()
 
 	p := Puller{