Deprecate scanner.Block & File

build.sh

@@ -96,7 +96,7 @@ setup() {
 }
 
 xdr() {
-	for f in discover/packets files/leveldb protocol/message scanner/blocks scanner/file ; do
+	for f in discover/packets files/leveldb protocol/message ; do
 		go run xdr/cmd/genxdr/main.go -- "${f}.go" > "${f}_xdr.go"
 	done
 }

config/config.go

@@ -18,7 +18,6 @@ import (
 	"code.google.com/p/go.crypto/bcrypt"
 	"github.com/calmh/syncthing/logger"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 )
 
 var l = logger.DefaultLogger
@@ -122,11 +121,11 @@ func (r *RepositoryConfiguration) NodeIDs() []protocol.NodeID {
 	return r.nodeIDs
 }
 
-func (r RepositoryConfiguration) FileRanker() func(scanner.File) int {
+func (r RepositoryConfiguration) FileRanker() func(protocol.FileInfo) int {
 	if len(r.SyncOrderPatterns) <= 0 {
 		return nil
 	}
-	return func(f scanner.File) int {
+	return func(f protocol.FileInfo) int {
 		ret := 0
 		for _, v := range r.SyncOrderPatterns {
 			if v.CompiledPattern().MatchString(f.Name) {

config/config_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 )
 
 var node1, node2, node3, node4 protocol.NodeID
@@ -274,7 +273,7 @@ func TestFileSorter(t *testing.T) {
 		},
 	}
 
-	f := []scanner.File{
+	f := []protocol.FileInfo{
 		{Name: "bar.mov"},
 		{Name: "baz.txt"},
 		{Name: "foo.jpg"},
@@ -290,7 +289,7 @@ func TestFileSorter(t *testing.T) {
 
 	files.SortBy(rcfg.FileRanker()).Sort(f)
 
-	expected := []scanner.File{
+	expected := []protocol.FileInfo{
 		{Name: "camera-uploads/foo.jpg"},
 		{Name: "camera-uploads/herp.mov"},
 		{Name: "camera-uploads/hurr.pl"},
@@ -314,7 +313,7 @@ func TestFileSorter(t *testing.T) {
 	}
 }
 
-func formatFiles(f []scanner.File) string {
+func formatFiles(f []protocol.FileInfo) string {
 	ret := ""
 
 	for _, v := range f {

files/cmd/pidx/main.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -29,7 +28,7 @@ func main() {
 
 	if *node == "" {
 		log.Printf("*** Global index for repo %q", *repo)
-		fs.WithGlobal(func(f scanner.File) bool {
+		fs.WithGlobal(func(f protocol.FileInfo) bool {
 			fmt.Println(f)
 			fmt.Println("\t", fs.Availability(f.Name))
 			return true
@@ -40,7 +39,7 @@ func main() {
 			log.Fatal(err)
 		}
 		log.Printf("*** Have index for repo %q node %q", *repo, n)
-		fs.WithHave(n, func(f scanner.File) bool {
+		fs.WithHave(n, func(f protocol.FileInfo) bool {
 			fmt.Println(f)
 			return true
 		})

files/leveldb.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -27,7 +26,7 @@ type versionList struct {
 	versions []fileVersion
 }
 
-type fileList []scanner.File
+type fileList []protocol.FileInfo
 
 func (l fileList) Len() int {
 	return len(l)
@@ -94,9 +93,9 @@ func globalKeyName(key []byte) []byte {
 
 type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool
 
-type fileIterator func(f scanner.File) bool
+type fileIterator func(f protocol.FileInfo) bool
 
-func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, deleteFn deletionHandler) bool {
+func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) bool {
 	sort.Sort(fileList(fs)) // sort list on name, same as on disk
 
 	start := nodeKey(repo, node, nil)                            // before all repo/node files
@@ -153,7 +152,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del
 
 		case cmp == 0:
 			// File exists on both sides - compare versions.
-			var ef scanner.File
+			var ef protocol.FileInfo
 			ef.UnmarshalXDR(dbi.Value())
 			if fs[fsi].Version > ef.Version {
 				ldbInsert(batch, repo, node, newName, fs[fsi])
@@ -182,7 +181,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del
 	return changed
 }
 
-func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
 	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
 		// Disk has files that we are missing. Remove it.
 		if debug {
@@ -194,9 +193,9 @@ func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
 	})
 }
 
-func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
 	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
-		var f scanner.File
+		var f protocol.FileInfo
 		err := f.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
@@ -216,7 +215,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File)
 	})
 }
 
-func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
 	batch := new(leveldb.Batch)
 	snap, err := db.GetSnapshot()
 	if err != nil {
@@ -234,7 +233,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
 			continue
 		}
 
-		var ef scanner.File
+		var ef protocol.FileInfo
 		err = ef.UnmarshalXDR(bs)
 		if err != nil {
 			panic(err)
@@ -253,7 +252,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
 	return true
 }
 
-func ldbInsert(batch dbWriter, repo, node, name []byte, file scanner.File) {
+func ldbInsert(batch dbWriter, repo, node, name []byte, file protocol.FileInfo) {
 	if debug {
 		l.Debugf("insert; repo=%q node=%x %v", repo, node, file)
 	}
@@ -362,7 +361,7 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 	defer dbi.Release()
 
 	for dbi.Next() {
-		var f scanner.File
+		var f protocol.FileInfo
 		err := f.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
@@ -373,17 +372,17 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 	}
 }
 
-func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File {
+func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
 	nk := nodeKey(repo, node, file)
 	bs, err := db.Get(nk, nil)
 	if err == leveldb.ErrNotFound {
-		return scanner.File{}
+		return protocol.FileInfo{}
 	}
 	if err != nil {
 		panic(err)
 	}
 
-	var f scanner.File
+	var f protocol.FileInfo
 	err = f.UnmarshalXDR(bs)
 	if err != nil {
 		panic(err)
@@ -391,7 +390,7 @@ func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File {
 	return f
 }
 
-func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
+func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo {
 	k := globalKey(repo, file)
 	snap, err := db.GetSnapshot()
 	if err != nil {
@@ -401,7 +400,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
 
 	bs, err := snap.Get(k, nil)
 	if err == leveldb.ErrNotFound {
-		return scanner.File{}
+		return protocol.FileInfo{}
 	}
 	if err != nil {
 		panic(err)
@@ -423,7 +422,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
 		panic(err)
 	}
 
-	var f scanner.File
+	var f protocol.FileInfo
 	err = f.UnmarshalXDR(bs)
 	if err != nil {
 		panic(err)
@@ -458,7 +457,7 @@ func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) {
 			panic(err)
 		}
 
-		var f scanner.File
+		var f protocol.FileInfo
 		err = f.UnmarshalXDR(bs)
 		if err != nil {
 			panic(err)
@@ -544,7 +543,7 @@ func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
 				panic(err)
 			}
 
-			var gf scanner.File
+			var gf protocol.FileInfo
 			err = gf.UnmarshalXDR(bs)
 			if err != nil {
 				panic(err)

files/set.go

@@ -9,12 +9,11 @@ import (
 	"sync"
 
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type fileRecord struct {
-	File   scanner.File
+	File   protocol.FileInfo
 	Usage  int
 	Global bool
 }
@@ -37,7 +36,7 @@ func NewSet(repo string, db *leveldb.DB) *Set {
 	return &s
 }
 
-func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) {
+func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
 	}
@@ -48,7 +47,7 @@ func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) {
 	}
 }
 
-func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) {
+func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
 	}
@@ -59,7 +58,7 @@ func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) {
 	}
 }
 
-func (s *Set) Update(node protocol.NodeID, fs []scanner.File) {
+func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) {
 	if debug {
 		l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
 	}
@@ -91,11 +90,11 @@ func (s *Set) WithGlobal(fn fileIterator) {
 	ldbWithGlobal(s.db, []byte(s.repo), fn)
 }
 
-func (s *Set) Get(node protocol.NodeID, file string) scanner.File {
+func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo {
 	return ldbGet(s.db, []byte(s.repo), node[:], []byte(file))
 }
 
-func (s *Set) GetGlobal(file string) scanner.File {
+func (s *Set) GetGlobal(file string) protocol.FileInfo {
 	return ldbGetGlobal(s.db, []byte(s.repo), []byte(file))
 }
 

files/set_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -23,48 +22,47 @@ func init() {
 	remoteNode, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
 }
 
-func genBlocks(n int) []scanner.Block {
-	b := make([]scanner.Block, n)
+func genBlocks(n int) []protocol.BlockInfo {
+	b := make([]protocol.BlockInfo, n)
 	for i := range b {
 		h := make([]byte, 32)
 		for j := range h {
 			h[j] = byte(i + j)
 		}
 		b[i].Size = uint32(i)
-		b[i].Offset = int64(i)
 		b[i].Hash = h
 	}
 	return b
 }
 
-func globalList(s *files.Set) []scanner.File {
-	var fs []scanner.File
-	s.WithGlobal(func(f scanner.File) bool {
+func globalList(s *files.Set) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithGlobal(func(f protocol.FileInfo) bool {
 		fs = append(fs, f)
 		return true
 	})
 	return fs
 }
 
-func haveList(s *files.Set, n protocol.NodeID) []scanner.File {
-	var fs []scanner.File
-	s.WithHave(n, func(f scanner.File) bool {
+func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithHave(n, func(f protocol.FileInfo) bool {
 		fs = append(fs, f)
 		return true
 	})
 	return fs
 }
 
-func needList(s *files.Set, n protocol.NodeID) []scanner.File {
-	var fs []scanner.File
-	s.WithNeed(n, func(f scanner.File) bool {
+func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithNeed(n, func(f protocol.FileInfo) bool {
 		fs = append(fs, f)
 		return true
 	})
 	return fs
 }
 
-type fileList []scanner.File
+type fileList []protocol.FileInfo
 
 func (l fileList) Len() int {
 	return len(l)
@@ -88,44 +86,44 @@ func TestGlobalSet(t *testing.T) {
 
 	m := files.NewSet("test", db)
 
-	local0 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)},
-		scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)},
-		scanner.File{Name: "z", Version: 1000, Blocks: genBlocks(8)},
+	local0 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)},
 	}
-	local1 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)},
-		scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
 	}
-	localTot := []scanner.File{
+	localTot := []protocol.FileInfo{
 		local0[0],
 		local0[1],
 		local0[2],
 		local0[3],
-		scanner.File{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
 	}
 
-	remote0 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
-		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
-		scanner.File{Name: "c", Version: 1002, Blocks: genBlocks(5)},
+	remote0 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)},
 	}
-	remote1 := []scanner.File{
-		scanner.File{Name: "b", Version: 1001, Blocks: genBlocks(6)},
-		scanner.File{Name: "e", Version: 1000, Blocks: genBlocks(7)},
+	remote1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)},
 	}
-	remoteTot := []scanner.File{
+	remoteTot := []protocol.FileInfo{
 		remote0[0],
 		remote1[0],
 		remote0[2],
 		remote1[1],
 	}
 
-	expectedGlobal := []scanner.File{
+	expectedGlobal := []protocol.FileInfo{
 		remote0[0],
 		remote1[0],
 		remote0[2],
@@ -134,13 +132,13 @@ func TestGlobalSet(t *testing.T) {
 		localTot[4],
 	}
 
-	expectedLocalNeed := []scanner.File{
+	expectedLocalNeed := []protocol.FileInfo{
 		remote1[0],
 		remote0[2],
 		remote1[1],
 	}
 
-	expectedRemoteNeed := []scanner.File{
+	expectedRemoteNeed := []protocol.FileInfo{
 		local0[3],
 	}
 
@@ -201,12 +199,12 @@ func TestGlobalSet(t *testing.T) {
 
 	f = m.Get(protocol.LocalNodeID, "zz")
 	if f.Name != "" {
-		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, scanner.File{})
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
 	}
 
 	f = m.GetGlobal("zz")
 	if f.Name != "" {
-		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, scanner.File{})
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
 	}
 
 	av := []protocol.NodeID{protocol.LocalNodeID, remoteNode}
@@ -232,41 +230,41 @@ func TestLocalDeleted(t *testing.T) {
 	m := files.NewSet("test", db)
 	lamport.Default = lamport.Clock{}
 
-	local1 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1000},
-		scanner.File{Name: "c", Version: 1000},
-		scanner.File{Name: "d", Version: 1000},
-		scanner.File{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
+		protocol.FileInfo{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
 
-	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
 		local1[0],
 		// [1] removed
 		local1[2],
 		local1[3],
 		local1[4],
 	})
-	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
 		local1[0],
 		local1[2],
 		// [3] removed
 		local1[4],
 	})
-	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
 		local1[0],
 		local1[2],
 		// [4] removed
 	})
 
-	expectedGlobal1 := []scanner.File{
+	expectedGlobal1 := []protocol.FileInfo{
 		local1[0],
-		scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
 		local1[2],
-		scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
-		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+		protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
 	g := globalList(m)
@@ -277,17 +275,17 @@ func TestLocalDeleted(t *testing.T) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
 	}
 
-	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
 		local1[0],
 		// [2] removed
 	})
 
-	expectedGlobal2 := []scanner.File{
+	expectedGlobal2 := []protocol.FileInfo{
 		local1[0],
-		scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
-		scanner.File{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
-		scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
-		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+		protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
 	g = globalList(m)
@@ -305,9 +303,9 @@ func Benchmark10kReplace(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	b.ResetTimer()
@@ -318,9 +316,9 @@ func Benchmark10kReplace(b *testing.B) {
 }
 
 func Benchmark10kUpdateChg(b *testing.B) {
-	var remote []scanner.File
+	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -331,9 +329,9 @@ func Benchmark10kUpdateChg(b *testing.B) {
 	m := files.NewSet("test", db)
 	m.Replace(remoteNode, remote)
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -350,9 +348,9 @@ func Benchmark10kUpdateChg(b *testing.B) {
 }
 
 func Benchmark10kUpdateSme(b *testing.B) {
-	var remote []scanner.File
+	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -362,9 +360,9 @@ func Benchmark10kUpdateSme(b *testing.B) {
 	m := files.NewSet("test", db)
 	m.Replace(remoteNode, remote)
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -376,9 +374,9 @@ func Benchmark10kUpdateSme(b *testing.B) {
 }
 
 func Benchmark10kNeed2k(b *testing.B) {
-	var remote []scanner.File
+	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -389,12 +387,12 @@ func Benchmark10kNeed2k(b *testing.B) {
 	m := files.NewSet("test", db)
 	m.Replace(remoteNode, remote)
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 8000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 	for i := 8000; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -409,9 +407,9 @@ func Benchmark10kNeed2k(b *testing.B) {
 }
 
 func Benchmark10kHaveFullList(b *testing.B) {
-	var remote []scanner.File
+	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -422,12 +420,12 @@ func Benchmark10kHaveFullList(b *testing.B) {
 	m := files.NewSet("test", db)
 	m.Replace(remoteNode, remote)
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 2000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 	for i := 2000; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -442,9 +440,9 @@ func Benchmark10kHaveFullList(b *testing.B) {
 }
 
 func Benchmark10kGlobal(b *testing.B) {
-	var remote []scanner.File
+	var remote []protocol.FileInfo
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
@@ -455,12 +453,12 @@ func Benchmark10kGlobal(b *testing.B) {
 	m := files.NewSet("test", db)
 	m.Replace(remoteNode, remote)
 
-	var local []scanner.File
+	var local []protocol.FileInfo
 	for i := 0; i < 2000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 	for i := 2000; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -482,18 +480,18 @@ func TestGlobalReset(t *testing.T) {
 
 	m := files.NewSet("test", db)
 
-	local := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1000},
-		scanner.File{Name: "c", Version: 1000},
-		scanner.File{Name: "d", Version: 1000},
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
 	}
 
-	remote := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+	remote := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -523,24 +521,24 @@ func TestNeed(t *testing.T) {
 
 	m := files.NewSet("test", db)
 
-	local := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1000},
-		scanner.File{Name: "c", Version: 1000},
-		scanner.File{Name: "d", Version: 1000},
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
 	}
 
-	remote := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+	remote := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
 	}
 
-	shouldNeed := []scanner.File{
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+	shouldNeed := []protocol.FileInfo{
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local)
@@ -564,19 +562,19 @@ func TestChanges(t *testing.T) {
 
 	m := files.NewSet("test", db)
 
-	local1 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1000},
-		scanner.File{Name: "c", Version: 1000},
-		scanner.File{Name: "d", Version: 1000},
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
 	}
 
-	local2 := []scanner.File{
+	local2 := []protocol.FileInfo{
 		local1[0],
 		// [1] deleted
 		local1[2],
-		scanner.File{Name: "d", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
 	}
 
 	m.ReplaceWithDelete(protocol.LocalNodeID, local1)

files/sort.go

@@ -2,13 +2,12 @@ package files
 
 import (
 	"sort"
-
-	"github.com/calmh/syncthing/scanner"
+	"github.com/calmh/syncthing/protocol"
 )
 
-type SortBy func(p scanner.File) int
+type SortBy func(p protocol.FileInfo) int
 
-func (by SortBy) Sort(files []scanner.File) {
+func (by SortBy) Sort(files []protocol.FileInfo) {
 	ps := &fileSorter{
 		files: files,
 		by:    by,
@@ -17,8 +16,8 @@ func (by SortBy) Sort(files []scanner.File) {
 }
 
 type fileSorter struct {
-	files []scanner.File
-	by    func(p1 scanner.File) int
+	files []protocol.FileInfo
+	by    func(p1 protocol.FileInfo) int
 }
 
 func (s *fileSorter) Len() int {

model/blockqueue.go

@@ -7,19 +7,19 @@ package model
 import (
 	"sync"
 
-	"github.com/calmh/syncthing/scanner"
+	"github.com/calmh/syncthing/protocol"
 )
 
 type bqAdd struct {
-	file scanner.File
-	have []scanner.Block
-	need []scanner.Block
+	file protocol.FileInfo
+	have []protocol.BlockInfo
+	need []protocol.BlockInfo
 }
 
 type bqBlock struct {
-	file  scanner.File
-	block scanner.Block   // get this block from the network
-	copy  []scanner.Block // copy these blocks from the old version of the file
+	file  protocol.FileInfo
+	block protocol.BlockInfo   // get this block from the network
+	copy  []protocol.BlockInfo // copy these blocks from the old version of the file
 	first bool
 	last  bool
 }

model/model.go

@@ -162,11 +162,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 		var have int64
 
 		for _, repo := range m.nodeRepos[node] {
-			m.repoFiles[repo].WithGlobal(func(f scanner.File) bool {
+			m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
 				if !protocol.IsDeleted(f.Flags) {
-					size := f.Size
+					var size int64
 					if protocol.IsDirectory(f.Flags) {
 						size = zeroEntrySize
+					} else {
+						size = f.Size()
 					}
 					tot += size
 					have += size
@@ -174,11 +176,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 				return true
 			})
 
-			m.repoFiles[repo].WithNeed(node, func(f scanner.File) bool {
+			m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
 				if !protocol.IsDeleted(f.Flags) {
-					size := f.Size
+					var size int64
 					if protocol.IsDirectory(f.Flags) {
 						size = zeroEntrySize
+					} else {
+						size = f.Size()
 					}
 					have -= size
 				}
@@ -209,7 +213,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 	return res
 }
 
-func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
+func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
 	for _, f := range fs {
 		fs, de, by := sizeOfFile(f)
 		files += fs
@@ -219,11 +223,11 @@ func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
 	return
 }
 
-func sizeOfFile(f scanner.File) (files, deleted int, bytes int64) {
+func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
 	if !protocol.IsDeleted(f.Flags) {
 		files++
 		if !protocol.IsDirectory(f.Flags) {
-			bytes += f.Size
+			bytes += f.Size()
 		} else {
 			bytes += zeroEntrySize
 		}
@@ -240,7 +244,7 @@ func (m *Model) GlobalSize(repo string) (files, deleted int, bytes int64) {
 	m.rmut.RLock()
 	defer m.rmut.RUnlock()
 	if rf, ok := m.repoFiles[repo]; ok {
-		rf.WithGlobal(func(f scanner.File) bool {
+		rf.WithGlobal(func(f protocol.FileInfo) bool {
 			fs, de, by := sizeOfFile(f)
 			files += fs
 			deleted += de
@@ -257,7 +261,7 @@ func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) {
 	m.rmut.RLock()
 	defer m.rmut.RUnlock()
 	if rf, ok := m.repoFiles[repo]; ok {
-		rf.WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
+		rf.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
 			fs, de, by := sizeOfFile(f)
 			files += fs
 			deleted += de
@@ -275,12 +279,12 @@ func (m *Model) NeedSize(repo string) (files int, bytes int64) {
 }
 
 // NeedFiles returns the list of currently needed files
-func (m *Model) NeedFilesRepo(repo string) []scanner.File {
+func (m *Model) NeedFilesRepo(repo string) []protocol.FileInfo {
 	m.rmut.RLock()
 	defer m.rmut.RUnlock()
 	if rf, ok := m.repoFiles[repo]; ok {
-		var fs []scanner.File
-		rf.WithNeed(protocol.LocalNodeID, func(f scanner.File) bool {
+		var fs []protocol.FileInfo
+		rf.WithNeed(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
 			fs = append(fs, f)
 			return true
 		})
@@ -304,23 +308,9 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf
 		return
 	}
 
-	var files = make([]scanner.File, len(fs))
-	for i := range fs {
-		f := fs[i]
-		lamport.Default.Tick(f.Version)
-		if debug {
-			var flagComment string
-			if protocol.IsDeleted(f.Flags) {
-				flagComment = " (deleted)"
-			}
-			l.Debugf("IDX(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
-		}
-		files[i] = fileFromFileInfo(f)
-	}
-
 	m.rmut.RLock()
 	if r, ok := m.repoFiles[repo]; ok {
-		r.Replace(nodeID, files)
+		r.Replace(nodeID, fs)
 	} else {
 		l.Fatalf("Index for nonexistant repo %q", repo)
 	}
@@ -339,23 +329,9 @@ func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.F
 		return
 	}
 
-	var files = make([]scanner.File, len(fs))
-	for i := range fs {
-		f := fs[i]
-		lamport.Default.Tick(f.Version)
-		if debug {
-			var flagComment string
-			if protocol.IsDeleted(f.Flags) {
-				flagComment = " (deleted)"
-			}
-			l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
-		}
-		files[i] = fileFromFileInfo(f)
-	}
-
 	m.rmut.RLock()
 	if r, ok := m.repoFiles[repo]; ok {
-		r.Update(nodeID, files)
+		r.Update(nodeID, fs)
 	} else {
 		l.Fatalf("IndexUpdate for nonexistant repo %q", repo)
 	}
@@ -432,14 +408,14 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
 	}
 
 	lf := r.Get(protocol.LocalNodeID, name)
-	if lf.Suppressed || protocol.IsDeleted(lf.Flags) {
+	if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
 		if debug {
 			l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
 		}
 		return nil, ErrInvalid
 	}
 
-	if offset > lf.Size {
+	if offset > lf.Size() {
 		if debug {
 			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
 		}
@@ -468,20 +444,20 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
 }
 
 // ReplaceLocal replaces the local repository index with the given list of files.
-func (m *Model) ReplaceLocal(repo string, fs []scanner.File) {
+func (m *Model) ReplaceLocal(repo string, fs []protocol.FileInfo) {
 	m.rmut.RLock()
 	m.repoFiles[repo].ReplaceWithDelete(protocol.LocalNodeID, fs)
 	m.rmut.RUnlock()
 }
 
-func (m *Model) CurrentRepoFile(repo string, file string) scanner.File {
+func (m *Model) CurrentRepoFile(repo string, file string) protocol.FileInfo {
 	m.rmut.RLock()
 	f := m.repoFiles[repo].Get(protocol.LocalNodeID, file)
 	m.rmut.RUnlock()
 	return f
 }
 
-func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File {
+func (m *Model) CurrentGlobalFile(repo string, file string) protocol.FileInfo {
 	m.rmut.RLock()
 	f := m.repoFiles[repo].GetGlobal(file)
 	m.rmut.RUnlock()
@@ -494,7 +470,7 @@ type cFiler struct {
 }
 
 // Implements scanner.CurrentFiler
-func (cf cFiler) CurrentFile(file string) scanner.File {
+func (cf cFiler) CurrentFile(file string) protocol.FileInfo {
 	return cf.m.CurrentRepoFile(cf.r, file)
 }
 
@@ -552,32 +528,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 
 // protocolIndex returns the current local index in protocol data types.
 func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
-	var index []protocol.FileInfo
-
-	var fs []scanner.File
-	m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
+	var fs []protocol.FileInfo
+	m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
 		fs = append(fs, f)
 		return true
 	})
 
-	for _, f := range fs {
-		mf := fileInfoFromFile(f)
-		if debug {
-			var flagComment string
-			if protocol.IsDeleted(mf.Flags) {
-				flagComment = " (deleted)"
-			}
-			l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
-		}
-		index = append(index, mf)
-	}
-
-	return index
+	return fs
 }
 
-func (m *Model) updateLocal(repo string, f scanner.File) {
+func (m *Model) updateLocal(repo string, f protocol.FileInfo) {
 	m.rmut.RLock()
-	m.repoFiles[repo].Update(protocol.LocalNodeID, []scanner.File{f})
+	m.repoFiles[repo].Update(protocol.LocalNodeID, []protocol.FileInfo{f})
 	m.rmut.RUnlock()
 }
 
@@ -736,11 +698,10 @@ func (m *Model) LoadIndexes(dir string) {
 	for repo := range m.repoCfgs {
 		fs := m.loadIndex(repo, dir)
 
-		var sfs = make([]scanner.File, len(fs))
+		var sfs = make([]protocol.FileInfo, len(fs))
 		for i := 0; i < len(fs); i++ {
 			lamport.Default.Tick(fs[i].Version)
-			sfs[i] = fileFromFileInfo(fs[i])
-			sfs[i].Suppressed = false // we might have saved an index with files that were suppressed; the should not be on startup
+			fs[i].Flags &= ^uint32(protocol.FlagInvalid) // we might have saved an index with files that were suppressed; the should not be on startup
 		}
 
 		m.repoFiles[repo].Replace(protocol.LocalNodeID, sfs)

model/model_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
@@ -25,27 +24,24 @@ func init() {
 	node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
 }
 
-var testDataExpected = map[string]scanner.File{
-	"foo": scanner.File{
+var testDataExpected = map[string]protocol.FileInfo{
+	"foo": protocol.FileInfo{
 		Name:     "foo",
 		Flags:    0,
 		Modified: 0,
-		Size:     7,
-		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
-	"empty": scanner.File{
+	"empty": protocol.FileInfo{
 		Name:     "empty",
 		Flags:    0,
 		Modified: 0,
-		Size:     0,
-		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
-	"bar": scanner.File{
+	"bar": protocol.FileInfo{
 		Name:     "bar",
 		Flags:    0,
 		Modified: 0,
-		Size:     10,
-		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
 }
 
@@ -89,7 +85,7 @@ func genFiles(n int) []protocol.FileInfo {
 		files[i] = protocol.FileInfo{
 			Name:     fmt.Sprintf("file%d", i),
 			Modified: t,
-			Blocks:   []protocol.BlockInfo{{100, []byte("some hash bytes")}},
+			Blocks:   []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
 		}
 	}
 
@@ -212,7 +208,7 @@ func BenchmarkRequest(b *testing.B) {
 		files[i] = protocol.FileInfo{
 			Name:     fmt.Sprintf("file%d", i),
 			Modified: t,
-			Blocks:   []protocol.BlockInfo{{100, []byte("some hash bytes")}},
+			Blocks:   []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
 		}
 	}
 

model/puller.go

@@ -21,7 +21,7 @@ import (
 
 type requestResult struct {
 	node     protocol.NodeID
-	file     scanner.File
+	file     protocol.FileInfo
 	filepath string // full filepath name
 	offset   int64
 	data     []byte
@@ -621,7 +621,7 @@ func (p *puller) queueNeededBlocks() {
 	}
 }
 
-func (p *puller) closeFile(f scanner.File) {
+func (p *puller) closeFile(f protocol.FileInfo) {
 	if debug {
 		l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
 	}

model/util.go

@@ -6,58 +6,12 @@ package model
 
 import (
 	"fmt"
-	"path/filepath"
 	"sync"
 	"time"
 
 	"github.com/calmh/syncthing/protocol"
-	"github.com/calmh/syncthing/scanner"
 )
 
-func fileFromFileInfo(f protocol.FileInfo) scanner.File {
-	var blocks = make([]scanner.Block, len(f.Blocks))
-	var offset int64
-	for i, b := range f.Blocks {
-		blocks[i] = scanner.Block{
-			Offset: offset,
-			Size:   b.Size,
-			Hash:   b.Hash,
-		}
-		offset += int64(b.Size)
-	}
-	return scanner.File{
-		// Name is with native separator and normalization
-		Name:       filepath.FromSlash(f.Name),
-		Size:       offset,
-		Flags:      f.Flags &^ protocol.FlagInvalid,
-		Modified:   f.Modified,
-		Version:    f.Version,
-		Blocks:     blocks,
-		Suppressed: f.Flags&protocol.FlagInvalid != 0,
-	}
-}
-
-func fileInfoFromFile(f scanner.File) protocol.FileInfo {
-	var blocks = make([]protocol.BlockInfo, len(f.Blocks))
-	for i, b := range f.Blocks {
-		blocks[i] = protocol.BlockInfo{
-			Size: b.Size,
-			Hash: b.Hash,
-		}
-	}
-	pf := protocol.FileInfo{
-		Name:     filepath.ToSlash(f.Name),
-		Flags:    f.Flags,
-		Modified: f.Modified,
-		Version:  f.Version,
-		Blocks:   blocks,
-	}
-	if f.Suppressed {
-		pf.Flags |= protocol.FlagInvalid
-	}
-	return pf
-}
-
 func cmMap(cm protocol.ClusterConfigMessage) map[string]map[protocol.NodeID]uint32 {
 	m := make(map[string]map[protocol.NodeID]uint32)
 	for _, repo := range cm.Repositories {

protocol/message.go

@@ -4,6 +4,8 @@
 
 package protocol
 
+import "fmt"
+
 type IndexMessage struct {
 	Repository string     // max:64
 	Files      []FileInfo // max:10000000
@@ -17,9 +19,34 @@ type FileInfo struct {
 	Blocks   []BlockInfo // max:1000000
 }
 
+func (f FileInfo) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
+}
+
+func (f FileInfo) Size() (bytes int64) {
+	for _, b := range f.Blocks {
+		bytes += int64(b.Size)
+	}
+	return
+}
+
+func (a FileInfo) Equals(b FileInfo) bool {
+	return a.Version == b.Version
+}
+
+func (a FileInfo) NewerThan(b FileInfo) bool {
+	return a.Version > b.Version
+}
+
 type BlockInfo struct {
-	Size uint32
-	Hash []byte // max:64
+	Offset int64 // noencode (cache only)
+	Size   uint32
+	Hash   []byte // max:64
+}
+
+func (b BlockInfo) String() string {
+	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
 }
 
 type RequestMessage struct {

scanner/blocks.go

@@ -7,25 +7,16 @@ package scanner
 import (
 	"bytes"
 	"crypto/sha256"
-	"fmt"
 	"io"
+
+	"github.com/calmh/syncthing/protocol"
 )
 
 const StandardBlockSize = 128 * 1024
 
-type Block struct {
-	Offset int64
-	Size   uint32
-	Hash   []byte
-}
-
-func (b Block) String() string {
-	return fmt.Sprintf("%d/%d/%x", b.Offset, b.Size, b.Hash)
-}
-
 // Blocks returns the blockwise hash of the reader.
-func Blocks(r io.Reader, blocksize int) ([]Block, error) {
-	var blocks []Block
+func Blocks(r io.Reader, blocksize int) ([]protocol.BlockInfo, error) {
+	var blocks []protocol.BlockInfo
 	var offset int64
 	for {
 		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
@@ -39,9 +30,9 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
 			break
 		}
 
-		b := Block{
-			Offset: offset,
+		b := protocol.BlockInfo{
 			Size:   uint32(n),
+			Offset: offset,
 			Hash:   hf.Sum(nil),
 		}
 		blocks = append(blocks, b)
@@ -50,7 +41,7 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
 
 	if len(blocks) == 0 {
 		// Empty file
-		blocks = append(blocks, Block{
+		blocks = append(blocks, protocol.BlockInfo{
 			Offset: 0,
 			Size:   0,
 			Hash:   []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55},
@@ -62,11 +53,18 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
 
 // BlockDiff returns lists of common and missing (to transform src into tgt)
 // blocks. Both block lists must have been created with the same block size.
-func BlockDiff(src, tgt []Block) (have, need []Block) {
+func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
 	if len(tgt) == 0 && len(src) != 0 {
 		return nil, nil
 	}
 
+	// Set the Offset field on each target block
+	var offset int64
+	for i := range tgt {
+		tgt[i].Offset = offset
+		offset += int64(tgt[i].Size)
+	}
+
 	if len(tgt) != 0 && len(src) == 0 {
 		// Copy the entire file
 		return nil, tgt

scanner/blocks_test.go

@@ -8,6 +8,8 @@ import (
 	"bytes"
 	"fmt"
 	"testing"
+
+	"github.com/calmh/syncthing/protocol"
 )
 
 var blocksTestData = []struct {
@@ -83,20 +85,20 @@ var diffTestData = []struct {
 	a string
 	b string
 	s int
-	d []Block
+	d []protocol.BlockInfo
 }{
-	{"contents", "contents", 1024, []Block{}},
-	{"", "", 1024, []Block{}},
-	{"contents", "contents", 3, []Block{}},
-	{"contents", "cantents", 3, []Block{{0, 3, nil}}},
-	{"contents", "contants", 3, []Block{{3, 3, nil}}},
-	{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}},
-	{"contents", "", 3, []Block{{0, 0, nil}}},
-	{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
-	{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
+	{"contents", "contents", 1024, []protocol.BlockInfo{}},
+	{"", "", 1024, []protocol.BlockInfo{}},
+	{"contents", "contents", 3, []protocol.BlockInfo{}},
+	{"contents", "cantents", 3, []protocol.BlockInfo{{0, 3, nil}}},
+	{"contents", "contants", 3, []protocol.BlockInfo{{3, 3, nil}}},
+	{"contents", "cantants", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}}},
+	{"contents", "", 3, []protocol.BlockInfo{{0, 0, nil}}},
+	{"", "contents", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
+	{"con", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
 	{"contents", "con", 3, nil},
-	{"contents", "cont", 3, []Block{{3, 1, nil}}},
-	{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
+	{"contents", "cont", 3, []protocol.BlockInfo{{3, 1, nil}}},
+	{"cont", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
 }
 
 func TestDiff(t *testing.T) {

scanner/blocks_xdr.go

@@ -1,49 +0,0 @@
-package scanner
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/syncthing/xdr"
-)
-
-func (o Block) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o Block) MarshalXDR() []byte {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o Block) AppendXDR(bs []byte) []byte {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
-}
-
-func (o Block) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(uint64(o.Offset))
-	xw.WriteUint32(o.Size)
-	xw.WriteBytes(o.Hash)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *Block) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *Block) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *Block) decodeXDR(xr *xdr.Reader) error {
-	o.Offset = int64(xr.ReadUint64())
-	o.Size = xr.ReadUint32()
-	o.Hash = xr.ReadBytes()
-	return xr.Error()
-}

scanner/file.go

@@ -1,30 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-// Use of this source code is governed by an MIT-style license that can be
-// found in the LICENSE file.
-
-package scanner
-
-import "fmt"
-
-type File struct {
-	Name       string
-	Flags      uint32
-	Modified   int64
-	Version    uint64
-	Size       int64
-	Blocks     []Block
-	Suppressed bool
-}
-
-func (f File) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v, Sup:%v}",
-		f.Name, f.Flags, f.Modified, f.Version, f.Size, f.Blocks, f.Suppressed)
-}
-
-func (f File) Equals(o File) bool {
-	return f.Modified == o.Modified && f.Version == o.Version
-}
-
-func (f File) NewerThan(o File) bool {
-	return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
-}

scanner/file_xdr.go

@@ -1,64 +0,0 @@
-package scanner
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/syncthing/xdr"
-)
-
-func (o File) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o File) MarshalXDR() []byte {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o File) AppendXDR(bs []byte) []byte {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
-}
-
-func (o File) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteString(o.Name)
-	xw.WriteUint32(o.Flags)
-	xw.WriteUint64(uint64(o.Modified))
-	xw.WriteUint64(o.Version)
-	xw.WriteUint64(uint64(o.Size))
-	xw.WriteUint32(uint32(len(o.Blocks)))
-	for i := range o.Blocks {
-		o.Blocks[i].encodeXDR(xw)
-	}
-	xw.WriteBool(o.Suppressed)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *File) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *File) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *File) decodeXDR(xr *xdr.Reader) error {
-	o.Name = xr.ReadString()
-	o.Flags = xr.ReadUint32()
-	o.Modified = int64(xr.ReadUint64())
-	o.Version = xr.ReadUint64()
-	o.Size = int64(xr.ReadUint64())
-	_BlocksSize := int(xr.ReadUint32())
-	o.Blocks = make([]Block, _BlocksSize)
-	for i := range o.Blocks {
-		(&o.Blocks[i]).decodeXDR(xr)
-	}
-	o.Suppressed = xr.ReadBool()
-	return xr.Error()
-}

scanner/walk.go

@@ -55,12 +55,12 @@ type Suppressor interface {
 
 type CurrentFiler interface {
 	// CurrentFile returns the file as seen at last scan.
-	CurrentFile(name string) File
+	CurrentFile(name string) protocol.FileInfo
 }
 
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
-func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
+func (w *Walker) Walk() (files []protocol.FileInfo, ignore map[string][]string, err error) {
 	if debug {
 		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
 	}
@@ -122,7 +122,7 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W
 	}
 }
 
-func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
+func (w *Walker) walkAndHashFiles(res *[]protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
@@ -183,7 +183,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 					} else {
 						flags |= uint32(info.Mode() & os.ModePerm)
 					}
-					f := File{
+					f := protocol.FileInfo{
 						Name:     rn,
 						Version:  lamport.Default.Tick(0),
 						Flags:    flags,
@@ -213,8 +213,8 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				if w.Suppressor != nil {
 					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
 						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
-						cf.Suppressed = true
-						cf.Version++
+						cf.Flags |= protocol.FlagInvalid
+						cf.Version = lamport.Default.Tick(cf.Version)
 						if debug {
 							l.Debugln("suppressed:", cf)
 						}
@@ -256,10 +256,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 			if w.IgnorePerms {
 				flags = protocol.FlagNoPermBits | 0666
 			}
-			f := File{
+			f := protocol.FileInfo{
 				Name:     rn,
 				Version:  lamport.Default.Tick(0),
-				Size:     info.Size(),
 				Flags:    flags,
 				Modified: info.ModTime().Unix(),
 				Blocks:   blocks,

xdr/cmd/genxdr/main.go

@@ -178,6 +178,9 @@ func handleStruct(name string, t *ast.StructType) {
 			if m := maxRe.FindStringSubmatch(c); m != nil {
 				max, _ = strconv.Atoi(m[1])
 			}
+			if strings.Contains(c, "noencode") {
+				continue
+			}
 		}
 
 		var f field