Move FileIntf to files package, expose Iterator type

This is where FileIntf is used, so it should be defined here (it's not
a protocol thing, really).

cmd/stindex/main.go

@@ -43,7 +43,7 @@ func main() {
 
 	if *device == "" {
 		log.Printf("*** Global index for folder %q", *folder)
-		fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
+		fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
 			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)
 			fmt.Println("\t", fs.Availability(f.Name))
@@ -55,7 +55,7 @@ func main() {
 			log.Fatal(err)
 		}
 		log.Printf("*** Have index for folder %q device %q", *folder, n)
-		fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
+		fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
 			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)
 			return true

internal/files/leveldb.go

@@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
 
 type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
 
-type fileIterator func(f protocol.FileIntf) bool
-
 func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
 	runtime.GC()
 
@@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 	}
 }
 
-func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
+func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
 	start := deviceKey(folder, device, nil)                            // before all folder/device files
 	limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 	snap, err := db.GetSnapshot()
@@ -666,7 +664,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
 	return f, true
 }
 
-func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
+func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
 	runtime.GC()
 
 	start := globalKey(folder, nil)
@@ -754,7 +752,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
 	return devices
 }
 
-func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
+func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
 	runtime.GC()
 
 	start := globalKey(folder, nil)
@@ -938,7 +936,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
 	dbi.Release()
 }
 
-func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
+func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	if truncate {
 		var tf protocol.FileInfoTruncated
 		err := tf.UnmarshalXDR(bs)

internal/files/set.go

@@ -38,6 +38,22 @@ type Set struct {
 	blockmap     *BlockMap
 }
 
+// FileIntf is the set of methods implemented by both protocol.FileInfo and
+// protocol.FileInfoTruncated.
+type FileIntf interface {
+	Size() int64
+	IsDeleted() bool
+	IsInvalid() bool
+	IsDirectory() bool
+	IsSymlink() bool
+	HasPermissionBits() bool
+}
+
+// The Iterator is called with either a protocol.FileInfo or a
+// protocol.FileInfoTruncated (depending on the method) and returns true to
+// continue iteration, false to stop.
+type Iterator func(f FileIntf) bool
+
 func NewSet(folder string, db *leveldb.DB) *Set {
 	var s = Set{
 		localVersion: make(map[protocol.DeviceID]uint64),
@@ -124,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	}
 }
 
-func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithNeed(%v)", s.folder, device)
 	}
 	ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
 	}
 	ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
-func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithHave(%v)", s.folder, device)
 	}
 	ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
 	}
 	ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
-func (s *Set) WithGlobal(fn fileIterator) {
+func (s *Set) WithGlobal(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobal()", s.folder)
 	}
 	ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithGlobalTruncated(fn fileIterator) {
+func (s *Set) WithGlobalTruncated(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobalTruncated()", s.folder)
 	}
@@ -210,8 +226,8 @@ func normalizeFilenames(fs []protocol.FileInfo) {
 	}
 }
 
-func nativeFileIterator(fn fileIterator) fileIterator {
-	return func(fi protocol.FileIntf) bool {
+func nativeFileIterator(fn Iterator) Iterator {
+	return func(fi FileIntf) bool {
 		switch f := fi.(type) {
 		case protocol.FileInfo:
 			f.Name = osutil.NativeFilename(f.Name)

internal/files/set_test.go

@@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
 
 func globalList(s *files.Set) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithGlobal(func(fi protocol.FileIntf) bool {
+	s.WithGlobal(func(fi FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
 
 func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithHave(n, func(fi protocol.FileIntf) bool {
+	s.WithHave(n, func(fi FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 
 func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithNeed(n, func(fi protocol.FileIntf) bool {
+	s.WithNeed(n, func(fi FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true

internal/model/model.go

@@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		return 0 // Folder doesn't exist, so we hardly have any of it
 	}
 
-	rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
+	rf.WithGlobalTruncated(func(f files.FileIntf) bool {
 		if !f.IsDeleted() {
 			tot += f.Size()
 		}
@@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 	}
 
 	var need int64
-	rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
+	rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
 		if !f.IsDeleted() {
 			need += f.Size()
 		}
@@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
 	return
 }
 
-func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
+func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
 	if !f.IsDeleted() {
 		files++
 	} else {
@@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
 
 // GlobalSize returns the number of files, deleted files and total bytes for all
 // files in the global model.
-func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
+func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
+		rf.WithGlobalTruncated(func(f files.FileIntf) bool {
 			fs, de, by := sizeOfFile(f)
-			files += fs
+			nfiles += fs
 			deleted += de
 			bytes += by
 			return true
@@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
 
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the local folder.
-func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
+func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+		rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 			if f.IsInvalid() {
 				return true
 			}
 			fs, de, by := sizeOfFile(f)
-			files += fs
+			nfiles += fs
 			deleted += de
 			bytes += by
 			return true
@@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
 }
 
 // NeedSize returns the number and total size of currently needed files.
-func (m *Model) NeedSize(folder string) (files int, bytes int64) {
+func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 			fs, de, by := sizeOfFile(f)
-			files += fs + de
+			nfiles += fs + de
 			bytes += by
 			return true
 		})
 	}
 	bytes -= m.progressEmitter.BytesCompleted(folder)
 	if debug {
-		l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes)
+		l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
 	}
 	return
 }
@@ -441,21 +441,21 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
 
 			for i, name := range progressNames {
 				if f, ok := rf.GetGlobal(name); ok {
-					progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
+					progress[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
 					seen[name] = true
 				}
 			}
 
 			for i, name := range queuedNames {
 				if f, ok := rf.GetGlobal(name); ok {
-					queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
+					queued[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
 					seen[name] = true
 				}
 			}
 		}
 		left := max - len(progress) - len(queued)
 		if max < 1 || left > 0 {
-			rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+			rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 				left--
 				ft := f.(protocol.FileInfoTruncated)
 				if !seen[ft.Name] {
@@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
 	maxLocalVer := uint64(0)
 	var err error
 
-	fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+	fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		if f.LocalVersion <= minLocalVer {
 			return true
@@ -1169,7 +1169,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 	batch = batch[:0]
 	// TODO: We should limit the Have scanning to start at sub
 	seenPrefix := false
-	fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+	fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfoTruncated)
 		if !strings.HasPrefix(f.Name, sub) {
 			// Return true so that we keep iterating, until we get to the part
@@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
 
 	m.setState(folder, FolderScanning)
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
-	fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+	fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
 		if len(batch) == indexBatchSize {
 			fs.Update(protocol.LocalDeviceID, batch)

internal/model/puller.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/files"
 	"github.com/syncthing/syncthing/internal/ignore"
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
@@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
 	}
 
 	p.model.fmut.RLock()
-	files := p.model.folderFiles[p.folder]
+	folderFiles := p.model.folderFiles[p.folder]
 	p.model.fmut.RUnlock()
 
 	// !!!
@@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
 
 	var deletions []protocol.FileInfo
 
-	files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
+	folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
 
 		// Needed items are delivered sorted lexicographically. This isn't
 		// really optimal from a performance point of view - it would be

internal/protocol/message.go

@@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool {
 	return f.Flags&FlagNoPermBits == 0
 }
 
-type FileIntf interface {
-	Size() int64
-	IsDeleted() bool
-	IsInvalid() bool
-	IsDirectory() bool
-	IsSymlink() bool
-	HasPermissionBits() bool
-}
-
 type BlockInfo struct {
 	Offset int64 // noencode (cache only)
 	Size   uint32