Merge pull request #1219 from syncthing/refactor-truncated

Refactor stuff around FileInfoTruncated

cmd/stindex/main.go

@@ -43,8 +43,8 @@ func main() {
 
 	if *device == "" {
 		log.Printf("*** Global index for folder %q", *folder)
-		fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
-			f := fi.(protocol.FileInfoTruncated)
+		fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
+			f := fi.(files.FileInfoTruncated)
 			fmt.Println(f)
 			fmt.Println("\t", fs.Availability(f.Name))
 			return true
@@ -55,8 +55,8 @@ func main() {
 			log.Fatal(err)
 		}
 		log.Printf("*** Have index for folder %q device %q", *folder, n)
-		fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
-			f := fi.(protocol.FileInfoTruncated)
+		fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
+			f := fi.(files.FileInfoTruncated)
 			fmt.Println(f)
 			return true
 		})

cmd/syncthing/gui.go

@@ -36,6 +36,7 @@ import (
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/discover"
 	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/files"
 	"github.com/syncthing/syncthing/internal/model"
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
@@ -782,9 +783,9 @@ func mimeTypeForFile(file string) string {
 	}
 }
 
-func toNeedSlice(files []protocol.FileInfoTruncated) []map[string]interface{} {
-	output := make([]map[string]interface{}, len(files))
-	for i, file := range files {
+func toNeedSlice(fs []files.FileInfoTruncated) []map[string]interface{} {
+	output := make([]map[string]interface{}, len(fs))
+	for i, file := range fs {
 		output[i] = map[string]interface{}{
 			"Name":         file.Name,
 			"Flags":        file.Flags,
@@ -792,7 +793,7 @@ func toNeedSlice(files []protocol.FileInfoTruncated) []map[string]interface{} {
 			"Version":      file.Version,
 			"LocalVersion": file.LocalVersion,
 			"NumBlocks":    file.NumBlocks,
-			"Size":         protocol.BlocksToSize(file.NumBlocks),
+			"Size":         files.BlocksToSize(file.NumBlocks),
 		}
 	}
 	return output

internal/files/leveldb.go

@@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
 
 type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
 
-type fileIterator func(f protocol.FileIntf) bool
-
 func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
 	runtime.GC()
 
@@ -246,7 +244,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			if debugDB {
 				l.Debugln("generic replace; exists - compare")
 			}
-			var ef protocol.FileInfoTruncated
+			var ef FileInfoTruncated
 			ef.UnmarshalXDR(dbi.Value())
 			if fs[fsi].Version > ef.Version ||
 				(fs[fsi].Version == ef.Version && fs[fsi].Flags != ef.Flags) {
@@ -308,7 +306,7 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) u
 
 func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 {
 	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 {
-		var tf protocol.FileInfoTruncated
+		var tf FileInfoTruncated
 		err := tf.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
@@ -378,7 +376,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) ui
 			continue
 		}
 
-		var ef protocol.FileInfoTruncated
+		var ef FileInfoTruncated
 		err = ef.UnmarshalXDR(bs)
 		if err != nil {
 			panic(err)
@@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
 	}
 }
 
-func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
+func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
 	start := deviceKey(folder, device, nil)                            // before all folder/device files
 	limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 	snap, err := db.GetSnapshot()
@@ -559,7 +557,7 @@ func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIt
 	}
 }
 
-func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f protocol.FileInfoTruncated) bool) {
+func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
 	runtime.GC()
 
 	start := deviceKey(folder, nil, nil)                                                  // before all folder/device files
@@ -583,7 +581,7 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
 
 	for dbi.Next() {
 		device := deviceKeyDevice(dbi.Key())
-		var f protocol.FileInfoTruncated
+		var f FileInfoTruncated
 		err := f.UnmarshalXDR(dbi.Value())
 		if err != nil {
 			panic(err)
@@ -612,7 +610,7 @@ func ldbGet(db *leveldb.DB, folder, device, file []byte) (protocol.FileInfo, boo
 	return f, true
 }
 
-func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool) {
+func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf, bool) {
 	k := globalKey(folder, file)
 	snap, err := db.GetSnapshot()
 	if err != nil {
@@ -633,7 +631,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
 	}
 	bs, err := snap.Get(k, nil)
 	if err == leveldb.ErrNotFound {
-		return protocol.FileInfo{}, false
+		return nil, false
 	}
 	if err != nil {
 		panic(err)
@@ -658,15 +656,14 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
 		panic(err)
 	}
 
-	var f protocol.FileInfo
-	err = f.UnmarshalXDR(bs)
+	fi, err := unmarshalTrunc(bs, truncate)
 	if err != nil {
 		panic(err)
 	}
-	return f, true
+	return fi, true
 }
 
-func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
+func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
 	runtime.GC()
 
 	start := globalKey(folder, nil)
@@ -754,7 +751,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
 	return devices
 }
 
-func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
+func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
 	runtime.GC()
 
 	start := globalKey(folder, nil)
@@ -938,9 +935,9 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
 	dbi.Release()
 }
 
-func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
+func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	if truncate {
-		var tf protocol.FileInfoTruncated
+		var tf FileInfoTruncated
 		err := tf.UnmarshalXDR(bs)
 		return tf, err
 	} else {

internal/files/set.go

@@ -30,14 +30,6 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-type fileRecord struct {
-	File   protocol.FileInfo
-	Usage  int
-	Global bool
-}
-
-type bitset uint64
-
 type Set struct {
 	localVersion map[protocol.DeviceID]uint64
 	mutex        sync.Mutex
@@ -46,6 +38,22 @@ type Set struct {
 	blockmap     *BlockMap
 }
 
+// FileIntf is the set of methods implemented by both protocol.FileInfo and
+// protocol.FileInfoTruncated.
+type FileIntf interface {
+	Size() int64
+	IsDeleted() bool
+	IsInvalid() bool
+	IsDirectory() bool
+	IsSymlink() bool
+	HasPermissionBits() bool
+}
+
+// The Iterator is called with either a protocol.FileInfo or a
+// protocol.FileInfoTruncated (depending on the method) and returns true to
+// continue iteration, false to stop.
+type Iterator func(f FileIntf) bool
+
 func NewSet(folder string, db *leveldb.DB) *Set {
 	var s = Set{
 		localVersion: make(map[protocol.DeviceID]uint64),
@@ -57,7 +65,7 @@ func NewSet(folder string, db *leveldb.DB) *Set {
 	ldbCheckGlobals(db, []byte(folder))
 
 	var deviceID protocol.DeviceID
-	ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f protocol.FileInfoTruncated) bool {
+	ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
 		copy(deviceID[:], device)
 		if f.LocalVersion > s.localVersion[deviceID] {
 			s.localVersion[deviceID] = f.LocalVersion
@@ -132,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	}
 }
 
-func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithNeed(%v)", s.folder, device)
 	}
 	ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
 	}
 	ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
-func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithHave(%v)", s.folder, device)
 	}
 	ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) {
+func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
 	if debug {
 		l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
 	}
 	ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
 }
 
-func (s *Set) WithGlobal(fn fileIterator) {
+func (s *Set) WithGlobal(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobal()", s.folder)
 	}
 	ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
 }
 
-func (s *Set) WithGlobalTruncated(fn fileIterator) {
+func (s *Set) WithGlobalTruncated(fn Iterator) {
 	if debug {
 		l.Debugf("%s WithGlobalTruncated()", s.folder)
 	}
@@ -181,9 +189,23 @@ func (s *Set) Get(device protocol.DeviceID, file string) (protocol.FileInfo, boo
 }
 
 func (s *Set) GetGlobal(file string) (protocol.FileInfo, bool) {
-	f, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
+	fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
+	if !ok {
+		return protocol.FileInfo{}, false
+	}
+	f := fi.(protocol.FileInfo)
 	f.Name = osutil.NativeFilename(f.Name)
-	return f, ok
+	return f, true
+}
+
+func (s *Set) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
+	fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
+	if !ok {
+		return FileInfoTruncated{}, false
+	}
+	f := fi.(FileInfoTruncated)
+	f.Name = osutil.NativeFilename(f.Name)
+	return f, true
 }
 
 func (s *Set) Availability(file string) []protocol.DeviceID {
@@ -218,13 +240,13 @@ func normalizeFilenames(fs []protocol.FileInfo) {
 	}
 }
 
-func nativeFileIterator(fn fileIterator) fileIterator {
-	return func(fi protocol.FileIntf) bool {
+func nativeFileIterator(fn Iterator) Iterator {
+	return func(fi FileIntf) bool {
 		switch f := fi.(type) {
 		case protocol.FileInfo:
 			f.Name = osutil.NativeFilename(f.Name)
 			return fn(f)
-		case protocol.FileInfoTruncated:
+		case FileInfoTruncated:
 			f.Name = osutil.NativeFilename(f.Name)
 			return fn(f)
 		default:

internal/files/set_test.go

@@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
 
 func globalList(s *files.Set) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithGlobal(func(fi protocol.FileIntf) bool {
+	s.WithGlobal(func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
 
 func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithHave(n, func(fi protocol.FileIntf) bool {
+	s.WithHave(n, func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 
 func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithNeed(n, func(fi protocol.FileIntf) bool {
+	s.WithNeed(n, func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true

internal/files/truncated.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
+//go:generate genxdr -o truncated_xdr.go truncated.go
+
+package files
+
+import (
+	"fmt"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+// Used for unmarshalling a FileInfo structure but skipping the block list.
+type FileInfoTruncated struct {
+	Name         string // max:8192
+	Flags        uint32
+	Modified     int64
+	Version      uint64
+	LocalVersion uint64
+	NumBlocks    uint32
+}
+
+func (f FileInfoTruncated) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks)
+}
+
+// Returns a statistical guess on the size, not the exact figure
+func (f FileInfoTruncated) Size() int64 {
+	if f.IsDeleted() || f.IsDirectory() {
+		return 128
+	}
+	return BlocksToSize(f.NumBlocks)
+}
+
+func (f FileInfoTruncated) IsDeleted() bool {
+	return f.Flags&protocol.FlagDeleted != 0
+}
+
+func (f FileInfoTruncated) IsInvalid() bool {
+	return f.Flags&protocol.FlagInvalid != 0
+}
+
+func (f FileInfoTruncated) IsDirectory() bool {
+	return f.Flags&protocol.FlagDirectory != 0
+}
+
+func (f FileInfoTruncated) IsSymlink() bool {
+	return f.Flags&protocol.FlagSymlink != 0
+}
+
+func (f FileInfoTruncated) HasPermissionBits() bool {
+	return f.Flags&protocol.FlagNoPermBits == 0
+}
+
+func BlocksToSize(num uint32) int64 {
+	if num < 2 {
+		return protocol.BlockSize / 2
+	}
+	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+}

internal/files/truncated_xdr.go

@@ -0,0 +1,112 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package files
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+FileInfoTruncated Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Name                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Name (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                      Modified (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                       Version (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                    Local Version (64 bits)                    +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                          Num Blocks                           |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct FileInfoTruncated {
+	string Name<8192>;
+	unsigned int Flags;
+	hyper Modified;
+	unsigned hyper Version;
+	unsigned hyper LocalVersion;
+	unsigned int NumBlocks;
+}
+
+*/
+
+func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o FileInfoTruncated) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o FileInfoTruncated) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
+}
+
+func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) {
+	if l := len(o.Name); l > 8192 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
+	}
+	xw.WriteString(o.Name)
+	xw.WriteUint32(o.Flags)
+	xw.WriteUint64(uint64(o.Modified))
+	xw.WriteUint64(o.Version)
+	xw.WriteUint64(o.LocalVersion)
+	xw.WriteUint32(o.NumBlocks)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error {
+	o.Name = xr.ReadStringMax(8192)
+	o.Flags = xr.ReadUint32()
+	o.Modified = int64(xr.ReadUint64())
+	o.Version = xr.ReadUint64()
+	o.LocalVersion = xr.ReadUint64()
+	o.NumBlocks = xr.ReadUint32()
+	return xr.Error()
+}

internal/model/model.go

@@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		return 0 // Folder doesn't exist, so we hardly have any of it
 	}
 
-	rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
+	rf.WithGlobalTruncated(func(f files.FileIntf) bool {
 		if !f.IsDeleted() {
 			tot += f.Size()
 		}
@@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 	}
 
 	var need int64
-	rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
+	rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
 		if !f.IsDeleted() {
 			need += f.Size()
 		}
@@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
 	return
 }
 
-func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
+func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
 	if !f.IsDeleted() {
 		files++
 	} else {
@@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
 
 // GlobalSize returns the number of files, deleted files and total bytes for all
 // files in the global model.
-func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
+func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
+		rf.WithGlobalTruncated(func(f files.FileIntf) bool {
 			fs, de, by := sizeOfFile(f)
-			files += fs
+			nfiles += fs
 			deleted += de
 			bytes += by
 			return true
@@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
 
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the local folder.
-func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
+func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+		rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 			if f.IsInvalid() {
 				return true
 			}
 			fs, de, by := sizeOfFile(f)
-			files += fs
+			nfiles += fs
 			deleted += de
 			bytes += by
 			return true
@@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
 }
 
 // NeedSize returns the number and total size of currently needed files.
-func (m *Model) NeedSize(folder string) (files int, bytes int64) {
+func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 			fs, de, by := sizeOfFile(f)
-			files += fs + de
+			nfiles += fs + de
 			bytes += by
 			return true
 		})
 	}
 	bytes -= m.progressEmitter.BytesCompleted(folder)
 	if debug {
-		l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes)
+		l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
 	}
 	return
 }
@@ -422,42 +422,42 @@ func (m *Model) NeedSize(folder string) (files int, bytes int64) {
 // NeedFiles returns the list of currently needed files in progress, queued,
 // and to be queued on next puller iteration. Also takes a soft cap which is
 // only respected when adding files from the model rather than the runner queue.
-func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated, []protocol.FileInfoTruncated) {
+func (m *Model) NeedFolderFiles(folder string, max int) ([]files.FileInfoTruncated, []files.FileInfoTruncated, []files.FileInfoTruncated) {
 	defer m.leveldbPanicWorkaround()
 
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
-		var progress, queued, rest []protocol.FileInfoTruncated
+		var progress, queued, rest []files.FileInfoTruncated
 		var seen map[string]bool
 
 		runner, ok := m.folderRunners[folder]
 		if ok {
 			progressNames, queuedNames := runner.Jobs()
 
-			progress = make([]protocol.FileInfoTruncated, len(progressNames))
-			queued = make([]protocol.FileInfoTruncated, len(queuedNames))
+			progress = make([]files.FileInfoTruncated, len(progressNames))
+			queued = make([]files.FileInfoTruncated, len(queuedNames))
 			seen = make(map[string]bool, len(progressNames)+len(queuedNames))
 
 			for i, name := range progressNames {
-				if f, ok := rf.GetGlobal(name); ok {
-					progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
+				if f, ok := rf.GetGlobalTruncated(name); ok {
+					progress[i] = f
 					seen[name] = true
 				}
 			}
 
 			for i, name := range queuedNames {
-				if f, ok := rf.GetGlobal(name); ok {
-					queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
+				if f, ok := rf.GetGlobalTruncated(name); ok {
+					queued[i] = f
 					seen[name] = true
 				}
 			}
 		}
 		left := max - len(progress) - len(queued)
 		if max < 1 || left > 0 {
-			rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
+			rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
 				left--
-				ft := f.(protocol.FileInfoTruncated)
+				ft := f.(files.FileInfoTruncated)
 				if !seen[ft.Name] {
 					rest = append(rest, ft)
 				}
@@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
 	maxLocalVer := uint64(0)
 	var err error
 
-	fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+	fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		if f.LocalVersion <= minLocalVer {
 			return true
@@ -1169,8 +1169,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 	batch = batch[:0]
 	// TODO: We should limit the Have scanning to start at sub
 	seenPrefix := false
-	fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
-		f := fi.(protocol.FileInfoTruncated)
+	fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
+		f := fi.(files.FileInfoTruncated)
 		if !strings.HasPrefix(f.Name, sub) {
 			// Return true so that we keep iterating, until we get to the part
 			// of the tree we are interested in. Then return false so we stop
@@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
 
 	m.setState(folder, FolderScanning)
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
-	fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+	fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
 		if len(batch) == indexBatchSize {
 			fs.Update(protocol.LocalDeviceID, batch)

internal/model/puller.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/files"
 	"github.com/syncthing/syncthing/internal/ignore"
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
@@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
 	}
 
 	p.model.fmut.RLock()
-	files := p.model.folderFiles[p.folder]
+	folderFiles := p.model.folderFiles[p.folder]
 	p.model.fmut.RUnlock()
 
 	// !!!
@@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
 
 	var deletions []protocol.FileInfo
 
-	files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
+	folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
 
 		// Needed items are delivered sorted lexicographically. This isn't
 		// really optimal from a performance point of view - it would be

internal/model/sharedpullerstate.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/syncthing/syncthing/internal/files"
 	"github.com/syncthing/syncthing/internal/protocol"
 )
 
@@ -259,7 +260,7 @@ func (s *sharedPullerState) Progress() *pullerProgress {
 		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
 		Pulled:              s.pullTotal - s.pullNeeded,
 		Pulling:             s.pullNeeded,
-		BytesTotal:          protocol.BlocksToSize(total),
-		BytesDone:           protocol.BlocksToSize(done),
+		BytesTotal:          files.BlocksToSize(total),
+		BytesDone:           files.BlocksToSize(done),
 	}
 }

internal/protocol/message.go

@@ -71,76 +71,6 @@ func (f FileInfo) HasPermissionBits() bool {
 	return f.Flags&FlagNoPermBits == 0
 }
 
-func (f FileInfo) ToTruncated() FileInfoTruncated {
-	return FileInfoTruncated{
-		Name:         f.Name,
-		Flags:        f.Flags,
-		Modified:     f.Modified,
-		Version:      f.Version,
-		LocalVersion: f.LocalVersion,
-		NumBlocks:    uint32(len(f.Blocks)),
-	}
-}
-
-// Used for unmarshalling a FileInfo structure but skipping the actual block list
-type FileInfoTruncated struct {
-	Name         string // max:8192
-	Flags        uint32
-	Modified     int64
-	Version      uint64
-	LocalVersion uint64
-	NumBlocks    uint32
-}
-
-func (f FileInfoTruncated) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
-		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks)
-}
-
-func BlocksToSize(num uint32) int64 {
-	if num < 2 {
-		return BlockSize / 2
-	}
-	return int64(num-1)*BlockSize + BlockSize/2
-}
-
-// Returns a statistical guess on the size, not the exact figure
-func (f FileInfoTruncated) Size() int64 {
-	if f.IsDeleted() || f.IsDirectory() {
-		return 128
-	}
-	return BlocksToSize(f.NumBlocks)
-}
-
-func (f FileInfoTruncated) IsDeleted() bool {
-	return f.Flags&FlagDeleted != 0
-}
-
-func (f FileInfoTruncated) IsInvalid() bool {
-	return f.Flags&FlagInvalid != 0
-}
-
-func (f FileInfoTruncated) IsDirectory() bool {
-	return f.Flags&FlagDirectory != 0
-}
-
-func (f FileInfoTruncated) IsSymlink() bool {
-	return f.Flags&FlagSymlink != 0
-}
-
-func (f FileInfoTruncated) HasPermissionBits() bool {
-	return f.Flags&FlagNoPermBits == 0
-}
-
-type FileIntf interface {
-	Size() int64
-	IsDeleted() bool
-	IsInvalid() bool
-	IsDirectory() bool
-	IsSymlink() bool
-	HasPermissionBits() bool
-}
-
 type BlockInfo struct {
 	Offset int64 // noencode (cache only)
 	Size   uint32

internal/protocol/message_xdr.go

@@ -245,106 +245,6 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
 
 /*
 
-FileInfoTruncated Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Length of Name                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                    Name (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Flags                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                      Modified (64 bits)                       +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                       Version (64 bits)                       +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                    Local Version (64 bits)                    +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                          Num Blocks                           |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct FileInfoTruncated {
-	string Name<8192>;
-	unsigned int Flags;
-	hyper Modified;
-	unsigned hyper Version;
-	unsigned hyper LocalVersion;
-	unsigned int NumBlocks;
-}
-
-*/
-
-func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o FileInfoTruncated) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o FileInfoTruncated) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
-	return []byte(aw), err
-}
-
-func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) {
-	if l := len(o.Name); l > 8192 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
-	}
-	xw.WriteString(o.Name)
-	xw.WriteUint32(o.Flags)
-	xw.WriteUint64(uint64(o.Modified))
-	xw.WriteUint64(o.Version)
-	xw.WriteUint64(o.LocalVersion)
-	xw.WriteUint32(o.NumBlocks)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error {
-	o.Name = xr.ReadStringMax(8192)
-	o.Flags = xr.ReadUint32()
-	o.Modified = int64(xr.ReadUint64())
-	o.Version = xr.ReadUint64()
-	o.LocalVersion = xr.ReadUint64()
-	o.NumBlocks = xr.ReadUint32()
-	return xr.Error()
-}
-
-/*
-
 BlockInfo Structure:
 
  0                   1                   2                   3