lib/model: Verify request content against weak (and possibly strong) hash (#4767)

cmd/syncthing/main.go

@@ -46,7 +46,6 @@ import (
 	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
-	"github.com/syncthing/syncthing/lib/weakhash"
 
 	"github.com/thejerf/suture"
 
@@ -697,26 +696,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		},
 	}
 
-	if opts := cfg.Options(); opts.WeakHashSelectionMethod == config.WeakHashAuto {
-		perfWithWeakHash := cpuBench(3, 150*time.Millisecond, true)
-		l.Infof("Hashing performance with rolling hash is %.02f MB/s", perfWithWeakHash)
-		perfWithoutWeakHash := cpuBench(3, 150*time.Millisecond, false)
-		l.Infof("Hashing performance without rolling hash is %.02f MB/s", perfWithoutWeakHash)
-
-		if perfWithoutWeakHash*0.8 > perfWithWeakHash {
-			l.Infof("Rolling hash disabled, as it has an unacceptable performance impact.")
-			weakhash.Enabled = false
-		} else {
-			l.Infof("Rolling hash enabled, as it has an acceptable performance impact.")
-			weakhash.Enabled = true
-		}
-	} else if opts.WeakHashSelectionMethod == config.WeakHashNever {
-		l.Infof("Disabling rolling hash")
-		weakhash.Enabled = false
-	} else if opts.WeakHashSelectionMethod == config.WeakHashAlways {
-		l.Infof("Enabling rolling hash")
-		weakhash.Enabled = true
-	}
+	perf := cpuBench(3, 150*time.Millisecond, true)
+	l.Infof("Hashing performance is %.02f MB/s", perf)
 
 	dbFile := locations[locDatabase]
 	ldb, err := db.Open(dbFile)

cmd/syncthing/usage_report.go

@@ -26,7 +26,6 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/upgrade"
-	"github.com/syncthing/syncthing/lib/weakhash"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
@@ -190,8 +189,6 @@ func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf,
 		res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
 		res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
 		res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
-		res["weakHashSelection"] = opts.WeakHashSelectionMethod.String()
-		res["weakHashEnabled"] = weakhash.Enabled
 		res["customTrafficClass"] = opts.TrafficClass != 0
 		res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
 		res["temporariesDisabled"] = opts.KeepTemporariesH == 0

lib/config/config_test.go

@@ -67,7 +67,6 @@ func TestDefaultValues(t *testing.T) {
 		OverwriteRemoteDevNames: false,
 		TempIndexMinBlocks:      10,
 		UnackedNotificationIDs:  []string{},
-		WeakHashSelectionMethod: WeakHashAuto,
 		DefaultFolderPath:       "~",
 		SetLowPriority:          true,
 	}
@@ -209,9 +208,8 @@ func TestOverriddenValues(t *testing.T) {
 			"channelNotification",   // added in 17->18 migration
 			"fsWatcherNotification", // added in 27->28 migration
 		},
-		WeakHashSelectionMethod: WeakHashNever,
-		DefaultFolderPath:       "/media/syncthing",
-		SetLowPriority:          false,
+		DefaultFolderPath: "/media/syncthing",
+		SetLowPriority:    false,
 	}
 
 	os.Unsetenv("STNOUPGRADE")

lib/config/optionsconfiguration.go

@@ -7,135 +7,50 @@
 package config
 
 import (
-	"encoding/json"
-	"encoding/xml"
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/util"
 )
 
-type WeakHashSelectionMethod int
-
-const (
-	WeakHashAuto WeakHashSelectionMethod = iota
-	WeakHashAlways
-	WeakHashNever
-)
-
-func (m WeakHashSelectionMethod) MarshalString() (string, error) {
-	switch m {
-	case WeakHashAuto:
-		return "auto", nil
-	case WeakHashAlways:
-		return "always", nil
-	case WeakHashNever:
-		return "never", nil
-	default:
-		return "", fmt.Errorf("unrecognized hash selection method")
-	}
-}
-
-func (m WeakHashSelectionMethod) String() string {
-	s, err := m.MarshalString()
-	if err != nil {
-		panic(err)
-	}
-	return s
-}
-
-func (m *WeakHashSelectionMethod) UnmarshalString(value string) error {
-	switch value {
-	case "auto":
-		*m = WeakHashAuto
-		return nil
-	case "always":
-		*m = WeakHashAlways
-		return nil
-	case "never":
-		*m = WeakHashNever
-		return nil
-	}
-	return fmt.Errorf("unrecognized hash selection method")
-}
-
-func (m WeakHashSelectionMethod) MarshalJSON() ([]byte, error) {
-	val, err := m.MarshalString()
-	if err != nil {
-		return nil, err
-	}
-	return json.Marshal(val)
-}
-
-func (m *WeakHashSelectionMethod) UnmarshalJSON(data []byte) error {
-	var value string
-	if err := json.Unmarshal(data, &value); err != nil {
-		return err
-	}
-	return m.UnmarshalString(value)
-}
-
-func (m WeakHashSelectionMethod) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	val, err := m.MarshalString()
-	if err != nil {
-		return err
-	}
-	return e.EncodeElement(val, start)
-}
-
-func (m *WeakHashSelectionMethod) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	var value string
-	if err := d.DecodeElement(&value, &start); err != nil {
-		return err
-	}
-	return m.UnmarshalString(value)
-}
-
-func (WeakHashSelectionMethod) ParseDefault(value string) (interface{}, error) {
-	var m WeakHashSelectionMethod
-	err := m.UnmarshalString(value)
-	return m, err
-}
-
 type OptionsConfiguration struct {
-	ListenAddresses         []string                `xml:"listenAddress" json:"listenAddresses" default:"default"`
-	GlobalAnnServers        []string                `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
-	GlobalAnnEnabled        bool                    `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
-	LocalAnnEnabled         bool                    `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
-	LocalAnnPort            int                     `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
-	LocalAnnMCAddr          string                  `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
-	MaxSendKbps             int                     `xml:"maxSendKbps" json:"maxSendKbps"`
-	MaxRecvKbps             int                     `xml:"maxRecvKbps" json:"maxRecvKbps"`
-	ReconnectIntervalS      int                     `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
-	RelaysEnabled           bool                    `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
-	RelayReconnectIntervalM int                     `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
-	StartBrowser            bool                    `xml:"startBrowser" json:"startBrowser" default:"true"`
-	NATEnabled              bool                    `xml:"natEnabled" json:"natEnabled" default:"true"`
-	NATLeaseM               int                     `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
-	NATRenewalM             int                     `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
-	NATTimeoutS             int                     `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
-	URAccepted              int                     `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
-	URSeen                  int                     `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
-	URUniqueID              string                  `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
-	URURL                   string                  `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
-	URPostInsecurely        bool                    `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
-	URInitialDelayS         int                     `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
-	RestartOnWakeup         bool                    `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
-	AutoUpgradeIntervalH    int                     `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
-	UpgradeToPreReleases    bool                    `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"`              // when auto upgrades are enabled
-	KeepTemporariesH        int                     `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`                        // 0 for off
-	CacheIgnoredFiles       bool                    `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
-	ProgressUpdateIntervalS int                     `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
-	LimitBandwidthInLan     bool                    `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
-	MinHomeDiskFree         Size                    `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
-	ReleasesURL             string                  `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
-	AlwaysLocalNets         []string                `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
-	OverwriteRemoteDevNames bool                    `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
-	TempIndexMinBlocks      int                     `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
-	UnackedNotificationIDs  []string                `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
-	TrafficClass            int                     `xml:"trafficClass" json:"trafficClass"`
-	WeakHashSelectionMethod WeakHashSelectionMethod `xml:"weakHashSelectionMethod" json:"weakHashSelectionMethod" restart:"true"`
-	DefaultFolderPath       string                  `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
-	SetLowPriority          bool                    `xml:"setLowPriority" json:"setLowPriority" default:"true"`
+	ListenAddresses         []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
+	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
+	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
+	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
+	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
+	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
+	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
+	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
+	RelaysEnabled           bool     `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
+	RelayReconnectIntervalM int      `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
+	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
+	NATEnabled              bool     `xml:"natEnabled" json:"natEnabled" default:"true"`
+	NATLeaseM               int      `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
+	NATRenewalM             int      `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
+	NATTimeoutS             int      `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
+	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	URSeen                  int      `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
+	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
+	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
+	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
+	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
+	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
+	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
+	UpgradeToPreReleases    bool     `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"`              // when auto upgrades are enabled
+	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`                        // 0 for off
+	CacheIgnoredFiles       bool     `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
+	ProgressUpdateIntervalS int      `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
+	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
+	MinHomeDiskFree         Size     `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
+	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
+	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
+	OverwriteRemoteDevNames bool     `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
+	TempIndexMinBlocks      int      `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
+	UnackedNotificationIDs  []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
+	TrafficClass            int      `xml:"trafficClass" json:"trafficClass"`
+	DefaultFolderPath       string   `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
+	SetLowPriority          bool     `xml:"setLowPriority" json:"setLowPriority" default:"true"`
 
 	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
 	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`

lib/config/testdata/overridenvalues.xml

@@ -34,7 +34,6 @@
         <releasesURL>https://localhost/releases</releasesURL>
         <overwriteRemoteDeviceNamesOnConnect>true</overwriteRemoteDeviceNamesOnConnect>
         <tempIndexMinBlocks>100</tempIndexMinBlocks>
-        <weakHashSelectionMethod>never</weakHashSelectionMethod>
         <defaultFolderPath>/media/syncthing</defaultFolderPath>
         <setLowPriority>false</setLowPriority>
     </options>

lib/db/blockmap.go

@@ -182,19 +182,6 @@ func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string,
 	return false
 }
 
-// Fix repairs incorrect blockmap entries, removing the old entry and
-// replacing it with a new entry for the given block
-func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error {
-	buf := make([]byte, 4)
-	binary.BigEndian.PutUint32(buf, uint32(index))
-
-	folderID := f.db.folderIdx.ID([]byte(folder))
-	batch := new(leveldb.Batch)
-	batch.Delete(blockKeyInto(nil, oldHash, folderID, file))
-	batch.Put(blockKeyInto(nil, newHash, folderID, file), buf)
-	return f.db.Write(batch, nil)
-}
-
 // m.blockKey returns a byte slice encoding the following information:
 //	   keyTypeBlock (1 byte)
 //	   folder (4 bytes)

lib/db/blockmap_test.go

@@ -219,34 +219,3 @@ func TestBlockFinderLookup(t *testing.T) {
 
 	f1.Deleted = false
 }
-
-func TestBlockFinderFix(t *testing.T) {
-	db, f := setup()
-
-	iterFn := func(folder, file string, index int32) bool {
-		return true
-	}
-
-	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
-	err := m.Add([]protocol.FileInfo{f1})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
-		t.Fatal("Block not found")
-	}
-
-	err = f.Fix("folder1", f1.Name, 0, f1.Blocks[0].Hash, f2.Blocks[0].Hash)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if f.Iterate(folders, f1.Blocks[0].Hash, iterFn) {
-		t.Fatal("Unexpected block")
-	}
-
-	if !f.Iterate(folders, f2.Blocks[0].Hash, iterFn) {
-		t.Fatal("Block not found")
-	}
-}

lib/model/model.go

@@ -7,6 +7,7 @@
 package model
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
@@ -34,7 +35,6 @@ import (
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/syncthing/syncthing/lib/versioner"
-	"github.com/syncthing/syncthing/lib/weakhash"
 	"github.com/thejerf/suture"
 )
 
@@ -1304,7 +1304,7 @@ func (m *Model) closeLocked(device protocol.DeviceID) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
 	if offset < 0 {
 		return protocol.ErrInvalid
 	}
@@ -1362,8 +1362,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 			// other than a regular file.
 			return protocol.ErrNoSuchFile
 		}
-
-		if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
+		err := readOffsetIntoBuf(folderFs, tempFn, offset, buf)
+		if err == nil && scanner.Validate(buf, hash, weakHash) {
 			return nil
 		}
 		// Fall through to reading from a non-temp file, just incase the temp
@@ -1382,9 +1382,55 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		return protocol.ErrGeneric
 	}
 
+	if !scanner.Validate(buf, hash, weakHash) {
+		m.recheckFile(deviceID, folderFs, folder, name, int(offset)/len(buf), hash)
+		return protocol.ErrNoSuchFile
+	}
+
 	return nil
 }
 
+func (m *Model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem, folder, name string, blockIndex int, hash []byte) {
+	cf, ok := m.CurrentFolderFile(folder, name)
+	if !ok {
+		l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
+		return
+	}
+
+	if cf.IsDeleted() || cf.IsInvalid() || cf.IsSymlink() || cf.IsDirectory() {
+		l.Debugf("%v recheckFile: %s: %q / %q: not a regular file", m, deviceID, folder, name)
+		return
+	}
+
+	if blockIndex > len(cf.Blocks) {
+		l.Debugf("%v recheckFile: %s: %q / %q i=%d: block index too far", m, deviceID, folder, name, blockIndex)
+		return
+	}
+
+	block := cf.Blocks[blockIndex]
+
+	// Seems to want a different version of the file, whatever.
+	if !bytes.Equal(block.Hash, hash) {
+		l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
+		return
+	}
+
+	// The hashes provided part of the request match what we expect to find according
+	// to what we have in the database, yet the content we've read off the filesystem doesn't
+	// Something is fishy, invalidate the file and rescan it.
+	cf.Invalidate(m.shortID)
+
+	// Update the index and tell others
+	// The file will temporarily become invalid, which is ok as the content is messed up.
+	m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
+
+	if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
+		l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
+	} else {
+		l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
+	}
+}
+
 func (m *Model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
 	m.fmut.RLock()
 	fs, ok := m.folderFiles[folder]
@@ -1836,7 +1882,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
 	}
 }
 
-func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
+func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
 	m.pmut.RLock()
 	nc, ok := m.conn[deviceID]
 	m.pmut.RUnlock()
@@ -1845,9 +1891,9 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
 		return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
 	}
 
-	l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t", m, deviceID, folder, name, offset, size, hash, fromTemporary)
+	l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, offset, size, hash, weakHash, fromTemporary)
 
-	return nc.Request(folder, name, offset, size, hash, fromTemporary)
+	return nc.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
 }
 
 func (m *Model) ScanFolders() map[string]error {
@@ -1973,7 +2019,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 		Hashers:               m.numHashers(folder),
 		ShortID:               m.shortID,
 		ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
-		UseWeakHashes:         weakhash.Enabled,
 		UseLargeBlocks:        folderCfg.UseLargeBlocks,
 	})
 

lib/model/model_test.go

@@ -181,7 +181,7 @@ func TestRequest(t *testing.T) {
 
 	// Existing, shared file
 	bs = bs[:6]
-	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
+	err := m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -190,32 +190,32 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
+	err = m.Request(device2, "default", "foo", 0, nil, 0, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Nonexistent file
-	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
+	err = m.Request(device1, "default", "nonexistent", 0, nil, 0, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Shared folder, but disallowed file name
-	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
+	err = m.Request(device1, "default", "../walk.go", 0, nil, 0, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Negative offset
-	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
+	err = m.Request(device1, "default", "foo", -4, nil, 0, false, bs[:0])
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Larger block than available
 	bs = bs[:42]
-	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
+	err = m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -357,7 +357,7 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
 	return nil
 }
 
-func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
+func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
 	f.mut.Lock()
 	defer f.mut.Unlock()
 	if f.requestFn != nil {
@@ -485,7 +485,7 @@ func BenchmarkRequestOut(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
+		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
 		if err != nil {
 			b.Error(err)
 		}
@@ -513,7 +513,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
+		if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, 0, false, buf); err != nil {
 			b.Error(err)
 		}
 	}

lib/model/requests_test.go

@@ -93,7 +93,7 @@ func TestSymlinkTraversalRead(t *testing.T) {
 
 	// Request a file by traversing the symlink
 	buf := make([]byte, 10)
-	err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, false, buf)
+	err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, 0, false, buf)
 	if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
 		t.Error("Managed to traverse symlink")
 	}
@@ -464,6 +464,73 @@ func TestIssue4841(t *testing.T) {
 	}
 }
 
+func TestRescanIfHaveInvalidContent(t *testing.T) {
+	m, fc, tmpDir := setupModelWithConnection()
+	defer m.Stop()
+	defer os.RemoveAll(tmpDir)
+
+	payload := []byte("hello")
+
+	if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	received := make(chan protocol.FileInfo)
+	fc.mut.Lock()
+	fc.indexFn = func(folder string, fs []protocol.FileInfo) {
+		if len(fs) != 1 {
+			t.Fatalf("Sent index with %d files, should be 1", len(fs))
+		}
+		if fs[0].Name != "foo" {
+			t.Fatalf(`Sent index with file %v, should be "foo"`, fs[0].Name)
+		}
+		received <- fs[0]
+		return
+	}
+	fc.mut.Unlock()
+
+	// Scan without ignore patterns with "foo" not existing locally
+	if err := m.ScanFolder("default"); err != nil {
+		t.Fatal("Failed scanning:", err)
+	}
+
+	f := <-received
+	if f.Blocks[0].WeakHash != 103547413 {
+		t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
+	}
+
+	buf := make([]byte, len(payload))
+
+	err := m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf, payload) {
+		t.Errorf("%s != %s", buf, payload)
+	}
+
+	payload = []byte("bye")
+	buf = make([]byte, len(payload))
+
+	if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	err = m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
+	if err == nil {
+		t.Fatalf("expected failure")
+	}
+
+	select {
+	case f := <-received:
+		if f.Blocks[0].WeakHash != 41943361 {
+			t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
+		}
+	case <-time.After(time.Second):
+		t.Fatalf("timed out")
+	}
+}
+
 func setupModelWithConnection() (*Model, *fakeConnection, string) {
 	tmpDir := createTmpDir()
 	cfg := defaultCfgWrapper.RawCopy()

lib/model/rwfolder.go

@@ -1186,36 +1186,32 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 		var file fs.File
 		var weakHashFinder *weakhash.Finder
 
-		if weakhash.Enabled {
-			blocksPercentChanged := 0
-			if tot := len(state.file.Blocks); tot > 0 {
-				blocksPercentChanged = (tot - state.have) * 100 / tot
-			}
+		blocksPercentChanged := 0
+		if tot := len(state.file.Blocks); tot > 0 {
+			blocksPercentChanged = (tot - state.have) * 100 / tot
+		}
 
-			if blocksPercentChanged >= f.WeakHashThresholdPct {
-				hashesToFind := make([]uint32, 0, len(state.blocks))
-				for _, block := range state.blocks {
-					if block.WeakHash != 0 {
-						hashesToFind = append(hashesToFind, block.WeakHash)
-					}
+		if blocksPercentChanged >= f.WeakHashThresholdPct {
+			hashesToFind := make([]uint32, 0, len(state.blocks))
+			for _, block := range state.blocks {
+				if block.WeakHash != 0 {
+					hashesToFind = append(hashesToFind, block.WeakHash)
 				}
+			}
 
-				if len(hashesToFind) > 0 {
-					file, err = f.fs.Open(state.file.Name)
-					if err == nil {
-						weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
-						if err != nil {
-							l.Debugln("weak hasher", err)
-						}
+			if len(hashesToFind) > 0 {
+				file, err = f.fs.Open(state.file.Name)
+				if err == nil {
+					weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
+					if err != nil {
+						l.Debugln("weak hasher", err)
 					}
-				} else {
-					l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
 				}
 			} else {
-				l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
+				l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
 			}
 		} else {
-			l.Debugf("not weak hashing %s. weak hashing disabled", state.file.Name)
+			l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
 		}
 
 		for _, block := range state.blocks {
@@ -1239,7 +1235,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 			}
 
 			found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
-				if _, err := verifyBuffer(buf, block); err != nil {
+				if verifyBuffer(buf, block) != nil {
 					return true
 				}
 
@@ -1274,17 +1270,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 						return false
 					}
 
-					hash, err := verifyBuffer(buf, block)
-					if err != nil {
-						if hash != nil {
-							l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
-							err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
-							if err != nil {
-								l.Warnln("finder fix:", err)
-							}
-						} else {
-							l.Debugln("Finder failed to verify buffer", err)
-						}
+					if err := verifyBuffer(buf, block); err != nil {
+						l.Debugln("Finder failed to verify buffer", err)
 						return false
 					}
 
@@ -1324,22 +1311,22 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 	}
 }
 
-func verifyBuffer(buf []byte, block protocol.BlockInfo) ([]byte, error) {
+func verifyBuffer(buf []byte, block protocol.BlockInfo) error {
 	if len(buf) != int(block.Size) {
-		return nil, fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
+		return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
 	}
 	hf := sha256.New()
 	_, err := hf.Write(buf)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	hash := hf.Sum(nil)
 
 	if !bytes.Equal(hash, block.Hash) {
-		return hash, fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
+		return fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
 	}
 
-	return hash, nil
+	return nil
 }
 
 func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
@@ -1411,7 +1398,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
 		// Fetch the block, while marking the selected device as in use so that
 		// leastBusy can select another device when someone else asks.
 		activity.using(selected)
-		buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
+		buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
 		activity.done(selected)
 		if lastError != nil {
 			l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
@@ -1420,7 +1407,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
 
 		// Verify that the received block matches the desired hash, if not
 		// try pulling it from another device.
-		_, lastError = verifyBuffer(buf, state.block)
+		lastError = verifyBuffer(buf, state.block)
 		if lastError != nil {
 			l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
 			continue

lib/model/rwfolder_test.go

@@ -434,52 +434,6 @@ func TestCopierCleanup(t *testing.T) {
 	}
 }
 
-// Make sure that the copier routine hashes the content when asked, and pulls
-// if it fails to find the block.
-func TestLastResortPulling(t *testing.T) {
-	// Add a file to index (with the incorrect block representation, as content
-	// doesn't actually match the block list)
-	file := setUpFile("empty", []int{0})
-	m := setUpModel(file)
-
-	// Pretend that we are handling a new file of the same content but
-	// with a different name (causing to copy that particular block)
-	file.Name = "newfile"
-
-	iterFn := func(folder, file string, index int32) bool {
-		return true
-	}
-
-	f := setUpSendReceiveFolder(m)
-
-	copyChan := make(chan copyBlocksState)
-	pullChan := make(chan pullBlockState, 1)
-	finisherChan := make(chan *sharedPullerState, 1)
-	dbUpdateChan := make(chan dbUpdateJob, 1)
-
-	// Run a single copier routine
-	go f.copierRoutine(copyChan, pullChan, finisherChan)
-
-	f.handleFile(file, copyChan, finisherChan, dbUpdateChan)
-
-	// Copier should hash empty file, realise that the region it has read
-	// doesn't match the hash which was advertised by the block map, fix it
-	// and ask to pull the block.
-	<-pullChan
-
-	// Verify that it did fix the incorrect hash.
-	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
-		t.Error("Found unexpected block")
-	}
-
-	if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
-		t.Error("Expected block not found")
-	}
-
-	(<-finisherChan).fd.Close()
-	os.Remove(filepath.Join("testdata", fs.TempName("newfile")))
-}
-
 func TestDeregisterOnFailInCopy(t *testing.T) {
 	file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
 	defer os.Remove("testdata/" + fs.TempName("filex"))

lib/protocol/benchmark_test.go

@@ -80,9 +80,9 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
 		// Use c0 and c1 for each alternating request, so we get as much
 		// data flowing in both directions.
 		if i%2 == 0 {
-			buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, false)
+			buf, err = c0.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
 		} else {
-			buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, false)
+			buf, err = c1.Request("folder", "file", int64(i), 128<<10, nil, 0, false)
 		}
 
 		if err != nil {
@@ -171,7 +171,7 @@ func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
+func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHAsh uint32, fromTemporary bool, buf []byte) error {
 	// We write the offset to the end of the buffer, so the receiver
 	// can verify that it did in fact get some data back over the
 	// connection.

lib/protocol/bep.pb.go

@@ -353,6 +353,7 @@ type Request struct {
 	Size          int32  `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
 	Hash          []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
 	FromTemporary bool   `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
+	WeakHash      uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
 }
 
 func (m *Request) Reset()                    { *m = Request{} }
@@ -1062,6 +1063,11 @@ func (m *Request) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
+	if m.WeakHash != 0 {
+		dAtA[i] = 0x40
+		i++
+		i = encodeVarintBep(dAtA, i, uint64(m.WeakHash))
+	}
 	return i, nil
 }
 
@@ -1519,6 +1525,9 @@ func (m *Request) ProtoSize() (n int) {
 	if m.FromTemporary {
 		n += 2
 	}
+	if m.WeakHash != 0 {
+		n += 1 + sovBep(uint64(m.WeakHash))
+	}
 	return n
 }
 
@@ -3515,6 +3524,25 @@ func (m *Request) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.FromTemporary = bool(v != 0)
+		case 8:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType)
+			}
+			m.WeakHash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBep
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.WeakHash |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipBep(dAtA[iNdEx:])
@@ -4192,115 +4220,116 @@ var (
 func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
 
 var fileDescriptorBep = []byte{
-	// 1757 bytes of a gzipped FileDescriptorProto
+	// 1762 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
-	0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1,
-	0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32,
-	0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6,
-	0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83,
-	0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed,
-	0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28,
-	0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5,
-	0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31,
-	0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0,
-	0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e,
-	0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06,
-	0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d,
-	0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3,
-	0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb,
-	0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6,
-	0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f,
-	0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16,
-	0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14,
-	0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6,
-	0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27,
-	0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02,
-	0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89,
-	0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8,
-	0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08,
-	0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89,
-	0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37,
-	0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3,
-	0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0,
-	0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40,
-	0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd,
-	0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0,
-	0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf,
-	0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f,
-	0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2,
-	0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d,
-	0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff,
-	0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb,
-	0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0,
-	0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66,
-	0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43,
-	0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36,
-	0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04,
-	0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8,
-	0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64,
-	0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a,
-	0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05,
-	0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9,
-	0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e,
-	0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a,
-	0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17,
-	0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad,
-	0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d,
-	0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3,
-	0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53,
-	0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54,
-	0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68,
-	0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32,
-	0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8,
-	0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d,
-	0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4,
-	0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05,
-	0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99,
-	0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb,
-	0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a,
-	0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40,
-	0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8,
-	0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f,
-	0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5,
-	0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74,
-	0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50,
-	0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19,
-	0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2,
-	0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93,
-	0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb,
-	0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86,
-	0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94,
-	0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6,
-	0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19,
-	0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45,
-	0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab,
-	0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca,
-	0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef,
-	0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5,
-	0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5,
-	0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71,
-	0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26,
-	0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41,
-	0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70,
-	0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42,
-	0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5,
-	0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29,
-	0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40,
-	0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a,
-	0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7,
-	0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6,
-	0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc,
-	0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad,
-	0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29,
-	0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d,
-	0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3,
-	0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b,
-	0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e,
-	0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd,
-	0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b,
-	0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4,
-	0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb,
-	0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab,
-	0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff,
-	0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00,
+	0x15, 0x5f, 0x4a, 0x94, 0x44, 0x3d, 0x69, 0x37, 0xdc, 0xb1, 0xbd, 0x65, 0x99, 0x8d, 0x44, 0x2b,
+	0x76, 0xbc, 0x59, 0x24, 0x6b, 0x37, 0x49, 0x5b, 0xb4, 0x68, 0x0b, 0xe8, 0x0f, 0x77, 0x2d, 0x54,
+	0xa6, 0xd4, 0x91, 0xd6, 0xa9, 0x73, 0x28, 0x41, 0x89, 0x23, 0x2d, 0x61, 0x8a, 0xa3, 0x92, 0xd4,
+	0xda, 0xca, 0x47, 0xd0, 0x27, 0xe8, 0x45, 0x40, 0x80, 0x9e, 0x0a, 0xf4, 0x83, 0xf8, 0x98, 0xf6,
+	0xd0, 0x43, 0x0f, 0x46, 0xb3, 0xbd, 0xf4, 0xd8, 0x4f, 0x50, 0x14, 0x9c, 0x21, 0x25, 0x6a, 0xd7,
+	0x0e, 0x7c, 0xc8, 0x89, 0x33, 0xef, 0xfd, 0xe6, 0x0d, 0xdf, 0x6f, 0x7e, 0xef, 0xcd, 0x40, 0x71,
+	0x48, 0x66, 0x27, 0x33, 0x9f, 0x86, 0x14, 0x49, 0xec, 0x33, 0xa2, 0xae, 0xfa, 0xe9, 0xc4, 0x09,
+	0x2f, 0xe6, 0xc3, 0x93, 0x11, 0x9d, 0x3e, 0x9c, 0xd0, 0x09, 0x7d, 0xc8, 0x3c, 0xc3, 0xf9, 0x98,
+	0xcd, 0xd8, 0x84, 0x8d, 0xf8, 0xc2, 0xda, 0x0c, 0x72, 0x8f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92,
+	0x4d, 0x2e, 0x9d, 0x11, 0x31, 0x3d, 0x6b, 0x4a, 0x14, 0x41, 0x13, 0x8e, 0x8a, 0x18, 0xb8, 0xc9,
+	0xb0, 0xa6, 0x24, 0x02, 0x8c, 0x5c, 0x87, 0x78, 0x21, 0x07, 0x64, 0x38, 0x80, 0x9b, 0x18, 0xe0,
+	0x3e, 0xec, 0xc5, 0x80, 0x4b, 0xe2, 0x07, 0x0e, 0xf5, 0x94, 0x2c, 0xc3, 0xec, 0x72, 0xeb, 0x53,
+	0x6e, 0xac, 0x05, 0x90, 0x7f, 0x4c, 0x2c, 0x9b, 0xf8, 0xe8, 0x63, 0x10, 0xc3, 0xc5, 0x8c, 0xef,
+	0xb5, 0xf7, 0xd9, 0x9d, 0x93, 0x24, 0x87, 0x93, 0x27, 0x24, 0x08, 0xac, 0x09, 0x19, 0x2c, 0x66,
+	0x04, 0x33, 0x08, 0xfa, 0x0d, 0x94, 0x46, 0x74, 0x3a, 0xf3, 0x49, 0xc0, 0x02, 0x67, 0xd8, 0x8a,
+	0xc3, 0x1b, 0x2b, 0x9a, 0x1b, 0x0c, 0x4e, 0x2f, 0xa8, 0xd5, 0x61, 0xb7, 0xe9, 0xce, 0x83, 0x90,
+	0xf8, 0x4d, 0xea, 0x8d, 0x9d, 0x09, 0x7a, 0x04, 0x85, 0x31, 0x75, 0x6d, 0xe2, 0x07, 0x8a, 0xa0,
+	0x65, 0x8f, 0x4a, 0x9f, 0xc9, 0x9b, 0x60, 0xa7, 0xcc, 0xd1, 0x10, 0x5f, 0xbd, 0xae, 0xee, 0xe0,
+	0x04, 0x56, 0xfb, 0x73, 0x06, 0xf2, 0xdc, 0x83, 0x0e, 0x20, 0xe3, 0xd8, 0x9c, 0xa2, 0x46, 0xfe,
+	0xea, 0x75, 0x35, 0xd3, 0x6e, 0xe1, 0x8c, 0x63, 0xa3, 0xdb, 0x90, 0x73, 0xad, 0x21, 0x71, 0x63,
+	0x72, 0xf8, 0x04, 0xbd, 0x0f, 0x45, 0x9f, 0x58, 0xb6, 0x49, 0x3d, 0x77, 0xc1, 0x28, 0x91, 0xb0,
+	0x14, 0x19, 0xba, 0x9e, 0xbb, 0x40, 0x9f, 0x02, 0x72, 0x26, 0x1e, 0xf5, 0x89, 0x39, 0x23, 0xfe,
+	0xd4, 0x61, 0x7f, 0x1b, 0x28, 0x22, 0x43, 0xed, 0x73, 0x4f, 0x6f, 0xe3, 0x40, 0x1f, 0xc2, 0x6e,
+	0x0c, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0xe4, 0x18, 0xb2, 0xcc, 0x8d, 0x2d, 0x66, 0x43, 0x8f, 0xe0,
+	0xb6, 0xed, 0x04, 0xd6, 0xd0, 0x25, 0x66, 0x48, 0xa6, 0x33, 0xd3, 0xf1, 0x6c, 0xf2, 0x92, 0x04,
+	0x4a, 0x9e, 0x61, 0x51, 0xec, 0x1b, 0x90, 0xe9, 0xac, 0xcd, 0x3d, 0xe8, 0x00, 0xf2, 0x33, 0x6b,
+	0x1e, 0x10, 0x5b, 0x29, 0x30, 0x4c, 0x3c, 0x8b, 0x58, 0xe2, 0x0a, 0x08, 0x14, 0xf9, 0x3a, 0x4b,
+	0x2d, 0xe6, 0x48, 0x58, 0x8a, 0x61, 0xb5, 0xff, 0x66, 0x20, 0xcf, 0x3d, 0xe8, 0xa3, 0x35, 0x4b,
+	0xe5, 0xc6, 0x41, 0x84, 0xfa, 0xe7, 0xeb, 0xaa, 0xc4, 0x7d, 0xed, 0x56, 0x8a, 0x35, 0x04, 0x62,
+	0x4a, 0x51, 0x6c, 0x8c, 0x0e, 0xa1, 0x68, 0xd9, 0x76, 0x74, 0x7a, 0x24, 0x50, 0xb2, 0x5a, 0xf6,
+	0xa8, 0x88, 0x37, 0x06, 0xf4, 0xf3, 0x6d, 0x35, 0x88, 0xd7, 0xf5, 0xf3, 0x36, 0x19, 0x44, 0x47,
+	0x31, 0x22, 0x7e, 0xac, 0xe0, 0x1c, 0xdb, 0x4f, 0x8a, 0x0c, 0x4c, 0xbf, 0x77, 0xa1, 0x3c, 0xb5,
+	0x5e, 0x9a, 0x01, 0xf9, 0xe3, 0x9c, 0x78, 0x23, 0xc2, 0xe8, 0xca, 0xe2, 0xd2, 0xd4, 0x7a, 0xd9,
+	0x8f, 0x4d, 0xa8, 0x02, 0xe0, 0x78, 0xa1, 0x4f, 0xed, 0xf9, 0x88, 0xf8, 0x31, 0x57, 0x29, 0x0b,
+	0xfa, 0x29, 0x48, 0x8c, 0x6c, 0xd3, 0xb1, 0x15, 0x49, 0x13, 0x8e, 0xc4, 0x86, 0x1a, 0x27, 0x5e,
+	0x60, 0x54, 0xb3, 0xbc, 0x93, 0x21, 0x2e, 0x30, 0x6c, 0xdb, 0x46, 0xbf, 0x02, 0x35, 0x78, 0xee,
+	0x44, 0x07, 0xc5, 0x23, 0x85, 0x0e, 0xf5, 0x4c, 0x9f, 0x4c, 0xe9, 0xa5, 0xe5, 0x06, 0x4a, 0x91,
+	0x6d, 0xa3, 0x44, 0x88, 0x76, 0x0a, 0x80, 0x63, 0x7f, 0xad, 0x0b, 0x39, 0x16, 0x31, 0x3a, 0x45,
+	0x2e, 0xd6, 0xb8, 0x7a, 0xe3, 0x19, 0x3a, 0x81, 0xdc, 0xd8, 0x71, 0x49, 0xa0, 0x64, 0xd8, 0x19,
+	0xa2, 0x94, 0xd2, 0x1d, 0x97, 0xb4, 0xbd, 0x31, 0x8d, 0x4f, 0x91, 0xc3, 0x6a, 0xe7, 0x50, 0x62,
+	0x01, 0xcf, 0x67, 0xb6, 0x15, 0x92, 0x1f, 0x2c, 0xec, 0x5f, 0x45, 0x90, 0x12, 0xcf, 0xfa, 0xd0,
+	0x85, 0xd4, 0xa1, 0x1f, 0xc7, 0xfd, 0x80, 0x57, 0xf7, 0xc1, 0xcd, 0x78, 0xa9, 0x86, 0x80, 0x40,
+	0x0c, 0x9c, 0xaf, 0x09, 0xab, 0xa7, 0x2c, 0x66, 0x63, 0xa4, 0x41, 0xe9, 0x7a, 0x11, 0xed, 0xe2,
+	0xb4, 0x09, 0x7d, 0x00, 0x30, 0xa5, 0xb6, 0x33, 0x76, 0x88, 0x6d, 0x06, 0x4c, 0x00, 0x59, 0x5c,
+	0x4c, 0x2c, 0x7d, 0xa4, 0x44, 0x72, 0x8f, 0x4a, 0xc8, 0x8e, 0x6b, 0x25, 0x99, 0x46, 0x1e, 0xc7,
+	0xbb, 0xb4, 0x5c, 0x27, 0xa9, 0x90, 0x64, 0x1a, 0x75, 0x3d, 0x8f, 0x6e, 0x15, 0xaf, 0xc4, 0x00,
+	0xbb, 0x1e, 0x4d, 0x17, 0xee, 0x23, 0x28, 0x24, 0x5d, 0x31, 0x3a, 0xcf, 0xad, 0x4a, 0x7a, 0x4a,
+	0x46, 0x21, 0x5d, 0xf7, 0x9b, 0x18, 0x86, 0x54, 0x90, 0xd6, 0x52, 0x04, 0xf6, 0xa7, 0xeb, 0x79,
+	0xd4, 0x8b, 0xd7, 0x79, 0x78, 0x81, 0x52, 0xd2, 0x84, 0xa3, 0x1c, 0x5e, 0xa7, 0x66, 0x44, 0xdb,
+	0x6d, 0x00, 0xc3, 0x85, 0x52, 0x66, 0x5a, 0x7c, 0x2f, 0xd1, 0x62, 0xff, 0x82, 0xfa, 0x61, 0xbb,
+	0xb5, 0x59, 0xd1, 0x58, 0xa0, 0x87, 0x00, 0x43, 0x97, 0x8e, 0x9e, 0x9b, 0x8c, 0xd6, 0xdd, 0x28,
+	0x62, 0x43, 0xbe, 0x7a, 0x5d, 0x2d, 0x63, 0xeb, 0x45, 0x23, 0x72, 0xf4, 0x9d, 0xaf, 0x09, 0x2e,
+	0x0e, 0x93, 0x21, 0xfa, 0x09, 0xe4, 0x99, 0x3d, 0x69, 0x0d, 0xb7, 0x36, 0x09, 0x31, 0x7b, 0x4a,
+	0x00, 0x31, 0x30, 0xe2, 0x2a, 0x58, 0x4c, 0x5d, 0xc7, 0x7b, 0x6e, 0x86, 0x96, 0x3f, 0x21, 0xa1,
+	0xb2, 0xcf, 0x6f, 0x88, 0xd8, 0x3a, 0x60, 0xc6, 0x5f, 0x8a, 0x7f, 0xfa, 0xa6, 0xba, 0x53, 0xf3,
+	0xa0, 0xb8, 0x8e, 0x13, 0x69, 0x90, 0x8e, 0xc7, 0x01, 0x09, 0x99, 0x60, 0xb2, 0x38, 0x9e, 0xad,
+	0x65, 0x90, 0x61, 0x0c, 0x70, 0x19, 0x20, 0x10, 0x2f, 0xac, 0xe0, 0x82, 0x49, 0xa3, 0x8c, 0xd9,
+	0x38, 0x2a, 0xfc, 0x17, 0xc4, 0x7a, 0x6e, 0x32, 0x07, 0x17, 0x86, 0x14, 0x19, 0x1e, 0x5b, 0xc1,
+	0x45, 0xbc, 0xdf, 0xaf, 0x21, 0xcf, 0x0f, 0x02, 0x7d, 0x0e, 0xd2, 0x88, 0xce, 0xbd, 0x70, 0x73,
+	0x39, 0xec, 0xa7, 0x7b, 0x0b, 0xf3, 0xc4, 0x99, 0xad, 0x81, 0xb5, 0x53, 0x28, 0xc4, 0x2e, 0x74,
+	0x7f, 0xdd, 0xf8, 0xc4, 0xc6, 0x9d, 0x6b, 0x9c, 0x6f, 0xdf, 0x16, 0x97, 0x96, 0x3b, 0xe7, 0x3f,
+	0x2f, 0x62, 0x3e, 0xa9, 0xfd, 0x4d, 0x80, 0x02, 0x8e, 0xce, 0x39, 0x08, 0x53, 0xf7, 0x4c, 0x6e,
+	0xeb, 0x9e, 0xd9, 0x54, 0x64, 0x66, 0xab, 0x22, 0x93, 0xa2, 0xca, 0xa6, 0x8a, 0x6a, 0xc3, 0x9c,
+	0xf8, 0x46, 0xe6, 0x72, 0x6f, 0x60, 0x2e, 0x9f, 0x62, 0xee, 0x3e, 0xec, 0x8d, 0x7d, 0x3a, 0x65,
+	0x37, 0x09, 0xf5, 0x2d, 0x7f, 0x11, 0x17, 0xc0, 0x6e, 0x64, 0x1d, 0x24, 0xc6, 0x6d, 0x82, 0xa5,
+	0x6d, 0x82, 0x6b, 0x26, 0x48, 0x98, 0x04, 0x33, 0xea, 0x05, 0xe4, 0xad, 0x39, 0x21, 0x10, 0x6d,
+	0x2b, 0xb4, 0x58, 0x46, 0x65, 0xcc, 0xc6, 0xe8, 0x01, 0x88, 0x23, 0x6a, 0xf3, 0x7c, 0xf6, 0xd2,
+	0x02, 0xd3, 0x7d, 0x9f, 0xfa, 0x4d, 0x6a, 0x13, 0xcc, 0x00, 0xb5, 0x19, 0xc8, 0x2d, 0xfa, 0xc2,
+	0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xda, 0xfd, 0x5b, 0xdb, 0x56, 0x0b, 0x0a, 0x73, 0xd6,
+	0xd8, 0x92, 0xc6, 0x75, 0x6f, 0xbb, 0xd1, 0x5c, 0x0f, 0xc4, 0xbb, 0x60, 0x52, 0x9d, 0xf1, 0xd2,
+	0xda, 0x3f, 0x04, 0x50, 0xdf, 0x8e, 0x46, 0x6d, 0x28, 0x71, 0xa4, 0x99, 0x7a, 0xe1, 0x1c, 0xbd,
+	0xcb, 0x46, 0xac, 0xc7, 0xc1, 0x7c, 0x3d, 0x7e, 0xe3, 0xf5, 0x98, 0xea, 0x26, 0xd9, 0x77, 0xeb,
+	0x26, 0x0f, 0x60, 0x97, 0x97, 0x77, 0xf2, 0x18, 0x10, 0xb5, 0xec, 0x51, 0xae, 0x91, 0x91, 0x77,
+	0x70, 0x79, 0xc8, 0xcb, 0x8c, 0xd9, 0x6b, 0x79, 0x10, 0x7b, 0x8e, 0x37, 0xa9, 0x55, 0x21, 0xd7,
+	0x74, 0x29, 0x3b, 0xb0, 0xbc, 0x4f, 0xac, 0x80, 0x7a, 0x09, 0x8f, 0x7c, 0x76, 0xfc, 0xf7, 0x0c,
+	0x94, 0x52, 0x0f, 0x35, 0xf4, 0x08, 0xf6, 0x9a, 0x9d, 0xf3, 0xfe, 0x40, 0xc7, 0x66, 0xb3, 0x6b,
+	0x9c, 0xb6, 0xcf, 0xe4, 0x1d, 0xf5, 0x70, 0xb9, 0xd2, 0x94, 0xe9, 0x06, 0xb4, 0xfd, 0x06, 0xab,
+	0x42, 0xae, 0x6d, 0xb4, 0xf4, 0xdf, 0xcb, 0x82, 0x7a, 0x7b, 0xb9, 0xd2, 0xe4, 0x14, 0x90, 0x5f,
+	0x68, 0x9f, 0x40, 0x99, 0x01, 0xcc, 0xf3, 0x5e, 0xab, 0x3e, 0xd0, 0xe5, 0x8c, 0xaa, 0x2e, 0x57,
+	0xda, 0xc1, 0x75, 0x5c, 0xcc, 0xf9, 0x87, 0x50, 0xc0, 0xfa, 0xef, 0xce, 0xf5, 0xfe, 0x40, 0xce,
+	0xaa, 0x07, 0xcb, 0x95, 0x86, 0x52, 0xc0, 0xa4, 0xa4, 0xee, 0x83, 0x84, 0xf5, 0x7e, 0xaf, 0x6b,
+	0xf4, 0x75, 0x59, 0x54, 0x7f, 0xb4, 0x5c, 0x69, 0xb7, 0xb6, 0x50, 0xb1, 0x4a, 0x7f, 0x06, 0xfb,
+	0xad, 0xee, 0x97, 0x46, 0xa7, 0x5b, 0x6f, 0x99, 0x3d, 0xdc, 0x3d, 0xc3, 0x7a, 0xbf, 0x2f, 0xe7,
+	0xd4, 0xea, 0x72, 0xa5, 0xbd, 0x9f, 0xc2, 0xdf, 0x10, 0xdd, 0x07, 0x20, 0xf6, 0xda, 0xc6, 0x99,
+	0x9c, 0x57, 0x6f, 0x2d, 0x57, 0xda, 0x7b, 0x29, 0x68, 0x44, 0x6a, 0x94, 0x71, 0xb3, 0xd3, 0xed,
+	0xeb, 0x72, 0xe1, 0x46, 0xc6, 0x8c, 0xec, 0xe3, 0x3f, 0x00, 0xba, 0xf9, 0x94, 0x45, 0xf7, 0x40,
+	0x34, 0xba, 0x86, 0x2e, 0xef, 0xf0, 0xfc, 0x6f, 0x22, 0x0c, 0xea, 0x11, 0x54, 0x83, 0x6c, 0xe7,
+	0xab, 0x2f, 0x64, 0x41, 0xfd, 0xf1, 0x72, 0xa5, 0xdd, 0xb9, 0x09, 0xea, 0x7c, 0xf5, 0xc5, 0x31,
+	0x85, 0x52, 0x3a, 0x70, 0x0d, 0xa4, 0x27, 0xfa, 0xa0, 0xde, 0xaa, 0x0f, 0xea, 0xf2, 0x0e, 0xff,
+	0xa5, 0xc4, 0xfd, 0x84, 0x84, 0x16, 0x2b, 0xc2, 0x43, 0xc8, 0x19, 0xfa, 0x53, 0x1d, 0xcb, 0x82,
+	0xba, 0xbf, 0x5c, 0x69, 0xbb, 0x09, 0xc0, 0x20, 0x97, 0xc4, 0x47, 0x15, 0xc8, 0xd7, 0x3b, 0x5f,
+	0xd6, 0x9f, 0xf5, 0xe5, 0x8c, 0x8a, 0x96, 0x2b, 0x6d, 0x2f, 0x71, 0xd7, 0xdd, 0x17, 0xd6, 0x22,
+	0x38, 0xfe, 0x9f, 0x00, 0xe5, 0xf4, 0xf5, 0x8d, 0x2a, 0x20, 0x9e, 0xb6, 0x3b, 0x7a, 0xb2, 0x5d,
+	0xda, 0x17, 0x8d, 0xd1, 0x11, 0x14, 0x5b, 0x6d, 0xac, 0x37, 0x07, 0x5d, 0xfc, 0x2c, 0xc9, 0x25,
+	0x0d, 0x6a, 0x39, 0x3e, 0x13, 0xf8, 0x02, 0xfd, 0x02, 0xca, 0xfd, 0x67, 0x4f, 0x3a, 0x6d, 0xe3,
+	0xb7, 0x26, 0x8b, 0x98, 0x51, 0x1f, 0x2c, 0x57, 0xda, 0xdd, 0x2d, 0x30, 0x99, 0xf9, 0x64, 0x64,
+	0x85, 0xc4, 0xee, 0xf3, 0x1b, 0x26, 0x72, 0x4a, 0x02, 0x6a, 0xc2, 0x7e, 0xb2, 0x74, 0xb3, 0x59,
+	0x56, 0xfd, 0x64, 0xb9, 0xd2, 0x3e, 0xfa, 0xde, 0xf5, 0xeb, 0xdd, 0x25, 0x01, 0xdd, 0x83, 0x42,
+	0x1c, 0x24, 0x51, 0x52, 0x7a, 0x69, 0xbc, 0xe0, 0xf8, 0x2f, 0x02, 0x14, 0xd7, 0xed, 0x2a, 0x22,
+	0xdc, 0xe8, 0x9a, 0x3a, 0xc6, 0x5d, 0x9c, 0x30, 0xb0, 0x76, 0x1a, 0x94, 0x0d, 0xd1, 0x5d, 0x28,
+	0x9c, 0xe9, 0x86, 0x8e, 0xdb, 0xcd, 0xa4, 0x30, 0xd6, 0x90, 0x33, 0xe2, 0x11, 0xdf, 0x19, 0xa1,
+	0x8f, 0xa1, 0x6c, 0x74, 0xcd, 0xfe, 0x79, 0xf3, 0x71, 0x92, 0x3a, 0xdb, 0x3f, 0x15, 0xaa, 0x3f,
+	0x1f, 0x5d, 0x30, 0x3e, 0x8f, 0xa3, 0x1a, 0x7a, 0x5a, 0xef, 0xb4, 0x5b, 0x1c, 0x9a, 0x55, 0x95,
+	0xe5, 0x4a, 0xbb, 0xbd, 0x86, 0xb6, 0xf9, 0x3b, 0x26, 0xc2, 0x1e, 0xdb, 0x50, 0xf9, 0xfe, 0xc6,
+	0x84, 0x34, 0xc8, 0xd7, 0x7b, 0x3d, 0xdd, 0x68, 0x25, 0x7f, 0xbf, 0xf1, 0xd5, 0x67, 0x33, 0xe2,
+	0xd9, 0x11, 0xe2, 0xb4, 0x8b, 0xcf, 0xf4, 0x41, 0xf2, 0xf3, 0x1b, 0xc4, 0x29, 0x8d, 0xae, 0xf7,
+	0xc6, 0xe1, 0xab, 0xef, 0x2a, 0x3b, 0xdf, 0x7e, 0x57, 0xd9, 0x79, 0x75, 0x55, 0x11, 0xbe, 0xbd,
+	0xaa, 0x08, 0xff, 0xba, 0xaa, 0xec, 0xfc, 0xe7, 0xaa, 0x22, 0x7c, 0xf3, 0xef, 0x8a, 0x30, 0xcc,
+	0xb3, 0x46, 0xf6, 0xf9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe5, 0x17, 0x2b, 0x62, 0xdd, 0x0e,
+	0x00, 0x00,
 }

lib/protocol/bep.proto

@@ -146,6 +146,7 @@ message Request {
     int32  size           = 5;
     bytes  hash           = 6;
     bool   from_temporary = 7;
+    uint32 weak_hash      = 8;
 }
 
 // Response

lib/protocol/common_test.go

@@ -11,6 +11,7 @@ type TestModel struct {
 	offset        int64
 	size          int
 	hash          []byte
+	weakHash      uint32
 	fromTemporary bool
 	closedCh      chan struct{}
 	closedErr     error
@@ -28,12 +29,13 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
 	t.size = len(buf)
 	t.hash = hash
+	t.weakHash = weakHash
 	t.fromTemporary = fromTemporary
 	copy(buf, t.data)
 	return nil

lib/protocol/nativemodel_darwin.go

@@ -26,7 +26,7 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 	m.Model.IndexUpdate(deviceID, folder, files)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
 	name = norm.NFD.String(name)
-	return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
+	return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
 }

lib/protocol/nativemodel_windows.go

@@ -25,14 +25,14 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 	m.Model.IndexUpdate(deviceID, folder, files)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
 	if strings.Contains(name, `\`) {
 		l.Warnf("Dropping request for %s, contains invalid path separator", name)
 		return ErrNoSuchFile
 	}
 
 	name = filepath.FromSlash(name)
-	return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
+	return m.Model.Request(deviceID, folder, name, offset, hash, weakHash, fromTemporary, buf)
 }
 
 func fixupFiles(files []FileInfo) []FileInfo {

lib/protocol/protocol.go

@@ -107,7 +107,7 @@ type Model interface {
 	// An index update was received from the peer device
 	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
 	// A request was made by the peer device
-	Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error
+	Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error
 	// A cluster configuration message was received
 	ClusterConfig(deviceID DeviceID, config ClusterConfig)
 	// The peer device closed the connection
@@ -122,7 +122,7 @@ type Connection interface {
 	Name() string
 	Index(folder string, files []FileInfo) error
 	IndexUpdate(folder string, files []FileInfo) error
-	Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
+	Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
 	ClusterConfig(config ClusterConfig)
 	DownloadProgress(folder string, updates []FileDownloadProgressUpdate)
 	Statistics() Statistics
@@ -254,7 +254,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
-func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
+func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
 	c.nextIDMut.Lock()
 	id := c.nextID
 	c.nextID++
@@ -275,6 +275,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
 		Offset:        offset,
 		Size:          int32(size),
 		Hash:          hash,
+		WeakHash:      weakHash,
 		FromTemporary: fromTemporary,
 	}, nil)
 	if !ok {
@@ -584,7 +585,7 @@ func (c *rawConnection) handleRequest(req Request) {
 		buf = make([]byte, size)
 	}
 
-	err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.FromTemporary, buf)
+	err := c.receiver.Request(c.id, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash, req.FromTemporary, buf)
 	if err != nil {
 		c.send(&Response{
 			ID:   req.ID,

lib/protocol/protocol_test.go

@@ -72,7 +72,7 @@ func TestClose(t *testing.T) {
 	c0.Index("default", nil)
 	c0.Index("default", nil)
 
-	if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
+	if _, err := c0.Request("default", "foo", 0, 0, nil, 0, false); err == nil {
 		t.Error("Request should return an error")
 	}
 }

lib/protocol/wireformat.go

@@ -34,7 +34,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
 	return c.Connection.IndexUpdate(folder, myFs)
 }
 
-func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
+func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
 	name = norm.NFC.String(filepath.ToSlash(name))
-	return c.Connection.Request(folder, name, offset, size, hash, fromTemporary)
+	return c.Connection.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
 }

lib/scanner/blockqueue.go

@@ -62,26 +62,24 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
 // workers are used in parallel. The outbox will become closed when the inbox
 // is closed and all items handled.
 type parallelHasher struct {
-	fs            fs.Filesystem
-	workers       int
-	outbox        chan<- protocol.FileInfo
-	inbox         <-chan protocol.FileInfo
-	counter       Counter
-	done          chan<- struct{}
-	useWeakHashes bool
-	wg            sync.WaitGroup
+	fs      fs.Filesystem
+	workers int
+	outbox  chan<- protocol.FileInfo
+	inbox   <-chan protocol.FileInfo
+	counter Counter
+	done    chan<- struct{}
+	wg      sync.WaitGroup
 }
 
-func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
+func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}) {
 	ph := &parallelHasher{
-		fs:            fs,
-		workers:       workers,
-		outbox:        outbox,
-		inbox:         inbox,
-		counter:       counter,
-		done:          done,
-		useWeakHashes: useWeakHashes,
-		wg:            sync.NewWaitGroup(),
+		fs:      fs,
+		workers: workers,
+		outbox:  outbox,
+		inbox:   inbox,
+		counter: counter,
+		done:    done,
+		wg:      sync.NewWaitGroup(),
 	}
 
 	for i := 0; i < workers; i++ {
@@ -106,7 +104,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
 				panic("Bug. Asked to hash a directory or a deleted file.")
 			}
 
-			blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes)
+			blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
 			if err != nil {
 				l.Debugln("hash error:", f.Name, err)
 				continue

lib/scanner/blocks.go

@@ -7,6 +7,7 @@
 package scanner
 
 import (
+	"bytes"
 	"context"
 	"hash"
 	"io"
@@ -107,6 +108,29 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
 	return blocks, nil
 }
 
+func Validate(buf, hash []byte, weakHash uint32) bool {
+	rd := bytes.NewReader(buf)
+	if weakHash != 0 {
+		whf := adler32.New()
+		if _, err := io.Copy(whf, rd); err == nil && whf.Sum32() == weakHash {
+			return true
+		}
+		// Copy error or mismatch, go to next algo.
+		rd.Seek(0, io.SeekStart)
+	}
+
+	if len(hash) > 0 {
+		hf := sha256.New()
+		if _, err := io.Copy(hf, rd); err == nil {
+			// Sum allocates, so let's hope we don't hit this often.
+			return bytes.Equal(hf.Sum(nil), hash)
+		}
+	}
+
+	// Both algos failed or no hashes were specified. Assume it's all good.
+	return true
+}
+
 type noopHash struct{}
 
 func (noopHash) Sum32() uint32             { return 0 }

lib/scanner/walk.go

@@ -64,8 +64,6 @@ type Config struct {
 	// Optional progress tick interval which defines how often FolderScanProgress
 	// events are emitted. Negative number means disabled.
 	ProgressTickIntervalS int
-	// Whether or not we should also compute weak hashes
-	UseWeakHashes bool
 	// Whether to use large blocks for large files or the old standard of 128KiB for everything.
 	UseLargeBlocks bool
 }
@@ -120,7 +118,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
 	// We're not required to emit scan progress events, just kick off hashers,
 	// and feed inputs directly from the walker.
 	if w.ProgressTickIntervalS < 0 {
-		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
 		return finishedChan
 	}
 
@@ -151,7 +149,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
 		done := make(chan struct{})
 		progress := newByteCounter()
 
-		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
 
 		// A routine which actually emits the FolderScanProgress events
 		// every w.ProgressTicker ticks, until the hasher routines terminate.

lib/weakhash/benchmark_test.go

@@ -11,6 +11,10 @@ import (
 	"testing"
 
 	"github.com/chmduquesne/rollinghash/adler32"
+	"github.com/chmduquesne/rollinghash/bozo32"
+	"github.com/chmduquesne/rollinghash/buzhash32"
+	"github.com/chmduquesne/rollinghash/buzhash64"
+	"github.com/chmduquesne/rollinghash/rabinkarp64"
 )
 
 const testFile = "../model/testdata/~syncthing~file.tmp"
@@ -59,3 +63,115 @@ func BenchmarkWeakHashAdler32Roll(b *testing.B) {
 
 	b.SetBytes(size)
 }
+
+func BenchmarkWeakHashRabinKarp64(b *testing.B) {
+	data := make([]byte, size)
+	hf := rabinkarp64.New()
+
+	for i := 0; i < b.N; i++ {
+		hf.Write(data)
+	}
+
+	_ = hf.Sum64()
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashRabinKarp64Roll(b *testing.B) {
+	data := make([]byte, size)
+	hf := rabinkarp64.New()
+	hf.Write(data)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		for i := 0; i <= size; i++ {
+			hf.Roll('a')
+		}
+	}
+
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBozo32(b *testing.B) {
+	data := make([]byte, size)
+	hf := bozo32.New()
+
+	for i := 0; i < b.N; i++ {
+		hf.Write(data)
+	}
+
+	_ = hf.Sum32()
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBozo32Roll(b *testing.B) {
+	data := make([]byte, size)
+	hf := bozo32.New()
+	hf.Write(data)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		for i := 0; i <= size; i++ {
+			hf.Roll('a')
+		}
+	}
+
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBuzhash32(b *testing.B) {
+	data := make([]byte, size)
+	hf := buzhash32.New()
+
+	for i := 0; i < b.N; i++ {
+		hf.Write(data)
+	}
+
+	_ = hf.Sum32()
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBuzhash32Roll(b *testing.B) {
+	data := make([]byte, size)
+	hf := buzhash32.New()
+	hf.Write(data)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		for i := 0; i <= size; i++ {
+			hf.Roll('a')
+		}
+	}
+
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBuzhash64(b *testing.B) {
+	data := make([]byte, size)
+	hf := buzhash64.New()
+
+	for i := 0; i < b.N; i++ {
+		hf.Write(data)
+	}
+
+	_ = hf.Sum64()
+	b.SetBytes(size)
+}
+
+func BenchmarkWeakHashBuzhash64Roll(b *testing.B) {
+	data := make([]byte, size)
+	hf := buzhash64.New()
+	hf.Write(data)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		for i := 0; i <= size; i++ {
+			hf.Roll('a')
+		}
+	}
+
+	b.SetBytes(size)
+}

lib/weakhash/weakhash.go

@@ -20,10 +20,6 @@ const (
 	maxWeakhashFinderHits = 10
 )
 
-var (
-	Enabled = true
-)
-
 // Find finds all the blocks of the given size within io.Reader that matches
 // the hashes provided, and returns a hash -> slice of offsets within reader
 // map, that produces the same weak hash.

vendor/manifest

@@ -94,7 +94,7 @@
 			"importpath": "github.com/chmduquesne/rollinghash",
 			"repository": "https://github.com/chmduquesne/rollinghash",
 			"vcs": "git",
-			"revision": "3dc7875a1f890f9bcf0619adb5571fc6f7d516bb",
+			"revision": "abb8cbaf9915e48ee20cae94bcd94221b61707a2",
 			"branch": "master",
 			"notests": true
 		},