lib/model, lib/scanner: Efficient inserts/deletes in the middle of the file

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3527

lib/config/folderconfiguration.go

@@ -40,6 +40,7 @@ type FolderConfiguration struct {
 	DisableSparseFiles    bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
 	Fsync                 bool                        `xml:"fsync" json:"fsync"`
+	DisableWeakHash       bool                        `xml:"disableWeakHash" json:"disableWeakHash"`
 
 	cachedPath string
 

lib/db/structs.pb.go

@@ -954,33 +954,33 @@ var (
 )
 
 var fileDescriptorStructs = []byte{
-	// 442 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x5d, 0x8b, 0xd3, 0x40,
-	0x14, 0x6d, 0xb7, 0xd9, 0x7e, 0xdc, 0xd8, 0xd5, 0x1d, 0x64, 0x19, 0x0a, 0xb6, 0x4b, 0x41, 0x10,
-	0xc1, 0x54, 0x57, 0x7c, 0xf1, 0x71, 0x1f, 0x16, 0x04, 0x11, 0x19, 0x97, 0xf5, 0xb1, 0x34, 0x99,
-	0xdb, 0xec, 0x60, 0x32, 0x53, 0x33, 0x93, 0x42, 0xfd, 0x25, 0xbe, 0xb9, 0x3f, 0xa7, 0x8f, 0xfe,
-	0x02, 0xd1, 0xfa, 0x47, 0x9c, 0xce, 0xa4, 0x31, 0x8f, 0xfb, 0x10, 0xb8, 0xe7, 0x9e, 0x73, 0xee,
-	0x3d, 0x93, 0x0b, 0x43, 0x6d, 0x8a, 0x32, 0x31, 0x3a, 0x5a, 0x15, 0xca, 0x28, 0x72, 0xc4, 0xe3,
-	0xd1, 0x8b, 0x54, 0x98, 0xdb, 0x32, 0x8e, 0x12, 0x95, 0xcf, 0x52, 0x95, 0xaa, 0x99, 0xa3, 0xe2,
-	0x72, 0xe9, 0x90, 0x03, 0xae, 0xf2, 0x96, 0xd1, 0x9b, 0x86, 0x5c, 0x6f, 0x64, 0x62, 0x6e, 0x85,
-	0x4c, 0x1b, 0x55, 0x26, 0x62, 0x3f, 0x21, 0x51, 0xd9, 0x2c, 0xc6, 0x95, 0xb7, 0x4d, 0x3f, 0x43,
-	0x78, 0x25, 0x32, 0xbc, 0xc1, 0x42, 0x0b, 0x25, 0xc9, 0x4b, 0xe8, 0xad, 0x7d, 0x49, 0xdb, 0xe7,
-	0xed, 0x67, 0xe1, 0xc5, 0xa3, 0xe8, 0x60, 0x8a, 0x6e, 0x30, 0x31, 0xaa, 0xb8, 0x0c, 0xb6, 0xbf,
-	0x26, 0x2d, 0x76, 0x90, 0x91, 0x33, 0xe8, 0x72, 0x5c, 0x8b, 0x04, 0xe9, 0x91, 0x35, 0x3c, 0x60,
-	0x15, 0x9a, 0x5e, 0x41, 0x58, 0x0d, 0x7d, 0x2f, 0xb4, 0x21, 0xaf, 0xa0, 0x5f, 0x39, 0xb4, 0x9d,
-	0xdc, 0xb1, 0x93, 0x1f, 0x46, 0x3c, 0x8e, 0x1a, 0xbb, 0xab, 0xc1, 0xb5, 0xec, 0x6d, 0xf0, 0xfd,
-	0x6e, 0xd2, 0x9a, 0xfe, 0xe8, 0xc0, 0xe9, 0x5e, 0xf5, 0x4e, 0x2e, 0xd5, 0x75, 0x51, 0xca, 0x64,
-	0x61, 0x90, 0x13, 0x02, 0x81, 0x5c, 0xe4, 0xe8, 0x42, 0x0e, 0x98, 0xab, 0xc9, 0x73, 0x08, 0xcc,
-	0x66, 0xe5, 0x73, 0x9c, 0x5c, 0x9c, 0xfd, 0x0f, 0x5e, 0xdb, 0x2d, 0xcb, 0x9c, 0x66, 0xef, 0xd7,
-	0xe2, 0x1b, 0xd2, 0x8e, 0xd5, 0x76, 0x98, 0xab, 0xc9, 0x39, 0x84, 0x2b, 0x2c, 0x72, 0xa1, 0x7d,
-	0xca, 0xc0, 0x52, 0x43, 0xd6, 0x6c, 0x91, 0x27, 0x00, 0xb9, 0xe2, 0x62, 0x29, 0x90, 0xcf, 0x35,
-	0x3d, 0x76, 0xde, 0xc1, 0xa1, 0xf3, 0x89, 0x50, 0xe8, 0x71, 0xcc, 0xd0, 0xe6, 0xa3, 0x5d, 0xcb,
-	0xf5, 0xd9, 0x01, 0xee, 0x19, 0x21, 0xd7, 0x8b, 0x4c, 0x70, 0xda, 0xf3, 0x4c, 0x05, 0xc9, 0x53,
-	0x38, 0x91, 0x6a, 0xde, 0xdc, 0xdb, 0x77, 0x82, 0xa1, 0x54, 0x1f, 0x1b, 0x9b, 0x1b, 0x77, 0x19,
-	0xdc, 0xef, 0x2e, 0x23, 0xe8, 0x6b, 0xfc, 0x5a, 0xa2, 0xb4, 0x97, 0x01, 0x97, 0xb4, 0xc6, 0x64,
-	0x02, 0x61, 0xfd, 0x0e, 0xbb, 0x31, 0xb4, 0xf4, 0x31, 0xab, 0x9f, 0xf6, 0x41, 0xef, 0x53, 0xe9,
-	0x4d, 0x9e, 0x09, 0xf9, 0x65, 0x6e, 0x16, 0x45, 0x8a, 0x86, 0x9e, 0xba, 0x1f, 0x3d, 0xac, 0xba,
-	0xd7, 0xae, 0xe9, 0x2f, 0x74, 0xf9, 0x78, 0xfb, 0x67, 0xdc, 0xda, 0xee, 0xc6, 0xed, 0x9f, 0xf6,
-	0xfb, 0xbd, 0x1b, 0xb7, 0xee, 0xfe, 0x8e, 0xdb, 0x71, 0xd7, 0xe5, 0x7b, 0xfd, 0x2f, 0x00, 0x00,
-	0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00,
+	// 446 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0xd1, 0x8a, 0xd3, 0x40,
+	0x14, 0xed, 0x6c, 0xb3, 0xdb, 0xf6, 0xc6, 0xae, 0xee, 0x20, 0xcb, 0x50, 0x30, 0x0d, 0x05, 0x21,
+	0x08, 0xa6, 0x5a, 0xf1, 0xc5, 0xc7, 0x7d, 0x58, 0x10, 0x44, 0x24, 0x2e, 0xeb, 0x63, 0x69, 0x32,
+	0xb7, 0xd9, 0xc1, 0x64, 0x26, 0x66, 0x26, 0x85, 0xfa, 0x25, 0xbe, 0xb9, 0x9f, 0xd3, 0x47, 0xbf,
+	0x40, 0xb4, 0xfe, 0x88, 0x74, 0x92, 0xc6, 0x3c, 0xfa, 0x76, 0xce, 0xbd, 0xe7, 0xdc, 0x7b, 0x66,
+	0x2e, 0x8c, 0xb5, 0x29, 0xab, 0xc4, 0xe8, 0xb0, 0x28, 0x95, 0x51, 0xf4, 0x84, 0xc7, 0x93, 0xe7,
+	0xa9, 0x30, 0x77, 0x55, 0x1c, 0x26, 0x2a, 0x9f, 0xa7, 0x2a, 0x55, 0x73, 0xdb, 0x8a, 0xab, 0xb5,
+	0x65, 0x96, 0x58, 0x54, 0x5b, 0x26, 0xaf, 0x3b, 0x72, 0xbd, 0x95, 0x89, 0xb9, 0x13, 0x32, 0xed,
+	0xa0, 0x4c, 0xc4, 0xf5, 0x84, 0x44, 0x65, 0xf3, 0x18, 0x8b, 0xda, 0x36, 0xfb, 0x04, 0xee, 0xb5,
+	0xc8, 0xf0, 0x16, 0x4b, 0x2d, 0x94, 0xa4, 0x2f, 0x60, 0xb0, 0xa9, 0x21, 0x23, 0x3e, 0x09, 0xdc,
+	0xc5, 0xa3, 0xf0, 0x68, 0x0a, 0x6f, 0x31, 0x31, 0xaa, 0xbc, 0x72, 0x76, 0x3f, 0xa7, 0xbd, 0xe8,
+	0x28, 0xa3, 0x97, 0x70, 0xc6, 0x71, 0x23, 0x12, 0x64, 0x27, 0x3e, 0x09, 0x1e, 0x44, 0x0d, 0x9b,
+	0x5d, 0x83, 0xdb, 0x0c, 0x7d, 0x27, 0xb4, 0xa1, 0x2f, 0x61, 0xd8, 0x38, 0x34, 0x23, 0x7e, 0x3f,
+	0x70, 0x17, 0x0f, 0x43, 0x1e, 0x87, 0x9d, 0xdd, 0xcd, 0xe0, 0x56, 0xf6, 0xc6, 0xf9, 0x76, 0x3f,
+	0xed, 0xcd, 0xbe, 0xf7, 0xe1, 0xe2, 0xa0, 0x7a, 0x2b, 0xd7, 0xea, 0xa6, 0xac, 0x64, 0xb2, 0x32,
+	0xc8, 0x29, 0x05, 0x47, 0xae, 0x72, 0xb4, 0x21, 0x47, 0x91, 0xc5, 0xf4, 0x19, 0x38, 0x66, 0x5b,
+	0xd4, 0x39, 0xce, 0x17, 0x97, 0xff, 0x82, 0xb7, 0xf6, 0x6d, 0x81, 0x91, 0xd5, 0x1c, 0xfc, 0x5a,
+	0x7c, 0x45, 0xd6, 0xf7, 0x49, 0xd0, 0x8f, 0x2c, 0xa6, 0x3e, 0xb8, 0x05, 0x96, 0xb9, 0xd0, 0x75,
+	0x4a, 0xc7, 0x27, 0xc1, 0x38, 0xea, 0x96, 0xe8, 0x13, 0x80, 0x5c, 0x71, 0xb1, 0x16, 0xc8, 0x97,
+	0x9a, 0x9d, 0x5a, 0xef, 0xe8, 0x58, 0xf9, 0x48, 0x19, 0x0c, 0x38, 0x66, 0x68, 0x90, 0xb3, 0x33,
+	0x9f, 0x04, 0xc3, 0xe8, 0x48, 0x0f, 0x1d, 0x21, 0x37, 0xab, 0x4c, 0x70, 0x36, 0xa8, 0x3b, 0x0d,
+	0xa5, 0x4f, 0xe1, 0x5c, 0xaa, 0x65, 0x77, 0xef, 0xd0, 0x0a, 0xc6, 0x52, 0x7d, 0xe8, 0x6c, 0xee,
+	0xdc, 0x65, 0xf4, 0x7f, 0x77, 0x99, 0xc0, 0x50, 0xe3, 0x97, 0x0a, 0x65, 0x82, 0x0c, 0x6c, 0xd2,
+	0x96, 0xd3, 0x29, 0xb8, 0xed, 0x3b, 0xa4, 0x66, 0xae, 0x4f, 0x82, 0xd3, 0xa8, 0x7d, 0xda, 0x7b,
+	0x7d, 0x48, 0xa5, 0xb7, 0x79, 0x26, 0xe4, 0xe7, 0xa5, 0x59, 0x95, 0x29, 0x1a, 0x76, 0x61, 0x3f,
+	0x7a, 0xdc, 0x54, 0x6f, 0x6c, 0xb1, 0xbe, 0xd0, 0xd5, 0xe3, 0xdd, 0x6f, 0xaf, 0xb7, 0xdb, 0x7b,
+	0xe4, 0xc7, 0xde, 0x23, 0xbf, 0xf6, 0x5e, 0xef, 0xfe, 0x8f, 0x47, 0xe2, 0x33, 0x9b, 0xef, 0xd5,
+	0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x2f, 0x12, 0xb6, 0xda, 0x02, 0x00, 0x00,
 }

lib/discover/local.pb.go

@@ -382,20 +382,21 @@ var (
 )
 
 var fileDescriptorLocal = []byte{
-	// 235 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
-	0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
-	0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
-	0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
-	0x69, 0x2d, 0x23, 0x17, 0x87, 0x63, 0x5e, 0x5e, 0x7e, 0x69, 0x5e, 0x72, 0xaa, 0x50, 0x10, 0x17,
-	0x53, 0x66, 0x8a, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x8f, 0x93, 0xd3, 0x89, 0x7b, 0xf2, 0x0c, 0xb7,
-	0xee, 0xc9, 0x9b, 0x20, 0x99, 0x57, 0x5c, 0x99, 0x97, 0x5c, 0x92, 0x91, 0x99, 0x97, 0x8e, 0xc4,
-	0xca, 0xc9, 0x4c, 0x82, 0x58, 0x91, 0x9c, 0x9f, 0xa3, 0xe7, 0x92, 0x5a, 0x96, 0x99, 0x9c, 0xea,
-	0xe9, 0xf2, 0xe8, 0x9e, 0x3c, 0x93, 0xa7, 0x4b, 0x10, 0xd0, 0x34, 0x21, 0x19, 0x2e, 0xce, 0xc4,
-	0x94, 0x94, 0xa2, 0xd4, 0xe2, 0xe2, 0xd4, 0x62, 0x09, 0x26, 0x05, 0x66, 0x0d, 0xce, 0x20, 0x84,
-	0x80, 0x90, 0x3e, 0x17, 0x77, 0x66, 0x5e, 0x71, 0x49, 0x22, 0xd0, 0xf6, 0x78, 0xa0, 0xd5, 0xcc,
-	0x40, 0xab, 0x99, 0x9d, 0xf8, 0x80, 0xda, 0xb9, 0x3c, 0xa1, 0xc2, 0x40, 0x63, 0xb8, 0x60, 0x4a,
-	0x3c, 0x53, 0x9c, 0x44, 0x4e, 0x3c, 0x94, 0x63, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0x02, 0x10, 0x3f,
-	0x78, 0x24, 0xc7, 0xb0, 0xe0, 0xb1, 0x1c, 0x63, 0x12, 0x1b, 0xd8, 0x05, 0xc6, 0x80, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00, 0x00,
+	// 241 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x8e, 0x4f, 0x4e, 0x84, 0x30,
+	0x14, 0xc6, 0x29, 0x24, 0x66, 0xa6, 0x63, 0x5c, 0x10, 0x17, 0xc4, 0x98, 0x42, 0x5c, 0xb1, 0x11,
+	0x16, 0x7a, 0x01, 0x09, 0x9b, 0x6e, 0xb9, 0x80, 0x81, 0xb6, 0x32, 0x2f, 0xc1, 0x3e, 0x43, 0x61,
+	0x12, 0x6f, 0xe3, 0x05, 0xbc, 0x07, 0x4b, 0xd7, 0x2e, 0x1a, 0xad, 0x17, 0x31, 0xe9, 0x68, 0x86,
+	0xdd, 0xf7, 0xfd, 0xf2, 0x7b, 0x7f, 0xe8, 0x6e, 0x40, 0xd1, 0x0e, 0xc5, 0xcb, 0x88, 0x13, 0xc6,
+	0x1b, 0x09, 0x46, 0xe0, 0x41, 0x8d, 0x57, 0xb7, 0x3d, 0x4c, 0xfb, 0xb9, 0x2b, 0x04, 0x3e, 0x97,
+	0x3d, 0xf6, 0x58, 0x7a, 0xa1, 0x9b, 0x9f, 0x7c, 0xf3, 0xc5, 0xa7, 0xe3, 0xe0, 0xcd, 0x3b, 0xa1,
+	0x9b, 0x07, 0xad, 0x71, 0xd6, 0x42, 0xc5, 0x0d, 0x0d, 0x41, 0x26, 0x24, 0x23, 0xf9, 0x79, 0x55,
+	0x2d, 0x36, 0x0d, 0x3e, 0x6d, 0x7a, 0xbf, 0xda, 0x67, 0x5e, 0xb5, 0x98, 0xf6, 0xa0, 0xfb, 0x55,
+	0x1a, 0xa0, 0x3b, 0x9e, 0x10, 0x38, 0x14, 0xb5, 0x3a, 0x80, 0x50, 0xbc, 0x76, 0x36, 0x0d, 0x79,
+	0xdd, 0x84, 0x20, 0xe3, 0x6b, 0xba, 0x6d, 0xa5, 0x1c, 0x95, 0x31, 0xca, 0x24, 0x61, 0x16, 0xe5,
+	0xdb, 0xe6, 0x04, 0xe2, 0x92, 0xee, 0x40, 0x9b, 0xa9, 0xd5, 0x42, 0x3d, 0x82, 0x4c, 0xa2, 0x8c,
+	0xe4, 0x51, 0x75, 0xe1, 0x6c, 0x4a, 0xf9, 0x1f, 0xe6, 0x75, 0x43, 0xff, 0x15, 0x2e, 0xab, 0xcb,
+	0xe5, 0x9b, 0x05, 0x8b, 0x63, 0xe4, 0xc3, 0x31, 0xf2, 0xe5, 0x58, 0xf0, 0xf6, 0xc3, 0x48, 0x77,
+	0xe6, 0x3f, 0xb8, 0xfb, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x46, 0x4f, 0x13, 0x14, 0x01, 0x00,
+	0x00,
 }

lib/model/rwfolder.go

@@ -28,6 +28,7 @@ import (
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/versioner"
+	"github.com/syncthing/syncthing/lib/weakhash"
 )
 
 func init() {
@@ -92,6 +93,7 @@ type rwFolder struct {
 	checkFreeSpace bool
 	ignoreDelete   bool
 	fsync          bool
+	useWeakHash    bool
 
 	copiers int
 	pullers int
@@ -128,6 +130,7 @@ func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Ver
 		checkFreeSpace: cfg.MinDiskFreePct != 0,
 		ignoreDelete:   cfg.IgnoreDelete,
 		fsync:          cfg.Fsync,
+		useWeakHash:    !cfg.DisableWeakHash,
 
 		queue:       newJobQueue(),
 		pullTimer:   time.NewTimer(time.Second),
@@ -1169,7 +1172,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		created:          time.Now(),
 	}
 
-	l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), reused)
+	l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused))
 
 	cs := copyBlocksState{
 		sharedPullerState: &s,
@@ -1231,6 +1234,21 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 		}
 		f.model.fmut.RUnlock()
 
+		var weakHashFinder *weakhash.Finder
+		if f.useWeakHash {
+			hashesToFind := make([]uint32, 0, len(state.blocks))
+			for _, block := range state.blocks {
+				if block.WeakHash != 0 {
+					hashesToFind = append(hashesToFind, block.WeakHash)
+				}
+			}
+
+			weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
+			if err != nil {
+				l.Debugln("weak hasher", err)
+			}
+		}
+
 		for _, block := range state.blocks {
 			if f.allowSparse && state.reused == 0 && block.IsEmpty() {
 				// The block is a block of all zeroes, and we are not reusing
@@ -1245,45 +1263,70 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 			}
 
 			buf = buf[:int(block.Size)]
-			found := f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
-				inFile, err := rootedJoinedPath(folderRoots[folder], file)
-				if err != nil {
-					return false
-				}
-				fd, err := os.Open(inFile)
-				if err != nil {
-					return false
-				}
 
-				_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
-				fd.Close()
-				if err != nil {
-					return false
-				}
-
-				hash, err := scanner.VerifyBuffer(buf, block)
-				if err != nil {
-					if hash != nil {
-						l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
-						err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
-						if err != nil {
-							l.Warnln("finder fix:", err)
-						}
-					} else {
-						l.Debugln("Finder failed to verify buffer", err)
-					}
-					return false
+			found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
+				if _, err := scanner.VerifyBuffer(buf, block); err != nil {
+					return true
 				}
 
 				_, err = dstFd.WriteAt(buf, block.Offset)
 				if err != nil {
 					state.fail("dst write", err)
+
 				}
-				if file == state.file.Name {
+				if offset == block.Offset {
 					state.copiedFromOrigin()
+				} else {
+					state.copiedFromOriginShifted()
 				}
-				return true
+
+				return false
 			})
+			if err != nil {
+				l.Debugln("weak hasher iter", err)
+			}
+
+			if !found {
+				found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
+					inFile, err := rootedJoinedPath(folderRoots[folder], file)
+					if err != nil {
+						return false
+					}
+					fd, err := os.Open(inFile)
+					if err != nil {
+						return false
+					}
+
+					_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
+					fd.Close()
+					if err != nil {
+						return false
+					}
+
+					hash, err := scanner.VerifyBuffer(buf, block)
+					if err != nil {
+						if hash != nil {
+							l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
+							err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
+							if err != nil {
+								l.Warnln("finder fix:", err)
+							}
+						} else {
+							l.Debugln("Finder failed to verify buffer", err)
+						}
+						return false
+					}
+
+					_, err = dstFd.WriteAt(buf, block.Offset)
+					if err != nil {
+						state.fail("dst write", err)
+					}
+					if file == state.file.Name {
+						state.copiedFromOrigin()
+					}
+					return true
+				})
+			}
 
 			if state.failed() != nil {
 				break
@@ -1300,6 +1343,7 @@ func (f *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
 				state.copyDone(block)
 			}
 		}
+		weakHashFinder.Close()
 		out <- state.sharedPullerState
 	}
 }

lib/model/rwfolder_test.go

@@ -7,12 +7,15 @@
 package model
 
 import (
+	"crypto/rand"
+	"io"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -72,6 +75,8 @@ func setUpRwFolder(model *Model) rwFolder {
 			stateTracker: newStateTracker("default"),
 			model:        model,
 		},
+
+		mtimeFS:   fs.NewMtimeFS(db.NewNamespacedKV(model.db, "mtime")),
 		dir:       "testdata",
 		queue:     newJobQueue(),
 		errors:    make(map[string]string),
@@ -199,7 +204,7 @@ func TestCopierFinder(t *testing.T) {
 
 	select {
 	case <-pullChan:
-		t.Fatal("Finisher channel has data to be read")
+		t.Fatal("Pull channel has data to be read")
 	case <-finisherChan:
 		t.Fatal("Finisher channel has data to be read")
 	default:
@@ -240,6 +245,132 @@ func TestCopierFinder(t *testing.T) {
 	os.Remove(tempFile)
 }
 
+func TestWeakHash(t *testing.T) {
+	tempFile := filepath.Join("testdata", defTempNamer.TempName("weakhash"))
+	var shift int64 = 10
+	var size int64 = 1 << 20
+	expectBlocks := int(size / protocol.BlockSize)
+	expectPulls := int(shift / protocol.BlockSize)
+	if shift > 0 {
+		expectPulls++
+	}
+
+	cleanup := func() {
+		for _, path := range []string{tempFile, "testdata/weakhash"} {
+			os.Remove(path)
+		}
+	}
+
+	cleanup()
+	defer cleanup()
+
+	f, err := os.Create("testdata/weakhash")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+	_, err = io.CopyN(f, rand.Reader, size)
+	if err != nil {
+		t.Error(err)
+	}
+	info, err := f.Stat()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Create two files, second file has `shifted` bytes random prefix, yet
+	// both are of the same length, for example:
+	// File 1: abcdefgh
+	// File 2: xyabcdef
+	f.Seek(0, os.SEEK_SET)
+	existing, err := scanner.Blocks(f, protocol.BlockSize, size, nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	f.Seek(0, os.SEEK_SET)
+	remainder := io.LimitReader(f, size-shift)
+	prefix := io.LimitReader(rand.Reader, shift)
+	nf := io.MultiReader(prefix, remainder)
+	desired, err := scanner.Blocks(nf, protocol.BlockSize, size, nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	existingFile := protocol.FileInfo{
+		Name:       "weakhash",
+		Blocks:     existing,
+		Size:       size,
+		ModifiedS:  info.ModTime().Unix(),
+		ModifiedNs: int32(info.ModTime().Nanosecond()),
+	}
+	desiredFile := protocol.FileInfo{
+		Name:      "weakhash",
+		Size:      size,
+		Blocks:    desired,
+		ModifiedS: info.ModTime().Unix() + 1,
+	}
+
+	// Setup the model/pull environment
+	m := setUpModel(existingFile)
+	fo := setUpRwFolder(m)
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState, expectBlocks)
+	finisherChan := make(chan *sharedPullerState, 1)
+
+	// Run a single fetcher routine
+	go fo.copierRoutine(copyChan, pullChan, finisherChan)
+
+	// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
+	fo.handleFile(desiredFile, copyChan, finisherChan)
+
+	var pulls []pullBlockState
+	for len(pulls) < expectBlocks {
+		select {
+		case pull := <-pullChan:
+			pulls = append(pulls, pull)
+		case <-time.After(time.Second):
+			t.Error("timed out")
+		}
+	}
+	finish := <-finisherChan
+
+	select {
+	case <-pullChan:
+		t.Fatal("Pull channel has data to be read")
+	case <-finisherChan:
+		t.Fatal("Finisher channel has data to be read")
+	default:
+	}
+
+	finish.fd.Close()
+	if err := os.Remove(tempFile); err != nil && !os.IsNotExist(err) {
+		t.Error(err)
+	}
+
+	// Test 2 - using weak hash, expectPulls blocks pulled.
+	fo.useWeakHash = true
+	fo.handleFile(desiredFile, copyChan, finisherChan)
+
+	pulls = pulls[:0]
+	for len(pulls) < expectPulls {
+		select {
+		case pull := <-pullChan:
+			pulls = append(pulls, pull)
+		case <-time.After(time.Second):
+			t.Error("timed out")
+		}
+	}
+
+	finish = <-finisherChan
+	finish.fd.Close()
+
+	expectShifted := expectBlocks - expectPulls
+	if finish.copyOriginShifted != expectShifted {
+		t.Errorf("did not copy %d shifted", expectShifted)
+	}
+}
+
 // Test that updating a file removes it's old blocks from the blockmap
 func TestCopierCleanup(t *testing.T) {
 	iterFn := func(folder, file string, index int32) bool {

lib/model/sharedpullerstate.go

@@ -31,30 +31,32 @@ type sharedPullerState struct {
 	created     time.Time
 
 	// Mutable, must be locked for access
-	err              error        // The first error we hit
-	fd               *os.File     // The fd of the temp file
-	copyTotal        int          // Total number of copy actions for the whole job
-	pullTotal        int          // Total number of pull actions for the whole job
-	copyOrigin       int          // Number of blocks copied from the original file
-	copyNeeded       int          // Number of copy actions still pending
-	pullNeeded       int          // Number of block pulls still pending
-	updated          time.Time    // Time when any of the counters above were last updated
-	closed           bool         // True if the file has been finalClosed.
-	available        []int32      // Indexes of the blocks that are available in the temporary file
-	availableUpdated time.Time    // Time when list of available blocks was last updated
-	mut              sync.RWMutex // Protects the above
+	err               error        // The first error we hit
+	fd                *os.File     // The fd of the temp file
+	copyTotal         int          // Total number of copy actions for the whole job
+	pullTotal         int          // Total number of pull actions for the whole job
+	copyOrigin        int          // Number of blocks copied from the original file
+	copyOriginShifted int          // Number of blocks copied from the original file but shifted
+	copyNeeded        int          // Number of copy actions still pending
+	pullNeeded        int          // Number of block pulls still pending
+	updated           time.Time    // Time when any of the counters above were last updated
+	closed            bool         // True if the file has been finalClosed.
+	available         []int32      // Indexes of the blocks that are available in the temporary file
+	availableUpdated  time.Time    // Time when list of available blocks was last updated
+	mut               sync.RWMutex // Protects the above
 }
 
 // A momentary state representing the progress of the puller
 type pullerProgress struct {
-	Total               int   `json:"total"`
-	Reused              int   `json:"reused"`
-	CopiedFromOrigin    int   `json:"copiedFromOrigin"`
-	CopiedFromElsewhere int   `json:"copiedFromElsewhere"`
-	Pulled              int   `json:"pulled"`
-	Pulling             int   `json:"pulling"`
-	BytesDone           int64 `json:"bytesDone"`
-	BytesTotal          int64 `json:"bytesTotal"`
+	Total                   int   `json:"total"`
+	Reused                  int   `json:"reused"`
+	CopiedFromOrigin        int   `json:"copiedFromOrigin"`
+	CopiedFromOriginShifted int   `json:"copiedFromOriginShifted"`
+	CopiedFromElsewhere     int   `json:"copiedFromElsewhere"`
+	Pulled                  int   `json:"pulled"`
+	Pulling                 int   `json:"pulling"`
+	BytesDone               int64 `json:"bytesDone"`
+	BytesTotal              int64 `json:"bytesTotal"`
 }
 
 // A lockedWriterAt synchronizes WriteAt calls with an external mutex.
@@ -241,6 +243,14 @@ func (s *sharedPullerState) copiedFromOrigin() {
 	s.mut.Unlock()
 }
 
+func (s *sharedPullerState) copiedFromOriginShifted() {
+	s.mut.Lock()
+	s.copyOrigin++
+	s.copyOriginShifted++
+	s.updated = time.Now()
+	s.mut.Unlock()
+}
+
 func (s *sharedPullerState) pullStarted() {
 	s.mut.Lock()
 	s.copyTotal--

lib/protocol/bep.pb.go

@@ -303,7 +303,7 @@ type FileInfo struct {
 	NoPermissions bool         `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
 	Version       Vector       `protobuf:"bytes,9,opt,name=version" json:"version"`
 	Sequence      int64        `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
-	Blocks        []BlockInfo  `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"`
+	Blocks        []BlockInfo  `protobuf:"bytes,16,rep,name=Blocks,json=blocks" json:"Blocks"`
 	SymlinkTarget string       `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
 }
 
@@ -312,9 +312,10 @@ func (*FileInfo) ProtoMessage()               {}
 func (*FileInfo) Descriptor() ([]byte, []int) { return fileDescriptorBep, []int{7} }
 
 type BlockInfo struct {
-	Offset int64  `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
-	Size   int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
-	Hash   []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
+	Offset   int64  `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
+	Size     int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
+	Hash     []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
+	WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
 }
 
 func (m *BlockInfo) Reset()                    { *m = BlockInfo{} }
@@ -913,6 +914,11 @@ func (m *BlockInfo) MarshalTo(data []byte) (int, error) {
 		i = encodeVarintBep(data, i, uint64(len(m.Hash)))
 		i += copy(data[i:], m.Hash)
 	}
+	if m.WeakHash != 0 {
+		data[i] = 0x20
+		i++
+		i = encodeVarintBep(data, i, uint64(m.WeakHash))
+	}
 	return i, nil
 }
 
@@ -1423,6 +1429,9 @@ func (m *BlockInfo) ProtoSize() (n int) {
 	if l > 0 {
 		n += 1 + l + sovBep(uint64(l))
 	}
+	if m.WeakHash != 0 {
+		n += 1 + sovBep(uint64(m.WeakHash))
+	}
 	return n
 }
 
@@ -3011,6 +3020,25 @@ func (m *BlockInfo) Unmarshal(data []byte) error {
 				m.Hash = []byte{}
 			}
 			iNdEx = postIndex
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field WeakHash", wireType)
+			}
+			m.WeakHash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBep
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.WeakHash |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipBep(data[iNdEx:])
@@ -4029,108 +4057,112 @@ var (
 )
 
 var fileDescriptorBep = []byte{
-	// 1645 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x73, 0xdb, 0x5a,
-	0x15, 0x8f, 0x6d, 0xf9, 0xdf, 0xb5, 0x93, 0xe7, 0xdc, 0xa6, 0x79, 0x46, 0xcd, 0x4b, 0x82, 0xde,
-	0x2b, 0x04, 0x0f, 0x4d, 0xa1, 0x05, 0x3a, 0xc3, 0x00, 0x33, 0x8e, 0xad, 0xa4, 0x9a, 0x3a, 0xb2,
-	0x2b, 0xdb, 0x29, 0x65, 0x81, 0x46, 0xb6, 0xae, 0x1d, 0x4d, 0x64, 0x5d, 0x23, 0xc9, 0x6d, 0xc3,
-	0x47, 0x80, 0x2f, 0xc0, 0x86, 0x99, 0x6e, 0xd9, 0xf3, 0x21, 0xca, 0xae, 0xd3, 0x25, 0x8b, 0x0e,
-	0x94, 0x0d, 0x4b, 0x36, 0xec, 0x39, 0xf7, 0x5e, 0x49, 0x96, 0xf3, 0x87, 0xe9, 0xe2, 0x2d, 0x32,
-	0xd6, 0x3d, 0xe7, 0x77, 0xcf, 0xb9, 0xe7, 0x77, 0xfe, 0xdc, 0x1b, 0x54, 0x1e, 0x91, 0xf9, 0xe1,
-	0xdc, 0xa7, 0x21, 0xc5, 0x25, 0xfe, 0x33, 0xa6, 0xae, 0xfc, 0x60, 0xea, 0x84, 0xe7, 0x8b, 0xd1,
-	0xe1, 0x98, 0xce, 0x1e, 0x4e, 0xe9, 0x94, 0x3e, 0xe4, 0x9a, 0xd1, 0x62, 0xc2, 0x57, 0x7c, 0xc1,
-	0xbf, 0xc4, 0x46, 0x65, 0x8e, 0xf2, 0x4f, 0x89, 0xeb, 0x52, 0xbc, 0x87, 0x2a, 0x36, 0x79, 0xe5,
-	0x8c, 0x89, 0xe9, 0x59, 0x33, 0x52, 0xcf, 0xec, 0x67, 0x0e, 0xca, 0x06, 0x12, 0x22, 0x1d, 0x24,
-	0x0c, 0x30, 0x76, 0x1d, 0xe2, 0x85, 0x02, 0x90, 0x15, 0x00, 0x21, 0xe2, 0x80, 0xfb, 0x68, 0x23,
-	0x02, 0xbc, 0x22, 0x7e, 0xe0, 0x50, 0xaf, 0x9e, 0xe3, 0x98, 0x75, 0x21, 0x3d, 0x13, 0x42, 0x25,
-	0x40, 0x85, 0xa7, 0xc4, 0xb2, 0x89, 0x8f, 0x7f, 0x80, 0xa4, 0xf0, 0x72, 0x2e, 0x7c, 0x6d, 0x3c,
-	0xba, 0x7b, 0x18, 0xc7, 0x70, 0x78, 0x4a, 0x82, 0xc0, 0x9a, 0x92, 0x01, 0x28, 0x0d, 0x0e, 0xc1,
-	0xbf, 0x02, 0xe7, 0x74, 0x36, 0xf7, 0x41, 0xc1, 0x0c, 0x67, 0xf9, 0x8e, 0x9d, 0x6b, 0x3b, 0x5a,
-	0x4b, 0x8c, 0x91, 0xde, 0xa0, 0x34, 0xd1, 0x7a, 0xcb, 0x5d, 0x04, 0x21, 0xf1, 0x5b, 0xd4, 0x9b,
-	0x38, 0x53, 0xfc, 0x23, 0x54, 0x9c, 0x50, 0x17, 0x4e, 0x11, 0x80, 0xfb, 0xdc, 0x41, 0xe5, 0x51,
-	0x6d, 0x69, 0xec, 0x98, 0x2b, 0x8e, 0xa4, 0x77, 0x1f, 0xf7, 0xd6, 0x8c, 0x18, 0xa6, 0xfc, 0x31,
-	0x8b, 0x0a, 0x42, 0x83, 0xb7, 0x51, 0xd6, 0xb1, 0x05, 0x45, 0x47, 0x85, 0x4f, 0x1f, 0xf7, 0xb2,
-	0x5a, 0xdb, 0x00, 0x09, 0xde, 0x42, 0x79, 0xd7, 0x1a, 0x11, 0x37, 0x22, 0x47, 0x2c, 0xf0, 0x3d,
-	0x54, 0xf6, 0x21, 0x60, 0x93, 0x7a, 0xee, 0x25, 0xa7, 0xa4, 0x64, 0x94, 0x98, 0xa0, 0x0b, 0x6b,
-	0xfc, 0x00, 0x61, 0x67, 0xea, 0x51, 0x9f, 0x98, 0x73, 0xe2, 0xcf, 0x1c, 0x7e, 0xda, 0xa0, 0x2e,
-	0x71, 0xd4, 0xa6, 0xd0, 0xf4, 0x96, 0x0a, 0xfc, 0x35, 0x5a, 0x8f, 0xe0, 0x36, 0x71, 0x49, 0x48,
-	0xea, 0x79, 0x8e, 0xac, 0x0a, 0x61, 0x9b, 0xcb, 0x20, 0xb6, 0x2d, 0xdb, 0x09, 0xac, 0x91, 0x4b,
-	0xcc, 0x90, 0xcc, 0xe6, 0xa6, 0xe3, 0xd9, 0xe4, 0x0d, 0x09, 0xea, 0x05, 0x8e, 0xc5, 0x91, 0x6e,
-	0x00, 0x2a, 0x4d, 0x68, 0x18, 0x1b, 0x22, 0xd3, 0x41, 0xbd, 0x76, 0x95, 0x8d, 0x36, 0x57, 0xc4,
-	0x6c, 0x44, 0x30, 0xe5, 0x3f, 0xc0, 0x86, 0xd0, 0xe0, 0xef, 0x25, 0x6c, 0x54, 0x8f, 0xb6, 0x19,
-	0xea, 0xef, 0x1f, 0xf7, 0x4a, 0x42, 0xa7, 0xb5, 0x53, 0xec, 0x60, 0x24, 0xa5, 0x2a, 0x87, 0x7f,
-	0xe3, 0x1d, 0x54, 0xb6, 0x6c, 0x9b, 0x65, 0x09, 0x5c, 0xe7, 0xc0, 0x75, 0xd9, 0x58, 0x0a, 0xf0,
-	0x93, 0xd5, 0xac, 0x4b, 0x57, 0xeb, 0xe4, 0xb6, 0x74, 0x33, 0xca, 0xc7, 0xc4, 0x8f, 0x2a, 0x35,
-	0xcf, 0xfd, 0x95, 0x98, 0x80, 0xd7, 0xe9, 0x77, 0x51, 0x75, 0x66, 0xbd, 0x31, 0x03, 0xf2, 0xbb,
-	0x05, 0xf1, 0xc6, 0x84, 0xd3, 0x92, 0x33, 0x2a, 0x20, 0xeb, 0x47, 0x22, 0xbc, 0x8b, 0x90, 0xe3,
-	0x85, 0x3e, 0xb5, 0x17, 0xb0, 0xab, 0x5e, 0xe4, 0xbc, 0xa5, 0x24, 0xf8, 0xa7, 0xa8, 0xc4, 0x49,
-	0x35, 0x21, 0xf0, 0x12, 0x68, 0xa5, 0x23, 0x39, 0x0a, 0xbc, 0xc8, 0x29, 0xe5, 0x71, 0xc7, 0x9f,
-	0x46, 0x91, 0x63, 0x35, 0x1b, 0xff, 0x02, 0xc9, 0xc1, 0x85, 0xc3, 0x12, 0x22, 0x2c, 0x85, 0x70,
-	0x56, 0xd3, 0x27, 0x33, 0xfa, 0xca, 0x72, 0x83, 0x7a, 0x99, 0xbb, 0xa9, 0x33, 0x84, 0x96, 0x02,
-	0x18, 0x91, 0x5e, 0xe9, 0xa2, 0x3c, 0xb7, 0x08, 0xe5, 0x57, 0x10, 0x45, 0x19, 0x75, 0x69, 0xb4,
-	0xc2, 0x87, 0x28, 0x3f, 0x71, 0x5c, 0x20, 0x32, 0xcb, 0x73, 0x88, 0x53, 0x15, 0x0d, 0x62, 0xcd,
-	0x9b, 0xd0, 0x28, 0x8b, 0x02, 0xa6, 0x0c, 0x51, 0x85, 0x1b, 0x1c, 0xce, 0x6d, 0x0b, 0xca, 0xe6,
-	0xdb, 0x32, 0xfb, 0xb7, 0x1c, 0x2a, 0xc5, 0x9a, 0x24, 0xe9, 0x99, 0x54, 0xd2, 0x1b, 0x51, 0xdf,
-	0x8b, 0x2e, 0xde, 0xbe, 0x6e, 0x2f, 0xd5, 0xf8, 0xb0, 0x3f, 0x70, 0x7e, 0x4f, 0x78, 0xdf, 0xe4,
-	0x0c, 0xfe, 0x8d, 0xf7, 0x51, 0xe5, 0x6a, 0xb3, 0xac, 0x1b, 0x69, 0x11, 0xfe, 0x0a, 0xa1, 0x19,
-	0xb5, 0x9d, 0x89, 0x43, 0x6c, 0x33, 0xe0, 0x05, 0x90, 0x33, 0xca, 0xb1, 0xa4, 0x8f, 0xeb, 0xac,
-	0xdc, 0x59, 0xab, 0xd8, 0x51, 0x4f, 0xc4, 0x4b, 0xa6, 0x71, 0x3c, 0x60, 0x1b, 0xf2, 0x2a, 0xb2,
-	0x1e, 0x2f, 0xd9, 0x74, 0xf3, 0xe8, 0x4a, 0x93, 0x96, 0x38, 0x60, 0xdd, 0xa3, 0xe9, 0x06, 0x85,
-	0x4e, 0x8a, 0xa7, 0x1f, 0xcb, 0xe7, 0x4a, 0x27, 0x9d, 0x91, 0x71, 0x48, 0x93, 0xb9, 0x12, 0xc1,
-	0xb0, 0x8c, 0x4a, 0x49, 0x29, 0x22, 0x7e, 0xd2, 0x64, 0xcd, 0x66, 0x6e, 0x12, 0x07, 0x78, 0xac,
-	0x80, 0x3a, 0x6f, 0x24, 0xa1, 0xe9, 0x01, 0xfe, 0x31, 0x2a, 0x1c, 0xb9, 0x74, 0x7c, 0x11, 0xf7,
-	0xed, 0x9d, 0xa5, 0x37, 0x2e, 0x4f, 0x65, 0x27, 0x02, 0xb2, 0x40, 0x82, 0xcb, 0x99, 0xeb, 0x78,
-	0x17, 0x66, 0x68, 0xf9, 0x53, 0x12, 0xd6, 0x37, 0xc5, 0x98, 0x8e, 0xa4, 0x03, 0x2e, 0xfc, 0xb9,
-	0xf4, 0xa7, 0xb7, 0x7b, 0x6b, 0xca, 0x73, 0x54, 0x4e, 0xec, 0xb0, 0x02, 0xa1, 0x93, 0x49, 0x00,
-	0x3b, 0x32, 0xfc, 0x9c, 0xd1, 0x2a, 0xc9, 0x51, 0x96, 0x1f, 0x4f, 0xe4, 0x08, 0x64, 0xe7, 0x56,
-	0x70, 0xce, 0xf3, 0x56, 0x35, 0xf8, 0x77, 0x64, 0xf2, 0x97, 0xa8, 0x20, 0x88, 0xc0, 0x8f, 0x51,
-	0x69, 0x4c, 0x17, 0x5e, 0xb8, 0x1c, 0xc2, 0x9b, 0xe9, 0xde, 0xe6, 0x9a, 0xe8, 0xf0, 0x09, 0x50,
-	0x39, 0x46, 0xc5, 0x48, 0x05, 0x91, 0xc4, 0x83, 0x47, 0x3a, 0xba, 0x1b, 0xf7, 0x5f, 0xff, 0x9c,
-	0xfa, 0xe1, 0xca, 0xdc, 0x81, 0xa9, 0x0c, 0x29, 0x5c, 0x88, 0xf3, 0x49, 0x86, 0x58, 0x28, 0x7f,
-	0xcd, 0xa0, 0xa2, 0xc1, 0x78, 0x0e, 0xc2, 0xd4, 0x3c, 0xcf, 0xaf, 0xcc, 0xf3, 0x65, 0x47, 0x64,
-	0x57, 0x3a, 0x22, 0x2e, 0xea, 0x5c, 0xaa, 0xa8, 0x97, 0xe4, 0x48, 0x37, 0x92, 0x93, 0xbf, 0x81,
-	0x9c, 0xc2, 0x92, 0x1c, 0x96, 0x96, 0x89, 0x4f, 0x67, 0x7c, 0x62, 0x53, 0xdf, 0xf2, 0x2f, 0xa3,
-	0x02, 0x5c, 0x67, 0xd2, 0x41, 0x2c, 0x54, 0x4c, 0x54, 0x32, 0x48, 0x30, 0x87, 0x52, 0x23, 0xb7,
-	0x1e, 0x1b, 0xcc, 0x43, 0x43, 0x5b, 0xfc, 0xd0, 0x60, 0x9e, 0x7d, 0xe3, 0xef, 0x23, 0x69, 0x4c,
-	0x6d, 0x71, 0xe4, 0x8d, 0x74, 0x99, 0xa8, 0xbe, 0x4f, 0xe1, 0x52, 0xb4, 0xa1, 0xe1, 0x18, 0x00,
-	0x1e, 0x04, 0xb5, 0x36, 0x7d, 0xed, 0xb9, 0xd4, 0xb2, 0x7b, 0x3e, 0x9d, 0xb2, 0x89, 0x7a, 0xeb,
-	0x64, 0x68, 0xa3, 0xe2, 0x82, 0xcf, 0x8e, 0x78, 0x36, 0x7c, 0xb3, 0xda, 0xcb, 0x57, 0x0d, 0x89,
-	0x41, 0x13, 0x37, 0x40, 0xb4, 0x55, 0xf9, 0x90, 0x41, 0xf2, 0xed, 0x68, 0xac, 0xa1, 0x8a, 0x40,
-	0x9a, 0xa9, 0xc7, 0xc2, 0xc1, 0xe7, 0x38, 0xe2, 0x63, 0x04, 0x2d, 0x92, 0xef, 0x1b, 0x6f, 0xa0,
-	0x54, 0xc3, 0xe6, 0x3e, 0xaf, 0x61, 0xe1, 0x0e, 0x1e, 0xb1, 0x9e, 0x48, 0xee, 0x55, 0x09, 0x62,
-	0xcf, 0x1b, 0xd5, 0x91, 0x68, 0x14, 0x2e, 0x53, 0x0a, 0x48, 0xea, 0x39, 0xde, 0x54, 0xd9, 0x43,
-	0xf9, 0x96, 0x4b, 0x79, 0xb2, 0x0a, 0x70, 0xe9, 0x07, 0xe0, 0x26, 0xe2, 0x50, 0xac, 0x1a, 0x1f,
-	0xb2, 0xa8, 0x92, 0x7a, 0xef, 0xc0, 0x79, 0x36, 0x5a, 0x9d, 0x61, 0x7f, 0xa0, 0x1a, 0x66, 0xab,
-	0xab, 0x1f, 0x6b, 0x27, 0xb5, 0x35, 0x79, 0xe7, 0x0f, 0x7f, 0xde, 0xaf, 0xcf, 0x96, 0xa0, 0xd5,
-	0xa7, 0x0c, 0xb8, 0xd0, 0xf4, 0xb6, 0xfa, 0xeb, 0x5a, 0x46, 0xde, 0x02, 0x60, 0x2d, 0x05, 0x14,
-	0xf7, 0xc5, 0x0f, 0x51, 0x95, 0x03, 0xcc, 0x61, 0xaf, 0xdd, 0x1c, 0xa8, 0xb5, 0xac, 0x2c, 0x03,
-	0x6e, 0xfb, 0x2a, 0x2e, 0xe2, 0xfb, 0x6b, 0xe8, 0x0b, 0xf5, 0xf9, 0x50, 0xed, 0x0f, 0x6a, 0x39,
-	0x79, 0x1b, 0x80, 0x38, 0x05, 0x8c, 0x3b, 0xe6, 0x3e, 0x94, 0xa1, 0xda, 0xef, 0x75, 0xf5, 0xbe,
-	0x5a, 0x93, 0xe4, 0x2f, 0x01, 0x75, 0x67, 0x05, 0x15, 0x55, 0xe8, 0xcf, 0xd0, 0x66, 0xbb, 0xfb,
-	0x42, 0xef, 0x74, 0x9b, 0x6d, 0xb3, 0x67, 0x74, 0x4f, 0x60, 0x4f, 0xbf, 0x96, 0x97, 0xf7, 0x00,
-	0x7f, 0x2f, 0x85, 0xbf, 0x56, 0x70, 0x5f, 0x01, 0x7b, 0x9a, 0x7e, 0x52, 0x2b, 0xc8, 0x77, 0x00,
-	0xfa, 0x45, 0x0a, 0xca, 0x48, 0x65, 0x11, 0xb7, 0x3a, 0x5d, 0x70, 0x5d, 0xbc, 0x16, 0x31, 0x27,
-	0xbb, 0xf1, 0x5b, 0x84, 0xaf, 0xbf, 0x08, 0xf1, 0x37, 0x48, 0xd2, 0xbb, 0xba, 0x0a, 0x84, 0xf2,
-	0xf8, 0xaf, 0x23, 0x74, 0xea, 0x11, 0xac, 0xa0, 0x5c, 0xe7, 0x37, 0x3f, 0x01, 0x32, 0xbf, 0x03,
-	0xa0, 0xbb, 0xd7, 0x41, 0xa0, 0x6c, 0x50, 0x54, 0x49, 0x1b, 0x56, 0x50, 0xe9, 0x54, 0x1d, 0x34,
-	0x81, 0xdc, 0x26, 0x18, 0xe7, 0x47, 0x8a, 0xd5, 0xa7, 0x24, 0xb4, 0x78, 0x03, 0xee, 0xa0, 0xbc,
-	0xae, 0x9e, 0xa9, 0x06, 0x18, 0xde, 0x04, 0xc0, 0x7a, 0x0c, 0xd0, 0x09, 0xd4, 0x15, 0x3c, 0x38,
-	0x0a, 0xcd, 0xce, 0x8b, 0xe6, 0xcb, 0x3e, 0x24, 0x07, 0x83, 0x7a, 0x23, 0x56, 0x37, 0xdd, 0xd7,
-	0xd6, 0x65, 0xd0, 0xf8, 0x6f, 0x06, 0x55, 0xd3, 0xb7, 0x23, 0x6c, 0x90, 0x8e, 0xb5, 0x8e, 0x1a,
-	0xbb, 0x4b, 0xeb, 0xd8, 0x37, 0x3e, 0x40, 0xe5, 0xb6, 0x66, 0xa8, 0xad, 0x41, 0xd7, 0x78, 0x19,
-	0xc7, 0x92, 0x06, 0xb5, 0x1d, 0x9f, 0x17, 0x37, 0x7b, 0x81, 0x56, 0xfb, 0x2f, 0x4f, 0x3b, 0x9a,
-	0xfe, 0xcc, 0xe4, 0x16, 0xb3, 0xf2, 0x3d, 0x00, 0x7f, 0x99, 0x06, 0xf7, 0xc5, 0xcd, 0xc0, 0x0d,
-	0x3f, 0x41, 0x9b, 0x31, 0x7c, 0xe9, 0x20, 0x27, 0xef, 0xc3, 0x9e, 0x9d, 0x1b, 0xf6, 0x2c, 0xfd,
-	0x3c, 0x46, 0x5f, 0xc4, 0x1b, 0x87, 0xfa, 0x33, 0x1d, 0xca, 0x02, 0x2a, 0x67, 0x17, 0xb6, 0xc9,
-	0x37, 0x6c, 0x1b, 0x7a, 0x17, 0x1e, 0x14, 0x45, 0xe3, 0x2f, 0x19, 0x54, 0x4e, 0x26, 0x14, 0xe3,
-	0x59, 0xef, 0x9a, 0xaa, 0x61, 0x74, 0x8d, 0x38, 0xf0, 0x44, 0xa9, 0x53, 0xfe, 0x09, 0xaf, 0xbb,
-	0xe2, 0x89, 0xaa, 0xab, 0x86, 0xd6, 0x8a, 0xfb, 0x21, 0x81, 0x9c, 0x10, 0x8f, 0xf8, 0xce, 0x18,
-	0xfe, 0xef, 0xa8, 0x82, 0x99, 0xfe, 0xb0, 0xf5, 0x34, 0x8e, 0x98, 0x17, 0x70, 0xca, 0x54, 0x7f,
-	0x31, 0x3e, 0xe7, 0xd1, 0x36, 0x58, 0xeb, 0x9c, 0x35, 0x3b, 0x5a, 0x5b, 0x40, 0x73, 0x72, 0x1d,
-	0xa0, 0x5b, 0x09, 0x54, 0x13, 0xaf, 0x03, 0x86, 0x6d, 0xd8, 0x68, 0xf7, 0xff, 0xcf, 0x22, 0x78,
-	0xb8, 0x14, 0x9a, 0xbd, 0x9e, 0xaa, 0xb7, 0xe3, 0xd3, 0x2f, 0x75, 0xcd, 0xf9, 0x9c, 0x78, 0x36,
-	0x43, 0x1c, 0x77, 0x8d, 0x13, 0x75, 0x10, 0x1f, 0x7e, 0x89, 0x38, 0xa6, 0xec, 0x5e, 0x3e, 0xda,
-	0x79, 0xf7, 0xcf, 0xdd, 0xb5, 0xf7, 0xf0, 0xf7, 0xee, 0xd3, 0x6e, 0xe6, 0x3d, 0xfc, 0xfd, 0xe3,
-	0xd3, 0xee, 0xda, 0xbf, 0xe1, 0xf7, 0xed, 0xbf, 0x76, 0x33, 0xa3, 0x02, 0x9f, 0x5d, 0x8f, 0xff,
-	0x17, 0x00, 0x00, 0xff, 0xff, 0x20, 0x74, 0xd7, 0x8f, 0x1b, 0x0e, 0x00, 0x00,
+	// 1700 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x73, 0xdb, 0xc6,
+	0x15, 0x17, 0x48, 0xf0, 0xdf, 0x23, 0xa5, 0x40, 0x6b, 0x59, 0x41, 0x61, 0x85, 0x42, 0xe0, 0xb8,
+	0x55, 0x34, 0x8d, 0xe2, 0xc6, 0x69, 0x33, 0xd3, 0x69, 0x3b, 0x43, 0x91, 0x90, 0xcc, 0x09, 0x0d,
+	0xb2, 0x4b, 0xca, 0xae, 0x73, 0x28, 0x06, 0x24, 0x96, 0x14, 0x46, 0x20, 0x96, 0x05, 0x40, 0xd9,
+	0xea, 0x47, 0x60, 0xbf, 0x40, 0x2f, 0x9c, 0xc9, 0xb5, 0xf7, 0x7e, 0x08, 0xf7, 0x96, 0xc9, 0xb1,
+	0x07, 0x4f, 0xa3, 0x5e, 0x7a, 0xec, 0xa5, 0xf7, 0x0e, 0x76, 0x01, 0x10, 0x94, 0xe4, 0x8e, 0x0f,
+	0x3d, 0x71, 0xf7, 0xbd, 0xdf, 0xbe, 0xdd, 0xf7, 0x7b, 0xef, 0xf7, 0x08, 0xa8, 0x0c, 0xc9, 0xec,
+	0x68, 0xe6, 0xd3, 0x90, 0xa2, 0x32, 0xfb, 0x19, 0x51, 0x57, 0xf9, 0x6c, 0xe2, 0x84, 0xe7, 0xf3,
+	0xe1, 0xd1, 0x88, 0x4e, 0x3f, 0x9f, 0xd0, 0x09, 0xfd, 0x9c, 0x79, 0x86, 0xf3, 0x31, 0xdb, 0xb1,
+	0x0d, 0x5b, 0xf1, 0x83, 0xda, 0x0c, 0x0a, 0x4f, 0x89, 0xeb, 0x52, 0xb4, 0x0f, 0x55, 0x9b, 0x5c,
+	0x3a, 0x23, 0x62, 0x7a, 0xd6, 0x94, 0xc8, 0x82, 0x2a, 0x1c, 0x54, 0x30, 0x70, 0x93, 0x61, 0x4d,
+	0x49, 0x04, 0x18, 0xb9, 0x0e, 0xf1, 0x42, 0x0e, 0xc8, 0x71, 0x00, 0x37, 0x31, 0xc0, 0x23, 0xd8,
+	0x8a, 0x01, 0x97, 0xc4, 0x0f, 0x1c, 0xea, 0xc9, 0x79, 0x86, 0xd9, 0xe4, 0xd6, 0xe7, 0xdc, 0xa8,
+	0x05, 0x50, 0x7c, 0x4a, 0x2c, 0x9b, 0xf8, 0xe8, 0x53, 0x10, 0xc3, 0xab, 0x19, 0xbf, 0x6b, 0xeb,
+	0x8b, 0xfb, 0x47, 0x49, 0x0e, 0x47, 0xcf, 0x48, 0x10, 0x58, 0x13, 0x32, 0xb8, 0x9a, 0x11, 0xcc,
+	0x20, 0xe8, 0x37, 0x50, 0x1d, 0xd1, 0xe9, 0xcc, 0x27, 0x01, 0x0b, 0x9c, 0x63, 0x27, 0xf6, 0x6e,
+	0x9d, 0x68, 0xae, 0x30, 0x38, 0x7b, 0x40, 0x6b, 0xc0, 0x66, 0xd3, 0x9d, 0x07, 0x21, 0xf1, 0x9b,
+	0xd4, 0x1b, 0x3b, 0x13, 0xf4, 0x18, 0x4a, 0x63, 0xea, 0xda, 0xc4, 0x0f, 0x64, 0x41, 0xcd, 0x1f,
+	0x54, 0xbf, 0x90, 0x56, 0xc1, 0x4e, 0x98, 0xe3, 0x58, 0x7c, 0xf3, 0x76, 0x7f, 0x03, 0x27, 0x30,
+	0xed, 0x4f, 0x39, 0x28, 0x72, 0x0f, 0xda, 0x85, 0x9c, 0x63, 0x73, 0x8a, 0x8e, 0x8b, 0xd7, 0x6f,
+	0xf7, 0x73, 0xed, 0x16, 0xce, 0x39, 0x36, 0xda, 0x81, 0x82, 0x6b, 0x0d, 0x89, 0x1b, 0x93, 0xc3,
+	0x37, 0xe8, 0x01, 0x54, 0x7c, 0x62, 0xd9, 0x26, 0xf5, 0xdc, 0x2b, 0x46, 0x49, 0x19, 0x97, 0x23,
+	0x43, 0xd7, 0x73, 0xaf, 0xd0, 0x67, 0x80, 0x9c, 0x89, 0x47, 0x7d, 0x62, 0xce, 0x88, 0x3f, 0x75,
+	0xd8, 0x6b, 0x03, 0x59, 0x64, 0xa8, 0x6d, 0xee, 0xe9, 0xad, 0x1c, 0xe8, 0x21, 0x6c, 0xc6, 0x70,
+	0x9b, 0xb8, 0x24, 0x24, 0x72, 0x81, 0x21, 0x6b, 0xdc, 0xd8, 0x62, 0x36, 0xf4, 0x18, 0x76, 0x6c,
+	0x27, 0xb0, 0x86, 0x2e, 0x31, 0x43, 0x32, 0x9d, 0x99, 0x8e, 0x67, 0x93, 0xd7, 0x24, 0x90, 0x8b,
+	0x0c, 0x8b, 0x62, 0xdf, 0x80, 0x4c, 0x67, 0x6d, 0xee, 0x89, 0xd8, 0xe0, 0x95, 0x0e, 0x64, 0xe9,
+	0x26, 0x1b, 0x2d, 0xe6, 0x48, 0xd8, 0x88, 0x61, 0xda, 0xbf, 0x73, 0x50, 0xe4, 0x1e, 0xf4, 0xe3,
+	0x94, 0x8d, 0xda, 0xf1, 0x6e, 0x84, 0xfa, 0xfb, 0xdb, 0xfd, 0x32, 0xf7, 0xb5, 0x5b, 0x19, 0x76,
+	0x10, 0x88, 0x99, 0xce, 0x61, 0x6b, 0xb4, 0x07, 0x15, 0xcb, 0xb6, 0xa3, 0x2a, 0x91, 0x40, 0xce,
+	0xab, 0xf9, 0x83, 0x0a, 0x5e, 0x19, 0xd0, 0x57, 0xeb, 0x55, 0x17, 0x6f, 0xf6, 0xc9, 0xbb, 0xca,
+	0x1d, 0x51, 0x3e, 0x22, 0x7e, 0xdc, 0xa9, 0x05, 0x76, 0x5f, 0x39, 0x32, 0xb0, 0x3e, 0xfd, 0x18,
+	0x6a, 0x53, 0xeb, 0xb5, 0x19, 0x90, 0x3f, 0xcc, 0x89, 0x37, 0x22, 0x8c, 0x96, 0x3c, 0xae, 0x4e,
+	0xad, 0xd7, 0xfd, 0xd8, 0x84, 0xea, 0x00, 0x8e, 0x17, 0xfa, 0xd4, 0x9e, 0x8f, 0x88, 0x2f, 0x97,
+	0x18, 0x6f, 0x19, 0x0b, 0xfa, 0x39, 0x94, 0x19, 0xa9, 0xa6, 0x63, 0xcb, 0x65, 0x55, 0x38, 0x10,
+	0x8f, 0x95, 0x38, 0xf1, 0x12, 0xa3, 0x94, 0xe5, 0x9d, 0x2c, 0x71, 0x89, 0x61, 0xdb, 0x36, 0xfa,
+	0x15, 0x28, 0xc1, 0x85, 0x13, 0x15, 0x84, 0x47, 0x0a, 0x1d, 0xea, 0x99, 0x3e, 0x99, 0xd2, 0x4b,
+	0xcb, 0x0d, 0xe4, 0x0a, 0xbb, 0x46, 0x8e, 0x10, 0xed, 0x0c, 0x00, 0xc7, 0x7e, 0xad, 0x0b, 0x05,
+	0x16, 0x11, 0xed, 0x42, 0x91, 0x37, 0x65, 0xac, 0xd2, 0x78, 0x87, 0x8e, 0xa0, 0x30, 0x76, 0x5c,
+	0x12, 0xc8, 0x39, 0x56, 0x43, 0x94, 0xe9, 0x68, 0xc7, 0x25, 0x6d, 0x6f, 0x4c, 0xe3, 0x2a, 0x72,
+	0x98, 0x76, 0x06, 0x55, 0x16, 0xf0, 0x6c, 0x66, 0x5b, 0x21, 0xf9, 0xbf, 0x85, 0xfd, 0x5b, 0x1e,
+	0xca, 0x89, 0x27, 0x2d, 0xba, 0x90, 0x29, 0xfa, 0x61, 0xac, 0x7b, 0xae, 0xe2, 0xdd, 0xdb, 0xf1,
+	0x32, 0xc2, 0x47, 0x20, 0x06, 0xce, 0x1f, 0x09, 0xd3, 0x4d, 0x1e, 0xb3, 0x35, 0x52, 0xa1, 0x7a,
+	0x53, 0x2c, 0x9b, 0x38, 0x6b, 0x42, 0x1f, 0x01, 0x4c, 0xa9, 0xed, 0x8c, 0x1d, 0x62, 0x9b, 0x01,
+	0x6b, 0x80, 0x3c, 0xae, 0x24, 0x96, 0x3e, 0x92, 0xa3, 0x76, 0x8f, 0xa4, 0x62, 0xc7, 0x9a, 0x48,
+	0xb6, 0x91, 0xc7, 0xf1, 0x2e, 0x2d, 0xd7, 0xb1, 0xe3, 0xaa, 0x27, 0xdb, 0x68, 0xba, 0x79, 0x74,
+	0x4d, 0xa4, 0x65, 0x06, 0xd8, 0xf4, 0x68, 0x56, 0xa0, 0x8f, 0xa1, 0x94, 0x4c, 0xbf, 0xa8, 0x9e,
+	0x6b, 0x4a, 0x7a, 0x4e, 0x46, 0x21, 0x4d, 0xe7, 0x4a, 0x0c, 0x43, 0x0a, 0x94, 0xd3, 0x56, 0x04,
+	0xf6, 0xd2, 0x74, 0x1f, 0xcd, 0xdc, 0x34, 0x0f, 0x2f, 0x90, 0xab, 0xaa, 0x70, 0x50, 0xc0, 0x69,
+	0x6a, 0x46, 0x80, 0x7e, 0x06, 0xc5, 0x63, 0x97, 0x8e, 0x2e, 0x12, 0xdd, 0xde, 0x5b, 0xdd, 0xc6,
+	0xec, 0x99, 0xea, 0x14, 0x87, 0x0c, 0x18, 0x25, 0x12, 0x5c, 0x4d, 0x5d, 0xc7, 0xbb, 0x30, 0x43,
+	0xcb, 0x9f, 0x90, 0x50, 0xde, 0xe6, 0x63, 0x3a, 0xb6, 0x0e, 0x98, 0xf1, 0x97, 0xe2, 0x9f, 0xbf,
+	0xdd, 0xdf, 0xd0, 0x3c, 0xa8, 0xa4, 0x71, 0xa2, 0x06, 0xa1, 0xe3, 0x71, 0x40, 0x42, 0x56, 0xcd,
+	0x3c, 0x8e, 0x77, 0x69, 0x8d, 0x72, 0xec, 0x79, 0xbc, 0x46, 0x08, 0xc4, 0x73, 0x2b, 0x38, 0x67,
+	0x75, 0xab, 0x61, 0xb6, 0x8e, 0x54, 0xf9, 0x8a, 0x58, 0x17, 0x26, 0x73, 0xf0, 0xaa, 0x95, 0x23,
+	0xc3, 0x53, 0x2b, 0x38, 0x8f, 0xef, 0xfb, 0x35, 0x14, 0x39, 0x4b, 0xe8, 0x09, 0x94, 0x47, 0x74,
+	0xee, 0x85, 0xab, 0x09, 0xbd, 0x9d, 0x15, 0x3e, 0xf3, 0xc4, 0x99, 0xa5, 0x40, 0xed, 0x04, 0x4a,
+	0xb1, 0x0b, 0x3d, 0x4a, 0xa7, 0x92, 0x78, 0x7c, 0x3f, 0x11, 0x67, 0xff, 0x9c, 0xfa, 0xe1, 0xda,
+	0x50, 0xda, 0x81, 0xc2, 0xa5, 0xe5, 0xce, 0xf9, 0xe3, 0x45, 0xcc, 0x37, 0xda, 0x5f, 0x05, 0x28,
+	0xe1, 0xa8, 0x08, 0x41, 0x98, 0x19, 0xf6, 0x85, 0xb5, 0x61, 0xbf, 0x92, 0x4b, 0x6e, 0x4d, 0x2e,
+	0x49, 0xc7, 0xe7, 0x33, 0x1d, 0xbf, 0x62, 0x4e, 0xbc, 0x93, 0xb9, 0xc2, 0x1d, 0xcc, 0x15, 0x33,
+	0xcc, 0x3d, 0x82, 0xad, 0xb1, 0x4f, 0xa7, 0x6c, 0x9c, 0x53, 0xdf, 0xf2, 0xaf, 0xe2, 0xee, 0xdc,
+	0x8c, 0xac, 0x83, 0xc4, 0xa8, 0x99, 0x50, 0xc6, 0x24, 0x98, 0x51, 0x2f, 0x20, 0xef, 0x7c, 0x36,
+	0x02, 0xd1, 0xb6, 0x42, 0x8b, 0x3d, 0xba, 0x86, 0xd9, 0x1a, 0xfd, 0x04, 0xc4, 0x11, 0xb5, 0xf9,
+	0x93, 0xb7, 0xb2, 0x3d, 0xa4, 0xfb, 0x3e, 0xf5, 0x9b, 0xd4, 0x26, 0x98, 0x01, 0xb4, 0x19, 0x48,
+	0x2d, 0xfa, 0xca, 0x73, 0xa9, 0x65, 0xf7, 0x7c, 0x3a, 0x89, 0xc6, 0xed, 0x3b, 0xc7, 0x46, 0x0b,
+	0x4a, 0x73, 0x36, 0x58, 0x92, 0xc1, 0xf1, 0xc9, 0xba, 0xd0, 0x6f, 0x06, 0xe2, 0x53, 0x28, 0x51,
+	0x47, 0x7c, 0x54, 0xfb, 0x5e, 0x00, 0xe5, 0xdd, 0x68, 0xd4, 0x86, 0x2a, 0x47, 0x9a, 0x99, 0x2f,
+	0x89, 0x83, 0xf7, 0xb9, 0x88, 0xcd, 0x18, 0x98, 0xa7, 0xeb, 0x3b, 0xff, 0x9e, 0x32, 0x6a, 0xce,
+	0xbf, 0x9f, 0x9a, 0x1f, 0xc2, 0x26, 0xd3, 0x59, 0xfa, 0xa7, 0x2b, 0xaa, 0xf9, 0x83, 0x02, 0xae,
+	0x0d, 0xb9, 0x8a, 0x98, 0x4d, 0x2b, 0x82, 0xd8, 0x73, 0xbc, 0x89, 0xb6, 0x0f, 0x85, 0xa6, 0x4b,
+	0x59, 0xb1, 0x8a, 0x3e, 0xb1, 0x02, 0xea, 0x25, 0x1c, 0xf2, 0xdd, 0xe1, 0xf7, 0x39, 0xa8, 0x66,
+	0x3e, 0x86, 0xd0, 0x63, 0xd8, 0x6a, 0x76, 0xce, 0xfa, 0x03, 0x1d, 0x9b, 0xcd, 0xae, 0x71, 0xd2,
+	0x3e, 0x95, 0x36, 0x94, 0xbd, 0xc5, 0x52, 0x95, 0xa7, 0x2b, 0xd0, 0xfa, 0x77, 0xce, 0x3e, 0x14,
+	0xda, 0x46, 0x4b, 0xff, 0x9d, 0x24, 0x28, 0x3b, 0x8b, 0xa5, 0x2a, 0x65, 0x80, 0xfc, 0xcf, 0xe4,
+	0xa7, 0x50, 0x63, 0x00, 0xf3, 0xac, 0xd7, 0x6a, 0x0c, 0x74, 0x29, 0xa7, 0x28, 0x8b, 0xa5, 0xba,
+	0x7b, 0x13, 0x17, 0xf3, 0xfd, 0x10, 0x4a, 0x58, 0xff, 0xed, 0x99, 0xde, 0x1f, 0x48, 0x79, 0x65,
+	0x77, 0xb1, 0x54, 0x51, 0x06, 0x98, 0x28, 0xe6, 0x11, 0x94, 0xb1, 0xde, 0xef, 0x75, 0x8d, 0xbe,
+	0x2e, 0x89, 0xca, 0x87, 0x8b, 0xa5, 0x7a, 0x6f, 0x0d, 0x15, 0x77, 0xe8, 0x2f, 0x60, 0xbb, 0xd5,
+	0x7d, 0x61, 0x74, 0xba, 0x8d, 0x96, 0xd9, 0xc3, 0xdd, 0x53, 0xac, 0xf7, 0xfb, 0x52, 0x41, 0xd9,
+	0x5f, 0x2c, 0xd5, 0x07, 0x19, 0xfc, 0xad, 0x86, 0xfb, 0x08, 0xc4, 0x5e, 0xdb, 0x38, 0x95, 0x8a,
+	0xca, 0xbd, 0xc5, 0x52, 0xfd, 0x20, 0x03, 0x8d, 0x48, 0x8d, 0x32, 0x6e, 0x76, 0xba, 0x7d, 0x5d,
+	0x2a, 0xdd, 0xca, 0x98, 0x91, 0x7d, 0xf8, 0x7b, 0x40, 0xb7, 0x3f, 0x17, 0xd1, 0x27, 0x20, 0x1a,
+	0x5d, 0x43, 0x97, 0x36, 0x78, 0xfe, 0xb7, 0x11, 0x06, 0xf5, 0x08, 0xd2, 0x20, 0xdf, 0xf9, 0xe6,
+	0x4b, 0x49, 0x50, 0x7e, 0xb4, 0x58, 0xaa, 0xf7, 0x6f, 0x83, 0x3a, 0xdf, 0x7c, 0x79, 0x48, 0xa1,
+	0x9a, 0x0d, 0xac, 0x41, 0xf9, 0x99, 0x3e, 0x68, 0xb4, 0x1a, 0x83, 0x86, 0xb4, 0xc1, 0x9f, 0x94,
+	0xb8, 0x9f, 0x91, 0xd0, 0x62, 0x02, 0xdc, 0x83, 0x82, 0xa1, 0x3f, 0xd7, 0xb1, 0x24, 0x28, 0xdb,
+	0x8b, 0xa5, 0xba, 0x99, 0x00, 0x0c, 0x72, 0x49, 0x7c, 0x54, 0x87, 0x62, 0xa3, 0xf3, 0xa2, 0xf1,
+	0xb2, 0x2f, 0xe5, 0x14, 0xb4, 0x58, 0xaa, 0x5b, 0x89, 0xbb, 0xe1, 0xbe, 0xb2, 0xae, 0x82, 0xc3,
+	0xff, 0x08, 0x50, 0xcb, 0xfe, 0x75, 0xa2, 0x3a, 0x88, 0x27, 0xed, 0x8e, 0x9e, 0x5c, 0x97, 0xf5,
+	0x45, 0x6b, 0x74, 0x00, 0x95, 0x56, 0x1b, 0xeb, 0xcd, 0x41, 0x17, 0xbf, 0x4c, 0x72, 0xc9, 0x82,
+	0x5a, 0x8e, 0xcf, 0x9a, 0x3b, 0xfa, 0x3c, 0xad, 0xf5, 0x5f, 0x3e, 0xeb, 0xb4, 0x8d, 0xaf, 0x4d,
+	0x16, 0x31, 0xa7, 0x3c, 0x58, 0x2c, 0xd5, 0x0f, 0xb3, 0xe0, 0x3e, 0xff, 0xdb, 0x60, 0x81, 0xbf,
+	0x82, 0xed, 0x04, 0xbe, 0xba, 0x20, 0xaf, 0xa8, 0x8b, 0xa5, 0xba, 0x77, 0xc7, 0x99, 0xd5, 0x3d,
+	0x4f, 0xe0, 0x83, 0xe4, 0xe0, 0x99, 0xf1, 0xb5, 0xd1, 0x7d, 0x61, 0x48, 0xa2, 0x52, 0x5f, 0x2c,
+	0x55, 0xe5, 0x8e, 0x63, 0x67, 0xde, 0x85, 0x47, 0x5f, 0x79, 0x87, 0x7f, 0x11, 0xa0, 0x92, 0x4e,
+	0xa8, 0x88, 0x67, 0xa3, 0x6b, 0xea, 0x18, 0x77, 0x71, 0x92, 0x78, 0xea, 0x34, 0x28, 0x5b, 0xa2,
+	0x8f, 0xa1, 0x74, 0xaa, 0x1b, 0x3a, 0x6e, 0x37, 0x13, 0x3d, 0xa4, 0x90, 0x53, 0xe2, 0x11, 0xdf,
+	0x19, 0xa1, 0x4f, 0xa1, 0x66, 0x74, 0xcd, 0xfe, 0x59, 0xf3, 0x69, 0x92, 0x31, 0x6b, 0xe0, 0x4c,
+	0xa8, 0xfe, 0x7c, 0x74, 0xce, 0xb2, 0x3d, 0x8c, 0xa4, 0xf3, 0xbc, 0xd1, 0x69, 0xb7, 0x38, 0x34,
+	0xaf, 0xc8, 0x8b, 0xa5, 0xba, 0x93, 0x42, 0xdb, 0xfc, 0xd3, 0x21, 0xc2, 0x1e, 0xda, 0x50, 0xff,
+	0xdf, 0xb3, 0x08, 0xa9, 0x50, 0x6c, 0xf4, 0x7a, 0xba, 0xd1, 0x4a, 0x5e, 0xbf, 0xf2, 0x35, 0x66,
+	0x33, 0xe2, 0xd9, 0x11, 0xe2, 0xa4, 0x8b, 0x4f, 0xf5, 0x41, 0xf2, 0xf8, 0x15, 0xe2, 0x84, 0x46,
+	0x7f, 0xda, 0xc7, 0x7b, 0x6f, 0x7e, 0xa8, 0x6f, 0x7c, 0xf7, 0x43, 0x7d, 0xe3, 0xcd, 0x75, 0x5d,
+	0xf8, 0xee, 0xba, 0x2e, 0xfc, 0xe3, 0xba, 0xbe, 0xf1, 0xaf, 0xeb, 0xba, 0xf0, 0xed, 0x3f, 0xeb,
+	0xc2, 0xb0, 0xc8, 0x66, 0xd7, 0x93, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xee, 0xe2, 0x0f, 0x00,
+	0x38, 0x0e, 0x00, 0x00,
 }

lib/protocol/bep.proto

@@ -120,9 +120,10 @@ enum FileInfoType {
 
 message BlockInfo {
     option (gogoproto.goproto_stringer) = false;
-    int64 offset = 1;
-    int32 size   = 2;
-    bytes hash   = 3;
+    int64  offset    = 1;
+    int32  size      = 2;
+    bytes  hash      = 3;
+    uint32 weak_hash = 4;
 }
 
 message Vector {

lib/protocol/bep_extensions.go

@@ -114,7 +114,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool {
 }
 
 func (b BlockInfo) String() string {
-	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
+	return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash)
 }
 
 // IsEmpty returns true if the block is a full block of zeroes.

lib/protocol/deviceid_test.pb.go

@@ -427,16 +427,16 @@ var (
 )
 
 var fileDescriptorDeviceidTest = []byte{
-	// 171 bytes of a gzipped FileDescriptorProto
+	// 176 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x12, 0x4e, 0x49, 0x2d, 0xcb,
 	0x4c, 0x4e, 0xcd, 0x4c, 0x89, 0x2f, 0x49, 0x2d, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17,
 	0xe2, 0x00, 0x53, 0xc9, 0xf9, 0x39, 0x52, 0xba, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9,
 	0xf9, 0xb9, 0xfa, 0xe9, 0xf9, 0xe9, 0xf9, 0xfa, 0x60, 0x99, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc,
-	0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x80, 0xc6, 0xf8, 0xe7, 0xa4, 0xb8, 0x80,
-	0x8d, 0xf5, 0x74, 0x11, 0x12, 0xe2, 0x62, 0x01, 0x99, 0x2c, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13,
-	0x04, 0x66, 0x2b, 0x99, 0x43, 0x94, 0xf9, 0xa5, 0x96, 0xc3, 0x95, 0xa9, 0x20, 0x2b, 0x73, 0x12,
-	0x38, 0x71, 0x4f, 0x9e, 0xe1, 0xd6, 0x3d, 0x79, 0x0e, 0x98, 0x3c, 0x44, 0xa3, 0x93, 0xcc, 0x89,
-	0x87, 0x72, 0x0c, 0x17, 0x80, 0xf8, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x40, 0xfc, 0xe0, 0x91, 0x1c,
-	0xc3, 0x0b, 0x20, 0x5e, 0xf0, 0x58, 0x8e, 0x31, 0x89, 0x0d, 0xec, 0x08, 0x63, 0x40, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00,
+	0x01, 0xb3, 0x20, 0x1a, 0x95, 0x54, 0xb9, 0xf8, 0x43, 0x52, 0x8b, 0x4b, 0xfc, 0x73, 0x52, 0x5c,
+	0xc0, 0xc6, 0x7a, 0xba, 0x08, 0x09, 0x71, 0xb1, 0x80, 0x4c, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0,
+	0x09, 0x02, 0xb3, 0x95, 0xcc, 0x21, 0xca, 0xfc, 0x52, 0xcb, 0xe1, 0xca, 0x54, 0x90, 0x95, 0x39,
+	0x09, 0x9c, 0xb8, 0x27, 0xcf, 0x70, 0xeb, 0x9e, 0x3c, 0x07, 0x4c, 0x1e, 0xa2, 0xd1, 0x49, 0xe6,
+	0xc4, 0x43, 0x39, 0x86, 0x0b, 0x0f, 0xe5, 0x18, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e,
+	0xf1, 0xc1, 0x23, 0x39, 0x86, 0x17, 0x8f, 0xe4, 0x18, 0x16, 0x3c, 0x96, 0x63, 0x4c, 0x62, 0x03,
+	0x3b, 0xc2, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x35, 0x9c, 0x00, 0x78, 0xd4, 0x00, 0x00, 0x00,
 }

lib/scanner/blocks.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/weakhash"
 )
 
 var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
@@ -25,6 +26,7 @@ type Counter interface {
 func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
 	hf := sha256.New()
 	hashLength := hf.Size()
+	whf := weakhash.NewHash(blocksize)
 
 	var blocks []protocol.BlockInfo
 	var hashes, thisHash []byte
@@ -44,7 +46,7 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
 	var offset int64
 	for {
 		lr := io.LimitReader(r, int64(blocksize))
-		n, err := copyBuffer(hf, lr, buf)
+		n, err := io.CopyBuffer(hf, io.TeeReader(lr, whf), buf)
 		if err != nil {
 			return nil, err
 		}
@@ -63,15 +65,17 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
 		thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
 
 		b := protocol.BlockInfo{
-			Size:   int32(n),
-			Offset: offset,
-			Hash:   thisHash,
+			Size:     int32(n),
+			Offset:   offset,
+			Hash:     thisHash,
+			WeakHash: whf.Sum32(),
 		}
 
 		blocks = append(blocks, b)
 		offset += int64(n)
 
 		hf.Reset()
+		whf.Reset()
 	}
 
 	if len(blocks) == 0 {
@@ -123,9 +127,12 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo)
 // list and actual reader contents
 func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
 	hf := sha256.New()
+	// A 32k buffer is used for copying into the hash function.
+	buf := make([]byte, 32<<10)
+
 	for i, block := range blocks {
 		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
-		_, err := io.Copy(hf, lr)
+		_, err := io.CopyBuffer(hf, lr, buf)
 		if err != nil {
 			return err
 		}
@@ -180,48 +187,3 @@ func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
 	}
 	return true
 }
-
-// This is a copy & paste of io.copyBuffer from the Go 1.5 standard library,
-// as we want this but also want to build with Go 1.3+.
-
-// copyBuffer is the actual implementation of Copy and CopyBuffer.
-// if buf is nil, one is allocated.
-func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
-	// If the reader has a WriteTo method, use it to do the copy.
-	// Avoids an allocation and a copy.
-	if wt, ok := src.(io.WriterTo); ok {
-		return wt.WriteTo(dst)
-	}
-	// Similarly, if the writer has a ReadFrom method, use it to do the copy.
-	if rt, ok := dst.(io.ReaderFrom); ok {
-		return rt.ReadFrom(src)
-	}
-	if buf == nil {
-		buf = make([]byte, 32*1024)
-	}
-	for {
-		nr, er := src.Read(buf)
-		if nr > 0 {
-			nw, ew := dst.Write(buf[0:nr])
-			if nw > 0 {
-				written += int64(nw)
-			}
-			if ew != nil {
-				err = ew
-				break
-			}
-			if nr != nw {
-				err = io.ErrShortWrite
-				break
-			}
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
-		}
-	}
-	return written, err
-}

lib/weakhash/benchmark_test.go

@@ -0,0 +1,30 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package weakhash
+
+import (
+	"os"
+	"testing"
+)
+
+const testFile = "../model/testdata/~syncthing~file.tmp"
+
+func BenchmarkFind1MFile(b *testing.B) {
+	b.ReportAllocs()
+	b.SetBytes(1 << 20)
+	for i := 0; i < b.N; i++ {
+		fd, err := os.Open(testFile)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = Find(fd, []uint32{0, 1, 2}, 128<<10)
+		if err != nil {
+			b.Fatal(err)
+		}
+		fd.Close()
+	}
+}

lib/weakhash/weakhash.go

@@ -0,0 +1,169 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package weakhash
+
+import (
+	"bufio"
+	"hash"
+	"io"
+	"os"
+)
+
+const (
+	Size = 4
+)
+
+func NewHash(size int) hash.Hash32 {
+	return &digest{
+		buf:  make([]byte, size),
+		size: size,
+	}
+}
+
+// Find finds all the blocks of the given size within io.Reader that matches
+// the hashes provided, and returns a hash -> slice of offsets within reader
+// map, that produces the same weak hash.
+func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) {
+	if ir == nil {
+		return nil, nil
+	}
+
+	r := bufio.NewReader(ir)
+	hf := NewHash(size)
+
+	n, err := io.CopyN(hf, r, int64(size))
+	if err == io.EOF {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if n != int64(size) {
+		return nil, io.ErrShortBuffer
+	}
+
+	offsets := make(map[uint32][]int64)
+	for _, hashToFind := range hashesToFind {
+		offsets[hashToFind] = nil
+	}
+
+	var i int64
+	var hash uint32
+	for {
+		hash = hf.Sum32()
+		if existing, ok := offsets[hash]; ok {
+			offsets[hash] = append(existing, i)
+		}
+		i++
+
+		bt, err := r.ReadByte()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return offsets, err
+		}
+		hf.Write([]byte{bt})
+	}
+	return offsets, nil
+}
+
+// Using this: http://tutorials.jenkov.com/rsync/checksums.html
+// Example implementations: https://gist.github.com/csabahenk/1096262/revisions
+// Alternative that could be used is adler32 http://blog.liw.fi/posts/rsync-in-python/#comment-fee8d5e07794fdba3fe2d76aa2706a13
+type digest struct {
+	buf  []byte
+	size int
+	a    uint16
+	b    uint16
+	j    int
+}
+
+func (d *digest) Write(data []byte) (int, error) {
+	for _, c := range data {
+		// TODO: Use this in Go 1.6
+		// d.a = d.a - uint16(d.buf[d.j]) + uint16(c)
+		// d.b = d.b - uint16(d.size)*uint16(d.buf[d.j]) + d.a
+		d.a -= uint16(d.buf[d.j])
+		d.a += uint16(c)
+		d.b -= uint16(d.size) * uint16(d.buf[d.j])
+		d.b += d.a
+
+		d.buf[d.j] = c
+		d.j = (d.j + 1) % d.size
+	}
+	return len(data), nil
+}
+
+func (d *digest) Reset() {
+	for i := range d.buf {
+		d.buf[i] = 0x0
+	}
+	d.a = 0
+	d.b = 0
+	d.j = 0
+}
+
+func (d *digest) Sum(b []byte) []byte {
+	r := d.Sum32()
+	return append(b, byte(r>>24), byte(r>>16), byte(r>>8), byte(r))
+}
+
+func (d *digest) Sum32() uint32 { return uint32(d.a) | (uint32(d.b) << 16) }
+func (digest) Size() int        { return Size }
+func (digest) BlockSize() int   { return 1 }
+
+func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	offsets, err := Find(file, hashesToFind, size)
+	if err != nil {
+		file.Close()
+		return nil, err
+	}
+
+	return &Finder{
+		file:    file,
+		size:    size,
+		offsets: offsets,
+	}, nil
+}
+
+type Finder struct {
+	file    *os.File
+	size    int
+	offsets map[uint32][]int64
+}
+
+// Iterate iterates all available blocks that matches the provided hash, reads
+// them into buf, and calls the iterator function. The iterator function should
+// return wether it wishes to continue interating.
+func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) {
+	if h == nil || hash == 0 || len(buf) != h.size {
+		return false, nil
+	}
+
+	for _, offset := range h.offsets[hash] {
+		_, err := h.file.ReadAt(buf, offset)
+		if err != nil {
+			return false, err
+		}
+		if !iterFunc(offset) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// Close releases any resource associated with the finder
+func (h *Finder) Close() {
+	if h != nil {
+		h.file.Close()
+	}
+}

lib/weakhash/weakhash_test.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// The existence of this file means we get 0% test coverage rather than no
+// test coverage at all. Remove when implementing an actual test.
+
+package weakhash
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+)
+
+var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz")
+var hashes = []uint32{
+	64225674,
+	64881038,
+	65536402,
+	66191766,
+	66847130,
+	67502494,
+	68157858,
+	68813222,
+	69468586,
+	70123950,
+	70779314,
+	71434678,
+	72090042,
+	72745406,
+	73400770,
+	74056134,
+	74711498,
+	75366862,
+	76022226,
+	76677590,
+	77332954,
+	77988318,
+	78643682,
+	77595084,
+	74842550,
+	70386080,
+	64225674,
+	64881038,
+	65536402,
+	66191766,
+	66847130,
+	67502494,
+	68157858,
+	68813222,
+	69468586,
+	70123950,
+	70779314,
+	71434678,
+	72090042,
+	72745406,
+	73400770,
+	74056134,
+	74711498,
+	75366862,
+	76022226,
+	76677590,
+	77332954,
+	77988318,
+	78643682,
+	77595084,
+	74842550,
+	70386080,
+	64225674,
+	64881038,
+	65536402,
+	66191766,
+	66847130,
+	67502494,
+	68157858,
+	68813222,
+	69468586,
+	70123950,
+	70779314,
+	71434678,
+	72090042,
+	72745406,
+	73400770,
+	74056134,
+	74711498,
+	75366862,
+	76022226,
+	76677590,
+	77332954,
+	77988318,
+	78643682,
+	77595084,
+	74842550,
+	70386080,
+	64225674,
+	64881038,
+	65536402,
+	66191766,
+	66847130,
+	67502494,
+	68157858,
+	68813222,
+	69468586,
+	70123950,
+	70779314,
+	71434678,
+	72090042,
+	72745406,
+	73400770,
+	74056134,
+	74711498,
+	75366862,
+	76022226,
+	76677590,
+	77332954,
+	77988318,
+	78643682,
+	71893365,
+	71893365,
+}
+
+// Tested using an alternative C implementation at https://gist.github.com/csabahenk/1096262
+func TestHashCorrect(t *testing.T) {
+	h := NewHash(Size)
+	pos := 0
+	for pos < Size {
+		h.Write([]byte{payload[pos]})
+		pos++
+	}
+
+	for i := 0; pos < len(payload); i++ {
+		if h.Sum32() != hashes[i] {
+			t.Errorf("mismatch at %d", i)
+		}
+		h.Write([]byte{payload[pos]})
+		pos++
+	}
+}
+
+func TestFinder(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+
+	if _, err := f.Write(payload); err != nil {
+		t.Error(err)
+	}
+
+	hashes := []uint32{64881038, 65536402}
+	finder, err := NewFinder(f.Name(), 4, hashes)
+	if err != nil {
+		t.Error(err)
+	}
+	defer finder.Close()
+
+	expected := map[uint32][]int64{
+		64881038: []int64{1, 27, 53, 79},
+		65536402: []int64{2, 28, 54, 80},
+	}
+	actual := make(map[uint32][]int64)
+
+	b := make([]byte, Size)
+
+	for _, hash := range hashes {
+		_, err := finder.Iterate(hash, b[:4], func(offset int64) bool {
+			if !bytes.Equal(b, payload[offset:offset+4]) {
+				t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4]))
+			}
+			actual[hash] = append(actual[hash], offset)
+			return true
+		})
+		if err != nil {
+			t.Error(err)
+		}
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Not equal: %#v != %#v", actual, expected)
+	}
+}

test/h2/config.xml

@@ -1,12 +1,10 @@
 <configuration version="16">
-    <folder id="default" label="" path="s2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
+    <folder id="default" label="" path="s2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <minDiskFreePct>1</minDiskFreePct>
-        <versioning type="trashcan">
-            <param key="cleanoutDays" val="1"></param>
-        </versioning>
+        <versioning></versioning>
         <copiers>1</copiers>
         <pullers>16</pullers>
         <hashers>0</hashers>
@@ -18,8 +16,9 @@
         <maxConflicts>-1</maxConflicts>
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
+        <disableWeakHash>false</disableWeakHash>
     </folder>
-    <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
+    <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -35,8 +34,9 @@
         <maxConflicts>-1</maxConflicts>
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
+        <disableWeakHash>false</disableWeakHash>
     </folder>
-    <folder id="s23" label="" path="s23-2/" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
+    <folder id="s23" label="" path="s23-2" type="readwrite" rescanIntervalS="15" ignorePerms="false" autoNormalize="true">
         <device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -52,6 +52,7 @@
         <maxConflicts>-1</maxConflicts>
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
+        <disableWeakHash>false</disableWeakHash>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
         <address>tcp://127.0.0.1:22001</address>

test/h3/config.xml

@@ -1,5 +1,5 @@
-<configuration version="15">
-    <folder id="s23" label="" path="s23-3/" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
+<configuration version="16">
+    <folder id="s23" label="" path="s23-3" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
         <device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <minDiskFreePct>1</minDiskFreePct>
@@ -15,8 +15,9 @@
         <maxConflicts>-1</maxConflicts>
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
+        <disableWeakHash>false</disableWeakHash>
     </folder>
-    <folder id="default" label="" path="s3/" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
+    <folder id="default" label="" path="s3" type="readwrite" rescanIntervalS="20" ignorePerms="false" autoNormalize="true">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="MRIW7OK-NETT3M4-N6SBWME-N25O76W-YJKVXPH-FUMQJ3S-P57B74J-GBITBAC"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
@@ -35,6 +36,7 @@
         <maxConflicts>-1</maxConflicts>
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
+        <disableWeakHash>false</disableWeakHash>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false">
         <address>tcp://127.0.0.1:22001</address>