lib/db: Deduplicate block lists in database (fixes #5898) (#6283)

* lib/db: Deduplicate block lists in database (fixes #5898)

This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.

Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:

- There is a larger risk of getting that wrong and either dropping a
  block list in error or keeping them around forever.

- It's tricky with our current database, as we don't have dirty reads.
  This means that if we update two FileInfos with identical block lists in
  the same transaction we can't just do read/modify/write for the ref
  counters as we wouldn't see our own first update. See above about
  tracking this and risks about getting it wrong.

GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.

The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.

This is per-key-class stats for a large database I'm currently working
with, under the current schema:

```
 0x00:  9138161 items, 870876 KB keys + 7397482 KB data, 95 B +  809 B avg, 1637651 B max
 0x01:   185656 items,  10388 KB keys + 1790909 KB data, 55 B + 9646 B avg,  924525 B max
 0x02:   916890 items,  84795 KB keys +    3667 KB data, 92 B +    4 B avg,     192 B max
 0x03:      384 items,     27 KB keys +       5 KB data, 72 B +   15 B avg,      87 B max
 0x04:     1109 items,     17 KB keys +      17 KB data, 15 B +   15 B avg,      69 B max
 0x06:      383 items,      3 KB keys +       0 KB data,  9 B +    2 B avg,      18 B max
 0x07:      510 items,      4 KB keys +      12 KB data,  9 B +   24 B avg,      41 B max
 0x08:     1349 items,     12 KB keys +      10 KB data,  9 B +    8 B avg,      17 B max
 0x09:      194 items,      0 KB keys +     123 KB data,  5 B +  634 B avg,   11484 B max
 0x0a:        3 items,      0 KB keys +       0 KB data, 14 B +    7 B avg,      30 B max
 0x0b:   181836 items,   2363 KB keys +   10694 KB data, 13 B +   58 B avg,     173 B max
 Total 10426475 items, 968490 KB keys + 9202925 KB data.
```

Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:

```
 0x00:  9138161 items, 870876 KB keys + 2611392 KB data, 95 B +  285 B avg,    4788 B max
 0x01:   185656 items,  10388 KB keys + 1790909 KB data, 55 B + 9646 B avg,  924525 B max
 0x02:   916890 items,  84795 KB keys +    3667 KB data, 92 B +    4 B avg,     192 B max
 0x03:      384 items,     27 KB keys +       5 KB data, 72 B +   15 B avg,      87 B max
 0x04:     1109 items,     17 KB keys +      17 KB data, 15 B +   15 B avg,      69 B max
 0x06:      383 items,      3 KB keys +       0 KB data,  9 B +    2 B avg,      18 B max
 0x07:      510 items,      4 KB keys +      12 KB data,  9 B +   24 B avg,      41 B max
 0x09:      194 items,      0 KB keys +     123 KB data,  5 B +  634 B avg,   11484 B max
 0x0a:        3 items,      0 KB keys +       0 KB data, 14 B +   17 B avg,      51 B max
 0x0b:   181836 items,   2363 KB keys +   10694 KB data, 13 B +   58 B avg,     173 B max
 0x0d:    44282 items,   1461 KB keys +   61081 KB data, 33 B + 1379 B avg, 1637399 B max
 Total 10469408 items, 969939 KB keys + 4477905 KB data.
```

Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.

There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.

On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:

```
 0x00:  297122 items,  20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
 0x01:  115299 items,   7738 KB keys +  17542 KB data, 67 B +  152 B avg,    419 B max
 0x02: 1430537 items, 121223 KB keys +   5722 KB data, 84 B +    4 B avg,    253 B max
 ...
 Total 1947412 items, 151268 KB keys + 337485 KB data.
```

to:

```
 0x00:  297122 items,  20894 KB keys +  37038 KB data, 70 B +  124 B avg,    520 B max
 0x01:  115299 items,   7738 KB keys +  17542 KB data, 67 B +  152 B avg,    419 B max
 0x02: 1430537 items, 121223 KB keys +   5722 KB data, 84 B +    4 B avg,    253 B max
 ...
 0x0d:   18041 items,    595 KB keys +  71964 KB data, 33 B + 3988 B avg, 101109 B max
 Total 1965447 items, 151863 KB keys + 139628 KB data.
```

* wip

* wip

* wip

* wip

cmd/stindex/accounting.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"text/tabwriter"
+
+	"github.com/syncthing/syncthing/lib/db/backend"
+)
+
+// account prints key and data size statistics per class
+func account(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var ksizes [256]int
+	var dsizes [256]int
+	var counts [256]int
+	var max [256]int
+
+	for it.Next() {
+		key := it.Key()
+		t := key[0]
+		ds := len(it.Value())
+		ks := len(key)
+		s := ks + ds
+
+		counts[t]++
+		ksizes[t] += ks
+		dsizes[t] += ds
+		if s > max[t] {
+			max[t] = s
+		}
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
+	toti, totds, totks := 0, 0, 0
+	for t := range ksizes {
+		if ksizes[t] > 0 {
+			// yes metric kilobytes 🤘
+			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
+			toti += counts[t]
+			totds += dsizes[t]
+			totks += ksizes[t]
+		}
+	}
+	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
+	tw.Flush()
+}

cmd/stindex/main.go

@@ -35,15 +35,18 @@ func main() {
 		log.Fatal(err)
 	}
 
-	if mode == "dump" {
+	switch mode {
+	case "dump":
 		dump(ldb)
-	} else if mode == "dumpsize" {
+	case "dumpsize":
 		dumpsize(ldb)
-	} else if mode == "idxck" {
+	case "idxck":
 		if !idxck(ldb) {
 			os.Exit(1)
 		}
-	} else {
+	case "account":
+		account(ldb)
+	default:
 		fmt.Println("Unknown mode")
 	}
 }

go.mod

@@ -35,11 +35,14 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 	github.com/sasha-s/go-deadlock v0.2.0
 	github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
 	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
 	github.com/thejerf/suture v3.0.2+incompatible
 	github.com/urfave/cli v1.22.2
 	github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
+	github.com/willf/bitset v1.1.10 // indirect
+	github.com/willf/bloom v2.0.3+incompatible
 	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
 	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
 	golang.org/x/text v0.3.2

go.sum

@@ -8,8 +8,10 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -178,7 +180,10 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -199,6 +204,10 @@ github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
+github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@@ -246,6 +255,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=

lib/db/backend/backend.go

@@ -100,6 +100,7 @@ type Backend interface {
 	NewReadTransaction() (ReadTransaction, error)
 	NewWriteTransaction() (WriteTransaction, error)
 	Close() error
+	Compact() error
 }
 
 type Tuning int

lib/db/backend/leveldb_backend.go

@@ -81,6 +81,10 @@ func (b *leveldbBackend) Delete(key []byte) error {
 	return wrapLeveldbErr(b.ldb.Delete(key, nil))
 }
 
+func (b *leveldbBackend) Compact() error {
+	return wrapLeveldbErr(b.ldb.CompactRange(util.Range{}))
+}
+
 // leveldbSnapshot implements backend.ReadTransaction
 type leveldbSnapshot struct {
 	snap *leveldb.Snapshot

lib/db/db_test.go

@@ -165,7 +165,7 @@ func TestUpdate0to3(t *testing.T) {
 
 	folder := []byte(update0to3Folder)
 
-	if err := updater.updateSchema0to1(); err != nil {
+	if err := updater.updateSchema0to1(0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -187,7 +187,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Invalid file wasn't added to global list")
 	}
 
-	if err := updater.updateSchema1to2(); err != nil {
+	if err := updater.updateSchema1to2(1); err != nil {
 		t.Fatal(err)
 	}
 
@@ -214,7 +214,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Local file wasn't added to sequence bucket", err)
 	}
 
-	if err := updater.updateSchema2to3(); err != nil {
+	if err := updater.updateSchema2to3(2); err != nil {
 		t.Fatal(err)
 	}
 

lib/db/keyer.go

@@ -59,6 +59,9 @@ const (
 
 	// KeyTypeNeed <int32 folder ID> <file name> = <nothing>
 	KeyTypeNeed = 12
+
+	// KeyTypeBlockList <block list hash> = BlockList
+	KeyTypeBlockList = 13
 )
 
 type keyer interface {
@@ -93,6 +96,9 @@ type keyer interface {
 
 	// Folder metadata
 	GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error)
+
+	// Block lists
+	GenerateBlockListKey(key []byte, hash []byte) blockListKey
 }
 
 // defaultKeyer implements our key scheme. It needs folder and device
@@ -281,6 +287,19 @@ func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey,
 	return key, nil
 }
 
+type blockListKey []byte
+
+func (k defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey {
+	key = resize(key, keyPrefixLen+len(hash))
+	key[0] = KeyTypeBlockList
+	copy(key[keyPrefixLen:], hash)
+	return key
+}
+
+func (k blockListKey) BlocksHash() []byte {
+	return k[keyPrefixLen:]
+}
+
 // resize returns a byte slice of the specified size, reusing bs if possible
 func resize(bs []byte, size int) []byte {
 	if cap(bs) < size {

lib/db/lowlevel.go

@@ -9,9 +9,23 @@ package db
 import (
 	"bytes"
 	"encoding/binary"
+	"time"
 
 	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/willf/bloom"
+)
+
+const (
+	// We set the bloom filter capacity to handle 100k individual block lists
+	// with a false positive probability of 1% for the first pass. Once we know
+	// how many block lists we have we will use that number instead, if it's
+	// more than 100k. For fewer than 100k block lists we will just get better
+	// false positive rate instead.
+	blockGCBloomCapacity          = 100000
+	blockGCBloomFalsePositiveRate = 0.01 // 1%
+	blockGCInterval               = 13 * time.Hour
 )
 
 // Lowlevel is the lowest level database interface. It has a very simple
@@ -21,9 +35,12 @@ import (
 // any given backend.
 type Lowlevel struct {
 	backend.Backend
-	folderIdx *smallIndex
-	deviceIdx *smallIndex
-	keyer     keyer
+	folderIdx  *smallIndex
+	deviceIdx  *smallIndex
+	keyer      keyer
+	gcMut      sync.RWMutex
+	gcKeyCount int
+	gcStop     chan struct{}
 }
 
 func NewLowlevel(backend backend.Backend) *Lowlevel {
@@ -31,11 +48,19 @@ func NewLowlevel(backend backend.Backend) *Lowlevel {
 		Backend:   backend,
 		folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}),
 		deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}),
+		gcMut:     sync.NewRWMutex(),
+		gcStop:    make(chan struct{}),
 	}
 	db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx)
+	go db.gcRunner()
 	return db
 }
 
+func (db *Lowlevel) Close() error {
+	close(db.gcStop)
+	return db.Backend.Close()
+}
+
 // ListFolders returns the list of folders currently in the database
 func (db *Lowlevel) ListFolders() []string {
 	return db.folderIdx.Values()
@@ -44,6 +69,9 @@ func (db *Lowlevel) ListFolders() []string {
 // updateRemoteFiles adds a list of fileinfos to the database and updates the
 // global versionlist and metadata.
 func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) error {
+	db.gcMut.RLock()
+	defer db.gcMut.RUnlock()
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -73,7 +101,7 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI
 		meta.addFile(devID, f)
 
 		l.Debugf("insert; folder=%q device=%v %v", folder, devID, f)
-		if err := t.Put(dk, mustMarshal(&f)); err != nil {
+		if err := t.putFile(dk, f); err != nil {
 			return err
 		}
 
@@ -97,6 +125,9 @@ func (db *Lowlevel) updateRemoteFiles(folder, device []byte, fs []protocol.FileI
 // updateLocalFiles adds fileinfos to the db, and updates the global versionlist,
 // metadata, sequence and blockmap buckets.
 func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta *metadataTracker) error {
+	db.gcMut.RLock()
+	defer db.gcMut.RUnlock()
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -151,7 +182,7 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
 		meta.addFile(protocol.LocalDeviceID, f)
 
 		l.Debugf("insert (local); folder=%q %v", folder, f)
-		if err := t.Put(dk, mustMarshal(&f)); err != nil {
+		if err := t.putFile(dk, f); err != nil {
 			return err
 		}
 
@@ -195,6 +226,9 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
 }
 
 func (db *Lowlevel) dropFolder(folder []byte) error {
+	db.gcMut.RLock()
+	defer db.gcMut.RUnlock()
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -250,6 +284,9 @@ func (db *Lowlevel) dropFolder(folder []byte) error {
 }
 
 func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracker) error {
+	db.gcMut.RLock()
+	defer db.gcMut.RUnlock()
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -425,16 +462,100 @@ func (db *Lowlevel) dropPrefix(prefix []byte) error {
 	return t.commit()
 }
 
-func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
-	if truncate {
-		var tf FileInfoTruncated
-		err := tf.Unmarshal(bs)
-		return tf, err
+func (db *Lowlevel) gcRunner() {
+	t := time.NewTicker(blockGCInterval)
+	defer t.Stop()
+	for {
+		select {
+		case <-db.gcStop:
+			return
+		case <-t.C:
+			if err := db.gcBlocks(); err != nil {
+				l.Warnln("Database block GC failed:", err)
+			}
+		}
+	}
+}
+
+func (db *Lowlevel) gcBlocks() error {
+	// The block GC uses a bloom filter to track used block lists. This means
+	// iterating over all items, adding their block lists to the filter, then
+	// iterating over the block lists and removing those that don't match the
+	// filter. The filter will give false positives so we will keep around one
+	// percent of block lists that we don't really need (at most).
+	//
+	// Block GC needs to run when there are no modifications to the FileInfos or
+	// block lists.
+
+	db.gcMut.Lock()
+	defer db.gcMut.Unlock()
+
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.Release()
+
+	// Set up the bloom filter with the initial capacity and false positive
+	// rate, or higher capacity if we've done this before and seen lots of block
+	// lists.
+
+	capacity := blockGCBloomCapacity
+	if db.gcKeyCount > capacity {
+		capacity = db.gcKeyCount
+	}
+	filter := bloom.NewWithEstimates(uint(capacity), blockGCBloomFalsePositiveRate)
+
+	// Iterate the FileInfos, unmarshal the blocks hashes and add them to
+	// the filter.
+
+	it, err := db.NewPrefixIterator([]byte{KeyTypeDevice})
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		var bl BlocksHashOnly
+		if err := bl.Unmarshal(it.Value()); err != nil {
+			return err
+		}
+		filter.Add(bl.BlocksHash)
+	}
+	it.Release()
+	if err := it.Error(); err != nil {
+		return err
+	}
+
+	// Iterate over block lists, removing keys with hashes that don't match
+	// the filter.
+
+	it, err = db.NewPrefixIterator([]byte{KeyTypeBlockList})
+	if err != nil {
+		return err
+	}
+	matched := 0
+	for it.Next() {
+		key := blockListKey(it.Key())
+		if filter.Test(key.BlocksHash()) {
+			matched++
+			continue
+		}
+		if err := t.Delete(key); err != nil {
+			return err
+		}
+	}
+	it.Release()
+	if err := it.Error(); err != nil {
+		return err
+	}
+
+	// Remember the number of unique keys we kept until the next pass.
+	db.gcKeyCount = matched
+
+	if err := t.Commit(); err != nil {
+		return err
 	}
 
-	var tf protocol.FileInfo
-	err := tf.Unmarshal(bs)
-	return tf, err
+	return db.Compact()
 }
 
 func unmarshalVersionList(data []byte) (VersionList, bool) {

lib/db/schemaupdater.go

@@ -22,9 +22,10 @@ import (
 //   5: v0.14.49
 //   6: v0.14.50
 //   7: v0.14.53
+//   8: v1.4.0
 const (
-	dbVersion             = 7
-	dbMinSyncthingVersion = "v0.14.53"
+	dbVersion             = 8
+	dbMinSyncthingVersion = "v1.4.0"
 )
 
 type databaseDowngradeError struct {
@@ -68,35 +69,26 @@ func (db *schemaUpdater) updateSchema() error {
 		return nil
 	}
 
-	if prevVersion < 1 {
-		if err := db.updateSchema0to1(); err != nil {
-			return err
-		}
-	}
-	if prevVersion < 2 {
-		if err := db.updateSchema1to2(); err != nil {
-			return err
-		}
-	}
-	if prevVersion < 3 {
-		if err := db.updateSchema2to3(); err != nil {
-			return err
-		}
-	}
-	// This update fixes problems existing in versions 3 and 4
-	if prevVersion == 3 || prevVersion == 4 {
-		if err := db.updateSchemaTo5(); err != nil {
-			return err
-		}
+	type migration struct {
+		schemaVersion int64
+		migration     func(prevVersion int) error
 	}
-	if prevVersion < 6 {
-		if err := db.updateSchema5to6(); err != nil {
-			return err
-		}
+	var migrations = []migration{
+		{1, db.updateSchema0to1},
+		{2, db.updateSchema1to2},
+		{3, db.updateSchema2to3},
+		{5, db.updateSchemaTo5},
+		{6, db.updateSchema5to6},
+		{7, db.updateSchema6to7},
+		{8, db.updateSchema7to8},
 	}
-	if prevVersion < 7 {
-		if err := db.updateSchema6to7(); err != nil {
-			return err
+
+	for _, m := range migrations {
+		if prevVersion < m.schemaVersion {
+			l.Infof("Migrating database to schema version %d...", m.schemaVersion)
+			if err := m.migration(int(prevVersion)); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -107,10 +99,11 @@ func (db *schemaUpdater) updateSchema() error {
 		return err
 	}
 
-	return nil
+	l.Infoln("Compacting database after migration...")
+	return db.Compact()
 }
 
-func (db *schemaUpdater) updateSchema0to1() error {
+func (db *schemaUpdater) updateSchema0to1(_ int) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -216,7 +209,7 @@ func (db *schemaUpdater) updateSchema0to1() error {
 
 // updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
 // to allow iteration in sequence order (simplifies sending indexes).
-func (db *schemaUpdater) updateSchema1to2() error {
+func (db *schemaUpdater) updateSchema1to2(_ int) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -251,7 +244,7 @@ func (db *schemaUpdater) updateSchema1to2() error {
 }
 
 // updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
-func (db *schemaUpdater) updateSchema2to3() error {
+func (db *schemaUpdater) updateSchema2to3(_ int) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -302,7 +295,11 @@ func (db *schemaUpdater) updateSchema2to3() error {
 // release candidates (dbVersion 3 and 4)
 // https://github.com/syncthing/syncthing/issues/5007
 // https://github.com/syncthing/syncthing/issues/5053
-func (db *schemaUpdater) updateSchemaTo5() error {
+func (db *schemaUpdater) updateSchemaTo5(prevVersion int) error {
+	if prevVersion != 3 && prevVersion != 4 {
+		return nil
+	}
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -321,10 +318,10 @@ func (db *schemaUpdater) updateSchemaTo5() error {
 		return err
 	}
 
-	return db.updateSchema2to3()
+	return db.updateSchema2to3(2)
 }
 
-func (db *schemaUpdater) updateSchema5to6() error {
+func (db *schemaUpdater) updateSchema5to6(_ int) error {
 	// For every local file with the Invalid bit set, clear the Invalid bit and
 	// set LocalFlags = FlagLocalIgnored.
 
@@ -369,7 +366,7 @@ func (db *schemaUpdater) updateSchema5to6() error {
 
 // updateSchema6to7 checks whether all currently locally needed files are really
 // needed and removes them if not.
-func (db *schemaUpdater) updateSchema6to7() error {
+func (db *schemaUpdater) updateSchema6to7(_ int) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -423,3 +420,36 @@ func (db *schemaUpdater) updateSchema6to7() error {
 	}
 	return t.commit()
 }
+
+func (db *schemaUpdater) updateSchema7to8(_ int) error {
+	// Loads and rewrites all files with blocks, to deduplicate block lists.
+
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.close()
+
+	it, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		var fi protocol.FileInfo
+		if err := fi.Unmarshal(it.Value()); err != nil {
+			return err
+		}
+		if fi.Blocks == nil {
+			continue
+		}
+		if err := t.putFile(it.Key(), fi); err != nil {
+			return err
+		}
+	}
+	it.Release()
+	if err := it.Error(); err != nil {
+		return err
+	}
+
+	return t.commit()
+}

lib/db/structs.pb.go

@@ -110,6 +110,7 @@ type FileInfoTruncated struct {
 	Sequence   int64                                               `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
 	// repeated BlockInfo Blocks         = 16
 	SymlinkTarget string                `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
+	BlocksHash    []byte                `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
 	Type          protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
 	Permissions   uint32                `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
 	ModifiedNs    int32                 `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"`
@@ -153,6 +154,82 @@ func (m *FileInfoTruncated) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_FileInfoTruncated proto.InternalMessageInfo
 
+// BlockList is the structure used to store block lists
+type BlockList struct {
+	Blocks []protocol.BlockInfo `protobuf:"bytes,1,rep,name=Blocks,proto3" json:"Blocks"`
+}
+
+func (m *BlockList) Reset()         { *m = BlockList{} }
+func (m *BlockList) String() string { return proto.CompactTextString(m) }
+func (*BlockList) ProtoMessage()    {}
+func (*BlockList) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e774e8f5f348d14d, []int{3}
+}
+func (m *BlockList) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *BlockList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_BlockList.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *BlockList) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_BlockList.Merge(m, src)
+}
+func (m *BlockList) XXX_Size() int {
+	return m.ProtoSize()
+}
+func (m *BlockList) XXX_DiscardUnknown() {
+	xxx_messageInfo_BlockList.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BlockList proto.InternalMessageInfo
+
+// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo
+type BlocksHashOnly struct {
+	BlocksHash []byte `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
+}
+
+func (m *BlocksHashOnly) Reset()         { *m = BlocksHashOnly{} }
+func (m *BlocksHashOnly) String() string { return proto.CompactTextString(m) }
+func (*BlocksHashOnly) ProtoMessage()    {}
+func (*BlocksHashOnly) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e774e8f5f348d14d, []int{4}
+}
+func (m *BlocksHashOnly) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *BlocksHashOnly) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_BlocksHashOnly.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *BlocksHashOnly) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_BlocksHashOnly.Merge(m, src)
+}
+func (m *BlocksHashOnly) XXX_Size() int {
+	return m.ProtoSize()
+}
+func (m *BlocksHashOnly) XXX_DiscardUnknown() {
+	xxx_messageInfo_BlocksHashOnly.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BlocksHashOnly proto.InternalMessageInfo
+
 // For each folder and device we keep one of these to track the current
 // counts and sequence. We also keep one for the global state of the folder.
 type Counts struct {
@@ -170,7 +247,7 @@ func (m *Counts) Reset()         { *m = Counts{} }
 func (m *Counts) String() string { return proto.CompactTextString(m) }
 func (*Counts) ProtoMessage()    {}
 func (*Counts) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e774e8f5f348d14d, []int{3}
+	return fileDescriptor_e774e8f5f348d14d, []int{5}
 }
 func (m *Counts) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -208,7 +285,7 @@ func (m *CountsSet) Reset()         { *m = CountsSet{} }
 func (m *CountsSet) String() string { return proto.CompactTextString(m) }
 func (*CountsSet) ProtoMessage()    {}
 func (*CountsSet) Descriptor() ([]byte, []int) {
-	return fileDescriptor_e774e8f5f348d14d, []int{4}
+	return fileDescriptor_e774e8f5f348d14d, []int{6}
 }
 func (m *CountsSet) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -241,6 +318,8 @@ func init() {
 	proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
 	proto.RegisterType((*VersionList)(nil), "db.VersionList")
 	proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
+	proto.RegisterType((*BlockList)(nil), "db.BlockList")
+	proto.RegisterType((*BlocksHashOnly)(nil), "db.BlocksHashOnly")
 	proto.RegisterType((*Counts)(nil), "db.Counts")
 	proto.RegisterType((*CountsSet)(nil), "db.CountsSet")
 }
@@ -248,50 +327,54 @@ func init() {
 func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) }
 
 var fileDescriptor_e774e8f5f348d14d = []byte{
-	// 683 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4f, 0x6f, 0xda, 0x4e,
-	0x10, 0xc5, 0xc1, 0x10, 0x18, 0x20, 0xbf, 0x64, 0x15, 0x45, 0x16, 0xd2, 0xcf, 0x58, 0x54, 0x95,
-	0xac, 0x1e, 0xa0, 0x4d, 0x6e, 0xed, 0x8d, 0x46, 0x91, 0x90, 0xaa, 0xb6, 0x5a, 0xa2, 0x9c, 0x2a,
-	0x21, 0xff, 0x59, 0xc8, 0x2a, 0xc6, 0x4b, 0xbc, 0x4b, 0x22, 0xf2, 0x29, 0x7a, 0xec, 0x31, 0x1f,
-	0x27, 0xc7, 0x1c, 0xab, 0x1e, 0x50, 0x0a, 0x3d, 0xf4, 0x63, 0x54, 0xbb, 0x6b, 0x1b, 0x37, 0xa7,
-	0xde, 0xe6, 0xbd, 0x19, 0x7b, 0x66, 0xde, 0xbc, 0x85, 0x16, 0x17, 0xc9, 0x22, 0x10, 0xbc, 0x37,
-	0x4f, 0x98, 0x60, 0x68, 0x27, 0xf4, 0xdb, 0x2f, 0x12, 0x32, 0x67, 0xbc, 0xaf, 0x08, 0x7f, 0x31,
-	0xe9, 0x4f, 0xd9, 0x94, 0x29, 0xa0, 0x22, 0x5d, 0xd8, 0x3e, 0x8a, 0xa8, 0xaf, 0x4b, 0x02, 0x16,
-	0xf5, 0x7d, 0x32, 0xd7, 0x7c, 0xf7, 0x1a, 0x1a, 0x67, 0x34, 0x22, 0x17, 0x24, 0xe1, 0x94, 0xc5,
-	0xe8, 0x35, 0xec, 0xde, 0xe8, 0xd0, 0x32, 0x1c, 0xc3, 0x6d, 0x1c, 0xef, 0xf7, 0xb2, 0x8f, 0x7a,
-	0x17, 0x24, 0x10, 0x2c, 0x19, 0x98, 0x0f, 0xab, 0x4e, 0x09, 0x67, 0x65, 0xe8, 0x08, 0xaa, 0x21,
-	0xb9, 0xa1, 0x01, 0xb1, 0x76, 0x1c, 0xc3, 0x6d, 0xe2, 0x14, 0x21, 0x0b, 0x76, 0x69, 0x7c, 0xe3,
-	0x45, 0x34, 0xb4, 0xca, 0x8e, 0xe1, 0xd6, 0x70, 0x06, 0xbb, 0x67, 0xd0, 0x48, 0xdb, 0x7d, 0xa0,
-	0x5c, 0xa0, 0x37, 0x50, 0x4b, 0xff, 0xc5, 0x2d, 0xc3, 0x29, 0xbb, 0x8d, 0xe3, 0xff, 0x7a, 0xa1,
-	0xdf, 0x2b, 0x4c, 0x95, 0xb6, 0xcc, 0xcb, 0xde, 0x9a, 0xdf, 0xee, 0x3b, 0xa5, 0xee, 0x93, 0x09,
-	0x07, 0xb2, 0x6a, 0x18, 0x4f, 0xd8, 0x79, 0xb2, 0x88, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63,
-	0x6f, 0x46, 0xd4, 0xf8, 0x75, 0xac, 0x62, 0xc9, 0x71, 0x7a, 0x47, 0xd4, 0x20, 0x65, 0xac, 0x62,
-	0xf4, 0x3f, 0xc0, 0x8c, 0x85, 0x74, 0x42, 0x49, 0x38, 0xe6, 0x56, 0x45, 0x65, 0xea, 0x19, 0x33,
-	0x42, 0x5f, 0xa0, 0x91, 0xa7, 0xfd, 0xa5, 0xd5, 0x74, 0x0c, 0xd7, 0x1c, 0xbc, 0x93, 0x73, 0xfc,
-	0x58, 0x75, 0x4e, 0xa6, 0x54, 0x5c, 0x2e, 0xfc, 0x5e, 0xc0, 0x66, 0x7d, 0xbe, 0x8c, 0x03, 0x71,
-	0x49, 0xe3, 0x69, 0x21, 0x2a, 0x6a, 0xdd, 0x1b, 0x5d, 0xb2, 0x44, 0x0c, 0x4f, 0x71, 0xde, 0x6e,
-	0xb0, 0x2c, 0xca, 0x5c, 0xff, 0x37, 0x99, 0xdb, 0x50, 0xe3, 0xe4, 0x7a, 0x41, 0xe2, 0x80, 0x58,
-	0xa0, 0x86, 0xcd, 0x31, 0x7a, 0x09, 0x7b, 0x7c, 0x39, 0x8b, 0x68, 0x7c, 0x35, 0x16, 0x5e, 0x32,
-	0x25, 0xc2, 0x3a, 0x50, 0xcb, 0xb7, 0x52, 0xf6, 0x5c, 0x91, 0xe8, 0x15, 0x98, 0x62, 0x39, 0xd7,
-	0x77, 0xda, 0x3b, 0x3e, 0xda, 0x76, 0xcc, 0x45, 0x5c, 0xce, 0x09, 0x56, 0x35, 0xc8, 0x81, 0xc6,
-	0x9c, 0x24, 0x33, 0xca, 0xf5, 0x5d, 0x4c, 0xc7, 0x70, 0x5b, 0xb8, 0x48, 0xa1, 0x4e, 0x41, 0xa0,
-	0x98, 0x5b, 0x0d, 0xc7, 0x70, 0x2b, 0xdb, 0x1d, 0x3f, 0x72, 0xd4, 0x07, 0xf0, 0x23, 0x16, 0x5c,
-	0x8d, 0x95, 0xf4, 0x2d, 0x99, 0x1f, 0xec, 0xaf, 0x57, 0x9d, 0x26, 0xf6, 0x6e, 0x07, 0x32, 0x31,
-	0xa2, 0x77, 0x04, 0xd7, 0xfd, 0x2c, 0x94, 0x3d, 0x23, 0x16, 0x78, 0xd1, 0x78, 0x12, 0x79, 0x53,
-	0x6e, 0xfd, 0xde, 0x55, 0x4d, 0x41, 0x71, 0x67, 0x92, 0x92, 0x9e, 0x0a, 0x49, 0x44, 0x04, 0x09,
-	0xad, 0xaa, 0xf6, 0x54, 0x0a, 0x91, 0xbb, 0x75, 0x9b, 0xfc, 0xac, 0x36, 0xd8, 0x5b, 0xaf, 0x3a,
-	0x80, 0xbd, 0xdb, 0xa1, 0x66, 0x73, 0xf7, 0x49, 0xb1, 0x62, 0x36, 0x2e, 0x2e, 0x57, 0x53, 0xbf,
-	0x6a, 0xc5, 0xec, 0xf3, 0x96, 0x4c, 0x2d, 0xf6, 0xcb, 0x80, 0xea, 0x7b, 0xb6, 0x88, 0x05, 0x47,
-	0x87, 0x50, 0x99, 0xd0, 0x88, 0x70, 0x65, 0xac, 0x0a, 0xd6, 0x40, 0xce, 0x1c, 0xd2, 0x44, 0x5d,
-	0x8c, 0x12, 0xae, 0xa4, 0xad, 0xe0, 0x22, 0xa5, 0x0e, 0xa7, 0xcf, 0xc0, 0x95, 0xff, 0x2a, 0x38,
-	0xc7, 0xc5, 0x7d, 0x4c, 0x95, 0xca, 0xf7, 0x39, 0x84, 0x8a, 0xbf, 0x14, 0x24, 0x33, 0xa6, 0x06,
-	0x7f, 0x99, 0xa0, 0xfa, 0xcc, 0x04, 0x6d, 0xa8, 0xe9, 0x97, 0x37, 0x3c, 0x55, 0xe7, 0x6f, 0xe2,
-	0x1c, 0x23, 0x1b, 0x0a, 0x2a, 0x5a, 0xe8, 0xb9, 0xae, 0xdd, 0x4f, 0x50, 0xd7, 0x5b, 0x8e, 0x88,
-	0x40, 0x2e, 0x54, 0x03, 0x05, 0xd2, 0xd7, 0x08, 0xf2, 0x35, 0xea, 0x74, 0x6a, 0xca, 0x34, 0x2f,
-	0xc7, 0x0f, 0x12, 0x22, 0x5f, 0x9d, 0x5a, 0xbc, 0x8c, 0x33, 0x38, 0x70, 0x1e, 0x7e, 0xda, 0xa5,
-	0x87, 0xb5, 0x6d, 0x3c, 0xae, 0x6d, 0xe3, 0x69, 0x6d, 0x97, 0xbe, 0x6e, 0xec, 0xd2, 0xfd, 0xc6,
-	0x36, 0x1e, 0x37, 0x76, 0xe9, 0xfb, 0xc6, 0x2e, 0xf9, 0x55, 0xe5, 0xbe, 0x93, 0x3f, 0x01, 0x00,
-	0x00, 0xff, 0xff, 0x7e, 0x87, 0x05, 0xb2, 0xd0, 0x04, 0x00, 0x00,
+	// 743 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44,
+	0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08,
+	0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5,
+	0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01,
+	0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6,
+	0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17,
+	0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96,
+	0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79,
+	0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84,
+	0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb,
+	0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70,
+	0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20,
+	0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8,
+	0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d,
+	0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f,
+	0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a,
+	0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e,
+	0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f,
+	0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7,
+	0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12,
+	0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35,
+	0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e,
+	0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26,
+	0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28,
+	0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40,
+	0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69,
+	0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6,
+	0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33,
+	0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3,
+	0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66,
+	0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5,
+	0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2,
+	0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54,
+	0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e,
+	0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29,
+	0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb,
+	0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84,
+	0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4,
+	0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5,
+	0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13,
+	0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19,
+	0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23,
+	0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4,
+	0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c,
+	0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74,
+	0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8,
+	0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00,
 }
 
 func (m *FileVersion) Marshal() (dAtA []byte, err error) {
@@ -408,6 +491,15 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 		i--
 		dAtA[i] = 0xc0
 	}
+	if len(m.BlocksHash) > 0 {
+		i -= len(m.BlocksHash)
+		copy(dAtA[i:], m.BlocksHash)
+		i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash)))
+		i--
+		dAtA[i] = 0x1
+		i--
+		dAtA[i] = 0x92
+	}
 	if len(m.SymlinkTarget) > 0 {
 		i -= len(m.SymlinkTarget)
 		copy(dAtA[i:], m.SymlinkTarget)
@@ -507,6 +599,75 @@ func (m *FileInfoTruncated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *BlockList) Marshal() (dAtA []byte, err error) {
+	size := m.ProtoSize()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *BlockList) MarshalTo(dAtA []byte) (int, error) {
+	size := m.ProtoSize()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *BlockList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Blocks) > 0 {
+		for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Blocks[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintStructs(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *BlocksHashOnly) Marshal() (dAtA []byte, err error) {
+	size := m.ProtoSize()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *BlocksHashOnly) MarshalTo(dAtA []byte) (int, error) {
+	size := m.ProtoSize()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *BlocksHashOnly) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.BlocksHash) > 0 {
+		i -= len(m.BlocksHash)
+		copy(dAtA[i:], m.BlocksHash)
+		i = encodeVarintStructs(dAtA, i, uint64(len(m.BlocksHash)))
+		i--
+		dAtA[i] = 0x1
+		i--
+		dAtA[i] = 0x92
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *Counts) Marshal() (dAtA []byte, err error) {
 	size := m.ProtoSize()
 	dAtA = make([]byte, size)
@@ -711,12 +872,44 @@ func (m *FileInfoTruncated) ProtoSize() (n int) {
 	if l > 0 {
 		n += 2 + l + sovStructs(uint64(l))
 	}
+	l = len(m.BlocksHash)
+	if l > 0 {
+		n += 2 + l + sovStructs(uint64(l))
+	}
 	if m.LocalFlags != 0 {
 		n += 2 + sovStructs(uint64(m.LocalFlags))
 	}
 	return n
 }
 
+func (m *BlockList) ProtoSize() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Blocks) > 0 {
+		for _, e := range m.Blocks {
+			l = e.ProtoSize()
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *BlocksHashOnly) ProtoSize() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.BlocksHash)
+	if l > 0 {
+		n += 2 + l + sovStructs(uint64(l))
+	}
+	return n
+}
+
 func (m *Counts) ProtoSize() (n int) {
 	if m == nil {
 		return 0
@@ -1340,6 +1533,40 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
 			}
 			m.SymlinkTarget = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 18:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
+			if m.BlocksHash == nil {
+				m.BlocksHash = []byte{}
+			}
+			iNdEx = postIndex
 		case 1000:
 			if wireType != 0 {
 				return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)
@@ -1383,6 +1610,180 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *BlockList) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: BlockList: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Blocks = append(m.Blocks, protocol.BlockInfo{})
+			if err := m.Blocks[len(m.Blocks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *BlocksHashOnly) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: BlocksHashOnly: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: BlocksHashOnly: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 18:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
+			if m.BlocksHash == nil {
+				m.BlocksHash = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *Counts) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0

lib/db/structs.proto

@@ -34,6 +34,7 @@ message FileInfoTruncated {
     int64                 sequence       = 10;
     // repeated BlockInfo Blocks         = 16
     string                symlink_target = 17;
+    bytes                 blocks_hash    = 18;
     protocol.FileInfoType type           = 2;
     uint32                permissions    = 4;
     int32                 modified_ns    = 11;
@@ -47,6 +48,16 @@ message FileInfoTruncated {
     bool no_permissions = 8;
 }
 
+// BlockList is the structure used to store block lists
+message BlockList {
+    repeated protocol.BlockInfo Blocks = 1 [(gogoproto.nullable) = false];
+}
+
+// BlocksHashOnly is used to only unmarshal the block list hash from a FileInfo
+message BlocksHashOnly {
+    bytes blocks_hash = 18;
+}
+
 // For each folder and device we keep one of these to track the current
 // counts and sequence. We also keep one for the global state of the folder.
 message Counts {

lib/db/transactions.go

@@ -58,13 +58,50 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo
 	if err != nil {
 		return nil, false, err
 	}
-	f, err := unmarshalTrunc(bs, trunc)
+	f, err := t.unmarshalTrunc(bs, trunc)
 	if err != nil {
 		return nil, false, err
 	}
 	return f, true, nil
 }
 
+func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) {
+	if trunc {
+		var tf FileInfoTruncated
+		err := tf.Unmarshal(bs)
+		if err != nil {
+			return nil, err
+		}
+		return tf, nil
+	}
+
+	var tf protocol.FileInfo
+	if err := tf.Unmarshal(bs); err != nil {
+		return nil, err
+	}
+	if err := t.fillBlockList(&tf); err != nil {
+		return nil, err
+	}
+	return tf, nil
+}
+
+func (t readOnlyTransaction) fillBlockList(fi *protocol.FileInfo) error {
+	if fi.BlocksHash == nil {
+		return nil
+	}
+	blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash)
+	bs, err := t.Get(blocksKey)
+	if err != nil {
+		return err
+	}
+	var bl BlockList
+	if err := bl.Unmarshal(bs); err != nil {
+		return err
+	}
+	fi.Blocks = bl.Blocks
+	return nil
+}
+
 func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) {
 	var err error
 	keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
@@ -132,7 +169,7 @@ func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate b
 			return nil
 		}
 
-		f, err := unmarshalTrunc(dbi.Value(), truncate)
+		f, err := t.unmarshalTrunc(dbi.Value(), truncate)
 		if err != nil {
 			l.Debugln("unmarshal error:", err)
 			continue
@@ -413,6 +450,28 @@ func (t readWriteTransaction) close() {
 	t.WriteTransaction.Release()
 }
 
+func (t readWriteTransaction) putFile(key []byte, fi protocol.FileInfo) error {
+	if fi.Blocks != nil {
+		if fi.BlocksHash == nil {
+			fi.BlocksHash = protocol.BlocksHash(fi.Blocks)
+		}
+		blocksKey := t.keyer.GenerateBlockListKey(nil, fi.BlocksHash)
+		if _, err := t.Get(blocksKey); backend.IsNotFound(err) {
+			// Marshal the block list and save it
+			blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks})
+			if err := t.Put(blocksKey, blocksBs); err != nil {
+				return err
+			}
+		} else if err != nil {
+			return err
+		}
+	}
+
+	fi.Blocks = nil
+	fiBs := mustMarshal(&fi)
+	return t.Put(key, fiBs)
+}
+
 // updateGlobal adds this device+version to the version list for the given
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.

lib/model/model.go

@@ -954,8 +954,8 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
 		files.Drop(deviceID)
 	}
 	for i := range fs {
-		// The local flags should never be transmitted over the wire. Make
-		// sure they look like they weren't.
+		// The local attributes should never be transmitted over the wire.
+		// Make sure they look like they weren't.
 		fs[i].LocalFlags = 0
 	}
 	files.Update(deviceID, fs)

lib/protocol/bep.pb.go

@@ -495,8 +495,9 @@ type FileInfo struct {
 	ModifiedBy    ShortID      `protobuf:"varint,12,opt,name=modified_by,json=modifiedBy,proto3,customtype=ShortID" json:"modified_by"`
 	Version       Vector       `protobuf:"bytes,9,opt,name=version,proto3" json:"version"`
 	Sequence      int64        `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
-	Blocks        []BlockInfo  `protobuf:"bytes,16,rep,name=Blocks,proto3" json:"Blocks"`
+	Blocks        []BlockInfo  `protobuf:"bytes,16,rep,name=blocks,proto3" json:"blocks"`
 	SymlinkTarget string       `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
+	BlocksHash    []byte       `protobuf:"bytes,18,opt,name=blocks_hash,json=blocksHash,proto3" json:"blocks_hash,omitempty"`
 	Type          FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
 	Permissions   uint32       `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
 	ModifiedNs    int32        `protobuf:"varint,11,opt,name=modified_ns,json=modifiedNs,proto3" json:"modified_ns,omitempty"`
@@ -920,120 +921,121 @@ func init() {
 func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) }
 
 var fileDescriptor_e3f59eb60afbbc6e = []byte{
-	// 1803 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6f, 0xdb, 0xc8,
-	0x15, 0x16, 0x25, 0xea, 0xd7, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92,
-	0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0x7e, 0xd0, 0x8e, 0x50, 0x47,
+	// 1816 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8,
+	0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92,
+	0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47,
 	0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76,
-	0xb4, 0x7f, 0x82, 0x4e, 0x3d, 0xf6, 0x22, 0x60, 0x81, 0x9e, 0xfa, 0x9f, 0xe4, 0x98, 0xf6, 0x50,
-	0x14, 0x3d, 0x18, 0x5d, 0xe7, 0xb2, 0xc7, 0xfe, 0x05, 0x45, 0x31, 0x33, 0xa4, 0x44, 0xd9, 0x9b,
-	0x45, 0x0e, 0x7b, 0xd2, 0xcc, 0x7b, 0x1f, 0xdf, 0xcc, 0x7c, 0xf3, 0xbd, 0x6f, 0x04, 0xc5, 0x13,
-	0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0xff, 0x0c, 0xa9, 0xab, 0xdd, 0xf3, 0xc9, 0x94,
-	0x06, 0x8f, 0xf9, 0xfc, 0x64, 0x36, 0x7a, 0x3c, 0xa6, 0x63, 0xca, 0x27, 0x7c, 0x24, 0xe0, 0xb5,
-	0x29, 0x64, 0x9f, 0x12, 0xd7, 0xa5, 0xa8, 0x0a, 0x25, 0x9b, 0x9c, 0x39, 0x43, 0x62, 0x7a, 0xd6,
-	0x84, 0xa8, 0x92, 0x2e, 0xd5, 0x8b, 0x18, 0x44, 0xa8, 0x6b, 0x4d, 0x08, 0x03, 0x0c, 0x5d, 0x87,
-	0x78, 0xa1, 0x00, 0xa4, 0x05, 0x40, 0x84, 0x38, 0xe0, 0x01, 0x6c, 0x47, 0x80, 0x33, 0xe2, 0x07,
-	0x0e, 0xf5, 0xd4, 0x0c, 0xc7, 0x6c, 0x89, 0xe8, 0x73, 0x11, 0xac, 0x05, 0x90, 0x7b, 0x4a, 0x2c,
-	0x9b, 0xf8, 0xe8, 0x63, 0x90, 0xc3, 0xf9, 0x54, 0xac, 0xb5, 0xfd, 0xe9, 0xad, 0x47, 0xf1, 0xce,
-	0x1f, 0x3d, 0x23, 0x41, 0x60, 0x8d, 0xc9, 0x60, 0x3e, 0x25, 0x98, 0x43, 0xd0, 0x6f, 0xa1, 0x34,
-	0xa4, 0x93, 0xa9, 0x4f, 0x02, 0x5e, 0x38, 0xcd, 0xbf, 0xb8, 0x73, 0xed, 0x8b, 0xd6, 0x1a, 0x83,
-	0x93, 0x1f, 0xd4, 0x1a, 0xb0, 0xd5, 0x72, 0x67, 0x41, 0x48, 0xfc, 0x16, 0xf5, 0x46, 0xce, 0x18,
-	0x3d, 0x81, 0xfc, 0x88, 0xba, 0x36, 0xf1, 0x03, 0x55, 0xd2, 0x33, 0xf5, 0xd2, 0xa7, 0xca, 0xba,
-	0xd8, 0x3e, 0x4f, 0x34, 0xe5, 0xd7, 0x17, 0xd5, 0x14, 0x8e, 0x61, 0xb5, 0xbf, 0xa6, 0x21, 0x27,
-	0x32, 0x68, 0x17, 0xd2, 0x8e, 0x2d, 0x28, 0x6a, 0xe6, 0x2e, 0x2f, 0xaa, 0xe9, 0x4e, 0x1b, 0xa7,
-	0x1d, 0x1b, 0xdd, 0x84, 0xac, 0x6b, 0x9d, 0x10, 0x37, 0x22, 0x47, 0x4c, 0xd0, 0x6d, 0x28, 0xfa,
-	0xc4, 0xb2, 0x4d, 0xea, 0xb9, 0x73, 0x4e, 0x49, 0x01, 0x17, 0x58, 0xa0, 0xe7, 0xb9, 0x73, 0xf4,
-	0x13, 0x40, 0xce, 0xd8, 0xa3, 0x3e, 0x31, 0xa7, 0xc4, 0x9f, 0x38, 0x7c, 0xb7, 0x81, 0x2a, 0x73,
-	0xd4, 0x8e, 0xc8, 0x1c, 0xad, 0x13, 0xe8, 0x1e, 0x6c, 0x45, 0x70, 0x9b, 0xb8, 0x24, 0x24, 0x6a,
-	0x96, 0x23, 0xcb, 0x22, 0xd8, 0xe6, 0x31, 0xf4, 0x04, 0x6e, 0xda, 0x4e, 0x60, 0x9d, 0xb8, 0xc4,
-	0x0c, 0xc9, 0x64, 0x6a, 0x3a, 0x9e, 0x4d, 0x5e, 0x91, 0x40, 0xcd, 0x71, 0x2c, 0x8a, 0x72, 0x03,
-	0x32, 0x99, 0x76, 0x44, 0x06, 0xed, 0x42, 0x6e, 0x6a, 0xcd, 0x02, 0x62, 0xab, 0x79, 0x8e, 0x89,
-	0x66, 0x8c, 0x25, 0xa1, 0x80, 0x40, 0x55, 0xae, 0xb2, 0xd4, 0xe6, 0x89, 0x98, 0xa5, 0x08, 0x56,
-	0xfb, 0x6f, 0x1a, 0x72, 0x22, 0x83, 0x3e, 0x5a, 0xb1, 0x54, 0x6e, 0xee, 0x32, 0xd4, 0xbf, 0x2f,
-	0xaa, 0x05, 0x91, 0xeb, 0xb4, 0x13, 0xac, 0x21, 0x90, 0x13, 0x8a, 0xe2, 0x63, 0x74, 0x07, 0x8a,
-	0x96, 0x6d, 0xb3, 0xdb, 0x23, 0x81, 0x9a, 0xd1, 0x33, 0xf5, 0x22, 0x5e, 0x07, 0xd0, 0x2f, 0x36,
-	0xd5, 0x20, 0x5f, 0xd5, 0xcf, 0xbb, 0x64, 0xc0, 0xae, 0x62, 0x48, 0xfc, 0x48, 0xc1, 0x59, 0xbe,
-	0x5e, 0x81, 0x05, 0xb8, 0x7e, 0xef, 0x42, 0x79, 0x62, 0xbd, 0x32, 0x03, 0xf2, 0xa7, 0x19, 0xf1,
-	0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xaa, 0x1f, 0x85, 0x50, 0x05, 0xc0, 0xf1, 0x42,
-	0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x11, 0xf4, 0x33, 0x28, 0x70, 0xb2, 0x4d, 0xc7,
-	0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2,
-	0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x6b, 0xd0, 0x82, 0x97, 0x0e, 0xbb, 0x28, 0x51, 0x29, 0x74, 0xa8,
-	0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x19, 0x95, 0x21, 0x3a, 0x09, 0x00,
-	0x8e, 0xf2, 0xb5, 0x1e, 0x64, 0x79, 0x45, 0x76, 0x8b, 0x42, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4,
-	0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x77, 0x88, 0x12, 0x4a, 0x77, 0x5c, 0xd2, 0xf1,
-	0x46, 0x34, 0xba, 0x45, 0x01, 0xab, 0x1d, 0x43, 0x89, 0x17, 0x3c, 0x9e, 0xda, 0x56, 0x48, 0x7e,
-	0xb0, 0xb2, 0x17, 0x32, 0x14, 0xe2, 0xcc, 0xea, 0xd2, 0xa5, 0xc4, 0xa5, 0x23, 0x90, 0x03, 0xe7,
-	0x2b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0x7d, 0x08, 0x30, 0xa1, 0xb6, 0x33, 0x72, 0x88, 0x6d,
-	0x06, 0xfc, 0xca, 0x32, 0xb8, 0x18, 0x47, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xe9, 0x93, 0xb9, 0x5a,
-	0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x89, 0xe6, 0x9c,
-	0x49, 0x3a, 0xb6, 0x27, 0x46, 0xec, 0x86, 0xa4, 0x9f, 0x93, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1,
-	0x90, 0x06, 0x85, 0x95, 0x26, 0x80, 0x6f, 0x60, 0x35, 0x47, 0x3f, 0x85, 0x5c, 0xd3, 0xa5, 0xc3,
-	0x97, 0x71, 0x7f, 0xdc, 0x58, 0x17, 0xe3, 0xf1, 0x04, 0x0b, 0x11, 0x90, 0xd9, 0x64, 0x30, 0x9f,
-	0xb8, 0x8e, 0xf7, 0xd2, 0x0c, 0x2d, 0x7f, 0x4c, 0x42, 0x75, 0x47, 0xd8, 0x64, 0x14, 0x1d, 0xf0,
-	0x20, 0xda, 0x8b, 0xcc, 0x51, 0x58, 0xdd, 0xee, 0x75, 0x72, 0x13, 0xee, 0xa8, 0x43, 0xe9, 0xaa,
-	0x7b, 0x6c, 0xe1, 0x64, 0x88, 0x99, 0xf7, 0x8a, 0x27, 0x2f, 0x50, 0x4b, 0xba, 0x54, 0xcf, 0xae,
-	0x69, 0xe9, 0x06, 0xe8, 0x31, 0xc0, 0x09, 0xdb, 0x9f, 0xc9, 0x6f, 0x60, 0x8b, 0xe5, 0x9b, 0xca,
-	0xe5, 0x45, 0xb5, 0x8c, 0xad, 0x73, 0xbe, 0xf1, 0xbe, 0xf3, 0x15, 0xc1, 0xc5, 0x93, 0x78, 0xc8,
-	0xd6, 0x74, 0xe9, 0xd0, 0x72, 0xcd, 0x91, 0x6b, 0x8d, 0x03, 0xf5, 0xdb, 0x3c, 0x5f, 0x14, 0x78,
-	0x6c, 0x9f, 0x85, 0x90, 0xca, 0xcc, 0x83, 0x19, 0x92, 0x1d, 0x39, 0x4f, 0x3c, 0x45, 0x75, 0xc8,
-	0x3b, 0xde, 0x99, 0xe5, 0x3a, 0x91, 0xdf, 0x34, 0xb7, 0x2f, 0x2f, 0xaa, 0x80, 0xad, 0xf3, 0x8e,
-	0x88, 0xe2, 0x38, 0xcd, 0xc8, 0xf2, 0xe8, 0x86, 0x35, 0x16, 0x78, 0xa9, 0x2d, 0x8f, 0x26, 0x6c,
-	0xf1, 0x57, 0xf2, 0x5f, 0xbe, 0xae, 0xa6, 0x6a, 0x1e, 0x14, 0x57, 0xa4, 0x33, 0x31, 0x9d, 0x5a,
-	0xc1, 0x29, 0x17, 0x53, 0x19, 0xf3, 0x31, 0x53, 0x32, 0x1d, 0x8d, 0x02, 0x12, 0x72, 0xd9, 0x65,
-	0x70, 0x34, 0x5b, 0x09, 0x2f, 0xcd, 0x69, 0x11, 0xc2, 0xbb, 0x0d, 0xc5, 0x73, 0x62, 0xbd, 0x34,
-	0x79, 0x11, 0xc1, 0x68, 0x81, 0x05, 0x9e, 0x5a, 0xc1, 0x69, 0xb4, 0xde, 0x6f, 0x20, 0x27, 0x14,
-	0x83, 0x3e, 0x83, 0xc2, 0x90, 0xce, 0xbc, 0x70, 0xfd, 0x9c, 0xec, 0x24, 0xdd, 0x88, 0x67, 0x22,
-	0x19, 0xac, 0x80, 0xb5, 0x7d, 0xc8, 0x47, 0x29, 0xf4, 0x60, 0x65, 0x95, 0x72, 0xf3, 0xd6, 0x15,
-	0xf5, 0x6e, 0xbe, 0x2f, 0x67, 0x96, 0x3b, 0x13, 0x1b, 0x95, 0xb1, 0x98, 0xd4, 0xfe, 0x2e, 0x41,
-	0x1e, 0x33, 0x41, 0x06, 0x61, 0xe2, 0x65, 0xca, 0x6e, 0xbc, 0x4c, 0xeb, 0x1e, 0x4e, 0x6f, 0xf4,
-	0x70, 0xdc, 0x86, 0x99, 0x44, 0x1b, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98,
-	0xe5, 0x5c, 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0xbf, 0x3d, 0xd4, 0xb7, 0xfc, 0x79,
-	0x64, 0x94, 0x5b, 0x2c, 0x3a, 0x88, 0x83, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0,
-	0x24, 0x98, 0x52, 0x2f, 0x20, 0xef, 0x3c, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c,
-	0xf9, 0x18, 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0x37, 0x1a, 0xbe, 0x4f, 0xfd,
-	0x16, 0xb5, 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e,
-	0x1d, 0xb3, 0x07, 0xe2, 0x9d, 0x46, 0xd7, 0x86, 0xfc, 0x8c, 0x5b, 0x61, 0x6c, 0x75, 0xf7, 0x37,
-	0xbb, 0xf1, 0x6a, 0x21, 0xe1, 0x9b, 0xb1, 0x8d, 0x44, 0x9f, 0xd6, 0xfe, 0x29, 0x81, 0xf6, 0x6e,
-	0x34, 0xea, 0x40, 0x49, 0x20, 0xcd, 0xc4, 0x7f, 0xa2, 0xfa, 0xfb, 0x2c, 0xc4, 0x8d, 0x00, 0x66,
-	0xab, 0xf1, 0x77, 0x3e, 0xa8, 0x09, 0xdb, 0xcb, 0xbc, 0x9f, 0xed, 0x3d, 0x84, 0x2d, 0xe1, 0x08,
-	0xf1, 0xdf, 0x07, 0x59, 0xcf, 0xd4, 0xb3, 0xcd, 0xb4, 0x92, 0xc2, 0xe5, 0x13, 0xd1, 0x66, 0x3c,
-	0x5e, 0xcb, 0x81, 0x7c, 0xe4, 0x78, 0xe3, 0x5a, 0x15, 0xb2, 0x2d, 0x97, 0xf2, 0x0b, 0xcb, 0xf9,
-	0xc4, 0x0a, 0xa8, 0x17, 0xf3, 0x28, 0x66, 0x7b, 0xff, 0x48, 0x43, 0x29, 0xf1, 0xd7, 0x0e, 0x3d,
-	0x81, 0xed, 0xd6, 0xe1, 0x71, 0x7f, 0x60, 0x60, 0xb3, 0xd5, 0xeb, 0xee, 0x77, 0x0e, 0x94, 0x94,
-	0x76, 0x67, 0xb1, 0xd4, 0xd5, 0xc9, 0x1a, 0xb4, 0xf9, 0xaf, 0xad, 0x0a, 0xd9, 0x4e, 0xb7, 0x6d,
-	0xfc, 0x41, 0x91, 0xb4, 0x9b, 0x8b, 0xa5, 0xae, 0x24, 0x80, 0xe2, 0x09, 0xfc, 0x04, 0xca, 0x1c,
-	0x60, 0x1e, 0x1f, 0xb5, 0x1b, 0x03, 0x43, 0x49, 0x6b, 0xda, 0x62, 0xa9, 0xef, 0x5e, 0xc5, 0x45,
-	0x9c, 0xdf, 0x83, 0x3c, 0x36, 0x7e, 0x7f, 0x6c, 0xf4, 0x07, 0x4a, 0x46, 0xdb, 0x5d, 0x2c, 0x75,
-	0x94, 0x00, 0xc6, 0x2d, 0xf5, 0x00, 0x0a, 0xd8, 0xe8, 0x1f, 0xf5, 0xba, 0x7d, 0x43, 0x91, 0xb5,
-	0x1f, 0x2d, 0x96, 0xfa, 0x8d, 0x0d, 0x54, 0xa4, 0xd2, 0x9f, 0xc3, 0x4e, 0xbb, 0xf7, 0x45, 0xf7,
-	0xb0, 0xd7, 0x68, 0x9b, 0x47, 0xb8, 0x77, 0x80, 0x8d, 0x7e, 0x5f, 0xc9, 0x6a, 0xd5, 0xc5, 0x52,
-	0xbf, 0x9d, 0xc0, 0x5f, 0x13, 0xdd, 0x87, 0x20, 0x1f, 0x75, 0xba, 0x07, 0x4a, 0x4e, 0xbb, 0xb1,
-	0x58, 0xea, 0x1f, 0x24, 0xa0, 0x8c, 0x54, 0x76, 0xe2, 0xd6, 0x61, 0xaf, 0x6f, 0x28, 0xf9, 0x6b,
-	0x27, 0xe6, 0x64, 0xef, 0xfd, 0x11, 0xd0, 0xf5, 0x3f, 0xbf, 0xe8, 0x3e, 0xc8, 0xdd, 0x5e, 0xd7,
-	0x50, 0x52, 0xe2, 0xfc, 0xd7, 0x11, 0x5d, 0xea, 0x11, 0x54, 0x83, 0xcc, 0xe1, 0x97, 0x9f, 0x2b,
-	0x92, 0xf6, 0xe3, 0xc5, 0x52, 0xbf, 0x75, 0x1d, 0x74, 0xf8, 0xe5, 0xe7, 0x7b, 0x14, 0x4a, 0xc9,
-	0xc2, 0x35, 0x28, 0x3c, 0x33, 0x06, 0x8d, 0x76, 0x63, 0xd0, 0x50, 0x52, 0x62, 0x4b, 0x71, 0xfa,
-	0x19, 0x09, 0x2d, 0xde, 0x84, 0x77, 0x20, 0xdb, 0x35, 0x9e, 0x1b, 0x58, 0x91, 0xb4, 0x9d, 0xc5,
-	0x52, 0xdf, 0x8a, 0x01, 0x5d, 0x72, 0x46, 0x7c, 0x54, 0x81, 0x5c, 0xe3, 0xf0, 0x8b, 0xc6, 0x8b,
-	0xbe, 0x92, 0xd6, 0xd0, 0x62, 0xa9, 0x6f, 0xc7, 0xe9, 0x86, 0x7b, 0x6e, 0xcd, 0x83, 0xbd, 0xff,
-	0x49, 0x50, 0x4e, 0xbe, 0x71, 0xa8, 0x02, 0xf2, 0x7e, 0xe7, 0xd0, 0x88, 0x97, 0x4b, 0xe6, 0xd8,
-	0x18, 0xd5, 0xa1, 0xd8, 0xee, 0x60, 0xa3, 0x35, 0xe8, 0xe1, 0x17, 0xf1, 0x59, 0x92, 0xa0, 0xb6,
-	0xe3, 0x73, 0x81, 0xcf, 0xd1, 0x2f, 0xa1, 0xdc, 0x7f, 0xf1, 0xec, 0xb0, 0xd3, 0xfd, 0x9d, 0xc9,
-	0x2b, 0xa6, 0xb5, 0x87, 0x8b, 0xa5, 0x7e, 0x77, 0x03, 0x4c, 0xa6, 0x3e, 0x19, 0x5a, 0x21, 0xb1,
-	0xfb, 0xe2, 0x39, 0x66, 0xc9, 0x82, 0x84, 0x5a, 0xb0, 0x13, 0x7f, 0xba, 0x5e, 0x2c, 0xa3, 0x7d,
-	0xb2, 0x58, 0xea, 0x1f, 0x7d, 0xef, 0xf7, 0xab, 0xd5, 0x0b, 0x12, 0xba, 0x0f, 0xf9, 0xa8, 0x48,
-	0xac, 0xa4, 0xe4, 0xa7, 0xd1, 0x07, 0x7b, 0x7f, 0x93, 0xa0, 0xb8, 0xb2, 0x2b, 0x46, 0x78, 0xb7,
-	0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0x2a, 0xd9, 0xa5, 0x7c, 0x88, 0xee, 0x42, 0xfe, 0xc0,
-	0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20, 0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x18,
-	0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8d, 0x8f, 0xce, 0xd7, 0x4f, 0x94, 0xea, 0xcf, 0x86,
-	0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0xcf, 0x1b, 0x87, 0x9d, 0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1,
-	0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x23, 0xcd, 0xb0, 0x7b, 0x36, 0x54, 0xbe, 0xdf, 0x98, 0x90, 0x0e,
-	0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0xbb, 0x5f, 0xe7, 0x1a, 0xd3, 0x29, 0xf1, 0x6c, 0x86,
-	0xd8, 0xef, 0xe1, 0x03, 0x63, 0x10, 0x6f, 0x7e, 0x8d, 0xd8, 0xa7, 0xec, 0xbf, 0x50, 0xb3, 0xfe,
-	0xfa, 0x9b, 0x4a, 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd,
-	0xe7, 0xb2, 0x92, 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf9, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4,
-	0xe6, 0x6d, 0x25, 0xf5, 0xaf, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x9b, 0xda, 0x67, 0xff, 0x0f, 0x00,
-	0x00, 0xff, 0xff, 0xd6, 0x0e, 0x8e, 0xa4, 0x11, 0x0f, 0x00, 0x00,
+	0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14,
+	0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3,
+	0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13,
+	0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a,
+	0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda,
+	0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b,
+	0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43,
+	0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03,
+	0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96,
+	0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4,
+	0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a,
+	0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70,
+	0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1,
+	0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83,
+	0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2,
+	0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda,
+	0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f,
+	0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f,
+	0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47,
+	0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66,
+	0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc,
+	0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20,
+	0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68,
+	0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a,
+	0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45,
+	0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2,
+	0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66,
+	0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe,
+	0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2,
+	0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85,
+	0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e,
+	0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4,
+	0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8,
+	0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00,
+	0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4,
+	0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1,
+	0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff,
+	0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce,
+	0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb,
+	0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a,
+	0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c,
+	0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04,
+	0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a,
+	0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83,
+	0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb,
+	0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61,
+	0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7,
+	0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba,
+	0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f,
+	0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2,
+	0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53,
+	0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5,
+	0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce,
+	0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a,
+	0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b,
+	0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83,
+	0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22,
+	0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67,
+	0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b,
+	0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37,
+	0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59,
+	0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc,
+	0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c,
+	0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4,
+	0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98,
+	0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18,
+	0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5,
+	0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3,
+	0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1,
+	0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75,
+	0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc,
+	0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f,
+	0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39,
+	0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80,
+	0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7,
+	0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b,
+	0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45,
+	0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c,
+	0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f,
+	0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18,
+	0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58,
+	0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46,
+	0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04,
+	0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe,
+	0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e,
+	0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12,
+	0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f,
+	0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41,
+	0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68,
+	0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56,
+	0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4,
+	0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72,
+	0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e,
+	0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17,
+	0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a,
+	0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde,
+	0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52,
+	0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25,
+	0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30,
+	0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70,
+	0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d,
+	0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f,
+	0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c,
+	0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3,
+	0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3,
+	0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a,
+	0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92,
+	0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25,
+	0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff,
+	0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00,
 }
 
 func (m *Hello) Marshal() (dAtA []byte, err error) {
@@ -1459,6 +1461,15 @@ func (m *FileInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 		i--
 		dAtA[i] = 0xc0
 	}
+	if len(m.BlocksHash) > 0 {
+		i -= len(m.BlocksHash)
+		copy(dAtA[i:], m.BlocksHash)
+		i = encodeVarintBep(dAtA, i, uint64(len(m.BlocksHash)))
+		i--
+		dAtA[i] = 0x1
+		i--
+		dAtA[i] = 0x92
+	}
 	if len(m.SymlinkTarget) > 0 {
 		i -= len(m.SymlinkTarget)
 		copy(dAtA[i:], m.SymlinkTarget)
@@ -2185,6 +2196,10 @@ func (m *FileInfo) ProtoSize() (n int) {
 	if l > 0 {
 		n += 2 + l + sovBep(uint64(l))
 	}
+	l = len(m.BlocksHash)
+	if l > 0 {
+		n += 2 + l + sovBep(uint64(l))
+	}
 	if m.LocalFlags != 0 {
 		n += 2 + sovBep(uint64(m.LocalFlags))
 	}
@@ -3835,6 +3850,40 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error {
 			}
 			m.SymlinkTarget = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 18:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field BlocksHash", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBep
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthBep
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthBep
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.BlocksHash = append(m.BlocksHash[:0], dAtA[iNdEx:postIndex]...)
+			if m.BlocksHash == nil {
+				m.BlocksHash = []byte{}
+			}
+			iNdEx = postIndex
 		case 1000:
 			if wireType != 0 {
 				return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)

lib/protocol/bep.proto

@@ -106,8 +106,9 @@ message FileInfo {
     uint64             modified_by    = 12 [(gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false];
     Vector             version        = 9 [(gogoproto.nullable) = false];
     int64              sequence       = 10;
-    repeated BlockInfo Blocks         = 16 [(gogoproto.nullable) = false];
+    repeated BlockInfo blocks         = 16 [(gogoproto.nullable) = false];
     string             symlink_target = 17;
+    bytes              blocks_hash    = 18;
     FileInfoType       type           = 2;
     uint32             permissions    = 4;
     int32              modified_ns    = 11;

lib/protocol/bep_extensions.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 const (
@@ -328,3 +329,11 @@ func (f Folder) Description() string {
 	}
 	return fmt.Sprintf("%q (%s)", f.Label, f.ID)
 }
+
+func BlocksHash(bs []BlockInfo) []byte {
+	h := sha256.New()
+	for _, b := range bs {
+		_, _ = h.Write(b.Hash)
+	}
+	return h.Sum(nil)
+}

lib/scanner/blockqueue.go

@@ -111,6 +111,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
 			}
 
 			f.Blocks = blocks
+			f.BlocksHash = protocol.BlocksHash(blocks)
 
 			// The size we saw when initially deciding to hash the file
 			// might not have been the size it actually had when we hashed