lib/db: Refactor key handling (ref #5198) (#5199)

This breaks out the key generation stuff into a separate type. It's
cleaner on its own, and it prepares for future stuff.

lib/db/keyer.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"encoding/binary"
+)
+
+const (
+	keyPrefixLen   = 1
+	keyFolderLen   = 4 // indexed
+	keyDeviceLen   = 4 // indexed
+	keySequenceLen = 8
+	keyHashLen     = 32
+
+	maxInt64 int64 = 1<<63 - 1
+)
+
+const (
+	KeyTypeDevice          = 0
+	KeyTypeGlobal          = 1
+	KeyTypeBlock           = 2
+	KeyTypeDeviceStatistic = 3
+	KeyTypeFolderStatistic = 4
+	KeyTypeVirtualMtime    = 5
+	KeyTypeFolderIdx       = 6
+	KeyTypeDeviceIdx       = 7
+	KeyTypeIndexID         = 8
+	KeyTypeFolderMeta      = 9
+	KeyTypeMiscData        = 10
+	KeyTypeSequence        = 11
+	KeyTypeNeed            = 12
+)
+
+type keyer interface {
+	// device file key stuff
+	GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey
+	NameFromDeviceFileKey(key []byte) []byte
+	DeviceFromDeviceFileKey(key []byte) ([]byte, bool)
+	FolderFromDeviceFileKey(key []byte) ([]byte, bool)
+
+	// global version key stuff
+	GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey
+	NameFromGlobalVersionKey(key []byte) []byte
+	FolderFromGlobalVersionKey(key []byte) ([]byte, bool)
+
+	// file need index
+	GenerateNeedFileKey(key, folder, name []byte) needFileKey
+
+	// file sequence index
+	GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey
+	SequenceFromSequenceKey(key []byte) int64
+
+	// index IDs
+	GenerateIndexIDKey(key, device, folder []byte) indexIDKey
+	DeviceFromIndexIDKey(key []byte) ([]byte, bool)
+
+	// Mtimes
+	GenerateMtimesKey(key, folder []byte) mtimesKey
+
+	// Folder metadata
+	GenerateFolderMetaKey(key, folder []byte) folderMetaKey
+}
+
+// defaultKeyer implements our key scheme. It needs folder and device
+// indexes.
+type defaultKeyer struct {
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+}
+
+func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer {
+	return defaultKeyer{
+		folderIdx: folderIdx,
+		deviceIdx: deviceIdx,
+	}
+}
+
+type deviceFileKey []byte
+
+func (k deviceFileKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen+keyDeviceLen]
+}
+
+func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey {
+	key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name))
+	key[0] = KeyTypeDevice
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], k.deviceIdx.ID(device))
+	copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name)
+	return key
+}
+
+func (k defaultKeyer) NameFromDeviceFileKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
+}
+
+func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) {
+	return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
+}
+
+func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) {
+	return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+}
+
+type globalVersionKey []byte
+
+func (k globalVersionKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey {
+	key = resize(key, keyPrefixLen+keyFolderLen+len(name))
+	key[0] = KeyTypeGlobal
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	copy(key[keyPrefixLen+keyFolderLen:], name)
+	return key
+}
+
+func (k defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen:]
+}
+
+func (k defaultKeyer) FolderFromGlobalVersionKey(key []byte) ([]byte, bool) {
+	return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+}
+
+type needFileKey []byte
+
+func (k needFileKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) needFileKey {
+	key = resize(key, keyPrefixLen+keyFolderLen+len(name))
+	key[0] = KeyTypeNeed
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	copy(key[keyPrefixLen+keyFolderLen:], name)
+	return key
+}
+
+type sequenceKey []byte
+
+func (k sequenceKey) WithoutSequence() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey {
+	key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen)
+	key[0] = KeyTypeSequence
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq))
+	return key
+}
+
+func (k defaultKeyer) SequenceFromSequenceKey(key []byte) int64 {
+	return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
+}
+
+type indexIDKey []byte
+
+func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) indexIDKey {
+	key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen)
+	key[0] = KeyTypeIndexID
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.deviceIdx.ID(device))
+	binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], k.folderIdx.ID(folder))
+	return key
+}
+
+func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) {
+	return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+}
+
+type mtimesKey []byte
+
+func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) mtimesKey {
+	key = resize(key, keyPrefixLen+keyFolderLen)
+	key[0] = KeyTypeVirtualMtime
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	return key
+}
+
+type folderMetaKey []byte
+
+func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) folderMetaKey {
+	key = resize(key, keyPrefixLen+keyFolderLen)
+	key[0] = KeyTypeFolderMeta
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
+	return key
+}
+
+// resize returns a byte slice of the specified size, reusing bs if possible
+func resize(bs []byte, size int) []byte {
+	if cap(bs) < size {
+		return make([]byte, size)
+	}
+	return bs[:size]
+}

lib/db/keyer_test.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestDeviceKey(t *testing.T) {
+	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
+	dev := []byte("device67890123456789012345678901")
+	name := []byte("name")
+
+	db := OpenMemory()
+
+	key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
+
+	fld2, ok := db.keyer.FolderFromDeviceFileKey(key)
+	if !ok {
+		t.Fatal("unexpectedly not found")
+	}
+	if !bytes.Equal(fld2, fld) {
+		t.Errorf("wrong folder %q != %q", fld2, fld)
+	}
+	dev2, ok := db.keyer.DeviceFromDeviceFileKey(key)
+	if !ok {
+		t.Fatal("unexpectedly not found")
+	}
+	if !bytes.Equal(dev2, dev) {
+		t.Errorf("wrong device %q != %q", dev2, dev)
+	}
+	name2 := db.keyer.NameFromDeviceFileKey(key)
+	if !bytes.Equal(name2, name) {
+		t.Errorf("wrong name %q != %q", name2, name)
+	}
+}
+
+func TestGlobalKey(t *testing.T) {
+	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
+	name := []byte("name")
+
+	db := OpenMemory()
+
+	key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
+
+	fld2, ok := db.keyer.FolderFromGlobalVersionKey(key)
+	if !ok {
+		t.Error("should have been found")
+	}
+	if !bytes.Equal(fld2, fld) {
+		t.Errorf("wrong folder %q != %q", fld2, fld)
+	}
+	name2 := db.keyer.NameFromGlobalVersionKey(key)
+	if !bytes.Equal(name2, name) {
+		t.Errorf("wrong name %q != %q", name2, name)
+	}
+
+	_, ok = db.keyer.FolderFromGlobalVersionKey([]byte{1, 2, 3, 4, 5})
+	if ok {
+		t.Error("should not have been found")
+	}
+}
+
+func TestSequenceKey(t *testing.T) {
+	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
+
+	db := OpenMemory()
+
+	const seq = 1234567890
+	key := db.keyer.GenerateSequenceKey(nil, fld, seq)
+	outSeq := db.keyer.SequenceFromSequenceKey(key)
+	if outSeq != seq {
+		t.Errorf("sequence number mangled, %d != %d", outSeq, seq)
+	}
+}

lib/db/leveldb.go

@@ -13,22 +13,6 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const (
-	KeyTypeDevice = iota
-	KeyTypeGlobal
-	KeyTypeBlock
-	KeyTypeDeviceStatistic
-	KeyTypeFolderStatistic
-	KeyTypeVirtualMtime
-	KeyTypeFolderIdx
-	KeyTypeDeviceIdx
-	KeyTypeIndexID
-	KeyTypeFolderMeta
-	KeyTypeMiscData
-	KeyTypeSequence
-	KeyTypeNeed
-)
-
 func (vl VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
@@ -86,7 +70,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *
 			// to determine the winner.)
 			//
 			// A surprise missing file entry here is counted as a win for us.
-			if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
+			if of, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
 				vl = vl.insertAt(i, nv)
 				return vl, removedFV, removedAt, i
 			}

lib/db/leveldb_dbinstance.go

@@ -16,7 +16,6 @@ import (
 	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
@@ -33,18 +32,9 @@ type Instance struct {
 	location  string
 	folderIdx *smallIndex
 	deviceIdx *smallIndex
+	keyer     keyer
 }
 
-const (
-	keyPrefixLen   = 1
-	keyFolderLen   = 4 // indexed
-	keyDeviceLen   = 4 // indexed
-	keySequenceLen = 8
-	keyHashLen     = 32
-
-	maxInt64 int64 = 1<<63 - 1
-)
-
 func Open(file string) (*Instance, error) {
 	opts := &opt.Options{
 		OpenFilesCacheCapacity: 100,
@@ -80,11 +70,12 @@ func OpenMemory() *Instance {
 
 func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
 	i := &Instance{
-		DB:       db,
-		location: location,
+		DB:        db,
+		location:  location,
+		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
 	}
-	i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
-	i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
+	i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
 	err := i.updateSchema()
 	return i, err
 }
@@ -107,7 +98,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
 	var gk []byte
 	for _, f := range fs {
 		name := []byte(f.Name)
-		fk = db.deviceKeyInto(fk, folder, device, name)
+		fk = db.keyer.GenerateDeviceFileKey(fk, folder, device, name)
 
 		// Get and unmarshal the file entry. If it doesn't exist or can't be
 		// unmarshalled we'll add it as a new entry.
@@ -131,7 +122,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
 
 		t.insertFile(fk, folder, device, f)
 
-		gk = db.globalKeyInto(gk, folder, name)
+		gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 		t.updateGlobal(gk, folder, device, f, meta)
 
 		// Write out and reuse the batch every few records, to avoid the batch
@@ -147,8 +138,8 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	var sk []byte
 	var dk []byte
 	for _, f := range fs {
-		sk = db.sequenceKeyInto(sk, folder, f.Sequence)
-		dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
+		sk = db.keyer.GenerateSequenceKey(sk, folder, f.Sequence)
+		dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
 		t.Put(sk, dk)
 		l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
 		t.checkFlush()
@@ -161,7 +152,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
 
 	var sk []byte
 	for _, f := range fs {
-		t.Delete(db.sequenceKeyInto(sk, folder, f.Sequence))
+		t.Delete(db.keyer.GenerateSequenceKey(sk, folder, f.Sequence))
 		l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
 		t.checkFlush()
 	}
@@ -176,7 +167,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 			prefix = append(prefix, '/')
 		}
 
-		if f, ok := db.getFileTrunc(db.deviceKey(folder, device, unslashedPrefix), true); ok && !fn(f) {
+		if f, ok := db.getFileTrunc(db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix), true); ok && !fn(f) {
 			return
 		}
 	}
@@ -184,11 +175,11 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, prefix)[:keyPrefixLen+keyFolderLen+keyDeviceLen+len(prefix)]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)), nil)
 	defer dbi.Release()
 
 	for dbi.Next() {
-		name := db.deviceKeyName(dbi.Key())
+		name := db.keyer.NameFromDeviceFileKey(dbi.Key())
 		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
 			return
 		}
@@ -212,13 +203,13 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(&util.Range{Start: db.sequenceKey(folder, startSeq), Limit: db.sequenceKey(folder, maxInt64)}, nil)
+	dbi := t.NewIterator(&util.Range{Start: db.keyer.GenerateSequenceKey(nil, folder, startSeq), Limit: db.keyer.GenerateSequenceKey(nil, folder, maxInt64)}, nil)
 	defer dbi.Release()
 
 	for dbi.Next() {
 		f, ok := db.getFile(dbi.Value())
 		if !ok {
-			l.Debugln("missing file for sequence number", db.sequenceKeySequence(dbi.Key()))
+			l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
 			continue
 		}
 
@@ -239,13 +230,19 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName()), nil)
 	defer dbi.Release()
 
 	var gk []byte
 
 	for dbi.Next() {
-		device := db.deviceKeyDevice(dbi.Key())
+		device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
+		if !ok {
+			// Not having the device in the index is bad. Clear it.
+			t.Delete(dbi.Key())
+			t.checkFlush()
+			continue
+		}
 		var f FileInfoTruncated
 		// The iterator function may keep a reference to the unmarshalled
 		// struct, which in turn references the buffer it was unmarshalled
@@ -261,7 +258,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 		case "", ".", "..", "/": // A few obviously invalid filenames
 			l.Infof("Dropping invalid filename %q from database", f.Name)
 			name := []byte(f.Name)
-			gk = db.globalKeyInto(gk, folder, name)
+			gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			t.removeFromGlobal(gk, folder, device, name, nil)
 			t.Delete(dbi.Key())
 			t.checkFlush()
@@ -308,7 +305,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 }
 
 func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
-	gk = db.globalKeyInto(gk, folder, file)
+	gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
 
 	bs, err := t.Get(gk, nil)
 	if err != nil {
@@ -320,7 +317,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
 		return gk, dk, nil, false
 	}
 
-	dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
+	dk = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, file)
 	if fi, ok := db.getFileTrunc(dk, truncate); ok {
 		return gk, dk, fi, true
 	}
@@ -345,12 +342,12 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, prefix)), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)), nil)
 	defer dbi.Release()
 
 	var fk []byte
 	for dbi.Next() {
-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
 			return
 		}
@@ -360,7 +357,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 			continue
 		}
 
-		fk = db.deviceKeyInto(fk, folder, vl.Versions[0].Device, name)
+		fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[0].Device, name)
 
 		f, ok := db.getFileTrunc(fk, truncate)
 		if !ok {
@@ -374,7 +371,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 }
 
 func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
-	k := db.globalKey(folder, file)
+	k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
 	bs, err := db.Get(k, nil)
 	if err == leveldb.ErrNotFound {
 		return nil
@@ -413,7 +410,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()
 
 	var fk []byte
@@ -431,7 +428,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 			continue
 		}
 
-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		needVersion := vl.Versions[0].Version
 		needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
 
@@ -446,7 +443,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 				continue
 			}
 
-			fk = db.deviceKeyInto(fk, folder, vl.Versions[i].Device, name)
+			fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[i].Device, name)
 			bs, err := t.Get(fk, nil)
 			if err != nil {
 				l.Debugln("surprise error:", err)
@@ -480,7 +477,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()
 
 	var dk []byte
@@ -488,7 +485,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	var f FileIntf
 	var ok bool
 	for dbi.Next() {
-		gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
+		gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
 		if !ok {
 			continue
 		}
@@ -507,7 +504,7 @@ func (db *Instance) ListFolders() []string {
 
 	folderExists := make(map[string]bool)
 	for dbi.Next() {
-		folder, ok := db.globalKeyFolder(dbi.Key())
+		folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
 		if ok && !folderExists[string(folder)] {
 			folderExists[string(folder)] = true
 		}
@@ -528,13 +525,13 @@ func (db *Instance) dropFolder(folder []byte) {
 
 	for _, key := range [][]byte{
 		// Remove all items related to the given folder from the device->file bucket
-		db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName(),
 		// Remove all sequences related to the folder
-		db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateSequenceKey(nil, []byte(folder), 0).WithoutSequence(),
 		// Remove all items related to the given folder from the global bucket
-		db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName(),
 		// Remove all needs related to the folder
-		db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName(),
 	} {
 		t.deleteKeyPrefix(key)
 	}
@@ -544,15 +541,15 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, nil)), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, nil)), nil)
 	defer dbi.Release()
 
 	var gk []byte
 
 	for dbi.Next() {
 		key := dbi.Key()
-		name := db.deviceKeyName(key)
-		gk = db.globalKeyInto(gk, folder, name)
+		name := db.keyer.NameFromDeviceFileKey(key)
+		gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 		t.removeFromGlobal(gk, folder, device, name, meta)
 		t.Delete(key)
 		t.checkFlush()
@@ -563,7 +560,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()
 
 	var fk []byte
@@ -578,10 +575,10 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 		// there are global entries pointing to no longer existing files. Here
 		// we find those and clear them out.
 
-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		var newVL VersionList
 		for i, version := range vl.Versions {
-			fk = db.deviceKeyInto(fk, folder, version.Device, name)
+			fk = db.keyer.GenerateDeviceFileKey(fk, folder, version.Device, name)
 			_, err := t.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
 				continue
@@ -607,110 +604,8 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	l.Debugf("db check completed for %q", folder)
 }
 
-// deviceKey returns a byte slice encoding the following information:
-//	   keyTypeDevice (1 byte)
-//	   folder (4 bytes)
-//	   device (4 bytes)
-//	   name (variable size)
-func (db *Instance) deviceKey(folder, device, file []byte) []byte {
-	return db.deviceKeyInto(nil, folder, device, file)
-}
-
-func (db *Instance) deviceKeyInto(k, folder, device, file []byte) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + keyDeviceLen + len(file)
-	k = resize(k, reqLen)
-	k[0] = KeyTypeDevice
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
-	binary.BigEndian.PutUint32(k[keyPrefixLen+keyFolderLen:], db.deviceIdx.ID(device))
-	copy(k[keyPrefixLen+keyFolderLen+keyDeviceLen:], file)
-	return k
-}
-
-// deviceKeyName returns the device ID from the key
-func (db *Instance) deviceKeyName(key []byte) []byte {
-	return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
-}
-
-// deviceKeyFolder returns the folder name from the key
-func (db *Instance) deviceKeyFolder(key []byte) []byte {
-	folder, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-	if !ok {
-		panic("bug: lookup of nonexistent folder ID")
-	}
-	return folder
-}
-
-// deviceKeyDevice returns the device ID from the key
-func (db *Instance) deviceKeyDevice(key []byte) []byte {
-	device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
-	if !ok {
-		panic("bug: lookup of nonexistent device ID")
-	}
-	return device
-}
-
-// globalKey returns a byte slice encoding the following information:
-//	   keyTypeGlobal (1 byte)
-//	   folder (4 bytes)
-//	   name (variable size)
-func (db *Instance) globalKey(folder, file []byte) []byte {
-	return db.globalKeyInto(nil, folder, file)
-}
-
-func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + len(file)
-	gk = resize(gk, reqLen)
-	gk[0] = KeyTypeGlobal
-	binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
-	copy(gk[keyPrefixLen+keyFolderLen:], file)
-	return gk
-}
-
-// globalKeyName returns the filename from the key
-func (db *Instance) globalKeyName(key []byte) []byte {
-	return key[keyPrefixLen+keyFolderLen:]
-}
-
-// globalKeyFolder returns the folder name from the key
-func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
-	return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-}
-
-// needKey is a globalKey with a different prefix
-func (db *Instance) needKey(folder, file []byte) []byte {
-	return db.needKeyInto(nil, folder, file)
-}
-
-func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
-	k = db.globalKeyInto(k, folder, file)
-	k[0] = KeyTypeNeed
-	return k
-}
-
-// sequenceKey returns a byte slice encoding the following information:
-//	   KeyTypeSequence (1 byte)
-//	   folder (4 bytes)
-//	   sequence number (8 bytes)
-func (db *Instance) sequenceKey(folder []byte, seq int64) []byte {
-	return db.sequenceKeyInto(nil, folder, seq)
-}
-
-func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + keySequenceLen
-	k = resize(k, reqLen)
-	k[0] = KeyTypeSequence
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
-	binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
-	return k
-}
-
-// sequenceKeySequence returns the sequence number from the key
-func (db *Instance) sequenceKeySequence(key []byte) int64 {
-	return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
-}
-
 func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
-	key := db.indexIDKey(device, folder)
+	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	cur, err := db.Get(key, nil)
 	if err != nil {
 		return 0
@@ -725,44 +620,13 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
 }
 
 func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
-	key := db.indexIDKey(device, folder)
+	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	bs, _ := id.Marshal() // marshalling can't fail
 	if err := db.Put(key, bs, nil); err != nil {
 		panic("storing index ID: " + err.Error())
 	}
 }
 
-func (db *Instance) indexIDKey(device, folder []byte) []byte {
-	k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen)
-	k[0] = KeyTypeIndexID
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device))
-	binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder))
-	return k
-}
-
-func (db *Instance) indexIDDevice(key []byte) []byte {
-	device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-	if !ok {
-		// uuh ...
-		return nil
-	}
-	return device
-}
-
-func (db *Instance) mtimesKey(folder []byte) []byte {
-	prefix := make([]byte, 5) // key type + 4 bytes folder idx number
-	prefix[0] = KeyTypeVirtualMtime
-	binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
-	return prefix
-}
-
-func (db *Instance) folderMetaKey(folder []byte) []byte {
-	prefix := make([]byte, 5) // key type + 4 bytes folder idx number
-	prefix[0] = KeyTypeFolderMeta
-	binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
-	return prefix
-}
-
 // DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
 // the database. This will cause a full index transmission on the next
 // connection.
@@ -785,7 +649,7 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
 	defer dbi.Release()
 
 	for dbi.Next() {
-		device := db.indexIDDevice(dbi.Key())
+		device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
 		if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
 			t.Delete(dbi.Key())
 		}
@@ -793,11 +657,11 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
 }
 
 func (db *Instance) dropMtimes(folder []byte) {
-	db.dropPrefix(db.mtimesKey(folder))
+	db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
 }
 
 func (db *Instance) dropFolderMeta(folder []byte) {
-	db.dropPrefix(db.folderMetaKey(folder))
+	db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
 }
 
 func (db *Instance) dropPrefix(prefix []byte) {
@@ -853,99 +717,6 @@ func leveldbIsCorrupted(err error) bool {
 	return false
 }
 
-// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
-// fast lookups in both directions and persists to the database. Don't use for
-// storing more items than fit comfortably in RAM.
-type smallIndex struct {
-	db     *Instance
-	prefix []byte
-	id2val map[uint32]string
-	val2id map[string]uint32
-	nextID uint32
-	mut    sync.Mutex
-}
-
-func newSmallIndex(db *Instance, prefix []byte) *smallIndex {
-	idx := &smallIndex{
-		db:     db,
-		prefix: prefix,
-		id2val: make(map[uint32]string),
-		val2id: make(map[string]uint32),
-		mut:    sync.NewMutex(),
-	}
-	idx.load()
-	return idx
-}
-
-// load iterates over the prefix space in the database and populates the in
-// memory maps.
-func (i *smallIndex) load() {
-	tr := i.db.newReadOnlyTransaction()
-	it := tr.NewIterator(util.BytesPrefix(i.prefix), nil)
-	for it.Next() {
-		val := string(it.Value())
-		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
-		i.id2val[id] = val
-		i.val2id[val] = id
-		if id >= i.nextID {
-			i.nextID = id + 1
-		}
-	}
-	it.Release()
-	tr.close()
-}
-
-// ID returns the index number for the given byte slice, allocating a new one
-// and persisting this to the database if necessary.
-func (i *smallIndex) ID(val []byte) uint32 {
-	i.mut.Lock()
-	// intentionally avoiding defer here as we want this call to be as fast as
-	// possible in the general case (folder ID already exists). The map lookup
-	// with the conversion of []byte to string is compiler optimized to not
-	// copy the []byte, which is why we don't assign it to a temp variable
-	// here.
-	if id, ok := i.val2id[string(val)]; ok {
-		i.mut.Unlock()
-		return id
-	}
-
-	id := i.nextID
-	i.nextID++
-
-	valStr := string(val)
-	i.val2id[valStr] = id
-	i.id2val[id] = valStr
-
-	key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
-	copy(key, i.prefix)
-	binary.BigEndian.PutUint32(key[len(i.prefix):], id)
-	i.db.Put(key, val, nil)
-
-	i.mut.Unlock()
-	return id
-}
-
-// Val returns the value for the given index number, or (nil, false) if there
-// is no such index number.
-func (i *smallIndex) Val(id uint32) ([]byte, bool) {
-	i.mut.Lock()
-	val, ok := i.id2val[id]
-	i.mut.Unlock()
-	if !ok {
-		return nil, false
-	}
-
-	return []byte(val), true
-}
-
-// resize returns a byte array of length reqLen, reusing k if possible
-func resize(k []byte, reqLen int) []byte {
-	if cap(k) < reqLen {
-		return make([]byte, reqLen)
-	}
-	return k[:reqLen]
-}
-
 type errorSuggestion struct {
 	inner      error
 	suggestion string

lib/db/leveldb_dbinstance_updateschema.go

@@ -91,16 +91,28 @@ func (db *Instance) updateSchema0to1() {
 	var gk []byte
 
 	for dbi.Next() {
-		folder := db.deviceKeyFolder(dbi.Key())
-		device := db.deviceKeyDevice(dbi.Key())
-		name := db.deviceKeyName(dbi.Key())
+		folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
+		if !ok {
+			// not having the folder in the index is bad; delete and continue
+			t.Delete(dbi.Key())
+			t.checkFlush()
+			continue
+		}
+		device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
+		if !ok {
+			// not having the device in the index is bad; delete and continue
+			t.Delete(dbi.Key())
+			t.checkFlush()
+			continue
+		}
+		name := db.keyer.NameFromDeviceFileKey(dbi.Key())
 
 		// Remove files with absolute path (see #4799)
 		if strings.HasPrefix(string(name), "/") {
 			if _, ok := changedFolders[string(folder)]; !ok {
 				changedFolders[string(folder)] = struct{}{}
 			}
-			gk = db.globalKeyInto(gk, folder, name)
+			gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			t.removeFromGlobal(gk, folder, device, nil, nil)
 			t.Delete(dbi.Key())
 			t.checkFlush()
@@ -130,7 +142,7 @@ func (db *Instance) updateSchema0to1() {
 
 		// Add invalid files to global list
 		if f.IsInvalid() {
-			gk = db.globalKeyInto(gk, folder, name)
+			gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if t.updateGlobal(gk, folder, device, f, meta) {
 				if _, ok := changedFolders[string(folder)]; !ok {
 					changedFolders[string(folder)] = struct{}{}
@@ -156,8 +168,8 @@ func (db *Instance) updateSchema1to2() {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
-			sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
-			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
+			sk = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
+			dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
 			t.Put(sk, dk)
 			t.checkFlush()
 			return true
@@ -176,7 +188,7 @@ func (db *Instance) updateSchema2to3() {
 		folder := []byte(folderStr)
 		db.withGlobal(folder, nil, true, func(f FileIntf) bool {
 			name := []byte(f.FileName())
-			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
+			dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
 			var v protocol.Vector
 			haveFile, ok := db.getFileTrunc(dk, true)
 			if ok {
@@ -185,7 +197,7 @@ func (db *Instance) updateSchema2to3() {
 			if !need(f, ok, v) {
 				return true
 			}
-			nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
+			nk = t.db.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
 			t.Put(nk, nil)
 			t.checkFlush()
 			return true
@@ -201,7 +213,7 @@ func (db *Instance) updateSchemaTo5() {
 	t := db.newReadWriteTransaction()
 	var nk []byte
 	for _, folderStr := range db.ListFolders() {
-		nk = db.needKeyInto(nk, []byte(folderStr), nil)
+		nk = db.keyer.GenerateNeedFileKey(nk, []byte(folderStr), nil)
 		t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen])
 	}
 	t.close()
@@ -230,7 +242,7 @@ func (db *Instance) updateSchema5to6() {
 			fi.LocalFlags = protocol.FlagLocalIgnored
 			bs, _ := fi.Marshal()
 
-			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
+			dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
 			t.Put(dk, bs)
 
 			t.checkFlush()

lib/db/leveldb_test.go

@@ -7,7 +7,6 @@
 package db
 
 import (
-	"bytes"
 	"os"
 	"testing"
 
@@ -15,58 +14,6 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func TestDeviceKey(t *testing.T) {
-	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
-	dev := []byte("device67890123456789012345678901")
-	name := []byte("name")
-
-	db := OpenMemory()
-	db.folderIdx.ID(fld)
-	db.deviceIdx.ID(dev)
-
-	key := db.deviceKey(fld, dev, name)
-
-	fld2 := db.deviceKeyFolder(key)
-	if !bytes.Equal(fld2, fld) {
-		t.Errorf("wrong folder %q != %q", fld2, fld)
-	}
-	dev2 := db.deviceKeyDevice(key)
-	if !bytes.Equal(dev2, dev) {
-		t.Errorf("wrong device %q != %q", dev2, dev)
-	}
-	name2 := db.deviceKeyName(key)
-	if !bytes.Equal(name2, name) {
-		t.Errorf("wrong name %q != %q", name2, name)
-	}
-}
-
-func TestGlobalKey(t *testing.T) {
-	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
-	name := []byte("name")
-
-	db := OpenMemory()
-	db.folderIdx.ID(fld)
-
-	key := db.globalKey(fld, name)
-
-	fld2, ok := db.globalKeyFolder(key)
-	if !ok {
-		t.Error("should have been found")
-	}
-	if !bytes.Equal(fld2, fld) {
-		t.Errorf("wrong folder %q != %q", fld2, fld)
-	}
-	name2 := db.globalKeyName(key)
-	if !bytes.Equal(name2, name) {
-		t.Errorf("wrong name %q != %q", name2, name)
-	}
-
-	_, ok = db.globalKeyFolder([]byte{1, 2, 3, 4, 5})
-	if ok {
-		t.Error("should not have been found")
-	}
-}
-
 func TestDropIndexIDs(t *testing.T) {
 	db := OpenMemory()
 
@@ -287,11 +234,11 @@ func TestUpdate0to3(t *testing.T) {
 
 	db.updateSchema0to1()
 
-	if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
+	if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
 		t.Error("File prefixed by '/' was not removed during transition to schema 1")
 	}
 
-	if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil {
+	if _, err := db.Get(db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)), nil); err != nil {
 		t.Error("Invalid file wasn't added to global list")
 	}
 

lib/db/leveldb_transactions.go

@@ -37,7 +37,7 @@ func (t readOnlyTransaction) close() {
 }
 
 func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
-	return t.db.getFile(t.db.deviceKey(folder, device, file))
+	return t.db.getFile(t.db.keyer.GenerateDeviceFileKey(nil, folder, device, file))
 }
 
 // A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
@@ -111,7 +111,7 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 	}
 
 	// Fixup the list of files we need.
-	nk := t.db.needKey(folder, name)
+	nk := t.db.keyer.GenerateNeedFileKey(nil, folder, name)
 	hasNeeded, _ := t.db.Has(nk, nil)
 	if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
 		if !hasNeeded {

lib/db/meta.go

@@ -56,7 +56,7 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
 // toDB saves the marshalled metadataTracker to the given db, under the key
 // corresponding to the given folder
 func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
-	key := db.folderMetaKey(folder)
+	key := db.keyer.GenerateFolderMetaKey(nil, folder)
 	bs, err := m.Marshal()
 	if err != nil {
 		return err
@@ -67,7 +67,7 @@ func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
 // fromDB initializes the metadataTracker from the marshalled data found in
 // the database under the key corresponding to the given folder
 func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
-	key := db.folderMetaKey(folder)
+	key := db.keyer.GenerateFolderMetaKey(nil, folder)
 	bs, err := db.Get(key, nil)
 	if err != nil {
 		return err

lib/db/set.go

@@ -160,7 +160,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	var dk []byte
 	folder := []byte(s.folder)
 	for _, nf := range oldFs {
-		dk = s.db.deviceKeyInto(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
+		dk = s.db.keyer.GenerateDeviceFileKey(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
 		ef, ok := s.db.getFile(dk)
 		if ok && ef.Version.Equal(nf.Version) && ef.IsInvalid() == nf.IsInvalid() {
 			continue
@@ -242,7 +242,7 @@ func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
 }
 
 func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
-	f, ok := s.db.getFile(s.db.deviceKey([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
+	f, ok := s.db.getFile(s.db.keyer.GenerateDeviceFileKey(nil, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
 	f.Name = osutil.NativeFilename(f.Name)
 	return f, ok
 }
@@ -309,7 +309,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
 }
 
 func (s *FileSet) MtimeFS() *fs.MtimeFS {
-	prefix := s.db.mtimesKey([]byte(s.folder))
+	prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
 	kv := NewNamespacedKV(s.db, string(prefix))
 	return fs.NewMtimeFS(s.fs, kv)
 }

lib/db/smallindex.go

@@ -0,0 +1,98 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"encoding/binary"
+
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
+// fast lookups in both directions and persists to the database. Don't use for
+// storing more items than fit comfortably in RAM.
+type smallIndex struct {
+	db     *leveldb.DB
+	prefix []byte
+	id2val map[uint32]string
+	val2id map[string]uint32
+	nextID uint32
+	mut    sync.Mutex
+}
+
+func newSmallIndex(db *leveldb.DB, prefix []byte) *smallIndex {
+	idx := &smallIndex{
+		db:     db,
+		prefix: prefix,
+		id2val: make(map[uint32]string),
+		val2id: make(map[string]uint32),
+		mut:    sync.NewMutex(),
+	}
+	idx.load()
+	return idx
+}
+
+// load iterates over the prefix space in the database and populates the in
+// memory maps.
+func (i *smallIndex) load() {
+	it := i.db.NewIterator(util.BytesPrefix(i.prefix), nil)
+	defer it.Release()
+	for it.Next() {
+		val := string(it.Value())
+		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
+		i.id2val[id] = val
+		i.val2id[val] = id
+		if id >= i.nextID {
+			i.nextID = id + 1
+		}
+	}
+}
+
+// ID returns the index number for the given byte slice, allocating a new one
+// and persisting this to the database if necessary.
+func (i *smallIndex) ID(val []byte) uint32 {
+	i.mut.Lock()
+	// intentionally avoiding defer here as we want this call to be as fast as
+	// possible in the general case (folder ID already exists). The map lookup
+	// with the conversion of []byte to string is compiler optimized to not
+	// copy the []byte, which is why we don't assign it to a temp variable
+	// here.
+	if id, ok := i.val2id[string(val)]; ok {
+		i.mut.Unlock()
+		return id
+	}
+
+	id := i.nextID
+	i.nextID++
+
+	valStr := string(val)
+	i.val2id[valStr] = id
+	i.id2val[id] = valStr
+
+	key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
+	copy(key, i.prefix)
+	binary.BigEndian.PutUint32(key[len(i.prefix):], id)
+	i.db.Put(key, val, nil)
+
+	i.mut.Unlock()
+	return id
+}
+
+// Val returns the value for the given index number, or (nil, false) if there
+// is no such index number.
+func (i *smallIndex) Val(id uint32) ([]byte, bool) {
+	i.mut.Lock()
+	val, ok := i.id2val[id]
+	i.mut.Unlock()
+	if !ok {
+		return nil, false
+	}
+
+	return []byte(val), true
+}