lib/db: Move database schema migration into its own file (#4985)

cmd/syncthing/main.go

@@ -751,9 +751,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		miscDB.PutString("prevVersion", Version)
 	}
 
-	// Potential database transitions
-	ldb.UpdateSchema()
-
 	m := model.NewModel(cfg, myID, "syncthing", Version, ldb, protectedFiles)
 
 	if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {

lib/db/leveldb.go

@@ -13,8 +13,6 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const dbVersion = 3
-
 const (
 	KeyTypeDevice = iota
 	KeyTypeGlobal

lib/db/leveldb_dbinstance.go

@@ -83,34 +83,10 @@ func newDBInstance(db *leveldb.DB, location string) *Instance {
 	}
 	i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
 	i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
+	i.updateSchema()
 	return i
 }
 
-// UpdateSchema does transitions to the current db version if necessary
-func (db *Instance) UpdateSchema() {
-	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
-	prevVersion, _ := miscDB.Int64("dbVersion")
-
-	if prevVersion >= dbVersion {
-		return
-	}
-
-	l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
-
-	if prevVersion == 0 {
-		db.updateSchema0to1()
-	}
-	if prevVersion <= 1 {
-		db.updateSchema1to2()
-	}
-	if prevVersion <= 2 {
-		db.updateSchema2to3()
-	}
-
-	l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion)
-	miscDB.PutInt64("dbVersion", dbVersion)
-}
-
 // Committed returns the number of items committed to the database since startup
 func (db *Instance) Committed() int64 {
 	return atomic.LoadInt64(&db.committed)
@@ -620,124 +596,6 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	l.Debugf("db check completed for %q", folder)
 }
 
-func (db *Instance) updateSchema0to1() {
-	t := db.newReadWriteTransaction()
-	defer t.close()
-
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
-	defer dbi.Release()
-
-	symlinkConv := 0
-	changedFolders := make(map[string]struct{})
-	ignAdded := 0
-	meta := newMetadataTracker() // dummy metadata tracker
-	var gk []byte
-
-	for dbi.Next() {
-		folder := db.deviceKeyFolder(dbi.Key())
-		device := db.deviceKeyDevice(dbi.Key())
-		name := db.deviceKeyName(dbi.Key())
-
-		// Remove files with absolute path (see #4799)
-		if strings.HasPrefix(string(name), "/") {
-			if _, ok := changedFolders[string(folder)]; !ok {
-				changedFolders[string(folder)] = struct{}{}
-			}
-			gk = db.globalKeyInto(gk, folder, name)
-			t.removeFromGlobal(gk, folder, device, nil, nil)
-			t.Delete(dbi.Key())
-			t.checkFlush()
-			continue
-		}
-
-		// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
-		// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
-		// local and remote, and does not reset delta indexes. It shouldn't really
-		// matter what the symlink type is, but this cleans it up for a possible
-		// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
-		var f protocol.FileInfo
-		if err := f.Unmarshal(dbi.Value()); err != nil {
-			// probably can't happen
-			continue
-		}
-		if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
-			f.Type = protocol.FileInfoTypeSymlink
-			bs, err := f.Marshal()
-			if err != nil {
-				panic("can't happen: " + err.Error())
-			}
-			t.Put(dbi.Key(), bs)
-			t.checkFlush()
-			symlinkConv++
-		}
-
-		// Add invalid files to global list
-		if f.Invalid {
-			gk = db.globalKeyInto(gk, folder, name)
-			if t.updateGlobal(gk, folder, device, f, meta) {
-				if _, ok := changedFolders[string(folder)]; !ok {
-					changedFolders[string(folder)] = struct{}{}
-				}
-				ignAdded++
-			}
-		}
-	}
-
-	for folder := range changedFolders {
-		db.dropFolderMeta([]byte(folder))
-	}
-
-	l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded)
-}
-
-// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
-// to allow iteration in sequence order (simplifies sending indexes).
-func (db *Instance) updateSchema1to2() {
-	t := db.newReadWriteTransaction()
-	defer t.close()
-
-	var sk []byte
-	var dk []byte
-	for _, folderStr := range db.ListFolders() {
-		folder := []byte(folderStr)
-		db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
-			sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
-			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
-			t.Put(sk, dk)
-			t.checkFlush()
-			return true
-		})
-	}
-}
-
-// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
-func (db *Instance) updateSchema2to3() {
-	t := db.newReadWriteTransaction()
-	defer t.close()
-
-	var nk []byte
-	var dk []byte
-	for _, folderStr := range db.ListFolders() {
-		folder := []byte(folderStr)
-		db.withGlobal(folder, nil, true, func(f FileIntf) bool {
-			name := []byte(f.FileName())
-			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
-			var v protocol.Vector
-			haveFile, ok := db.getFileTrunc(dk, true)
-			if ok {
-				v = haveFile.FileVersion()
-			}
-			if !need(f, ok, v) {
-				return true
-			}
-			nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
-			t.Put(nk, nil)
-			t.checkFlush()
-			return true
-		})
-	}
-}
-
 // deviceKey returns a byte slice encoding the following information:
 //	   keyTypeDevice (1 byte)
 //	   folder (4 bytes)

lib/db/leveldb_dbinstance_updateschema.go

@@ -0,0 +1,155 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const dbVersion = 3
+
+func (db *Instance) updateSchema() {
+	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
+	prevVersion, _ := miscDB.Int64("dbVersion")
+
+	if prevVersion >= dbVersion {
+		return
+	}
+
+	l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
+
+	if prevVersion < 1 {
+		db.updateSchema0to1()
+	}
+	if prevVersion < 2 {
+		db.updateSchema1to2()
+	}
+	if prevVersion < 3 {
+		db.updateSchema2to3()
+	}
+
+	miscDB.PutInt64("dbVersion", dbVersion)
+}
+
+func (db *Instance) updateSchema0to1() {
+	t := db.newReadWriteTransaction()
+	defer t.close()
+
+	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
+	defer dbi.Release()
+
+	symlinkConv := 0
+	changedFolders := make(map[string]struct{})
+	ignAdded := 0
+	meta := newMetadataTracker() // dummy metadata tracker
+	var gk []byte
+
+	for dbi.Next() {
+		folder := db.deviceKeyFolder(dbi.Key())
+		device := db.deviceKeyDevice(dbi.Key())
+		name := db.deviceKeyName(dbi.Key())
+
+		// Remove files with absolute path (see #4799)
+		if strings.HasPrefix(string(name), "/") {
+			if _, ok := changedFolders[string(folder)]; !ok {
+				changedFolders[string(folder)] = struct{}{}
+			}
+			gk = db.globalKeyInto(gk, folder, name)
+			t.removeFromGlobal(gk, folder, device, nil, nil)
+			t.Delete(dbi.Key())
+			t.checkFlush()
+			continue
+		}
+
+		// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
+		// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
+		// local and remote, and does not reset delta indexes. It shouldn't really
+		// matter what the symlink type is, but this cleans it up for a possible
+		// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
+		var f protocol.FileInfo
+		if err := f.Unmarshal(dbi.Value()); err != nil {
+			// probably can't happen
+			continue
+		}
+		if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
+			f.Type = protocol.FileInfoTypeSymlink
+			bs, err := f.Marshal()
+			if err != nil {
+				panic("can't happen: " + err.Error())
+			}
+			t.Put(dbi.Key(), bs)
+			t.checkFlush()
+			symlinkConv++
+		}
+
+		// Add invalid files to global list
+		if f.Invalid {
+			gk = db.globalKeyInto(gk, folder, name)
+			if t.updateGlobal(gk, folder, device, f, meta) {
+				if _, ok := changedFolders[string(folder)]; !ok {
+					changedFolders[string(folder)] = struct{}{}
+				}
+				ignAdded++
+			}
+		}
+	}
+
+	for folder := range changedFolders {
+		db.dropFolderMeta([]byte(folder))
+	}
+}
+
+// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
+// to allow iteration in sequence order (simplifies sending indexes).
+func (db *Instance) updateSchema1to2() {
+	t := db.newReadWriteTransaction()
+	defer t.close()
+
+	var sk []byte
+	var dk []byte
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+		db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
+			sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
+			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
+			t.Put(sk, dk)
+			t.checkFlush()
+			return true
+		})
+	}
+}
+
+// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
+func (db *Instance) updateSchema2to3() {
+	t := db.newReadWriteTransaction()
+	defer t.close()
+
+	var nk []byte
+	var dk []byte
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+		db.withGlobal(folder, nil, true, func(f FileIntf) bool {
+			name := []byte(f.FileName())
+			dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
+			var v protocol.Vector
+			haveFile, ok := db.getFileTrunc(dk, true)
+			if ok {
+				v = haveFile.FileVersion()
+			}
+			if !need(f, ok, v) {
+				return true
+			}
+			nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
+			t.Put(nk, nil)
+			t.checkFlush()
+			return true
+		})
+	}
+}