Browse Source

feat: add `syncthing debug database-statistics` command (#10117)

This adds a command that shows database statistics. Currently it
requires a fork of the sqlite package to add the dbstats virtual table;
the modernc variant already has it.

This also provides the canonical mapping between folder ID and database
file, for tinkerers...

```
% ./bin/syncthing debug database-statistics
DATABASE                 FOLDER ID    TABLE                                  SIZE     FILL
========                 ====== ==    =====                                  ====     ====
main.db                  -            folders                               4 KiB    8.4 %
main.db                  -            folders_database_name                 4 KiB    6.0 %
main.db                  -            kv                                    4 KiB   41.1 %
main.db                  -            schemamigrations                      4 KiB    3.9 %
main.db                  -            sqlite_autoindex_folders_1            4 KiB    3.7 %
...
folder.0007-txpxsvyd.db  w3ejt-fn4dm  indexids                              4 KiB    1.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  kv                                    4 KiB    0.8 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  mtimes                              608 KiB   81.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  schemamigrations                      4 KiB    3.9 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_blocklists_1      4108 KiB   89.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_blocks_1        700020 KiB   88.1 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_devices_1            4 KiB    3.6 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_kv_1                 4 KiB    0.6 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_schema                        12 KiB   45.9 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_sequence                       4 KiB    1.0 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_stat1                          4 KiB   12.2 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_stat4                          4 KiB    0.2 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  (total)                         1906020 KiB   92.8 %
main.db + children       -            (total)                         2205888 KiB   92.0 %
```
Jakob Borg 5 months ago
parent
commit
085455d72e
6 changed files with 121 additions and 24 deletions
  1. 1 1
      .github/workflows/build-syncthing.yaml
  2. 44 20
      cmd/syncthing/main.go
  3. 4 1
      go.mod
  4. 2 2
      go.sum
  5. 69 0
      internal/db/sqlite/db_stats.go
  6. 1 0
      lib/build/build.go

+ 1 - 1
.github/workflows/build-syncthing.yaml

@@ -22,7 +22,7 @@ env:
   BUILD_USER: builder
   BUILD_HOST: github.syncthing.net
 
-  TAGS: "netgo osusergo sqlite_omit_load_extension"
+  TAGS: "netgo osusergo sqlite_omit_load_extension sqlite_dbstat"
 
 # A note on actions and third party code... The actions under actions/ (like
 # `uses: actions/checkout`) are maintained by GitHub, and we need to trust

+ 44 - 20
cmd/syncthing/main.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -26,6 +27,7 @@ import (
 	"sort"
 	"strconv"
 	"syscall"
+	"text/tabwriter"
 	"time"
 
 	"github.com/alecthomas/kong"
@@ -37,6 +39,7 @@ import (
 	"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
 	"github.com/syncthing/syncthing/cmd/syncthing/generate"
 	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/db/sqlite"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
@@ -825,24 +828,6 @@ func exitCodeForUpgrade(err error) int {
 	return svcutil.ExitError.AsInt()
 }
 
-// convertLegacyArgs returns the slice of arguments with single dash long
-// flags converted to double dash long flags.
-func convertLegacyArgs(args []string) []string {
-	// Legacy args begin with a single dash, followed by two or more characters.
-	legacyExp := regexp.MustCompile(`^-\w{2,}`)
-
-	res := make([]string, len(args))
-	for i, arg := range args {
-		if legacyExp.MatchString(arg) {
-			res[i] = "-" + arg
-		} else {
-			res[i] = arg
-		}
-	}
-
-	return res
-}
-
 type versionCmd struct{}
 
 func (versionCmd) Run() error {
@@ -900,7 +885,8 @@ func (u upgradeCmd) Run() error {
 	release, err := checkUpgrade()
 	if err == nil {
 		lf := flock.New(locations.Get(locations.LockFile))
-		locked, err := lf.TryLock()
+		var locked bool
+		locked, err = lf.TryLock()
 		if err != nil {
 			l.Warnln("Upgrade:", err)
 			os.Exit(1)
@@ -930,7 +916,8 @@ func (browserCmd) Run() error {
 }
 
 type debugCmd struct {
-	ResetDatabase resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
+	ResetDatabase      resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
+	DatabaseStatistics databaseStatsCmd `cmd:"" help:"Display database size statistics"`
 }
 
 type resetDatabaseCmd struct{}
@@ -945,6 +932,43 @@ func (resetDatabaseCmd) Run() error {
 	return nil
 }
 
+type databaseStatsCmd struct{}
+
+func (c databaseStatsCmd) Run() error {
+	db, err := sqlite.Open(locations.Get(locations.Database))
+	if err != nil {
+		return err
+	}
+	ds, err := db.Statistics()
+	if err != nil {
+		return err
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 2, 2, 2, ' ', 0)
+	hdr := fmt.Sprintf("%s\t%s\t%s\t%12s\t%7s\n", "DATABASE", "FOLDER ID", "TABLE", "SIZE", "FILL")
+	fmt.Fprint(tw, hdr)
+	fmt.Fprint(tw, regexp.MustCompile(`[A-Z]`).ReplaceAllString(hdr, "="))
+	c.printStat(tw, ds)
+	return tw.Flush()
+}
+
+func (c databaseStatsCmd) printStat(w io.Writer, s *sqlite.DatabaseStatistics) {
+	for _, table := range s.Tables {
+		fmt.Fprintf(w, "%s\t%s\t%s\t%8d KiB\t%5.01f %%\n", s.Name, cmp.Or(s.FolderID, "-"), table.Name, table.Size/1024, float64(table.Size-table.Unused)*100/float64(table.Size))
+	}
+	for _, next := range s.Children {
+		c.printStat(w, &next)
+		s.Total.Size += next.Total.Size
+		s.Total.Unused += next.Total.Unused
+	}
+
+	totalName := s.Name
+	if len(s.Children) > 0 {
+		totalName += " + children"
+	}
+	fmt.Fprintf(w, "%s\t%s\t%s\t%8d KiB\t%5.01f %%\n", totalName, cmp.Or(s.FolderID, "-"), "(total)", s.Total.Size/1024, float64(s.Total.Size-s.Total.Unused)*100/float64(s.Total.Size))
+}
+
 func setConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
 	homeSet := homeDir != ""
 	confSet := confDir != ""

+ 4 - 1
go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/maruel/panicparse/v2 v2.5.0
-	github.com/mattn/go-sqlite3 v1.14.27
+	github.com/mattn/go-sqlite3 v1.14.28
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
 	github.com/maxmind/geoipupdate/v6 v6.1.0
 	github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
@@ -108,3 +108,6 @@ require (
 
 // https://github.com/gobwas/glob/pull/55
 replace github.com/gobwas/glob v0.2.3 => github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b
+
+// https://github.com/mattn/go-sqlite3/pull/1338
+replace github.com/mattn/go-sqlite3 v1.14.28 => github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8

+ 2 - 2
go.sum

@@ -31,6 +31,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b h1:Fjm4GuJ+TGMgqfGHN42IQArJb77CfD/mAwLbDUoJe6g=
 github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b/go.mod h1:91K7jfEsgJSyfSrX+gmrRfZMtntx6JsHolWubGXDopg=
+github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8 h1:oNVrBJGXkD334ToEmxJz8G6LhzD1/sMA4twMHsMLzQo=
+github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/calmh/incontainer v1.0.0 h1:g2cTUtZuFGmMGX8GoykPkN1Judj2uw8/3/aEtq4Z/rg=
 github.com/calmh/incontainer v1.0.0/go.mod h1:eOhqnw15c9X+4RNBe0W3HlUZFfX16O0EDsCOInTndHY=
 github.com/calmh/xdr v1.2.0 h1:GaGSNH4ZDw9kNdYqle6+RcAENiaQ8/611Ok+jQbBEeU=
@@ -164,8 +166,6 @@ github.com/maruel/panicparse/v2 v2.5.0/go.mod h1:DA2fDiBk63bKfBf4CVZP9gb4fuvzdPb
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU=
-github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ=
 github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=

+ 69 - 0
internal/db/sqlite/db_stats.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+type DatabaseStatistics struct {
+	Name     string               `json:"name"`
+	FolderID string               `json:"folderID,omitempty"`
+	Tables   []TableStatistics    `json:"tables"`
+	Total    TableStatistics      `json:"total"`
+	Children []DatabaseStatistics `json:"children,omitempty"`
+}
+
+type TableStatistics struct {
+	Name   string `json:"name,omitempty"`
+	Size   int64  `json:"size"`
+	Unused int64  `json:"unused"`
+}
+
+func (s *DB) Statistics() (*DatabaseStatistics, error) {
+	ts, total, err := s.tableStats()
+	if err != nil {
+		return nil, wrap(err)
+	}
+	ds := DatabaseStatistics{
+		Name:   s.baseName,
+		Tables: ts,
+		Total:  total,
+	}
+
+	err = s.forEachFolder(func(fdb *folderDB) error {
+		tables, total, err := fdb.tableStats()
+		if err != nil {
+			return wrap(err)
+		}
+		ds.Children = append(ds.Children, DatabaseStatistics{
+			Name:     fdb.baseName,
+			FolderID: fdb.folderID,
+			Tables:   tables,
+			Total:    total,
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, wrap(err)
+	}
+
+	return &ds, nil
+}
+
+func (s *baseDB) tableStats() ([]TableStatistics, TableStatistics, error) {
+	var stats []TableStatistics
+	if err := s.stmt(`
+		SELECT name, pgsize AS size, unused FROM dbstat
+		WHERE aggregate=true
+		ORDER BY name
+	`).Select(&stats); err != nil {
+		return nil, TableStatistics{}, wrap(err)
+	}
+	var total TableStatistics
+	for _, s := range stats {
+		total.Size += s.Size
+		total.Unused += s.Unused
+	}
+	return stats, total, nil
+}

+ 1 - 0
lib/build/build.go

@@ -48,6 +48,7 @@ var (
 	}
 	replaceTags = map[string]string{
 		"sqlite_omit_load_extension": "",
+		"sqlite_dbstat":              "",
 		"osusergo":                   "",
 		"netgo":                      "",
 	}