feat: add debug commands for folder counts and files (#10206)

This adds two debugging commands that print information directly from
the database; one for folder counts, and one for file metadata for files
matching a pattern in a folder. E.g.,

```
% syncthing debug database-counts p3jms-73gps
DEVICE   TYPE       FLAGS    DELETED  COUNT  SIZE
-local-  FILE       -------  ---      0      0
-local-  FILE       --G----  ---      2473   70094796496
-local-  DIRECTORY  -------  ---      0      0
-local-  DIRECTORY  --G----  ---      19     2432
PSEUDOP  FILE       -------  ---      2473   70094796496
PSEUDOP  FILE       -nG----  ---      0      0
PSEUDOP  DIRECTORY  -------  ---      19     2432
PSEUDOP  DIRECTORY  -nG----  ---      0      0
```

```
% syncthing debug database-file p3jms-73gps 20240929-DSCF1387
DSCF1387
DEVICE   TYPE  NAME                          SEQUENCE  DELETED  MODIFIED                        SIZE      FLAGS    VERSION             BLOCKLIST
-local-  FILE  Austin/20240929-DSCF1387.raf  1204      ---      2024-09-29T01:10:54Z            48911888  --G----  HX2ELNU:1744213700  fsQdMvUL
PSEUDOP  FILE  Austin/20240929-DSCF1387.raf  22279     ---      2024-09-29T01:10:54Z            48911888  -------  HX2ELNU:1744213700  fsQdMvUL
-local-  FILE  Austin/20240929-DSCF1387.xmp  1196      ---      2024-10-16T08:08:35.137501751Z  5579      --G----  HX2ELNU:1744213700  xDGMnepi
PSEUDOP  FILE  Austin/20240929-DSCF1387.xmp  19910     ---      2024-10-16T08:08:35.137501751Z  5579      -------  HX2ELNU:1744213700  xDGMnepi
```

The local flag bits get a string representation for the bitmask,

```
	FlagLocalUnsupported:   "u",
	FlagLocalIgnored:       "i",
	FlagLocalMustRescan:    "r",
	FlagLocalReceiveOnly:   "e",
	FlagLocalGlobal:        "G",
	FlagLocalNeeded:        "n",
	FlagLocalRemoteInvalid: "v",
```

.golangci.yml

@@ -27,6 +27,7 @@ linters:
     - musttag
     - nestif
     - nlreturn
+    - noinlineerr
     - nonamedreturns
     - paralleltest
     - prealloc

cmd/syncthing/main.go

@@ -920,8 +920,10 @@ func (browserCmd) Run() error {
 }
 
 type debugCmd struct {
-	ResetDatabase      resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
-	DatabaseStatistics databaseStatsCmd `cmd:"" help:"Display database size statistics"`
+	ResetDatabase      resetDatabaseCmd  `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
+	DatabaseStatistics databaseStatsCmd  `cmd:"" help:"Display database size statistics"`
+	DatabaseCounts     databaseCountsCmd `cmd:"" help:"Display database folder counts"`
+	DatabaseFile       databaseFileCmd   `cmd:"" help:"Display database file metadata"`
 }
 
 type resetDatabaseCmd struct{}
@@ -956,6 +958,33 @@ func (c databaseStatsCmd) Run() error {
 	return tw.Flush()
 }
 
+type databaseCountsCmd struct {
+	Folder string `arg:"" required:""`
+}
+
+func (c databaseCountsCmd) Run() error {
+	db, err := sqlite.Open(locations.Get(locations.Database))
+	if err != nil {
+		return err
+	}
+
+	return db.DebugCounts(os.Stdout, c.Folder)
+}
+
+type databaseFileCmd struct {
+	Folder string `arg:"" required:""`
+	File   string `arg:"" required:""`
+}
+
+func (c databaseFileCmd) Run() error {
+	db, err := sqlite.Open(locations.Get(locations.Database))
+	if err != nil {
+		return err
+	}
+
+	return db.DebugFilePattern(os.Stdout, c.Folder, c.File)
+}
+
 func (c databaseStatsCmd) printStat(w io.Writer, s *sqlite.DatabaseStatistics) {
 	for _, table := range s.Tables {
 		fmt.Fprintf(w, "%s\t%s\t%s\t%8d KiB\t%5.01f %%\n", s.Name, cmp.Or(s.FolderID, "-"), table.Name, table.Size/1024, float64(table.Size-table.Unused)*100/float64(table.Size))

internal/db/sqlite/db_folderdb.go

@@ -10,6 +10,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"io"
 	"iter"
 	"path/filepath"
 	"strings"
@@ -376,6 +377,22 @@ func (s *DB) DropDevice(device protocol.DeviceID) error {
 	})
 }
 
+func (s *DB) DebugCounts(out io.Writer, folder string) error {
+	fdb, err := s.getFolderDB(folder, false)
+	if err != nil {
+		return err
+	}
+	return fdb.DebugCounts(out)
+}
+
+func (s *DB) DebugFilePattern(out io.Writer, folder, name string) error {
+	fdb, err := s.getFolderDB(folder, false)
+	if err != nil {
+		return err
+	}
+	return fdb.DebugFilePattern(out, name)
+}
+
 // forEachFolder runs the function for each currently open folderDB,
 // returning the first error that was encountered.
 func (s *DB) forEachFolder(fn func(fdb *folderDB) error) error {

internal/db/sqlite/folderdb_counts.go

@@ -16,7 +16,7 @@ type countsRow struct {
 	Count      int
 	Size       int64
 	Deleted    bool
-	LocalFlags int64 `db:"local_flags"`
+	LocalFlags protocol.FlagLocal `db:"local_flags"`
 }
 
 func (s *folderDB) CountLocal(device protocol.DeviceID) (db.Counts, error) {

internal/db/sqlite/folderdb_local.go

@@ -8,9 +8,14 @@ package sqlite
 
 import (
 	"database/sql"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
 	"iter"
+	"strings"
+	"text/tabwriter"
+	"time"
 
 	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/internal/itererr"
@@ -126,3 +131,82 @@ func (s *folderDB) ListDevicesForFolder() ([]protocol.DeviceID, error) {
 	}
 	return devs, nil
 }
+
+func (s *folderDB) DebugCounts(out io.Writer) error {
+	type deviceCountsRow struct {
+		countsRow
+
+		DeviceID string
+	}
+
+	delMap := map[bool]string{
+		true:  "del",
+		false: "---",
+	}
+
+	var res []deviceCountsRow
+	if err := s.stmt(`
+		SELECT d.device_id as deviceid, s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
+		INNER JOIN devices d ON d.idx = s.device_idx
+	`).Select(&res); err != nil {
+		return wrap(err)
+	}
+
+	tw := tabwriter.NewWriter(out, 2, 2, 2, ' ', 0)
+	fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", "DEVICE", "TYPE", "FLAGS", "DELETED", "COUNT", "SIZE")
+	for _, row := range res {
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%d\n", shortDevice(row.DeviceID), shortType(row.Type), row.LocalFlags.HumanString(), delMap[row.Deleted], row.Count, row.Size)
+	}
+	return tw.Flush()
+}
+
+func (s *folderDB) DebugFilePattern(out io.Writer, name string) error {
+	type hashFileMetadata struct {
+		db.FileMetadata
+
+		Version       dbVector
+		BlocklistHash []byte
+		DeviceID      string
+	}
+	name = "%" + name + "%"
+	res := itererr.Zip(iterStructs[hashFileMetadata](s.stmt(`
+		SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.local_flags as localflags, f.version, f.blocklist_hash as blocklisthash, d.device_id as deviceid FROM files f
+		INNER JOIN devices d ON d.idx = f.device_idx
+		WHERE f.name LIKE ?
+		ORDER BY f.name, f.device_idx
+	`).Queryx(name)))
+
+	delMap := map[bool]string{
+		true:  "del",
+		false: "---",
+	}
+
+	tw := tabwriter.NewWriter(out, 2, 2, 2, ' ', 0)
+	fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", "DEVICE", "TYPE", "NAME", "SEQUENCE", "DELETED", "MODIFIED", "SIZE", "FLAGS", "VERSION", "BLOCKLIST")
+	for row, err := range res {
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%s\t%s\t%d\t%s\t%s\t%s\n", shortDevice(row.DeviceID), shortType(row.Type), row.Name, row.Sequence, delMap[row.Deleted], row.ModTime().UTC().Format(time.RFC3339Nano), row.Size, row.LocalFlags.HumanString(), row.Version.HumanString(), shortHash(row.BlocklistHash))
+	}
+	return tw.Flush()
+}
+
+func shortDevice(s string) string {
+	if dev, err := protocol.DeviceIDFromString(s); err == nil && dev == protocol.LocalDeviceID {
+		return "-local-"
+	}
+	short, _, _ := strings.Cut(s, "-")
+	return short
+}
+
+func shortType(t protocol.FileInfoType) string {
+	return strings.TrimPrefix(t.String(), "FILE_INFO_TYPE_")
+}
+
+func shortHash(bs []byte) string {
+	if len(bs) == 0 {
+		return "-nil-"
+	}
+	return base64.RawStdEncoding.EncodeToString(bs)[:8]
+}

lib/protocol/bep_fileinfo.go

@@ -11,6 +11,8 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
+	"slices"
+	"strings"
 	"time"
 
 	"github.com/syncthing/syncthing/internal/gen/bep"
@@ -40,10 +42,52 @@ const (
 	LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded | FlagLocalRemoteInvalid
 )
 
+// localFlagBitNames maps flag values to characters which can be used to
+// build a permission-like bit string for easier reading.
+var localFlagBitNames = map[FlagLocal]string{
+	FlagLocalUnsupported:   "u",
+	FlagLocalIgnored:       "i",
+	FlagLocalMustRescan:    "r",
+	FlagLocalReceiveOnly:   "e",
+	FlagLocalGlobal:        "G",
+	FlagLocalNeeded:        "n",
+	FlagLocalRemoteInvalid: "v",
+}
+
 func (f FlagLocal) IsInvalid() bool {
 	return f&LocalInvalidFlags != 0
 }
 
+// HumanString returns a permission-like string representation of the flag bits
+func (f FlagLocal) HumanString() string {
+	if f == 0 {
+		return strings.Repeat("-", len(localFlagBitNames))
+	}
+
+	bit := FlagLocal(1)
+	var res bytes.Buffer
+	var extra strings.Builder
+	for f != 0 {
+		if f&bit != 0 {
+			if name, ok := localFlagBitNames[bit]; ok {
+				res.WriteString(name)
+			} else {
+				fmt.Fprintf(&extra, "+0x%x", bit)
+			}
+		} else {
+			res.WriteString("-")
+		}
+		f &^= bit
+		bit <<= 1
+	}
+	if res.Len() < len(localFlagBitNames) {
+		res.WriteString(strings.Repeat("-", len(localFlagBitNames)-res.Len()))
+	}
+	base := res.Bytes()
+	slices.Reverse(base)
+	return string(base) + extra.String()
+}
+
 // BlockSizes is the list of valid block sizes, from min to max
 var BlockSizes []int
 

lib/protocol/vector.go

@@ -38,6 +38,17 @@ func (v *Vector) String() string {
 	return buf.String()
 }
 
+func (v *Vector) HumanString() string {
+	var buf strings.Builder
+	for i, c := range v.Counters {
+		if i > 0 {
+			buf.WriteRune(',')
+		}
+		fmt.Fprintf(&buf, "%s:%d", c.ID, c.Value)
+	}
+	return buf.String()
+}
+
 func (v *Vector) ToWire() *bep.Vector {
 	counters := make([]*bep.Counter, len(v.Counters))
 	for i, c := range v.Counters {