cmd/stcrashreceiver: Ignore patterns, improve metrics

cmd/stcrashreceiver/main.go

@@ -21,11 +21,14 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strings"
 
 	"github.com/alecthomas/kong"
 	raven "github.com/getsentry/raven-go"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/ur"
 )
@@ -33,13 +36,15 @@ import (
 const maxRequestSize = 1 << 20 // 1 MiB
 
 type cli struct {
-	Dir           string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
-	DSN           string `help:"Sentry DSN" env:"SENTRY_DSN"`
-	Listen        string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
-	MaxDiskFiles  int    `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
-	MaxDiskSizeMB int64  `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
-	SentryQueue   int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
-	DiskQueue     int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
+	Dir            string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
+	DSN            string `help:"Sentry DSN" env:"SENTRY_DSN"`
+	Listen         string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
+	MaxDiskFiles   int    `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
+	MaxDiskSizeMB  int64  `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
+	SentryQueue    int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
+	DiskQueue      int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
+	MetricsListen  string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
+	IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
 }
 
 func main() {
@@ -62,19 +67,38 @@ func main() {
 	}
 	go ss.Serve(context.Background())
 
+	var ip *ignorePatterns
+	if params.IngorePatterns != "" {
+		var err error
+		ip, err = loadIgnorePatterns(params.IngorePatterns)
+		if err != nil {
+			log.Fatalf("Failed to load ignore patterns: %v", err)
+		}
+	}
+
 	cr := &crashReceiver{
 		store:  ds,
 		sentry: ss,
+		ignore: ip,
 	}
 
 	mux.Handle("/", cr)
 	mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
 		w.Write([]byte("OK"))
 	})
-	mux.Handle("/metrics", promhttp.Handler())
+
+	if params.MetricsListen != "" {
+		mmux := http.NewServeMux()
+		mmux.Handle("/metrics", promhttp.Handler())
+		go func() {
+			if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
+				log.Fatalln("HTTP serve metrics:", err)
+			}
+		}()
+	}
 
 	if params.DSN != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip))
 	}
 
 	log.SetOutput(os.Stdout)
@@ -83,7 +107,7 @@ func main() {
 	}
 }
 
-func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
+func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
 		result := "failure"
 		defer func() {
@@ -98,6 +122,11 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
 			return
 		}
 
+		if ignore.match(bs) {
+			result = "ignored"
+			return
+		}
+
 		var reports []ur.FailureReport
 		err = json.Unmarshal(bs, &reports)
 		if err != nil {
@@ -110,7 +139,7 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
 			return
 		}
 
-		version, err := parseVersion(reports[0].Version)
+		version, err := build.ParseVersion(reports[0].Version)
 		if err != nil {
 			http.Error(w, err.Error(), 400)
 			return
@@ -158,3 +187,42 @@ func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string,
 	}
 	return reportServer + path, nil
 }
+
+type ignorePatterns struct {
+	patterns []*regexp.Regexp
+}
+
+func loadIgnorePatterns(path string) (*ignorePatterns, error) {
+	bs, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var patterns []*regexp.Regexp
+	for _, line := range strings.Split(string(bs), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		re, err := regexp.Compile(line)
+		if err != nil {
+			return nil, err
+		}
+		patterns = append(patterns, re)
+	}
+
+	log.Printf("Loaded %d ignore patterns", len(patterns))
+	return &ignorePatterns{patterns: patterns}, nil
+}
+
+func (i *ignorePatterns) match(report []byte) bool {
+	if i == nil {
+		return false
+	}
+	for _, re := range i.patterns {
+		if re.Match(report) {
+			return true
+		}
+	}
+	return false
+}

cmd/stcrashreceiver/sentry.go

@@ -18,6 +18,7 @@ import (
 
 	raven "github.com/getsentry/raven-go"
 	"github.com/maruel/panicparse/v2/stack"
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 const reportServer = "https://crash.syncthing.net/report/"
@@ -105,7 +106,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 		return nil, errors.New("no first line")
 	}
 
-	version, err := parseVersion(string(parts[0]))
+	version, err := build.ParseVersion(string(parts[0]))
 	if err != nil {
 		return nil, err
 	}
@@ -143,12 +144,12 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 	}
 
 	// Lock the source code loader to the version we are processing here.
-	if version.commit != "" {
+	if version.Commit != "" {
 		// We have a commit hash, so we know exactly which source to use
-		loader.LockWithVersion(version.commit)
-	} else if strings.HasPrefix(version.tag, "v") {
+		loader.LockWithVersion(version.Commit)
+	} else if strings.HasPrefix(version.Tag, "v") {
 		// Lets hope the tag is close enough
-		loader.LockWithVersion(version.tag)
+		loader.LockWithVersion(version.Tag)
 	} else {
 		// Last resort
 		loader.LockWithVersion("main")
@@ -215,106 +216,26 @@ func crashReportFingerprint(message string) []string {
 	return []string{"{{ default }}", message}
 }
 
-// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC [foo, bar]
-// or, somewhere along the way the "+" in the version tag disappeared:
-// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) [email protected] 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
-var (
-	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
-	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
-	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
-)
-
-type version struct {
-	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
-	tag      string   // "v1.1.4-rc.1"
-	commit   string   // "6aaae618", blank when absent
-	codename string   // "Erbium Earthworm"
-	runtime  string   // "go1.12.5"
-	goos     string   // "darwin"
-	goarch   string   // "amd64"
-	builder  string   // "[email protected]"
-	extra    []string // "foo", "bar"
-}
-
-func (v version) environment() string {
-	if v.commit != "" {
-		return "Development"
-	}
-	if strings.Contains(v.tag, "-rc.") {
-		return "Candidate"
-	}
-	if strings.Contains(v.tag, "-") {
-		return "Beta"
-	}
-	return "Stable"
-}
-
-func parseVersion(line string) (version, error) {
-	m := longVersionRE.FindStringSubmatch(line)
-	if len(m) == 0 {
-		return version{}, errors.New("unintelligeble version string")
-	}
-
-	v := version{
-		version:  m[1],
-		codename: m[2],
-		runtime:  m[3],
-		goos:     m[4],
-		goarch:   m[5],
-		builder:  m[6],
-	}
-
-	// Split the version tag into tag and commit. This is old style
-	// v1.2.3-something.4+11-g12345678 or newer with just dots
-	// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
-	parts := []string{v.version}
-	if strings.Contains(v.version, "+") {
-		parts = strings.Split(v.version, "+")
-	} else {
-		idxs := gitExtraRE.FindStringIndex(v.version)
-		if len(idxs) > 0 {
-			parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
-		}
-	}
-	v.tag = parts[0]
-	if len(parts) > 1 {
-		fields := gitExtraSepRE.Split(parts[1], -1)
-		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
-			v.commit = fields[1][1:]
-		}
-	}
-
-	if len(m) >= 8 && m[7] != "" {
-		tags := strings.Split(m[7], ",")
-		for i := range tags {
-			tags[i] = strings.TrimSpace(tags[i])
-		}
-		v.extra = tags
-	}
-
-	return v, nil
-}
-
-func packet(version version, reportType string) *raven.Packet {
+func packet(version build.VersionParts, reportType string) *raven.Packet {
 	pkt := &raven.Packet{
 		Platform:    "go",
-		Release:     version.tag,
-		Environment: version.environment(),
+		Release:     version.Tag,
+		Environment: version.Environment(),
 		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.version},
-			raven.Tag{Key: "tag", Value: version.tag},
-			raven.Tag{Key: "codename", Value: version.codename},
-			raven.Tag{Key: "runtime", Value: version.runtime},
-			raven.Tag{Key: "goos", Value: version.goos},
-			raven.Tag{Key: "goarch", Value: version.goarch},
-			raven.Tag{Key: "builder", Value: version.builder},
+			raven.Tag{Key: "version", Value: version.Version},
+			raven.Tag{Key: "tag", Value: version.Tag},
+			raven.Tag{Key: "codename", Value: version.Codename},
+			raven.Tag{Key: "runtime", Value: version.Runtime},
+			raven.Tag{Key: "goos", Value: version.GOOS},
+			raven.Tag{Key: "goarch", Value: version.GOARCH},
+			raven.Tag{Key: "builder", Value: version.Builder},
 			raven.Tag{Key: "report_type", Value: reportType},
 		},
 	}
-	if version.commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	if version.Commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit})
 	}
-	for _, tag := range version.extra {
+	for _, tag := range version.Extra {
 		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
 	}
 	return pkt

cmd/stcrashreceiver/sentry_test.go

@@ -12,66 +12,6 @@ import (
 	"testing"
 )
 
-func TestParseVersion(t *testing.T) {
-	cases := []struct {
-		longVersion string
-		parsed      version
-	}{
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "[email protected]",
-			},
-		},
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC [foo, bar]`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "[email protected]",
-				extra:    []string{"foo", "bar"},
-			},
-		},
-		{
-			longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) [email protected] 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
-			parsed: version{
-				version:  "v1.23.7-dev.26.gdf7b56ae-stversionextra",
-				tag:      "v1.23.7-dev",
-				commit:   "df7b56ae",
-				codename: "Fermium Flea",
-				runtime:  "go1.20.5",
-				goos:     "darwin",
-				goarch:   "arm64",
-				builder:  "[email protected]",
-				extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
-			},
-		},
-	}
-
-	for _, tc := range cases {
-		v, err := parseVersion(tc.longVersion)
-		if err != nil {
-			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
-			continue
-		}
-		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
-			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
-		}
-	}
-}
-
 func TestParseReport(t *testing.T) {
 	bs, err := os.ReadFile("_testdata/panic.log")
 	if err != nil {

cmd/stcrashreceiver/stcrashreceiver.go

@@ -12,11 +12,16 @@ import (
 	"net/http"
 	"path"
 	"strings"
+	"sync"
 )
 
 type crashReceiver struct {
 	store  *diskStore
 	sentry *sentryService
+	ignore *ignorePatterns
+
+	ignoredMut sync.RWMutex
+	ignored    map[string]struct{}
 }
 
 func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -64,6 +69,12 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
 func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
+	r.ignoredMut.RLock()
+	_, ignored := r.ignored[reportID]
+	r.ignoredMut.RUnlock()
+	if ignored {
+		return // found
+	}
 	if !r.store.Exists(reportID) {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
@@ -76,6 +87,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		metricCrashReportsTotal.WithLabelValues(result).Inc()
 	}()
 
+	r.ignoredMut.RLock()
+	_, ignored := r.ignored[reportID]
+	r.ignoredMut.RUnlock()
+	if ignored {
+		result = "ignored_cached"
+		io.Copy(io.Discard, req.Body)
+		return // found
+	}
+
 	// Read at most maxRequestSize of report data.
 	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
@@ -86,6 +106,17 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		return
 	}
 
+	if r.ignore.match(bs) {
+		r.ignoredMut.Lock()
+		if r.ignored == nil {
+			r.ignored = make(map[string]struct{})
+		}
+		r.ignored[reportID] = struct{}{}
+		r.ignoredMut.Unlock()
+		result = "ignored"
+		return
+	}
+
 	result = "success"
 
 	// Store the report

lib/build/parse.go

@@ -0,0 +1,93 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package build
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+)
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC [foo, bar]
+// or, somewhere along the way the "+" in the version tag disappeared:
+// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) [email protected] 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
+var (
+	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
+	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
+)
+
+type VersionParts struct {
+	Version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	Tag      string   // "v1.1.4-rc.1"
+	Commit   string   // "6aaae618", blank when absent
+	Codename string   // "Erbium Earthworm"
+	Runtime  string   // "go1.12.5"
+	GOOS     string   // "darwin"
+	GOARCH   string   // "amd64"
+	Builder  string   // "[email protected]"
+	Extra    []string // "foo", "bar"
+}
+
+func (v VersionParts) Environment() string {
+	if v.Commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.Tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.Tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func ParseVersion(line string) (VersionParts, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return VersionParts{}, errors.New("unintelligeble version string")
+	}
+
+	v := VersionParts{
+		Version:  m[1],
+		Codename: m[2],
+		Runtime:  m[3],
+		GOOS:     m[4],
+		GOARCH:   m[5],
+		Builder:  m[6],
+	}
+
+	// Split the version tag into tag and commit. This is old style
+	// v1.2.3-something.4+11-g12345678 or newer with just dots
+	// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
+	parts := []string{v.Version}
+	if strings.Contains(v.Version, "+") {
+		parts = strings.Split(v.Version, "+")
+	} else {
+		idxs := gitExtraRE.FindStringIndex(v.Version)
+		if len(idxs) > 0 {
+			parts = []string{v.Version[:idxs[0]], v.Version[idxs[0]+1:]}
+		}
+	}
+	v.Tag = parts[0]
+	if len(parts) > 1 {
+		fields := gitExtraSepRE.Split(parts[1], -1)
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.Commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.Extra = tags
+	}
+
+	return v, nil
+}

lib/build/parse_test.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package build
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      VersionParts
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC`,
+			parsed: VersionParts{
+				Version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				Tag:      "v1.1.4-rc.1",
+				Commit:   "6aaae618",
+				Codename: "Erbium Earthworm",
+				Runtime:  "go1.12.5",
+				GOOS:     "darwin",
+				GOARCH:   "amd64",
+				Builder:  "[email protected]",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) [email protected] 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: VersionParts{
+				Version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				Tag:      "v1.1.4-rc.1",
+				Commit:   "6aaae618",
+				Codename: "Erbium Earthworm",
+				Runtime:  "go1.12.5",
+				GOOS:     "darwin",
+				GOARCH:   "amd64",
+				Builder:  "[email protected]",
+				Extra:    []string{"foo", "bar"},
+			},
+		},
+		{
+			longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) [email protected] 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
+			parsed: VersionParts{
+				Version:  "v1.23.7-dev.26.gdf7b56ae-stversionextra",
+				Tag:      "v1.23.7-dev",
+				Commit:   "df7b56ae",
+				Codename: "Fermium Flea",
+				Runtime:  "go1.20.5",
+				GOOS:     "darwin",
+				GOARCH:   "arm64",
+				Builder:  "[email protected]",
+				Extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := ParseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}