Run benchmarks when running tests

Godeps/Godeps.json

@@ -11,7 +11,7 @@
 		},
 		{
 			"ImportPath": "github.com/calmh/logger",
-			"Rev": "4d4e2801954c5581e4c2a80a3d3beb3b3645fd04"
+			"Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2"
 		},
 		{
 			"ImportPath": "github.com/calmh/luhn",

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -6,6 +6,7 @@ package logger
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -37,6 +38,13 @@ type Logger struct {
 var DefaultLogger = New()
 
 func New() *Logger {
+	if os.Getenv("LOGGER_DISCARD") != "" {
+		// Hack to completely disable logging, for example when running benchmarks.
+		return &Logger{
+			logger: log.New(ioutil.Discard, "", 0),
+		}
+	}
+
 	return &Logger{
 		logger: log.New(os.Stdout, "", log.Ltime),
 	}

benchfilter.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+// Neatly format benchmarking output which otherwise looks like crap.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"text/tabwriter"
+)
+
+var (
+	benchRe   = regexp.MustCompile(`^Bench`)
+	spacesRe  = regexp.MustCompile(`\s+`)
+	numbersRe = regexp.MustCompile(`\b[\d\.]+\b`)
+)
+
+func main() {
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0)
+	br := bufio.NewScanner(os.Stdin)
+	n := 0
+
+	for br.Scan() {
+		line := br.Bytes()
+
+		if benchRe.Match(line) {
+			n++
+			line = spacesRe.ReplaceAllLiteral(line, []byte("\t"))
+			line = numbersRe.ReplaceAllFunc(line, func(n []byte) []byte {
+				return []byte(fmt.Sprintf("%12s", n))
+			})
+			tw.Write(line)
+			tw.Write([]byte("\n"))
+		} else if n > 0 && bytes.HasPrefix(line, []byte("ok")) {
+			n = 0
+			tw.Flush()
+			fmt.Printf("%s\n\n", line)
+		}
+	}
+	tw.Flush()
+}

build.go

@@ -110,6 +110,9 @@ func main() {
 		case "test":
 			test("./...")
 
+		case "bench":
+			bench("./...")
+
 		case "assets":
 			assets()
 
@@ -182,6 +185,11 @@ func test(pkg string) {
 	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
 }
 
+func bench(pkg string) {
+	setBuildEnv()
+	runPrint("go", "test", "-run", "NONE", "-bench", ".", pkg)
+}
+
 func install(pkg string, tags []string) {
 	os.Setenv("GOBIN", "./bin")
 	args := []string{"install", "-v", "-ldflags", ldflags()}

build.sh

@@ -18,7 +18,9 @@ case "${1:-default}" in
 		ulimit -d 512000 &>/dev/null || true
 		ulimit -m 512000 &>/dev/null || true
 
-		go run build.go "$1"
+		go run build.go test
+		echo
+		LOGGER_DISCARD=1 go run build.go bench | go run benchfilter.go
 		;;
 
 	tar)

internal/model/model_test.go

@@ -176,74 +176,58 @@ func genFiles(n int) []protocol.FileInfo {
 	return files
 }
 
-func BenchmarkIndex10000(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genFiles(10000)
+func BenchmarkIndex_10000(b *testing.B) {
+	benchmarkIndex(b, 10000)
+}
 
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files, 0, nil)
-	}
+func BenchmarkIndex_100(b *testing.B) {
+	benchmarkIndex(b, 100)
 }
 
-func BenchmarkIndex00100(b *testing.B) {
+func benchmarkIndex(b *testing.B, nfiles int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genFiles(100)
+	m.StartFolderRO("default")
+
+	files := genFiles(nfiles)
+	m.Index(device1, "default", files, 0, nil)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		m.Index(device1, "default", files, 0, nil)
 	}
+	b.ReportAllocs()
 }
 
-func BenchmarkIndexUpdate10000f10000(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genFiles(10000)
-	m.Index(device1, "default", files, 0, nil)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", files, 0, nil)
-	}
+func BenchmarkIndexUpdate_10000_10000(b *testing.B) {
+	benchmarkIndexUpdate(b, 10000, 10000)
 }
 
-func BenchmarkIndexUpdate10000f00100(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genFiles(10000)
-	m.Index(device1, "default", files, 0, nil)
+func BenchmarkIndexUpdate_10000_100(b *testing.B) {
+	benchmarkIndexUpdate(b, 10000, 100)
+}
 
-	ufiles := genFiles(100)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles, 0, nil)
-	}
+func BenchmarkIndexUpdate_10000_1(b *testing.B) {
+	benchmarkIndexUpdate(b, 10000, 1)
 }
 
-func BenchmarkIndexUpdate10000f00001(b *testing.B) {
+func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genFiles(10000)
+	m.StartFolderRO("default")
+
+	files := genFiles(nfiles)
+	ufiles := genFiles(nufiles)
+
 	m.Index(device1, "default", files, 0, nil)
 
-	ufiles := genFiles(1)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		m.IndexUpdate(device1, "default", ufiles, 0, nil)
 	}
+	b.ReportAllocs()
 }
 
 type FakeConnection struct {
@@ -1166,56 +1150,23 @@ func genDeepFiles(n, d int) []protocol.FileInfo {
 }
 
 func BenchmarkTree_10000_50(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genDeepFiles(10000, 50)
-
-	m.Index(device1, "default", files, 0, nil)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.GlobalDirectoryTree("default", "", -1, false)
-	}
+	benchmarkTree(b, 10000, 50)
 }
 
-func BenchmarkTree_10000_10(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genDeepFiles(10000, 10)
-
-	m.Index(device1, "default", files, 0, nil)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.GlobalDirectoryTree("default", "", -1, false)
-	}
+func BenchmarkTree_100_50(b *testing.B) {
+	benchmarkTree(b, 100, 50)
 }
 
-func BenchmarkTree_00100_50(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
-	m.AddFolder(defaultFolderConfig)
-	m.ScanFolder("default")
-	files := genDeepFiles(100, 50)
-
-	m.Index(device1, "default", files, 0, nil)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.GlobalDirectoryTree("default", "", -1, false)
-	}
+func BenchmarkTree_100_10(b *testing.B) {
+	benchmarkTree(b, 100, 10)
 }
 
-func BenchmarkTree_00100_10(b *testing.B) {
+func benchmarkTree(b *testing.B, n1, n2 int) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
 	m.AddFolder(defaultFolderConfig)
 	m.ScanFolder("default")
-	files := genDeepFiles(100, 10)
+	files := genDeepFiles(n1, n2)
 
 	m.Index(device1, "default", files, 0, nil)
 
@@ -1223,4 +1174,5 @@ func BenchmarkTree_00100_10(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		m.GlobalDirectoryTree("default", "", -1, false)
 	}
+	b.ReportAllocs()
 }