Clean up the scripts a bit (...)

- Move the Go files into script/ instead of random places
- Rewrite check-contrib.sh into check-authors.go and check-copyright.go
- Clean up build.sh a little bit

build.go

@@ -181,6 +181,8 @@ func setup() {
 	runPrint("go", "get", "-v", "golang.org/x/tools/cmd/vet")
 	runPrint("go", "get", "-v", "golang.org/x/net/html")
 	runPrint("go", "get", "-v", "github.com/tools/godep")
+	runPrint("go", "get", "-v", "github.com/axw/gocov/gocov")
+	runPrint("go", "get", "-v", "github.com/AlekSi/gocov-xml")
 }
 
 func test(pkg string) {
@@ -405,7 +407,7 @@ func setBuildEnv() {
 
 func assets() {
 	setBuildEnv()
-	runPipe("lib/auto/gui.files.go", "go", "run", "cmd/genassets/main.go", "gui")
+	runPipe("lib/auto/gui.files.go", "go", "run", "script/genassets.go", "gui")
 }
 
 func xdr() {
@@ -414,7 +416,7 @@ func xdr() {
 
 func translate() {
 	os.Chdir("gui/assets/lang")
-	runPipe("lang-en-new.json", "go", "run", "../../../cmd/translate/main.go", "lang-en.json", "../../")
+	runPipe("lang-en-new.json", "go", "run", "../../../script/translate.go", "lang-en.json", "../../")
 	os.Remove("lang-en.json")
 	err := os.Rename("lang-en-new.json", "lang-en.json")
 	if err != nil {
@@ -425,7 +427,7 @@ func translate() {
 
 func transifex() {
 	os.Chdir("gui/assets/lang")
-	runPrint("go", "run", "../../../cmd/transifexdl/main.go")
+	runPrint("go", "run", "../../../script/transifexdl.go")
 	os.Chdir("../../..")
 	assets()
 }

build.sh

@@ -4,90 +4,97 @@ IFS=$'\n\t'
 
 STTRACE=${STTRACE:-}
 
+script() {
+	name="$1"
+	shift
+	go run "script/$name.go" "$@"
+}
+
+build() {
+	go run build.go "$@"
+}
+
 case "${1:-default}" in
 	default)
-		go run build.go
+		build
 		;;
 
 	clean)
-		go run build.go "$1"
-		;;
-
-	test)
-		ulimit -t 60 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-		go run build.go test
-		;;
-
-	bench)
-		LOGGER_DISCARD=1 go run build.go bench | go run benchfilter.go
+		build "$@"
 		;;
 
 	tar)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	deps)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	assets)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	xdr)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	translate)
-		go run build.go "$1"
+		build "$@"
+		;;
+
+	deb)
+		build "$@"
+		;;
+
+	setup)
+		build "$@"
+		;;
+
+	test)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
+		LOGGER_DISCARD=1 build test
+		;;
+
+	bench)
+		LOGGER_DISCARD=1 build bench | script benchfilter
 		;;
 
 	prerelease)
-		go run build.go transifex
+		build transifex
 		git add -A gui/assets/ lib/auto/
 		pushd man ; ./refresh.sh ; popd
 		git add -A man
-		echo
-		echo Changelog:
-		go run changelog.go
-		;;
-
-	deb)
-		go run build.go "$1"
 		;;
 
 	noupgrade)
-		go run build.go -no-upgrade tar
+		build -no-upgrade tar
 		;;
 
 	all)
-		go run build.go -goos darwin -goarch amd64 tar
+		build -goos darwin -goarch amd64 tar
 
-		go run build.go -goos dragonfly -goarch amd64 tar
+		build -goos dragonfly -goarch amd64 tar
 
-		go run build.go -goos freebsd -goarch 386 tar
-		go run build.go -goos freebsd -goarch amd64 tar
+		build -goos freebsd -goarch 386 tar
+		build -goos freebsd -goarch amd64 tar
 
-		go run build.go -goos linux -goarch 386 tar
-		go run build.go -goos linux -goarch amd64 tar
-		go run build.go -goos linux -goarch arm tar
+		build -goos linux -goarch 386 tar
+		build -goos linux -goarch amd64 tar
+		build -goos linux -goarch arm tar
 
-		go run build.go -goos netbsd -goarch 386 tar
-		go run build.go -goos netbsd -goarch amd64 tar
+		build -goos netbsd -goarch 386 tar
+		build -goos netbsd -goarch amd64 tar
 
-		go run build.go -goos openbsd -goarch 386 tar
-		go run build.go -goos openbsd -goarch amd64 tar
+		build -goos openbsd -goarch 386 tar
+		build -goos openbsd -goarch amd64 tar
 
-		go run build.go -goos solaris -goarch amd64 tar
+		build -goos solaris -goarch amd64 tar
 
-		go run build.go -goos windows -goarch 386 zip
-		go run build.go -goos windows -goarch amd64 zip
-		;;
-
-	setup)
-		echo "Don't worry, just build."
+		build -goos windows -goarch 386 zip
+		build -goos windows -goarch amd64 zip
 		;;
 
 	test-cov)
@@ -95,9 +102,6 @@ case "${1:-default}" in
 		ulimit -d 512000 &>/dev/null || true
 		ulimit -m 512000 &>/dev/null || true
 
-		go get github.com/axw/gocov/gocov
-		go get github.com/AlekSi/gocov-xml
-
 		echo "mode: set" > coverage.out
 		fail=0
 

check-contrib.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-missing-authors() {
-	for email in $(git log --format=%ae HEAD | sort | uniq) ; do
-		grep -q "$email" AUTHORS || echo $email
-	done
-}
-
-no-docs-typos() {
-	# Commits that are known to not change code
-	grep -v 63bd0136fb40a91efaa279cb4b4159d82e8e6904 |\
-	grep -v 4e2feb6fbc791bb8a2daf0ab8efb10775d66343e |\
-	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
-	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
-	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\
-	grep -v f1120d7aa936c0658429edef0037792520b46334 |\
-	grep -v a9339d0627fff439879d157c75077f02c9fac61b |\
-	grep -v 254c63763a3ad42fd82259f1767db526cff94a14 |\
-	grep -v 4b76ec40c07078beaa2c5e250ed7d9bd6276a718 |\
-	grep -v ffc39dfbcb34eacc3ea12327a02b6e7741a2c207 |\
-	grep -v 32a76901a91ff0f663db6f0830e0aedec946e4d0 |\
-	grep -v af3288043a49bcc28f8ae3060852a09de552fe5f |\
-	grep -v 3626003f680bad3e63677982576d3a05421e88e9
-}
-
-print-missing-authors() {
-	for email in $(missing-authors) ; do
-		git log --author="$email" --format="%H %ae %s" | no-docs-typos
-	done
-}
-
-print-missing-copyright() {
-	find . -name \*.go | xargs egrep -L 'Copyright|automatically generated' | grep -v Godeps | grep -v lib/auto/
-}
-
-authors=$(print-missing-authors)
-if [[ ! -z $authors ]] ; then
-	echo '***'
-	echo Author emails not in AUTHORS:
-	echo $authors
-	echo '***'
-	exit 1
-fi
-
-copy=$(print-missing-copyright)
-if [[ ! -z $copy ]] ; then
-	echo ***
-	echo Files missing copyright notice:
-	echo $copy
-	echo ***
-	exit 1
-fi
-

benchfilter.go → script/benchfilter.go

@@ -11,17 +11,15 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"os"
 	"regexp"
+	"strings"
 	"text/tabwriter"
 )
 
 var (
-	benchRe   = regexp.MustCompile(`^Bench`)
-	spacesRe  = regexp.MustCompile(`\s+`)
-	numbersRe = regexp.MustCompile(`\b[\d\.]+\b`)
+	benchRe = regexp.MustCompile(`^(Bench[^\s]+)\s+(\d+)\s+(\d+ ns/op)\s*(\d+ B/op)?\s*(\d+ allocs/op)?`)
 )
 
 func main() {
@@ -30,17 +28,15 @@ func main() {
 	n := 0
 
 	for br.Scan() {
-		line := br.Bytes()
+		line := br.Text()
 
-		if benchRe.Match(line) {
+		if match := benchRe.FindStringSubmatch(line); match != nil {
 			n++
-			line = spacesRe.ReplaceAllLiteral(line, []byte("\t"))
-			line = numbersRe.ReplaceAllFunc(line, func(n []byte) []byte {
-				return []byte(fmt.Sprintf("%12s", n))
-			})
-			tw.Write(line)
-			tw.Write([]byte("\n"))
-		} else if n > 0 && bytes.HasPrefix(line, []byte("ok")) {
+			for i := range match[2:] {
+				match[2+i] = fmt.Sprintf("%16s", match[2+i])
+			}
+			tw.Write([]byte(strings.Join(match[1:], "\t") + "\n"))
+		} else if n > 0 && strings.HasPrefix(line, "ok") {
 			n = 0
 			tw.Flush()
 			fmt.Printf("%s\n\n", line)

script/check-authors.go

@@ -0,0 +1,132 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+// Checks for authors that are not mentioned in AUTHORS
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// list of commits that we don't include in our checks; because they are
+// legacy things that don't check code, are committed with incorrect address,
+// or for other reasons.
+var excludeCommits = stringSetFromStrings([]string{
+	"63bd0136fb40a91efaa279cb4b4159d82e8e6904",
+	"4e2feb6fbc791bb8a2daf0ab8efb10775d66343e",
+	"f2459ef3319b2f060dbcdacd0c35a1788a94b8bd",
+	"b61f418bf2d1f7d5a9d7088a20a2a448e5e66801",
+	"a9339d0627fff439879d157c75077f02c9fac61b",
+	"254c63763a3ad42fd82259f1767db526cff94a14",
+	"4b76ec40c07078beaa2c5e250ed7d9bd6276a718",
+	"32a76901a91ff0f663db6f0830e0aedec946e4d0",
+	"3626003f680bad3e63677982576d3a05421e88e9",
+})
+
+func init() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+}
+
+func main() {
+	actual := actualAuthorEmails()
+	listed := listedAuthorEmails()
+	missing := actual.except(listed)
+	if len(missing) > 0 {
+		log.Println("Missing authors:")
+		for author := range missing {
+			log.Println(" ", author)
+		}
+		os.Exit(1)
+	}
+}
+
+// actualAuthorEmails returns the set of author emails found in the actual git
+// commit log, except those in excluded commits.
+func actualAuthorEmails() stringSet {
+	cmd := exec.Command("git", "log", "--format=%H %ae")
+	bs, err := cmd.Output()
+	if err != nil {
+		log.Fatal("authorEmails:", err)
+	}
+
+	authors := newStringSet()
+	for _, line := range bytes.Split(bs, []byte{'\n'}) {
+		fields := strings.Fields(string(line))
+		if len(fields) != 2 {
+			continue
+		}
+
+		hash, author := fields[0], fields[1]
+		if excludeCommits.has(hash) {
+			continue
+		}
+
+		authors.add(author)
+	}
+
+	return authors
+}
+
+// listedAuthorEmails returns the set of author emails mentioned in AUTHORS
+func listedAuthorEmails() stringSet {
+	bs, err := ioutil.ReadFile("AUTHORS")
+	if err != nil {
+		log.Fatal("listedAuthorEmails:", err)
+	}
+
+	emailRe := regexp.MustCompile(`<([^>]+)>`)
+	matches := emailRe.FindAllStringSubmatch(string(bs), -1)
+
+	authors := newStringSet()
+	for _, match := range matches {
+		authors.add(match[1])
+	}
+	return authors
+}
+
+// A simple string set type
+
+type stringSet map[string]struct{}
+
+func newStringSet() stringSet {
+	return make(stringSet)
+}
+
+func stringSetFromStrings(ss []string) stringSet {
+	s := newStringSet()
+	for _, e := range ss {
+		s.add(e)
+	}
+	return s
+}
+
+func (s stringSet) add(e string) {
+	s[e] = struct{}{}
+}
+
+func (s stringSet) has(e string) bool {
+	_, ok := s[e]
+	return ok
+}
+
+func (s stringSet) except(other stringSet) stringSet {
+	diff := newStringSet()
+	for e := range s {
+		if !other.has(e) {
+			diff.add(e)
+		}
+	}
+	return diff
+}

script/check-copyright.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+// Checks for files missing copyright notice
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// File extensions to check
+var checkExts = map[string]bool{
+	".go": true,
+}
+
+// Valid copyright headers, searched for in the top five lines in each file.
+var copyrightRegexps = []string{
+	`Copyright`,
+	`package auto`,
+	`automatically generated by genxdr`,
+}
+
+var copyrightRe = regexp.MustCompile(strings.Join(copyrightRegexps, "|"))
+
+func main() {
+	flag.Parse()
+	for _, dir := range flag.Args() {
+		err := filepath.Walk(dir, checkCopyright)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+	}
+}
+
+func checkCopyright(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		return err
+	}
+	if !info.Mode().IsRegular() {
+		return nil
+	}
+	if !checkExts[filepath.Ext(path)] {
+		return nil
+	}
+
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	scanner := bufio.NewScanner(fd)
+	for i := 0; scanner.Scan() && i < 5; i++ {
+		if copyrightRe.MatchString(scanner.Text()) {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Missing copyright in %s?", path)
+}