syncthing/build.go

// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

// +build ignore

package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	goarch        string
	goos          string
	noupgrade     bool
	version       string
	goCmd         string
	race          bool
	debug         = os.Getenv("BUILDDEBUG") != ""
	extraTags     string
	installSuffix string
	pkgdir        string
	cc            string
	run           string
	benchRun      string
	debugBinary   bool
	coverage      bool
	timeout       = "120s"
	numVersions   = 5
)

type target struct {
	name              string
	debname           string
	debdeps           []string
	debpre            string
	debpost           string
	description       string
	buildPkgs         []string
	binaryName        string
	archiveFiles      []archiveFile
	systemdServices   []string
	installationFiles []archiveFile
	tags              []string
}

type archiveFile struct {
	src  string
	dst  string
	perm os.FileMode
}

var targets = map[string]target{
	"all": {
		// Only valid for the "build" and "install" commands as it lacks all
		// the archive creation stuff. buildPkgs gets filled out in init()
		tags: []string{"purego"},
	},
	"syncthing": {
		// The default target for "build", "install", "tar", "zip", "deb", etc.
		name:        "syncthing",
		debname:     "syncthing",
		debdeps:     []string{"libc6", "procps"},
		debpost:     "script/post-upgrade",
		description: "Open Source Continuous File Synchronization",
		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
		archiveFiles: []archiveFile{
			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
			{src: "README.md", dst: "README.txt", perm: 0644},
			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
			// All files from etc/ and extra/ added automatically in init().
		},
		installationFiles: []archiveFile{
			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
			{src: "etc/linux-systemd/system/[email protected]", dst: "deb/lib/systemd/system/[email protected]", perm: 0644},
			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
		},
	},
	"stdiscosrv": {
		name:        "stdiscosrv",
		debname:     "syncthing-discosrv",
		debdeps:     []string{"libc6"},
		debpre:      "cmd/stdiscosrv/scripts/preinst",
		description: "Syncthing Discovery Server",
		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
		archiveFiles: []archiveFile{
			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
		},
		systemdServices: []string{
			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
		},
		installationFiles: []archiveFile{
			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
		},
		tags: []string{"purego"},
	},
	"strelaysrv": {
		name:        "strelaysrv",
		debname:     "syncthing-relaysrv",
		debdeps:     []string{"libc6"},
		debpre:      "cmd/strelaysrv/scripts/preinst",
		description: "Syncthing Relay Server",
		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
		archiveFiles: []archiveFile{
			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
		},
		systemdServices: []string{
			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
		},
		installationFiles: []archiveFile{
			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
		},
	},
	"strelaypoolsrv": {
		name:        "strelaypoolsrv",
		debname:     "syncthing-relaypoolsrv",
		debdeps:     []string{"libc6"},
		description: "Syncthing Relay Pool Server",
		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
		archiveFiles: []archiveFile{
			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
			{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
		},
		installationFiles: []archiveFile{
			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
		},
	},
}

// These are repos we need to clone to run "go generate"

type dependencyRepo struct {
	path   string
	repo   string
	commit string
}

var dependencyRepos = []dependencyRepo{
	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
}

func init() {
	all := targets["all"]
	pkgs, _ := filepath.Glob("cmd/*")
	for _, pkg := range pkgs {
		pkg = filepath.Base(pkg)
		if strings.HasPrefix(pkg, ".") {
			// ignore dotfiles
			continue
		}
		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
	}
	targets["all"] = all

	// The "syncthing" target includes a few more files found in the "etc"
	// and "extra" dirs.
	syncthingPkg := targets["syncthing"]
	for _, file := range listFiles("etc") {
		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
	}
	for _, file := range listFiles("extra") {
		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
	}
	for _, file := range listFiles("extra") {
		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
	}
	targets["syncthing"] = syncthingPkg
}

func main() {
	log.SetFlags(0)

	parseFlags()

	if debug {
		t0 := time.Now()
		defer func() {
			log.Println("... build completed in", time.Since(t0))
		}()
	}

	// Invoking build.go with no parameters at all builds everything (incrementally),
	// which is what you want for maximum error checking during development.
	if flag.NArg() == 0 {
		runCommand("install", targets["all"])
	} else {
		// with any command given but not a target, the target is
		// "syncthing". So "go run build.go install" is "go run build.go install
		// syncthing" etc.
		targetName := "syncthing"
		if flag.NArg() > 1 {
			targetName = flag.Arg(1)
		}
		target, ok := targets[targetName]
		if !ok {
			log.Fatalln("Unknown target", target)
		}

		runCommand(flag.Arg(0), target)
	}
}

func runCommand(cmd string, target target) {
	switch cmd {
	case "install":
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		tags = append(tags, strings.Fields(extraTags)...)
		install(target, tags)
		metalintShort()

	case "build":
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		tags = append(tags, strings.Fields(extraTags)...)
		build(target, tags)

	case "test":
		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")

	case "bench":
		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")

	case "integration":
		integration(false)

	case "integrationbench":
		integration(true)

	case "assets":
		rebuildAssets()

	case "proto":
		proto()

	case "translate":
		translate()

	case "transifex":
		transifex()

	case "tar":
		buildTar(target)

	case "zip":
		buildZip(target)

	case "deb":
		buildDeb(target)

	case "vet":
		metalintShort()

	case "lint":
		metalintShort()

	case "metalint":
		metalint()

	case "version":
		fmt.Println(getVersion())

	case "changelog":
		vers, err := currentAndLatestVersions(numVersions)
		if err != nil {
			log.Fatal(err)
		}
		for _, ver := range vers {
			underline := strings.Repeat("=", len(ver))
			msg, err := tagMessage(ver)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Printf("%s\n%s\n\n%s\n\n", ver, underline, msg)
		}

	default:
		log.Fatalf("Unknown command %q", cmd)
	}
}

func parseFlags() {
	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.StringVar(&goCmd, "gocmd", "go", "Specify `go` command")
	flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
	flag.BoolVar(&race, "race", race, "Use race detector")
	flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
	flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
	flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
	flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
	flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
	flag.StringVar(&run, "run", "", "Specify which tests to run")
	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
	flag.Parse()
}

func test(tags []string, pkgs ...string) {
	lazyRebuildAssets()

	tags = append(tags, "purego")
	args := []string{"test", "-short", "-timeout", timeout, "-tags", strings.Join(tags, " ")}

	if runtime.GOARCH == "amd64" {
		switch runtime.GOOS {
		case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
			args = append(args, "-race")
		}
	}

	if coverage {
		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
	}

	args = append(args, runArgs()...)

	runPrint(goCmd, append(args, pkgs...)...)
}

func bench(tags []string, pkgs ...string) {
	lazyRebuildAssets()
	args := append([]string{"test", "-run", "NONE", "-tags", strings.Join(tags, " ")}, benchArgs()...)
	runPrint(goCmd, append(args, pkgs...)...)
}

func integration(bench bool) {
	lazyRebuildAssets()
	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
	tags := "purego,integration"
	if bench {
		tags += ",benchmark"
	}
	args = append(args, tags)
	args = append(args, runArgs()...)
	if bench {
		if run == "" {
			args = append(args, "-run", "Benchmark")
		}
		args = append(args, benchArgs()...)
	}
	args = append(args, "./test")
	fmt.Println(args)
	runPrint(goCmd, args...)
}

func runArgs() []string {
	if run == "" {
		return nil
	}
	return []string{"-run", run}
}

func benchArgs() []string {
	if benchRun == "" {
		return []string{"-bench", "."}
	}
	return []string{"-bench", benchRun}
}

func install(target target, tags []string) {
	lazyRebuildAssets()

	tags = append(target.tags, tags...)

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))

	setBuildEnvVars()

	// On Windows generate a special file which the Go compiler will
	// automatically use when generating Windows binaries to set things like
	// the file icon, version, etc.
	if goos == "windows" {
		sysoPath, err := shouldBuildSyso(cwd)
		if err != nil {
			log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
		}
		defer shouldCleanupSyso(sysoPath)
	}

	args := []string{"install", "-v", "-trimpath"}
	args = appendParameters(args, tags, target.buildPkgs...)
	runPrint(goCmd, args...)
}

func build(target target, tags []string) {
	lazyRebuildAssets()
	tags = append(target.tags, tags...)

	rmr(target.BinaryName())

	setBuildEnvVars()

	// On Windows generate a special file which the Go compiler will
	// automatically use when generating Windows binaries to set things like
	// the file icon, version, etc.
	if goos == "windows" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		sysoPath, err := shouldBuildSyso(cwd)
		if err != nil {
			log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
		}
		defer shouldCleanupSyso(sysoPath)
	}

	args := []string{"build", "-v", "-trimpath"}
	args = appendParameters(args, tags, target.buildPkgs...)
	runPrint(goCmd, args...)
}

func setBuildEnvVars() {
	os.Setenv("GOOS", goos)
	os.Setenv("GOARCH", goarch)
	os.Setenv("CC", cc)
	if os.Getenv("CGO_ENABLED") == "" {
		switch goos {
		case "darwin", "solaris":
		default:
			os.Setenv("CGO_ENABLED", "0")
		}
	}
}

func appendParameters(args []string, tags []string, pkgs ...string) []string {
	if pkgdir != "" {
		args = append(args, "-pkgdir", pkgdir)
	}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, " "))
	}
	if installSuffix != "" {
		args = append(args, "-installsuffix", installSuffix)
	}
	if race {
		args = append(args, "-race")
	}

	if !debugBinary {
		// Regular binaries get version tagged and skip some debug symbols
		args = append(args, "-ldflags", ldflags(tags))
	} else {
		// -gcflags to disable optimizations and inlining. Skip -ldflags
		// because `Could not launch program: decoding dwarf section info at
		// offset 0x0: too short` on 'dlv exec ...' see
		// https://github.com/derekparker/delve/issues/79
		args = append(args, "-gcflags", "-N -l")
	}

	return append(args, pkgs...)
}

func buildTar(target target) {
	name := archiveName(target)
	filename := name + ".tar.gz"

	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}

	build(target, tags)
	codesign(target)

	for i := range target.archiveFiles {
		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
		target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
	}

	tarGz(filename, target.archiveFiles)
	fmt.Println(filename)
}

func buildZip(target target) {
	name := archiveName(target)
	filename := name + ".zip"

	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}

	build(target, tags)
	codesign(target)

	for i := range target.archiveFiles {
		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
		target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
	}

	zipFile(filename, target.archiveFiles)
	fmt.Println(filename)
}

func buildDeb(target target) {
	os.RemoveAll("deb")

	// "goarch" here is set to whatever the Debian packages expect. We correct
	// it to what we actually know how to build and keep the Debian variant
	// name in "debarch".
	debarch := goarch
	switch goarch {
	case "i386":
		goarch = "386"
	case "armel", "armhf":
		goarch = "arm"
	}

	build(target, []string{"noupgrade"})

	for i := range target.installationFiles {
		target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.BinaryName(), 1)
		target.installationFiles[i].dst = strings.Replace(target.installationFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
	}

	for _, af := range target.installationFiles {
		if err := copyFile(af.src, af.dst, af.perm); err != nil {
			log.Fatal(err)
		}
	}

	maintainer := "Syncthing Release Management <[email protected]>"
	debver := version
	if strings.HasPrefix(debver, "v") {
		debver = debver[1:]
		// Debian interprets dashes as separator between main version and
		// Debian package version, and thus thinks 0.14.26-rc.1 is better
		// than just 0.14.26. This rectifies that.
		debver = strings.Replace(debver, "-", "~", -1)
	}
	args := []string{
		"-t", "deb",
		"-s", "dir",
		"-C", "deb",
		"-n", target.debname,
		"-v", debver,
		"-a", debarch,
		"-m", maintainer,
		"--vendor", maintainer,
		"--description", target.description,
		"--url", "https://syncthing.net/",
		"--license", "MPL-2",
	}
	for _, dep := range target.debdeps {
		args = append(args, "-d", dep)
	}
	for _, service := range target.systemdServices {
		args = append(args, "--deb-systemd", service)
	}
	if target.debpost != "" {
		args = append(args, "--after-upgrade", target.debpost)
	}
	if target.debpre != "" {
		args = append(args, "--before-install", target.debpre)
	}
	runPrint("fpm", args...)
}

func shouldBuildSyso(dir string) (string, error) {
	type M map[string]interface{}
	version := getVersion()
	version = strings.TrimPrefix(version, "v")
	major, minor, patch := semanticVersion()
	bs, err := json.Marshal(M{
		"FixedFileInfo": M{
			"FileVersion": M{
				"Major": major,
				"Minor": minor,
				"Patch": patch,
			},
			"ProductVersion": M{
				"Major": major,
				"Minor": minor,
				"Patch": patch,
			},
		},
		"StringFileInfo": M{
			"FileDescription": "Open Source Continuous File Synchronization",
			"LegalCopyright":  "The Syncthing Authors",
			"FileVersion":     version,
			"ProductVersion":  version,
			"ProductName":     "Syncthing",
		},
		"IconPath": "assets/logo.ico",
	})
	if err != nil {
		return "", err
	}

	jsonPath := filepath.Join(dir, "versioninfo.json")
	err = ioutil.WriteFile(jsonPath, bs, 0644)
	if err != nil {
		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
	}

	defer func() {
		if err := os.Remove(jsonPath); err != nil {
			log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
		}
	}()

	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")

	if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
	}

	return sysoPath, nil
}

func shouldCleanupSyso(sysoFilePath string) {
	if sysoFilePath == "" {
		return
	}
	if err := os.Remove(sysoFilePath); err != nil {
		log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
	}
}

// copyFile copies a file from src to dst, ensuring the containing directory
// exists. The permission bits are copied as well. If dst already exists and
// the contents are identical to src the modification time is not updated.
func copyFile(src, dst string, perm os.FileMode) error {
	in, err := ioutil.ReadFile(src)
	if err != nil {
		return err
	}

	out, err := ioutil.ReadFile(dst)
	if err != nil {
		// The destination probably doesn't exist, we should create
		// it.
		goto copy
	}

	if bytes.Equal(in, out) {
		// The permission bits may have changed without the contents
		// changing so we always mirror them.
		os.Chmod(dst, perm)
		return nil
	}

copy:
	os.MkdirAll(filepath.Dir(dst), 0777)
	if err := ioutil.WriteFile(dst, in, perm); err != nil {
		return err
	}

	return nil
}

func listFiles(dir string) []string {
	var res []string
	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if fi.Mode().IsRegular() {
			res = append(res, path)
		}
		return nil
	})
	return res
}

func rebuildAssets() {
	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
}

func lazyRebuildAssets() {
	if shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui") {
		rebuildAssets()
	}
}

func shouldRebuildAssets(target, srcdir string) bool {
	info, err := os.Stat(target)
	if err != nil {
		// If the file doesn't exist, we must rebuild it
		return true
	}

	// Check if any of the files in gui/ are newer than the asset file. If
	// so we should rebuild it.
	currentBuild := info.ModTime()
	assetsAreNewer := false
	stop := errors.New("no need to iterate further")
	filepath.Walk(srcdir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.ModTime().After(currentBuild) {
			assetsAreNewer = true
			return stop
		}
		return nil
	})

	return assetsAreNewer
}

func proto() {
	pv := protobufVersion()
	dependencyRepos = append(dependencyRepos,
		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
	)

	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
	os.MkdirAll("repos", 0755)
	for _, dep := range dependencyRepos {
		path := filepath.Join("repos", dep.path)
		if _, err := os.Stat(path); err != nil {
			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
		} else {
			runPrintInDir(path, "git", "fetch")
		}
		runPrintInDir(path, "git", "checkout", dep.commit)
	}
	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
	runPrint(goCmd, "generate", "proto/generate.go")
}

func translate() {
	os.Chdir("gui/default/assets/lang")
	runPipe("lang-en-new.json", goCmd, "run", "../../../../script/translate.go", "lang-en.json", "../../../")
	os.Remove("lang-en.json")
	err := os.Rename("lang-en-new.json", "lang-en.json")
	if err != nil {
		log.Fatal(err)
	}
	os.Chdir("../../../..")
}

func transifex() {
	os.Chdir("gui/default/assets/lang")
	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
}

func ldflags(tags []string) string {
	b := new(strings.Builder)
	b.WriteString("-w")
	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Tags=%s", strings.Join(tags, ","))
	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
		fmt.Fprintf(b, " %s", v)
	}
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		if debug {
			log.Println("rm -r", path)
		}
		os.RemoveAll(path)
	}
}

func getReleaseVersion() (string, error) {
	bs, err := ioutil.ReadFile("RELEASE")
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(bs)), nil
}

func getGitVersion() (string, error) {
	// The current version as Git sees it
	bs, err := runError("git", "describe", "--always", "--dirty", "--abbrev=8")
	if err != nil {
		return "", err
	}
	vcur := string(bs)

	// The closest current tag name
	bs, err = runError("git", "describe", "--always", "--abbrev=0")
	if err != nil {
		return "", err
	}
	v0 := string(bs)

	// To be more semantic-versionish and ensure proper ordering in our
	// upgrade process, we make sure there's only one hypen in the version.

	versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
	if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
		suffix := strings.ReplaceAll(m[1], "-", ".")

		if strings.Contains(v0, "-") {
			// We're based of a tag with a prerelease string. We can just
			// add our dev stuff directly.
			return fmt.Sprintf("%s.dev.%s", v0, suffix), nil
		}

		// We're based on a release version. We need to bump the patch
		// version and then add a -dev prerelease string.
		next := nextPatchVersion(v0)
		return fmt.Sprintf("%s-dev.%s", next, suffix), nil
	}
	return vcur, nil
}

func getVersion() string {
	// First try for a RELEASE file,
	if ver, err := getReleaseVersion(); err == nil {
		return ver
	}
	// ... then see if we have a Git tag.
	if ver, err := getGitVersion(); err == nil {
		if strings.Contains(ver, "-") {
			// The version already contains a hash and stuff. See if we can
			// find a current branch name to tack onto it as well.
			return ver + getBranchSuffix()
		}
		return ver
	}
	// This seems to be a dev build.
	return "unknown-dev"
}

func semanticVersion() (major, minor, patch int) {
	r := regexp.MustCompile(`v(\d+)\.(\d+).(\d+)`)
	matches := r.FindStringSubmatch(getVersion())
	if len(matches) != 4 {
		return 0, 0, 0
	}

	var ints [3]int
	for i, s := range matches[1:] {
		ints[i], _ = strconv.Atoi(s)
	}
	return ints[0], ints[1], ints[2]
}

func getBranchSuffix() string {
	bs, err := runError("git", "branch", "-a", "--contains")
	if err != nil {
		return ""
	}

	branches := strings.Split(string(bs), "\n")
	if len(branches) == 0 {
		return ""
	}

	branch := ""
	for i, candidate := range branches {
		if strings.HasPrefix(candidate, "*") {
			// This is the current branch. Select it!
			branch = strings.TrimLeft(candidate, " \t*")
			break
		} else if i == 0 {
			// Otherwise the first branch in the list will do.
			branch = strings.TrimSpace(branch)
		}
	}

	if branch == "" {
		return ""
	}

	// The branch name may be on the form "remotes/origin/foo" from which we
	// just want "foo".
	parts := strings.Split(branch, "/")
	if len(parts) == 0 || len(parts[len(parts)-1]) == 0 {
		return ""
	}

	branch = parts[len(parts)-1]
	switch branch {
	case "master", "release", "main":
		// these are not special
		return ""
	}

	validBranchRe := regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
	if !validBranchRe.MatchString(branch) {
		// There's some odd stuff in the branch name. Better skip it.
		return ""
	}

	return "-" + branch
}

func buildStamp() int64 {
	// If SOURCE_DATE_EPOCH is set, use that.
	if s, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); s > 0 {
		return s
	}

	// Try to get the timestamp of the latest commit.
	bs, err := runError("git", "show", "-s", "--format=%ct")
	if err != nil {
		// Fall back to "now".
		return time.Now().Unix()
	}

	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildUser() string {
	if v := os.Getenv("BUILD_USER"); v != "" {
		return v
	}

	u, err := user.Current()
	if err != nil {
		return "unknown-user"
	}
	return strings.Replace(u.Username, " ", "-", -1)
}

func buildHost() string {
	if v := os.Getenv("BUILD_HOST"); v != "" {
		return v
	}

	h, err := os.Hostname()
	if err != nil {
		return "unknown-host"
	}
	return h
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macos"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func archiveName(target target) string {
	return fmt.Sprintf("%s-%s-%s", target.name, buildArch(), version)
}

func runError(cmd string, args ...string) ([]byte, error) {
	if debug {
		t0 := time.Now()
		log.Println("runError:", cmd, strings.Join(args, " "))
		defer func() {
			log.Println("... in", time.Since(t0))
		}()
	}
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	return bytes.TrimSpace(bs), err
}

func runPrint(cmd string, args ...string) {
	runPrintInDir(".", cmd, args...)
}

func runPrintInDir(dir string, cmd string, args ...string) {
	if debug {
		t0 := time.Now()
		log.Println("runPrint:", cmd, strings.Join(args, " "))
		defer func() {
			log.Println("... in", time.Since(t0))
		}()
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	ecmd.Dir = dir
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func runPipe(file, cmd string, args ...string) {
	if debug {
		t0 := time.Now()
		log.Println("runPipe:", cmd, strings.Join(args, " "))
		defer func() {
			log.Println("... in", time.Since(t0))
		}()
	}
	fd, err := os.Create(file)
	if err != nil {
		log.Fatal(err)
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = fd
	ecmd.Stderr = os.Stderr
	err = ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	fd.Close()
}

func tarGz(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	gw, err := gzip.NewWriterLevel(fd, gzip.BestCompression)
	if err != nil {
		log.Fatal(err)
	}
	tw := tar.NewWriter(gw)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}
		h := &tar.Header{
			Name:    f.dst,
			Size:    info.Size(),
			Mode:    int64(info.Mode()),
			ModTime: info.ModTime(),
		}

		err = tw.WriteHeader(h)
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(tw, sf)
		if err != nil {
			log.Fatal(err)
		}
		sf.Close()
	}

	err = tw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = gw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func zipFile(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	zw := zip.NewWriter(fd)

	var fw *flate.Writer

	// Register the deflator.
	zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
		var err error
		if fw == nil {
			// Creating a flate compressor for every file is
			// expensive, create one and reuse it.
			fw, err = flate.NewWriter(out, flate.BestCompression)
		} else {
			fw.Reset(out)
		}
		return fw, err
	})

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}

		fh, err := zip.FileInfoHeader(info)
		if err != nil {
			log.Fatal(err)
		}
		fh.Name = filepath.ToSlash(f.dst)
		fh.Method = zip.Deflate

		if strings.HasSuffix(f.dst, ".txt") {
			// Text file. Read it and convert line endings.
			bs, err := ioutil.ReadAll(sf)
			if err != nil {
				log.Fatal(err)
			}
			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
			fh.UncompressedSize = uint32(len(bs))
			fh.UncompressedSize64 = uint64(len(bs))

			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			of.Write(bs)
		} else {
			// Binary file. Copy verbatim.
			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(of, sf)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	err = zw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func codesign(target target) {
	switch goos {
	case "windows":
		windowsCodesign(target.BinaryName())
	case "darwin":
		macosCodesign(target.BinaryName())
	}
}

func macosCodesign(file string) {
	if pass := os.Getenv("CODESIGN_KEYCHAIN_PASS"); pass != "" {
		bs, err := runError("security", "unlock-keychain", "-p", pass)
		if err != nil {
			log.Println("Codesign: unlocking keychain failed:", string(bs))
			return
		}
	}

	if id := os.Getenv("CODESIGN_IDENTITY"); id != "" {
		bs, err := runError("codesign", "--options=runtime", "-s", id, file)
		if err != nil {
			log.Println("Codesign: signing failed:", string(bs))
			return
		}
		log.Println("Codesign: successfully signed", file)
	}
}

func windowsCodesign(file string) {
	st := "signtool.exe"

	if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
		st = path
	}

	for i, algo := range []string{"sha1", "sha256"} {
		args := []string{"sign", "/fd", algo}
		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
			args = append(args, "/f", f)
		}
		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
			args = append(args, "/p", p)
		}
		if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
			switch algo {
			case "sha256":
				args = append(args, "/tr", tr, "/td", algo)
			default:
				args = append(args, "/t", tr)
			}
		}
		if i > 0 {
			args = append(args, "/as")
		}
		args = append(args, file)

		bs, err := runError(st, args...)
		if err != nil {
			log.Println("Codesign: signing failed:", string(bs))
			return
		}
		log.Println("Codesign: successfully signed", file, "using", algo)
	}
}

func metalint() {
	lazyRebuildAssets()
	runPrint(goCmd, "test", "-run", "Metalint", "./meta")
}

func metalintShort() {
	lazyRebuildAssets()
	runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
}

func temporaryBuildDir() (string, error) {
	// The base of our temp dir is "syncthing-xxxxxxxx" where the x:es
	// are eight bytes from the sha256 of our working directory. We do
	// this because we want a name in the global temp dir that doesn't
	// conflict with someone else building syncthing on the same
	// machine, yet is persistent between runs from the same source
	// directory.
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	hash := sha256.Sum256([]byte(wd))
	base := fmt.Sprintf("syncthing-%x", hash[:4])

	// The temp dir is taken from $STTMPDIR if set, otherwise the system
	// default (potentially infrluenced by $TMPDIR on unixes).
	var tmpDir string
	if t := os.Getenv("STTMPDIR"); t != "" {
		tmpDir = t
	} else {
		tmpDir = os.TempDir()
	}

	return filepath.Join(tmpDir, base), nil
}

func (t target) BinaryName() string {
	if goos == "windows" {
		return t.binaryName + ".exe"
	}
	return t.binaryName
}

func protobufVersion() string {
	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
	if err != nil {
		log.Fatal("Getting protobuf version:", err)
	}
	return string(bs)
}

func currentAndLatestVersions(n int) ([]string, error) {
	bs, err := runError("git", "tag", "--sort", "taggerdate")
	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(bs), "\n")
	reverseStrings(lines)

	// The one at the head is the latest version. We always keep that one.
	// Then we filter out remaining ones with dashes (pre-releases etc).

	latest := lines[:1]
	nonPres := filterStrings(lines[1:], func(s string) bool { return !strings.Contains(s, "-") })
	vers := append(latest, nonPres...)
	return vers[:n], nil
}

func reverseStrings(ss []string) {
	for i := 0; i < len(ss)/2; i++ {
		ss[i], ss[len(ss)-1-i] = ss[len(ss)-1-i], ss[i]
	}
}

func filterStrings(ss []string, op func(string) bool) []string {
	n := ss[:0]
	for _, s := range ss {
		if op(s) {
			n = append(n, s)
		}
	}
	return n
}

func tagMessage(tag string) (string, error) {
	hash, err := runError("git", "rev-parse", tag)
	if err != nil {
		return "", err
	}
	obj, err := runError("git", "cat-file", "-p", string(hash))
	if err != nil {
		return "", err
	}
	return trimTagMessage(string(obj), tag), nil
}

func trimTagMessage(msg, tag string) string {
	firstBlank := strings.Index(msg, "\n\n")
	if firstBlank > 0 {
		msg = msg[firstBlank+2:]
	}
	msg = strings.TrimPrefix(msg, tag)
	beginSig := strings.Index(msg, "-----BEGIN PGP")
	if beginSig > 0 {
		msg = msg[:beginSig]
	}
	return strings.TrimSpace(msg)
}

func nextPatchVersion(ver string) string {
	parts := strings.SplitN(ver, "-", 2)
	digits := strings.Split(parts[0], ".")
	n, _ := strconv.Atoi(digits[len(digits)-1])
	digits[len(digits)-1] = strconv.Itoa(n + 1)
	return strings.Join(digits, ".")
}