build.go 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559
  1. // Copyright (C) 2014 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. //go:build ignore
  7. // +build ignore
  8. package main
  9. import (
  10. "archive/tar"
  11. "archive/zip"
  12. "bytes"
  13. "compress/flate"
  14. "compress/gzip"
  15. "encoding/base64"
  16. "encoding/json"
  17. "errors"
  18. "flag"
  19. "fmt"
  20. "io"
  21. "log"
  22. "os"
  23. "os/exec"
  24. "os/user"
  25. "path/filepath"
  26. "regexp"
  27. "runtime"
  28. "strconv"
  29. "strings"
  30. "text/template"
  31. "time"
  32. buildpkg "github.com/syncthing/syncthing/lib/build"
  33. )
  34. var (
  35. goarch string
  36. goos string
  37. noupgrade bool
  38. version string
  39. goCmd string
  40. race bool
  41. debug = os.Getenv("BUILDDEBUG") != ""
  42. extraTags string
  43. installSuffix string
  44. pkgdir string
  45. cc string
  46. run string
  47. benchRun string
  48. buildOut string
  49. debugBinary bool
  50. coverage bool
  51. long bool
  52. timeout = "120s"
  53. longTimeout = "600s"
  54. numVersions = 5
  55. withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
  56. )
  57. type target struct {
  58. name string
  59. debname string
  60. debdeps []string
  61. debpre string
  62. description string
  63. buildPkgs []string
  64. binaryName string
  65. archiveFiles []archiveFile
  66. systemdService string
  67. installationFiles []archiveFile
  68. tags []string
  69. }
  70. type archiveFile struct {
  71. src string
  72. dst string
  73. perm os.FileMode
  74. }
  75. var targets = map[string]target{
  76. "all": {
  77. // Only valid for the "build" and "install" commands as it lacks all
  78. // the archive creation stuff. buildPkgs gets filled out in init()
  79. tags: []string{"purego"},
  80. },
  81. "syncthing": {
  82. // The default target for "build", "install", "tar", "zip", "deb", etc.
  83. name: "syncthing",
  84. debname: "syncthing",
  85. debdeps: []string{"libc6", "procps"},
  86. description: "Open Source Continuous File Synchronization",
  87. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/syncthing"},
  88. binaryName: "syncthing", // .exe will be added automatically for Windows builds
  89. archiveFiles: []archiveFile{
  90. {src: "{{binary}}", dst: "{{binary}}", perm: 0755},
  91. {src: "README.md", dst: "README.txt", perm: 0644},
  92. {src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
  93. {src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
  94. // All files from etc/ and extra/ added automatically in init().
  95. },
  96. systemdService: "syncthing@*.service",
  97. installationFiles: []archiveFile{
  98. {src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
  99. {src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
  100. {src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
  101. {src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
  102. {src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
  103. {src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
  104. {src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
  105. {src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
  106. {src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
  107. {src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
  108. {src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
  109. {src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
  110. {src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
  111. {src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
  112. {src: "etc/linux-systemd/system/[email protected]", dst: "deb/lib/systemd/system/[email protected]", perm: 0644},
  113. {src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
  114. {src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
  115. {src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
  116. {src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
  117. {src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
  118. {src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
  119. {src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
  120. {src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
  121. {src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
  122. {src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
  123. {src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
  124. },
  125. },
  126. "stdiscosrv": {
  127. name: "stdiscosrv",
  128. debname: "syncthing-discosrv",
  129. debdeps: []string{"libc6"},
  130. debpre: "cmd/stdiscosrv/scripts/preinst",
  131. description: "Syncthing Discovery Server",
  132. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
  133. binaryName: "stdiscosrv", // .exe will be added automatically for Windows builds
  134. archiveFiles: []archiveFile{
  135. {src: "{{binary}}", dst: "{{binary}}", perm: 0755},
  136. {src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
  137. {src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
  138. {src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
  139. },
  140. systemdService: "stdiscosrv.service",
  141. installationFiles: []archiveFile{
  142. {src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
  143. {src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
  144. {src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
  145. {src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
  146. {src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
  147. {src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0644},
  148. {src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
  149. {src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
  150. },
  151. tags: []string{"purego"},
  152. },
  153. "strelaysrv": {
  154. name: "strelaysrv",
  155. debname: "syncthing-relaysrv",
  156. debdeps: []string{"libc6"},
  157. debpre: "cmd/strelaysrv/scripts/preinst",
  158. description: "Syncthing Relay Server",
  159. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
  160. binaryName: "strelaysrv", // .exe will be added automatically for Windows builds
  161. archiveFiles: []archiveFile{
  162. {src: "{{binary}}", dst: "{{binary}}", perm: 0755},
  163. {src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
  164. {src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
  165. {src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
  166. {src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
  167. },
  168. systemdService: "strelaysrv.service",
  169. installationFiles: []archiveFile{
  170. {src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
  171. {src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
  172. {src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
  173. {src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
  174. {src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
  175. {src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
  176. {src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0644},
  177. {src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
  178. {src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
  179. },
  180. },
  181. "strelaypoolsrv": {
  182. name: "strelaypoolsrv",
  183. debname: "syncthing-relaypoolsrv",
  184. debdeps: []string{"libc6"},
  185. description: "Syncthing Relay Pool Server",
  186. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
  187. binaryName: "strelaypoolsrv", // .exe will be added automatically for Windows builds
  188. archiveFiles: []archiveFile{
  189. {src: "{{binary}}", dst: "{{binary}}", perm: 0755},
  190. {src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
  191. {src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
  192. {src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
  193. },
  194. installationFiles: []archiveFile{
  195. {src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
  196. {src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
  197. {src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
  198. {src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
  199. },
  200. },
  201. "stupgrades": {
  202. name: "stupgrades",
  203. description: "Syncthing Upgrade Check Server",
  204. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stupgrades"},
  205. binaryName: "stupgrades",
  206. },
  207. "stcrashreceiver": {
  208. name: "stcrashreceiver",
  209. description: "Syncthing Crash Server",
  210. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stcrashreceiver"},
  211. binaryName: "stcrashreceiver",
  212. },
  213. "ursrv": {
  214. name: "ursrv",
  215. description: "Syncthing Usage Reporting Server",
  216. buildPkgs: []string{"github.com/syncthing/syncthing/cmd/ursrv"},
  217. binaryName: "ursrv",
  218. },
  219. }
  220. func initTargets() {
  221. all := targets["all"]
  222. pkgs, _ := filepath.Glob("cmd/*")
  223. for _, pkg := range pkgs {
  224. pkg = filepath.Base(pkg)
  225. if strings.HasPrefix(pkg, ".") {
  226. // ignore dotfiles
  227. continue
  228. }
  229. if noupgrade && pkg == "stupgrades" {
  230. continue
  231. }
  232. all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
  233. }
  234. targets["all"] = all
  235. // The "syncthing" target includes a few more files found in the "etc"
  236. // and "extra" dirs.
  237. syncthingPkg := targets["syncthing"]
  238. for _, file := range listFiles("etc") {
  239. syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
  240. }
  241. for _, file := range listFiles("extra") {
  242. syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
  243. }
  244. for _, file := range listFiles("extra") {
  245. syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
  246. }
  247. targets["syncthing"] = syncthingPkg
  248. }
  249. func main() {
  250. log.SetFlags(0)
  251. parseFlags()
  252. if debug {
  253. t0 := time.Now()
  254. defer func() {
  255. log.Println("... build completed in", time.Since(t0))
  256. }()
  257. }
  258. initTargets()
  259. // Invoking build.go with no parameters at all builds everything (incrementally),
  260. // which is what you want for maximum error checking during development.
  261. if flag.NArg() == 0 {
  262. runCommand("install", targets["all"])
  263. } else {
  264. // with any command given but not a target, the target is
  265. // "syncthing". So "go run build.go install" is "go run build.go install
  266. // syncthing" etc.
  267. targetName := "syncthing"
  268. if flag.NArg() > 1 {
  269. targetName = flag.Arg(1)
  270. }
  271. target, ok := targets[targetName]
  272. if !ok {
  273. log.Fatalln("Unknown target", target)
  274. }
  275. runCommand(flag.Arg(0), target)
  276. }
  277. }
  278. func runCommand(cmd string, target target) {
  279. var tags []string
  280. if noupgrade {
  281. tags = []string{"noupgrade"}
  282. }
  283. tags = append(tags, strings.Fields(extraTags)...)
  284. switch cmd {
  285. case "install":
  286. install(target, tags)
  287. metalintShort()
  288. case "build":
  289. build(target, tags)
  290. case "test":
  291. test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
  292. case "bench":
  293. bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
  294. case "integration":
  295. integration(false)
  296. case "integrationbench":
  297. integration(true)
  298. case "assets":
  299. rebuildAssets()
  300. case "update-deps":
  301. updateDependencies()
  302. case "proto":
  303. proto()
  304. case "testmocks":
  305. testmocks()
  306. case "translate":
  307. translate()
  308. case "transifex":
  309. transifex()
  310. case "weblate":
  311. weblate()
  312. case "tar":
  313. buildTar(target, tags)
  314. case "zip":
  315. buildZip(target, tags)
  316. case "deb":
  317. buildDeb(target)
  318. case "vet":
  319. metalintShort()
  320. case "lint":
  321. metalintShort()
  322. case "metalint":
  323. metalint()
  324. case "version":
  325. fmt.Println(getVersion())
  326. case "changelog":
  327. vers, err := currentAndLatestVersions(numVersions)
  328. if err != nil {
  329. log.Fatal(err)
  330. }
  331. for _, ver := range vers {
  332. underline := strings.Repeat("=", len(ver))
  333. msg, err := tagMessage(ver)
  334. if err != nil {
  335. log.Fatal(err)
  336. }
  337. fmt.Printf("%s\n%s\n\n%s\n\n", ver, underline, msg)
  338. }
  339. default:
  340. log.Fatalf("Unknown command %q", cmd)
  341. }
  342. }
  343. func parseFlags() {
  344. flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
  345. flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
  346. flag.StringVar(&goCmd, "gocmd", "go", "Specify `go` command")
  347. flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
  348. flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
  349. flag.BoolVar(&race, "race", race, "Use race detector")
  350. flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
  351. flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
  352. flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
  353. flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
  354. flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
  355. flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
  356. flag.BoolVar(&long, "long", long, "Run tests without the -short flag")
  357. flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
  358. flag.StringVar(&run, "run", "", "Specify which tests to run")
  359. flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
  360. flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
  361. flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
  362. flag.Parse()
  363. }
  364. func test(tags []string, pkgs ...string) {
  365. lazyRebuildAssets()
  366. tags = append(tags, "purego")
  367. args := []string{"test", "-tags", strings.Join(tags, " ")}
  368. if long {
  369. timeout = longTimeout
  370. } else {
  371. args = append(args, "-short")
  372. }
  373. args = append(args, "-timeout", timeout)
  374. if runtime.GOARCH == "amd64" {
  375. switch runtime.GOOS {
  376. case buildpkg.Darwin, buildpkg.Linux, buildpkg.FreeBSD: // , "windows": # See https://github.com/golang/go/issues/27089
  377. args = append(args, "-race")
  378. }
  379. }
  380. if coverage {
  381. args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
  382. }
  383. args = append(args, runArgs()...)
  384. runPrint(goCmd, append(args, pkgs...)...)
  385. }
  386. func bench(tags []string, pkgs ...string) {
  387. lazyRebuildAssets()
  388. args := append([]string{"test", "-run", "NONE", "-tags", strings.Join(tags, " ")}, benchArgs()...)
  389. runPrint(goCmd, append(args, pkgs...)...)
  390. }
  391. func integration(bench bool) {
  392. lazyRebuildAssets()
  393. args := []string{"test", "-v", "-timeout", "60m", "-tags"}
  394. tags := "purego,integration"
  395. if bench {
  396. tags += ",benchmark"
  397. }
  398. args = append(args, tags)
  399. args = append(args, runArgs()...)
  400. if bench {
  401. if run == "" {
  402. args = append(args, "-run", "Benchmark")
  403. }
  404. args = append(args, benchArgs()...)
  405. }
  406. args = append(args, "./test")
  407. fmt.Println(args)
  408. runPrint(goCmd, args...)
  409. }
  410. func runArgs() []string {
  411. if run == "" {
  412. return nil
  413. }
  414. return []string{"-run", run}
  415. }
  416. func benchArgs() []string {
  417. if benchRun == "" {
  418. return []string{"-bench", "."}
  419. }
  420. return []string{"-bench", benchRun}
  421. }
  422. func install(target target, tags []string) {
  423. if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
  424. log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
  425. }
  426. lazyRebuildAssets()
  427. tags = append(target.tags, tags...)
  428. cwd, err := os.Getwd()
  429. if err != nil {
  430. log.Fatal(err)
  431. }
  432. os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
  433. setBuildEnvVars()
  434. // On Windows generate a special file which the Go compiler will
  435. // automatically use when generating Windows binaries to set things like
  436. // the file icon, version, etc.
  437. if goos == "windows" {
  438. sysoPath, err := shouldBuildSyso(cwd)
  439. if err != nil {
  440. log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
  441. }
  442. defer shouldCleanupSyso(sysoPath)
  443. }
  444. args := []string{"install", "-v"}
  445. args = appendParameters(args, tags, target.buildPkgs...)
  446. runPrint(goCmd, args...)
  447. }
  448. func build(target target, tags []string) {
  449. if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
  450. log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
  451. }
  452. lazyRebuildAssets()
  453. tags = append(target.tags, tags...)
  454. rmr(target.BinaryName())
  455. setBuildEnvVars()
  456. // On Windows generate a special file which the Go compiler will
  457. // automatically use when generating Windows binaries to set things like
  458. // the file icon, version, etc.
  459. if goos == "windows" {
  460. cwd, err := os.Getwd()
  461. if err != nil {
  462. log.Fatal(err)
  463. }
  464. sysoPath, err := shouldBuildSyso(cwd)
  465. if err != nil {
  466. log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
  467. }
  468. defer shouldCleanupSyso(sysoPath)
  469. }
  470. args := []string{"build", "-v"}
  471. if buildOut != "" {
  472. args = append(args, "-o", buildOut)
  473. }
  474. args = appendParameters(args, tags, target.buildPkgs...)
  475. runPrint(goCmd, args...)
  476. }
  477. func setBuildEnvVars() {
  478. os.Setenv("GOOS", goos)
  479. os.Setenv("GOARCH", goarch)
  480. os.Setenv("CC", cc)
  481. if os.Getenv("CGO_ENABLED") == "" {
  482. switch goos {
  483. case "darwin", "solaris":
  484. default:
  485. os.Setenv("CGO_ENABLED", "0")
  486. }
  487. }
  488. }
  489. func appendParameters(args []string, tags []string, pkgs ...string) []string {
  490. if pkgdir != "" {
  491. args = append(args, "-pkgdir", pkgdir)
  492. }
  493. if len(tags) > 0 {
  494. args = append(args, "-tags", strings.Join(tags, " "))
  495. }
  496. if installSuffix != "" {
  497. args = append(args, "-installsuffix", installSuffix)
  498. }
  499. if race {
  500. args = append(args, "-race")
  501. }
  502. if !debugBinary {
  503. // Regular binaries get version tagged and skip some debug symbols
  504. args = append(args, "-trimpath", "-ldflags", ldflags(tags))
  505. } else {
  506. // -gcflags to disable optimizations and inlining. Skip -ldflags
  507. // because `Could not launch program: decoding dwarf section info at
  508. // offset 0x0: too short` on 'dlv exec ...' see
  509. // https://github.com/go-delve/delve/issues/79
  510. args = append(args, "-gcflags", "all=-N -l")
  511. }
  512. return append(args, pkgs...)
  513. }
  514. func buildTar(target target, tags []string) {
  515. name := archiveName(target)
  516. filename := name + ".tar.gz"
  517. for _, tag := range tags {
  518. if tag == "noupgrade" {
  519. name += "-noupgrade"
  520. break
  521. }
  522. }
  523. build(target, tags)
  524. codesign(target)
  525. for i := range target.archiveFiles {
  526. target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
  527. target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
  528. target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
  529. }
  530. tarGz(filename, target.archiveFiles)
  531. fmt.Println(filename)
  532. }
  533. func buildZip(target target, tags []string) {
  534. name := archiveName(target)
  535. filename := name + ".zip"
  536. for _, tag := range tags {
  537. if tag == "noupgrade" {
  538. name += "-noupgrade"
  539. break
  540. }
  541. }
  542. build(target, tags)
  543. codesign(target)
  544. for i := range target.archiveFiles {
  545. target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
  546. target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
  547. target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
  548. }
  549. zipFile(filename, target.archiveFiles)
  550. fmt.Println(filename)
  551. }
  552. func buildDeb(target target) {
  553. os.RemoveAll("deb")
  554. // "goarch" here is set to whatever the Debian packages expect. We correct
  555. // it to what we actually know how to build and keep the Debian variant
  556. // name in "debarch".
  557. debarch := goarch
  558. switch goarch {
  559. case "i386":
  560. goarch = "386"
  561. case "armel", "armhf":
  562. goarch = "arm"
  563. }
  564. build(target, []string{"noupgrade"})
  565. for i := range target.installationFiles {
  566. target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.BinaryName(), 1)
  567. target.installationFiles[i].dst = strings.Replace(target.installationFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
  568. }
  569. for _, af := range target.installationFiles {
  570. if err := copyFile(af.src, af.dst, af.perm); err != nil {
  571. log.Fatal(err)
  572. }
  573. }
  574. maintainer := "Syncthing Release Management <[email protected]>"
  575. debver := version
  576. if strings.HasPrefix(debver, "v") {
  577. debver = debver[1:]
  578. // Debian interprets dashes as separator between main version and
  579. // Debian package version, and thus thinks 0.14.26-rc.1 is better
  580. // than just 0.14.26. This rectifies that.
  581. debver = strings.Replace(debver, "-", "~", -1)
  582. }
  583. args := []string{
  584. "-t", "deb",
  585. "-s", "dir",
  586. "-C", "deb",
  587. "-n", target.debname,
  588. "-v", debver,
  589. "-a", debarch,
  590. "-m", maintainer,
  591. "--vendor", maintainer,
  592. "--description", target.description,
  593. "--url", "https://syncthing.net/",
  594. "--license", "MPL-2",
  595. }
  596. for _, dep := range target.debdeps {
  597. args = append(args, "-d", dep)
  598. }
  599. if target.systemdService != "" {
  600. debpost, err := createPostInstScript(target)
  601. defer os.Remove(debpost)
  602. if err != nil {
  603. log.Fatal(err)
  604. }
  605. args = append(args, "--after-upgrade", debpost)
  606. }
  607. if target.debpre != "" {
  608. args = append(args, "--before-install", target.debpre)
  609. }
  610. runPrint("fpm", args...)
  611. }
  612. func createPostInstScript(target target) (string, error) {
  613. scriptname := filepath.Join("script", "deb-post-inst.template")
  614. t, err := template.ParseFiles(scriptname)
  615. if err != nil {
  616. return "", err
  617. }
  618. scriptname = strings.TrimSuffix(scriptname, ".template")
  619. w, err := os.Create(scriptname)
  620. if err != nil {
  621. return "", err
  622. }
  623. defer w.Close()
  624. if err = t.Execute(w, struct {
  625. Service, Command string
  626. }{
  627. target.systemdService, target.binaryName,
  628. }); err != nil {
  629. return "", err
  630. }
  631. return scriptname, nil
  632. }
  633. func shouldBuildSyso(dir string) (string, error) {
  634. type M map[string]interface{}
  635. version := getVersion()
  636. version = strings.TrimPrefix(version, "v")
  637. major, minor, patch := semanticVersion()
  638. bs, err := json.Marshal(M{
  639. "FixedFileInfo": M{
  640. "FileVersion": M{
  641. "Major": major,
  642. "Minor": minor,
  643. "Patch": patch,
  644. },
  645. "ProductVersion": M{
  646. "Major": major,
  647. "Minor": minor,
  648. "Patch": patch,
  649. },
  650. },
  651. "StringFileInfo": M{
  652. "CompanyName": "The Syncthing Authors",
  653. "FileDescription": "Syncthing - Open Source Continuous File Synchronization",
  654. "FileVersion": version,
  655. "InternalName": "syncthing",
  656. "LegalCopyright": "The Syncthing Authors",
  657. "OriginalFilename": "syncthing",
  658. "ProductName": "Syncthing",
  659. "ProductVersion": version,
  660. },
  661. "IconPath": "assets/logo.ico",
  662. })
  663. if err != nil {
  664. return "", err
  665. }
  666. jsonPath := filepath.Join(dir, "versioninfo.json")
  667. err = os.WriteFile(jsonPath, bs, 0644)
  668. if err != nil {
  669. return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
  670. }
  671. defer func() {
  672. if err := os.Remove(jsonPath); err != nil {
  673. log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
  674. }
  675. }()
  676. sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
  677. // See https://github.com/josephspurrier/goversioninfo#command-line-flags
  678. armOption := ""
  679. if strings.Contains(goarch, "arm") {
  680. armOption = "-arm=true"
  681. }
  682. if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
  683. return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
  684. }
  685. return sysoPath, nil
  686. }
  687. func shouldCleanupSyso(sysoFilePath string) {
  688. if sysoFilePath == "" {
  689. return
  690. }
  691. if err := os.Remove(sysoFilePath); err != nil {
  692. log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
  693. }
  694. }
  695. // copyFile copies a file from src to dst, ensuring the containing directory
  696. // exists. The permission bits are copied as well. If dst already exists and
  697. // the contents are identical to src the modification time is not updated.
  698. func copyFile(src, dst string, perm os.FileMode) error {
  699. in, err := os.ReadFile(src)
  700. if err != nil {
  701. return err
  702. }
  703. out, err := os.ReadFile(dst)
  704. if err != nil {
  705. // The destination probably doesn't exist, we should create
  706. // it.
  707. goto copy
  708. }
  709. if bytes.Equal(in, out) {
  710. // The permission bits may have changed without the contents
  711. // changing so we always mirror them.
  712. os.Chmod(dst, perm)
  713. return nil
  714. }
  715. copy:
  716. os.MkdirAll(filepath.Dir(dst), 0777)
  717. if err := os.WriteFile(dst, in, perm); err != nil {
  718. return err
  719. }
  720. return nil
  721. }
  722. func listFiles(dir string) []string {
  723. var res []string
  724. filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
  725. if err != nil {
  726. return err
  727. }
  728. if fi.Mode().IsRegular() {
  729. res = append(res, path)
  730. }
  731. return nil
  732. })
  733. return res
  734. }
  735. func rebuildAssets() {
  736. os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
  737. runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
  738. }
  739. func lazyRebuildAssets() {
  740. shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
  741. shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
  742. if withNextGenGUI {
  743. shouldRebuild = buildNextGenGUI() || shouldRebuild
  744. }
  745. if shouldRebuild {
  746. rebuildAssets()
  747. }
  748. }
  749. func buildNextGenGUI() bool {
  750. // Check if we need to run the npm process, and if so also set the flag
  751. // to rebuild Go assets afterwards. The index.html is regenerated every
  752. // time by the build process. This assumes the new GUI ends up in
  753. // next-gen-gui/dist/next-gen-gui.
  754. if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
  755. // The GUI is up to date.
  756. return false
  757. }
  758. runPrintInDir("next-gen-gui", "npm", "install")
  759. runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
  760. rmr("gui/tech-ui")
  761. for _, src := range listFiles("next-gen-gui/dist") {
  762. rel, _ := filepath.Rel("next-gen-gui/dist", src)
  763. dst := filepath.Join("gui", rel)
  764. if err := copyFile(src, dst, 0644); err != nil {
  765. fmt.Println("copy:", err)
  766. os.Exit(1)
  767. }
  768. }
  769. return true
  770. }
  771. func shouldRebuildAssets(target, srcdir string) bool {
  772. info, err := os.Stat(target)
  773. if err != nil {
  774. // If the file doesn't exist, we must rebuild it
  775. return true
  776. }
  777. // Check if any of the files in gui/ are newer than the asset file. If
  778. // so we should rebuild it.
  779. currentBuild := info.ModTime()
  780. assetsAreNewer := false
  781. stop := errors.New("no need to iterate further")
  782. filepath.Walk(srcdir, func(path string, info os.FileInfo, err error) error {
  783. if err != nil {
  784. return err
  785. }
  786. if info.ModTime().After(currentBuild) {
  787. assetsAreNewer = true
  788. return stop
  789. }
  790. return nil
  791. })
  792. return assetsAreNewer
  793. }
  794. func updateDependencies() {
  795. // Figure out desired Go version
  796. bs, err := os.ReadFile("go.mod")
  797. if err != nil {
  798. log.Fatal(err)
  799. }
  800. re := regexp.MustCompile(`(?m)^go\s+([0-9.]+)`)
  801. matches := re.FindSubmatch(bs)
  802. if len(matches) != 2 {
  803. log.Fatal("failed to parse go.mod")
  804. }
  805. goVersion := string(matches[1])
  806. runPrint(goCmd, "get", "-u", "./...")
  807. runPrint(goCmd, "mod", "tidy", "-go="+goVersion, "-compat="+goVersion)
  808. // We might have updated the protobuf package and should regenerate to match.
  809. proto()
  810. }
  811. func proto() {
  812. pv := protobufVersion()
  813. repo := "https://github.com/gogo/protobuf.git"
  814. path := filepath.Join("repos", "protobuf")
  815. runPrint(goCmd, "install", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
  816. os.MkdirAll("repos", 0755)
  817. if _, err := os.Stat(path); err != nil {
  818. runPrint("git", "clone", repo, path)
  819. } else {
  820. runPrintInDir(path, "git", "fetch")
  821. }
  822. runPrintInDir(path, "git", "checkout", pv)
  823. runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
  824. runPrint(goCmd, "generate", "proto/generate.go")
  825. }
  826. func testmocks() {
  827. args := []string{
  828. "generate",
  829. "github.com/syncthing/syncthing/lib/config",
  830. "github.com/syncthing/syncthing/lib/connections",
  831. "github.com/syncthing/syncthing/lib/discover",
  832. "github.com/syncthing/syncthing/lib/events",
  833. "github.com/syncthing/syncthing/lib/logger",
  834. "github.com/syncthing/syncthing/lib/model",
  835. "github.com/syncthing/syncthing/lib/protocol",
  836. }
  837. runPrint(goCmd, args...)
  838. }
  839. func translate() {
  840. os.Chdir("gui/default/assets/lang")
  841. runPipe("lang-en-new.json", goCmd, "run", "../../../../script/translate.go", "lang-en.json", "../../../")
  842. os.Remove("lang-en.json")
  843. err := os.Rename("lang-en-new.json", "lang-en.json")
  844. if err != nil {
  845. log.Fatal(err)
  846. }
  847. os.Chdir("../../../..")
  848. }
  849. func transifex() {
  850. os.Chdir("gui/default/assets/lang")
  851. runPrint(goCmd, "run", "../../../../script/transifexdl.go")
  852. }
  853. func weblate() {
  854. os.Chdir("gui/default/assets/lang")
  855. runPrint(goCmd, "run", "../../../../script/weblatedl.go")
  856. }
  857. func ldflags(tags []string) string {
  858. b := new(strings.Builder)
  859. b.WriteString("-w")
  860. fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
  861. fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
  862. fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
  863. fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
  864. fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Tags=%s", strings.Join(tags, ","))
  865. if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
  866. fmt.Fprintf(b, " %s", v)
  867. }
  868. return b.String()
  869. }
  870. func rmr(paths ...string) {
  871. for _, path := range paths {
  872. if debug {
  873. log.Println("rm -r", path)
  874. }
  875. os.RemoveAll(path)
  876. }
  877. }
  878. func getReleaseVersion() (string, error) {
  879. bs, err := os.ReadFile("RELEASE")
  880. if err != nil {
  881. return "", err
  882. }
  883. return string(bytes.TrimSpace(bs)), nil
  884. }
  885. func getGitVersion() (string, error) {
  886. // The current version as Git sees it
  887. bs, err := runError("git", "describe", "--always", "--dirty", "--abbrev=8")
  888. if err != nil {
  889. return "", err
  890. }
  891. vcur := string(bs)
  892. // The closest current tag name
  893. bs, err = runError("git", "describe", "--always", "--abbrev=0")
  894. if err != nil {
  895. return "", err
  896. }
  897. v0 := string(bs)
  898. // To be more semantic-versionish and ensure proper ordering in our
  899. // upgrade process, we make sure there's only one hyphen in the version.
  900. versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
  901. if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
  902. suffix := strings.ReplaceAll(m[1], "-", ".")
  903. if strings.Contains(v0, "-") {
  904. // We're based of a tag with a prerelease string. We can just
  905. // add our dev stuff directly.
  906. return fmt.Sprintf("%s.dev.%s", v0, suffix), nil
  907. }
  908. // We're based on a release version. We need to bump the patch
  909. // version and then add a -dev prerelease string.
  910. next := nextPatchVersion(v0)
  911. return fmt.Sprintf("%s-dev.%s", next, suffix), nil
  912. }
  913. return vcur, nil
  914. }
  915. func getVersion() string {
  916. // First try for a RELEASE file,
  917. if ver, err := getReleaseVersion(); err == nil {
  918. return ver
  919. }
  920. // ... then see if we have a Git tag.
  921. if ver, err := getGitVersion(); err == nil {
  922. if strings.Contains(ver, "-") {
  923. // The version already contains a hash and stuff. See if we can
  924. // find a current branch name to tack onto it as well.
  925. return ver + getBranchSuffix()
  926. }
  927. return ver
  928. }
  929. // This seems to be a dev build.
  930. return "unknown-dev"
  931. }
  932. func semanticVersion() (major, minor, patch int) {
  933. r := regexp.MustCompile(`v(\d+)\.(\d+).(\d+)`)
  934. matches := r.FindStringSubmatch(getVersion())
  935. if len(matches) != 4 {
  936. return 0, 0, 0
  937. }
  938. var ints [3]int
  939. for i, s := range matches[1:] {
  940. ints[i], _ = strconv.Atoi(s)
  941. }
  942. return ints[0], ints[1], ints[2]
  943. }
  944. func getBranchSuffix() string {
  945. bs, err := runError("git", "branch", "-a", "--contains")
  946. if err != nil {
  947. return ""
  948. }
  949. branches := strings.Split(string(bs), "\n")
  950. if len(branches) == 0 {
  951. return ""
  952. }
  953. branch := ""
  954. for i, candidate := range branches {
  955. if strings.HasPrefix(candidate, "*") {
  956. // This is the current branch. Select it!
  957. branch = strings.TrimLeft(candidate, " \t*")
  958. break
  959. } else if i == 0 {
  960. // Otherwise the first branch in the list will do.
  961. branch = strings.TrimSpace(branch)
  962. }
  963. }
  964. if branch == "" {
  965. return ""
  966. }
  967. // The branch name may be on the form "remotes/origin/foo" from which we
  968. // just want "foo".
  969. parts := strings.Split(branch, "/")
  970. if len(parts) == 0 || len(parts[len(parts)-1]) == 0 {
  971. return ""
  972. }
  973. branch = parts[len(parts)-1]
  974. switch branch {
  975. case "release", "main":
  976. // these are not special
  977. return ""
  978. }
  979. if strings.HasPrefix(branch, "release-") {
  980. // release branches are not special
  981. return ""
  982. }
  983. validBranchRe := regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
  984. if !validBranchRe.MatchString(branch) {
  985. // There's some odd stuff in the branch name. Better skip it.
  986. return ""
  987. }
  988. return "-" + branch
  989. }
  990. func buildStamp() int64 {
  991. // If SOURCE_DATE_EPOCH is set, use that.
  992. if s, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); s > 0 {
  993. return s
  994. }
  995. // Try to get the timestamp of the latest commit.
  996. bs, err := runError("git", "show", "-s", "--format=%ct")
  997. if err != nil {
  998. // Fall back to "now".
  999. return time.Now().Unix()
  1000. }
  1001. s, _ := strconv.ParseInt(string(bs), 10, 64)
  1002. return s
  1003. }
  1004. func buildUser() string {
  1005. if v := os.Getenv("BUILD_USER"); v != "" {
  1006. return v
  1007. }
  1008. u, err := user.Current()
  1009. if err != nil {
  1010. return "unknown-user"
  1011. }
  1012. return strings.Replace(u.Username, " ", "-", -1)
  1013. }
  1014. func buildHost() string {
  1015. if v := os.Getenv("BUILD_HOST"); v != "" {
  1016. return v
  1017. }
  1018. h, err := os.Hostname()
  1019. if err != nil {
  1020. return "unknown-host"
  1021. }
  1022. return h
  1023. }
  1024. func buildArch() string {
  1025. os := goos
  1026. if os == "darwin" {
  1027. os = "macos"
  1028. }
  1029. return fmt.Sprintf("%s-%s", os, goarch)
  1030. }
  1031. func archiveName(target target) string {
  1032. return fmt.Sprintf("%s-%s-%s", target.name, buildArch(), version)
  1033. }
  1034. func runError(cmd string, args ...string) ([]byte, error) {
  1035. if debug {
  1036. t0 := time.Now()
  1037. log.Println("runError:", cmd, strings.Join(args, " "))
  1038. defer func() {
  1039. log.Println("... in", time.Since(t0))
  1040. }()
  1041. }
  1042. ecmd := exec.Command(cmd, args...)
  1043. bs, err := ecmd.CombinedOutput()
  1044. return bytes.TrimSpace(bs), err
  1045. }
  1046. func runPrint(cmd string, args ...string) {
  1047. runPrintInDir(".", cmd, args...)
  1048. }
  1049. func runPrintInDir(dir string, cmd string, args ...string) {
  1050. if debug {
  1051. t0 := time.Now()
  1052. log.Println("runPrint:", cmd, strings.Join(args, " "))
  1053. defer func() {
  1054. log.Println("... in", time.Since(t0))
  1055. }()
  1056. }
  1057. ecmd := exec.Command(cmd, args...)
  1058. ecmd.Stdout = os.Stdout
  1059. ecmd.Stderr = os.Stderr
  1060. ecmd.Dir = dir
  1061. err := ecmd.Run()
  1062. if err != nil {
  1063. log.Fatal(err)
  1064. }
  1065. }
  1066. func runPipe(file, cmd string, args ...string) {
  1067. if debug {
  1068. t0 := time.Now()
  1069. log.Println("runPipe:", cmd, strings.Join(args, " "))
  1070. defer func() {
  1071. log.Println("... in", time.Since(t0))
  1072. }()
  1073. }
  1074. fd, err := os.Create(file)
  1075. if err != nil {
  1076. log.Fatal(err)
  1077. }
  1078. ecmd := exec.Command(cmd, args...)
  1079. ecmd.Stdout = fd
  1080. ecmd.Stderr = os.Stderr
  1081. err = ecmd.Run()
  1082. if err != nil {
  1083. log.Fatal(err)
  1084. }
  1085. fd.Close()
  1086. }
  1087. func tarGz(out string, files []archiveFile) {
  1088. fd, err := os.Create(out)
  1089. if err != nil {
  1090. log.Fatal(err)
  1091. }
  1092. gw, err := gzip.NewWriterLevel(fd, gzip.BestCompression)
  1093. if err != nil {
  1094. log.Fatal(err)
  1095. }
  1096. tw := tar.NewWriter(gw)
  1097. for _, f := range files {
  1098. sf, err := os.Open(f.src)
  1099. if err != nil {
  1100. log.Fatal(err)
  1101. }
  1102. info, err := sf.Stat()
  1103. if err != nil {
  1104. log.Fatal(err)
  1105. }
  1106. h := &tar.Header{
  1107. Name: f.dst,
  1108. Size: info.Size(),
  1109. Mode: int64(info.Mode()),
  1110. ModTime: info.ModTime(),
  1111. }
  1112. err = tw.WriteHeader(h)
  1113. if err != nil {
  1114. log.Fatal(err)
  1115. }
  1116. _, err = io.Copy(tw, sf)
  1117. if err != nil {
  1118. log.Fatal(err)
  1119. }
  1120. sf.Close()
  1121. }
  1122. err = tw.Close()
  1123. if err != nil {
  1124. log.Fatal(err)
  1125. }
  1126. err = gw.Close()
  1127. if err != nil {
  1128. log.Fatal(err)
  1129. }
  1130. err = fd.Close()
  1131. if err != nil {
  1132. log.Fatal(err)
  1133. }
  1134. }
  1135. func zipFile(out string, files []archiveFile) {
  1136. fd, err := os.Create(out)
  1137. if err != nil {
  1138. log.Fatal(err)
  1139. }
  1140. zw := zip.NewWriter(fd)
  1141. var fw *flate.Writer
  1142. // Register the deflator.
  1143. zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
  1144. var err error
  1145. if fw == nil {
  1146. // Creating a flate compressor for every file is
  1147. // expensive, create one and reuse it.
  1148. fw, err = flate.NewWriter(out, flate.BestCompression)
  1149. } else {
  1150. fw.Reset(out)
  1151. }
  1152. return fw, err
  1153. })
  1154. for _, f := range files {
  1155. sf, err := os.Open(f.src)
  1156. if err != nil {
  1157. log.Fatal(err)
  1158. }
  1159. info, err := sf.Stat()
  1160. if err != nil {
  1161. log.Fatal(err)
  1162. }
  1163. fh, err := zip.FileInfoHeader(info)
  1164. if err != nil {
  1165. log.Fatal(err)
  1166. }
  1167. fh.Name = filepath.ToSlash(f.dst)
  1168. fh.Method = zip.Deflate
  1169. if strings.HasSuffix(f.dst, ".txt") {
  1170. // Text file. Read it and convert line endings.
  1171. bs, err := io.ReadAll(sf)
  1172. if err != nil {
  1173. log.Fatal(err)
  1174. }
  1175. bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\r', '\n'}, -1)
  1176. fh.UncompressedSize = uint32(len(bs))
  1177. fh.UncompressedSize64 = uint64(len(bs))
  1178. of, err := zw.CreateHeader(fh)
  1179. if err != nil {
  1180. log.Fatal(err)
  1181. }
  1182. of.Write(bs)
  1183. } else {
  1184. // Binary file. Copy verbatim.
  1185. of, err := zw.CreateHeader(fh)
  1186. if err != nil {
  1187. log.Fatal(err)
  1188. }
  1189. _, err = io.Copy(of, sf)
  1190. if err != nil {
  1191. log.Fatal(err)
  1192. }
  1193. }
  1194. }
  1195. err = zw.Close()
  1196. if err != nil {
  1197. log.Fatal(err)
  1198. }
  1199. err = fd.Close()
  1200. if err != nil {
  1201. log.Fatal(err)
  1202. }
  1203. }
  1204. func codesign(target target) {
  1205. switch goos {
  1206. case "windows":
  1207. windowsCodesign(target.BinaryName())
  1208. case "darwin":
  1209. macosCodesign(target.BinaryName())
  1210. }
  1211. }
  1212. func macosCodesign(file string) {
  1213. if pass := os.Getenv("CODESIGN_KEYCHAIN_PASS"); pass != "" {
  1214. bs, err := runError("security", "unlock-keychain", "-p", pass)
  1215. if err != nil {
  1216. log.Println("Codesign: unlocking keychain failed:", string(bs))
  1217. return
  1218. }
  1219. }
  1220. if id := os.Getenv("CODESIGN_IDENTITY"); id != "" {
  1221. bs, err := runError("codesign", "--options=runtime", "-s", id, file)
  1222. if err != nil {
  1223. log.Println("Codesign: signing failed:", string(bs))
  1224. return
  1225. }
  1226. log.Println("Codesign: successfully signed", file)
  1227. }
  1228. }
  1229. func windowsCodesign(file string) {
  1230. st := "signtool.exe"
  1231. if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
  1232. st = path
  1233. }
  1234. for i, algo := range []string{"sha1", "sha256"} {
  1235. args := []string{"sign", "/fd", algo}
  1236. if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
  1237. args = append(args, "/f", f)
  1238. } else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
  1239. // Decode the PFX certificate from base64.
  1240. bs, err := base64.RawStdEncoding.DecodeString(b)
  1241. if err != nil {
  1242. log.Println("Codesign: signing failed: decoding base64:", err)
  1243. return
  1244. }
  1245. // Write it to a temporary file
  1246. f, err := os.CreateTemp("", "codesign-*.pfx")
  1247. if err != nil {
  1248. log.Println("Codesign: signing failed: creating temp file:", err)
  1249. return
  1250. }
  1251. _ = f.Chmod(0600) // best effort remove other users' access
  1252. defer os.Remove(f.Name())
  1253. if _, err := f.Write(bs); err != nil {
  1254. log.Println("Codesign: signing failed: writing temp file:", err)
  1255. return
  1256. }
  1257. if err := f.Close(); err != nil {
  1258. log.Println("Codesign: signing failed: closing temp file:", err)
  1259. return
  1260. }
  1261. // Use that when signing
  1262. args = append(args, "/f", f.Name())
  1263. }
  1264. if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
  1265. args = append(args, "/p", p)
  1266. }
  1267. if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
  1268. switch algo {
  1269. case "sha256":
  1270. args = append(args, "/tr", tr, "/td", algo)
  1271. default:
  1272. args = append(args, "/t", tr)
  1273. }
  1274. }
  1275. if i > 0 {
  1276. args = append(args, "/as")
  1277. }
  1278. args = append(args, file)
  1279. bs, err := runError(st, args...)
  1280. if err != nil {
  1281. log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
  1282. return
  1283. }
  1284. log.Println("Codesign: successfully signed", file, "using", algo)
  1285. }
  1286. }
  1287. func metalint() {
  1288. lazyRebuildAssets()
  1289. runPrint(goCmd, "test", "-run", "Metalint", "./meta")
  1290. }
  1291. func metalintShort() {
  1292. lazyRebuildAssets()
  1293. runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
  1294. }
  1295. func (t target) BinaryName() string {
  1296. if goos == "windows" {
  1297. return t.binaryName + ".exe"
  1298. }
  1299. return t.binaryName
  1300. }
  1301. func protobufVersion() string {
  1302. bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
  1303. if err != nil {
  1304. log.Fatal("Getting protobuf version:", err)
  1305. }
  1306. return string(bs)
  1307. }
  1308. func currentAndLatestVersions(n int) ([]string, error) {
  1309. bs, err := runError("git", "tag", "--sort", "taggerdate")
  1310. if err != nil {
  1311. return nil, err
  1312. }
  1313. lines := strings.Split(string(bs), "\n")
  1314. reverseStrings(lines)
  1315. // The one at the head is the latest version. We always keep that one.
  1316. // Then we filter out remaining ones with dashes (pre-releases etc).
  1317. latest := lines[:1]
  1318. nonPres := filterStrings(lines[1:], func(s string) bool { return !strings.Contains(s, "-") })
  1319. vers := append(latest, nonPres...)
  1320. return vers[:n], nil
  1321. }
  1322. func reverseStrings(ss []string) {
  1323. for i := 0; i < len(ss)/2; i++ {
  1324. ss[i], ss[len(ss)-1-i] = ss[len(ss)-1-i], ss[i]
  1325. }
  1326. }
  1327. func filterStrings(ss []string, op func(string) bool) []string {
  1328. n := ss[:0]
  1329. for _, s := range ss {
  1330. if op(s) {
  1331. n = append(n, s)
  1332. }
  1333. }
  1334. return n
  1335. }
  1336. func tagMessage(tag string) (string, error) {
  1337. hash, err := runError("git", "rev-parse", tag)
  1338. if err != nil {
  1339. return "", err
  1340. }
  1341. obj, err := runError("git", "cat-file", "-p", string(hash))
  1342. if err != nil {
  1343. return "", err
  1344. }
  1345. return trimTagMessage(string(obj), tag), nil
  1346. }
  1347. func trimTagMessage(msg, tag string) string {
  1348. firstBlank := strings.Index(msg, "\n\n")
  1349. if firstBlank > 0 {
  1350. msg = msg[firstBlank+2:]
  1351. }
  1352. msg = strings.TrimPrefix(msg, tag)
  1353. beginSig := strings.Index(msg, "-----BEGIN PGP")
  1354. if beginSig > 0 {
  1355. msg = msg[:beginSig]
  1356. }
  1357. return strings.TrimSpace(msg)
  1358. }
  1359. func nextPatchVersion(ver string) string {
  1360. parts := strings.SplitN(ver, "-", 2)
  1361. digits := strings.Split(parts[0], ".")
  1362. n, _ := strconv.Atoi(digits[len(digits)-1])
  1363. digits[len(digits)-1] = strconv.Itoa(n + 1)
  1364. return strings.Join(digits, ".")
  1365. }