New global discovery protocol over HTTPS (fixes #628, fixes #1907)

Godeps/Godeps.json

@@ -43,11 +43,11 @@
 		},
 		{
 			"ImportPath": "github.com/syncthing/relaysrv/client",
-			"Rev": "7fe1fdd8c751df165ea825bc8d3e895f118bb236"
+			"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
 		},
 		{
 			"ImportPath": "github.com/syncthing/relaysrv/protocol",
-			"Rev": "7fe1fdd8c751df165ea825bc8d3e895f118bb236"
+			"Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
@@ -55,7 +55,7 @@
 		},
 		{
 			"ImportPath": "github.com/thejerf/suture",
-			"Rev": "fc7aaeabdc43fe41c5328efa1479ffea0b820978"
+			"Rev": "860b44045335c64a6d54ac7eed22a3aedfc687c9"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",

Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go

@@ -32,6 +32,7 @@ type ProtocolClient struct {
 
 	mut       sync.RWMutex
 	connected bool
+	latency   time.Duration
 }
 
 func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient {
@@ -168,6 +169,13 @@ func (c *ProtocolClient) StatusOK() bool {
 	return con
 }
 
+func (c *ProtocolClient) Latency() time.Duration {
+	c.mut.RLock()
+	lat := c.latency
+	c.mut.RUnlock()
+	return lat
+}
+
 func (c *ProtocolClient) String() string {
 	return fmt.Sprintf("ProtocolClient@%p", c)
 }
@@ -177,11 +185,21 @@ func (c *ProtocolClient) connect() error {
 		return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme)
 	}
 
-	conn, err := tls.Dial("tcp", c.URI.Host, c.config)
+	t0 := time.Now()
+	tcpConn, err := net.Dial("tcp", c.URI.Host)
 	if err != nil {
 		return err
 	}
 
+	c.mut.Lock()
+	c.latency = time.Since(t0)
+	c.mut.Unlock()
+
+	conn := tls.Client(tcpConn, c.config)
+	if err = conn.Handshake(); err != nil {
+		return err
+	}
+
 	if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
 		conn.Close()
 		return err

Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	syncthingprotocol "github.com/syncthing/protocol"
@@ -20,10 +21,10 @@ func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs [
 	}
 
 	conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs))
-	conn.SetDeadline(time.Now().Add(10 * time.Second))
 	if err != nil {
 		return protocol.SessionInvitation{}, err
 	}
+	conn.SetDeadline(time.Now().Add(10 * time.Second))
 
 	if err := performHandshakeAndValidation(conn, uri); err != nil {
 		return protocol.SessionInvitation{}, err
@@ -97,6 +98,29 @@ func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) {
 	}
 }
 
+func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool {
+	id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0])
+	invs := make(chan protocol.SessionInvitation, 1)
+	c := NewProtocolClient(uri, certs, invs)
+	go c.Serve()
+	defer func() {
+		close(invs)
+		c.Stop()
+	}()
+
+	for i := 0; i < times; i++ {
+		_, err := GetInvitationFromRelay(uri, id, certs)
+		if err == nil {
+			return true
+		}
+		if !strings.Contains(err.Error(), "Incorrect response code") {
+			return false
+		}
+		time.Sleep(sleep)
+	}
+	return false
+}
+
 func configForCerts(certs []tls.Certificate) *tls.Config {
 	return &tls.Config{
 		Certificates:           certs,

Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go

@@ -7,8 +7,9 @@ package protocol
 
 import (
 	"fmt"
-	syncthingprotocol "github.com/syncthing/protocol"
 	"net"
+
+	syncthingprotocol "github.com/syncthing/protocol"
 )
 
 const (

Godeps/_workspace/src/github.com/thejerf/suture/README.md

@@ -6,10 +6,8 @@ Suture
 Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
 "sutree" -> "suture" -> holds your code together when it's trying to die.
 
-This is intended to be a production-quality library going into code that I
-will be very early on the phone tree to support when it goes down. However,
-it has not been deployed into something quite that serious yet. (I will
-update this statement when that changes.)
+This library has hit maturity, and isn't expected to be changed
+radically. This can also be imported via gopkg.in/thejerf/suture.v1 .
 
 It is intended to deal gracefully with the real failure cases that can
 occur with supervision trees (such as burning all your CPU time endlessly
@@ -24,10 +22,6 @@ This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/su
 including an example, usage, and everything else you might expect from a
 README.md on GitHub. (DRY.)
 
-This is not currently tagged with particular git tags for Go as this is
-currently considered to be alpha code. As I move this into production and
-feel more confident about it, I'll give it relevant tags.
-
 Code Signing
 ------------
 

cmd/stfinddevice/main.go

@@ -7,41 +7,105 @@
 package main
 
 import (
+	"crypto/tls"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"net/url"
 	"os"
+	"time"
 
 	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
 )
 
-func main() {
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
+var timeout = 5 * time.Second
 
+func main() {
 	var server string
 
-	flag.StringVar(&server, "server", "udp4://announce.syncthing.net:22027", "Announce server")
+	flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
+	flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
+	flag.Usage = usage
 	flag.Parse()
 
-	if len(flag.Args()) != 1 || server == "" {
-		log.Printf("Usage: %s [-server=\"udp4://announce.syncthing.net:22027\"] <device>", os.Args[0])
+	if flag.NArg() != 1 {
+		flag.Usage()
 		os.Exit(64)
 	}
 
 	id, err := protocol.DeviceIDFromString(flag.Args()[0])
 	if err != nil {
-		log.Println(err)
+		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	discoverer := discover.NewDiscoverer(protocol.LocalDeviceID, nil, nil)
-	discoverer.StartGlobal([]string{server}, nil)
-	addresses, relays := discoverer.Lookup(id)
-	for _, addr := range addresses {
-		log.Println("address:", addr)
+	if server != "" {
+		checkServers(id, server)
+	} else {
+		checkServers(id, config.DefaultDiscoveryServers...)
+	}
+}
+
+type checkResult struct {
+	server string
+	direct []string
+	relays []discover.Relay
+	error
+}
+
+func checkServers(deviceID protocol.DeviceID, servers ...string) {
+	t0 := time.Now()
+	resc := make(chan checkResult)
+	for _, srv := range servers {
+		srv := srv
+		go func() {
+			res := checkServer(deviceID, srv)
+			res.server = srv
+			resc <- res
+		}()
+	}
+
+	for _ = range servers {
+		res := <-resc
+
+		u, _ := url.Parse(res.server)
+		fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
+
+		if res.error != nil {
+			fmt.Println("  " + res.error.Error())
+		}
+		for _, addr := range res.direct {
+			fmt.Println("  address:", addr)
+		}
+		for _, rel := range res.relays {
+			fmt.Printf("  relay: %s (%d ms)\n", rel.URL, rel.Latency)
+		}
 	}
-	for _, addr := range relays {
-		log.Println("relay:", addr)
+}
+
+func checkServer(deviceID protocol.DeviceID, server string) checkResult {
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return checkResult{error: err}
 	}
+
+	res := make(chan checkResult, 1)
+
+	time.AfterFunc(timeout, func() {
+		res <- checkResult{error: errors.New("timeout")}
+	})
+
+	go func() {
+		direct, relays, err := disco.Lookup(deviceID)
+		res <- checkResult{direct: direct, relays: relays, error: err}
+	}()
+
+	return <-res
+}
+
+func usage() {
+	fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
+	flag.PrintDefaults()
 }

cmd/syncthing/externaladdr.go → cmd/syncthing/addresslister.go

@@ -32,6 +32,17 @@ func newAddressLister(upnpSvc *upnpSvc, cfg *config.Wrapper) *addressLister {
 // port number - this means that the outside address of a NAT gateway should
 // be substituted.
 func (e *addressLister) ExternalAddresses() []string {
+	return e.addresses(false)
+}
+
+// AllAddresses returns a list of addresses that are our best guess for where
+// we are reachable from the local network. Same conditions as
+// ExternalAddresses, but private IPv4 addresses are included.
+func (e *addressLister) AllAddresses() []string {
+	return e.addresses(true)
+}
+
+func (e *addressLister) addresses(includePrivateIPV4 bool) []string {
 	var addrs []string
 
 	// Grab our listen addresses from the config. Unspecified ones are passed
@@ -56,6 +67,9 @@ func (e *addressLister) ExternalAddresses() []string {
 		} else if isPublicIPv4(addr.IP) || isPublicIPv6(addr.IP) {
 			// A public address; include as is.
 			addrs = append(addrs, "tcp://"+addr.String())
+		} else if includePrivateIPV4 && addr.IP.To4().IsGlobalUnicast() {
+			// A private IPv4 address.
+			addrs = append(addrs, "tcp://"+addr.String())
 		}
 	}
 
@@ -67,8 +81,6 @@ func (e *addressLister) ExternalAddresses() []string {
 		}
 	}
 
-	l.Infoln("External addresses:", addrs)
-
 	return addrs
 }
 

cmd/syncthing/connections.go

@@ -43,7 +43,7 @@ type connectionSvc struct {
 	myID       protocol.DeviceID
 	model      *model.Model
 	tlsCfg     *tls.Config
-	discoverer *discover.Discoverer
+	discoverer discover.Finder
 	conns      chan model.IntermediateConnection
 	relaySvc   *relay.Svc
 
@@ -54,7 +54,7 @@ type connectionSvc struct {
 	relaysEnabled bool
 }
 
-func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl *model.Model, tlsCfg *tls.Config, discoverer *discover.Discoverer, relaySvc *relay.Svc) *connectionSvc {
+func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, mdl *model.Model, tlsCfg *tls.Config, discoverer discover.Finder, relaySvc *relay.Svc) *connectionSvc {
 	svc := &connectionSvc{
 		Supervisor: suture.NewSimple("connectionSvc"),
 		cfg:        cfg,
@@ -264,13 +264,14 @@ func (s *connectionSvc) connect() {
 			}
 
 			var addrs []string
-			var relays []string
+			var relays []discover.Relay
 			for _, addr := range deviceCfg.Addresses {
 				if addr == "dynamic" {
 					if s.discoverer != nil {
-						t, r := s.discoverer.Lookup(deviceID)
-						addrs = append(addrs, t...)
-						relays = append(relays, r...)
+						if t, r, err := s.discoverer.Lookup(deviceID); err == nil {
+							addrs = append(addrs, t...)
+							relays = append(relays, r...)
+						}
 					}
 				} else {
 					addrs = append(addrs, addr)
@@ -333,7 +334,7 @@ func (s *connectionSvc) connect() {
 			s.lastRelayCheck[deviceID] = time.Now()
 
 			for _, addr := range relays {
-				uri, err := url.Parse(addr)
+				uri, err := url.Parse(addr.URL)
 				if err != nil {
 					l.Infoln("Failed to parse relay connection url:", addr, err)
 					continue

cmd/syncthing/gui.go

@@ -33,6 +33,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
@@ -58,14 +59,15 @@ type apiSvc struct {
 	assetDir        string
 	model           *model.Model
 	eventSub        *events.BufferedSubscription
-	discoverer      *discover.Discoverer
+	discoverer      *discover.CachingMux
+	relaySvc        *relay.Svc
 	listener        net.Listener
 	fss             *folderSummarySvc
 	stop            chan struct{}
 	systemConfigMut sync.Mutex
 }
 
-func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.Discoverer) (*apiSvc, error) {
+func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) (*apiSvc, error) {
 	svc := &apiSvc{
 		id:              id,
 		cfg:             cfg,
@@ -73,6 +75,7 @@ func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir strin
 		model:           m,
 		eventSub:        eventSub,
 		discoverer:      discoverer,
+		relaySvc:        relaySvc,
 		systemConfigMut: sync.NewMutex(),
 	}
 
@@ -164,7 +167,6 @@ func (s *apiSvc) Serve() {
 	postRestMux.HandleFunc("/rest/db/override", s.postDBOverride)              // folder
 	postRestMux.HandleFunc("/rest/db/scan", s.postDBScan)                      // folder [sub...] [delay]
 	postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig)          // <body>
-	postRestMux.HandleFunc("/rest/system/discovery", s.postSystemDiscovery)    // device addr
 	postRestMux.HandleFunc("/rest/system/error", s.postSystemError)            // <body>
 	postRestMux.HandleFunc("/rest/system/error/clear", s.postSystemErrorClear) // -
 	postRestMux.HandleFunc("/rest/system/ping", s.restPing)                    // -
@@ -630,11 +632,30 @@ func (s *apiSvc) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys - m.HeapReleased
 	res["tilde"] = tilde
-	if cfg.Options().GlobalAnnEnabled && s.discoverer != nil {
-		res["extAnnounceOK"] = s.discoverer.ExtAnnounceOK()
+	if cfg.Options().LocalAnnEnabled || cfg.Options().GlobalAnnEnabled {
+		res["discoveryEnabled"] = true
+		discoErrors := make(map[string]string)
+		discoMethods := 0
+		for disco, err := range s.discoverer.ChildErrors() {
+			discoMethods++
+			if err != nil {
+				discoErrors[disco] = err.Error()
+			}
+		}
+		res["discoveryMethods"] = discoMethods
+		res["discoveryErrors"] = discoErrors
 	}
-	if relaySvc != nil {
-		res["relayClientStatus"] = relaySvc.ClientStatus()
+	if s.relaySvc != nil {
+		res["relaysEnabled"] = true
+		relayClientStatus := make(map[string]bool)
+		relayClientLatency := make(map[string]int)
+		for _, relay := range s.relaySvc.Relays() {
+			latency, ok := s.relaySvc.RelayStatus(relay)
+			relayClientStatus[relay] = ok
+			relayClientLatency[relay] = int(latency / time.Millisecond)
+		}
+		res["relayClientStatus"] = relayClientStatus
+		res["relayClientLatency"] = relayClientLatency
 	}
 	cpuUsageLock.RLock()
 	var cpusum float64
@@ -679,25 +700,16 @@ func (s *apiSvc) showGuiError(l logger.LogLevel, err string) {
 	guiErrorsMut.Unlock()
 }
 
-func (s *apiSvc) postSystemDiscovery(w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var device = qs.Get("device")
-	var addr = qs.Get("addr")
-	if len(device) != 0 && len(addr) != 0 && s.discoverer != nil {
-		s.discoverer.Hint(device, []string{addr})
-	}
-}
-
 func (s *apiSvc) getSystemDiscovery(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	devices := map[string][]discover.CacheEntry{}
+	devices := make(map[string]discover.CacheEntry)
 
 	if s.discoverer != nil {
 		// Device ids can't be marshalled as keys so we need to manually
 		// rebuild this map using strings. Discoverer may be nil if discovery
 		// has not started yet.
-		for device, entries := range s.discoverer.All() {
-			devices[device.String()] = entries
+		for device, entry := range s.discoverer.Cache() {
+			devices[device.String()] = entry
 		}
 	}
 

cmd/syncthing/main.go

@@ -114,7 +114,6 @@ var (
 	writeRateLimit *ratelimit.Bucket
 	readRateLimit  *ratelimit.Bucket
 	stop           = make(chan int)
-	relaySvc       *relay.Svc
 	cert           tls.Certificate
 	lans           []*net.IPNet
 )
@@ -689,8 +688,7 @@ func syncthingMain() {
 
 	var addrList *addressLister
 
-	// Start UPnP. The UPnP service will restart global discovery if the
-	// external port changes.
+	// Start UPnP
 
 	if opts.UPnPEnabled {
 		upnpSvc := newUPnPSvc(cfg, addr.Port)
@@ -703,14 +701,6 @@ func syncthingMain() {
 		addrList = newAddressLister(nil, cfg)
 	}
 
-	// Start discovery
-
-	discoverer := discovery(addrList, relaySvc)
-
-	// GUI
-
-	setupGUI(mainSvc, cfg, m, apiSub, discoverer)
-
 	// Start relay management
 
 	var relaySvc *relay.Svc
@@ -719,9 +709,51 @@ func syncthingMain() {
 		mainSvc.Add(relaySvc)
 	}
 
+	// Start discovery
+
+	cachedDiscovery := discover.NewCachingMux()
+	mainSvc.Add(cachedDiscovery)
+
+	if cfg.Options().GlobalAnnEnabled {
+		for _, srv := range cfg.GlobalDiscoveryServers() {
+			l.Infoln("Using discovery server", srv)
+			gd, err := discover.NewGlobal(srv, cert, addrList, relaySvc)
+			if err != nil {
+				l.Warnln("Global discovery:", err)
+				continue
+			}
+
+			// Each global discovery server gets its results cached for five
+			// minutes, and is not asked again for a minute when it's returned
+			// unsuccessfully.
+			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute)
+		}
+	}
+
+	if cfg.Options().LocalAnnEnabled {
+		// v4 broadcasts
+		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv4 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(bcd, 0, 0)
+		}
+		// v6 multicasts
+		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, addrList, relaySvc)
+		if err != nil {
+			l.Warnln("IPv6 local discovery:", err)
+		} else {
+			cachedDiscovery.Add(mcd, 0, 0)
+		}
+	}
+
+	// GUI
+
+	setupGUI(mainSvc, cfg, m, apiSub, cachedDiscovery, relaySvc)
+
 	// Start connection management
 
-	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg, discoverer, relaySvc)
+	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg, cachedDiscovery, relaySvc)
 	mainSvc.Add(connectionSvc)
 
 	if cpuProfile {
@@ -844,7 +876,7 @@ func startAuditing(mainSvc *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.Discoverer) {
+func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription, discoverer *discover.CachingMux, relaySvc *relay.Svc) {
 	opts := cfg.Options()
 	guiCfg := overrideGUIConfig(cfg.GUI(), guiAddress, guiAuthentication, guiAPIKey)
 
@@ -873,7 +905,7 @@ func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, a
 
 			urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
 			l.Infoln("Starting web GUI on", urlShow)
-			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub, discoverer)
+			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub, discoverer, relaySvc)
 			if err != nil {
 				l.Fatalln("Cannot start GUI:", err)
 			}
@@ -944,28 +976,6 @@ func shutdown() {
 	stop <- exitSuccess
 }
 
-func discovery(addrList *addressLister, relaySvc *relay.Svc) *discover.Discoverer {
-	opts := cfg.Options()
-	disc := discover.NewDiscoverer(myID, opts.ListenAddress, relaySvc)
-	if opts.LocalAnnEnabled {
-		l.Infoln("Starting local discovery announcements")
-		disc.StartLocal(opts.LocalAnnPort, opts.LocalAnnMCAddr)
-	}
-
-	if opts.GlobalAnnEnabled {
-		go func() {
-			// Defer starting global announce server, giving time to connect
-			// to relay servers.
-			time.Sleep(5 * time.Second)
-			l.Infoln("Starting global discovery announcements")
-			disc.StartGlobal(opts.GlobalAnnServers, addrList)
-		}()
-
-	}
-
-	return disc
-}
-
 func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {

cmd/syncthing/verbose.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/syncthing/syncthing/lib/events"
 )
@@ -139,6 +140,16 @@ func (s *verboseSvc) formatEvent(ev events.Event) string {
 		data := ev.Data.(map[string]string)
 		device := data["device"]
 		return fmt.Sprintf("Device %v was resumed", device)
+
+	case events.ExternalPortMappingChanged:
+		data := ev.Data.(map[string]int)
+		port := data["port"]
+		return fmt.Sprintf("External port mapping changed; new port is %d.", port)
+	case events.RelayStateChanged:
+		data := ev.Data.(map[string][]string)
+		newRelays := data["new"]
+		return fmt.Sprintf("Relay state changed; connected relay(s) are %s.", strings.Join(newRelays, ", "))
+
 	}
 
 	return fmt.Sprintf("%s %#v", ev.Type, ev)

gui/assets/css/overrides.css

@@ -249,3 +249,7 @@ ul.three-columns li, ul.two-columns li {
         position: static;
     }
 }
+
+.popover {
+    min-width: 250px;
+}

gui/index.html

@@ -379,28 +379,28 @@
                     <th><span class="fa fa-fw fa-tachometer"></span>&nbsp;<span translate>CPU Utilization</span></th>
                     <td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
                   </tr>
-                  <tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
-                    <th><span class="fa fa-fw fa-bullhorn"></span>&nbsp;<span translate>Global Discovery</span></th>
+                  <tr ng-if="system.discoveryEnabled">
+                    <th><span class="fa fa-fw fa-map-signs"></span>&nbsp;<span translate>Discovery</span></th>
                     <td class="text-right">
-                      <span ng-if="announceServersFailed.length == 0" class="data text-success">
-                        <span>OK</span>
+                      <span ng-if="discoveryFailed.length == 0" class="data text-success">
+                        <span>{{discoveryTotal}}/{{discoveryTotal}}</span>
                       </span>
-                      <span ng-if="announceServersFailed.length != 0" class="data" ng-class="{'text-danger': announceServersFailed.length == announceServersTotal}">
-                        <span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
-                          {{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
+                      <span ng-if="discoveryFailed.length != 0" class="data" ng-class="{'text-danger': discoveryFailed.length == discoveryTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{discoveryFailed.join('<br>\n')}}">
+                          {{discoveryTotal-discoveryFailed.length}}/{{discoveryTotal}}
                         </span>
                       </span>
                     </td>
                   </tr>
-                  <tr ng-if="system.relayClientStatus != undefined && relayClientsTotal > 0">
+                  <tr ng-if="system.relaysEnabled">
                     <th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Relays</span></th>
                     <td class="text-right">
-                      <span ng-if="relayClientsFailed.length == 0" class="data text-success">
-                        <span>OK</span>
+                      <span ng-if="relaysFailed.length == 0" class="data text-success">
+                        <span>{{relaysTotal}}/{{relaysTotal}}</span>
                       </span>
-                      <span ng-if="relayClientsFailed.length != 0" class="data" ng-class="{'text-danger': relayClientsFailed.length == relayClientsTotal}">
-                        <span popover data-trigger="hover" data-placement="bottom" data-content="{{relayClientsFailed.join('\n')}}">
-                          {{relayClientsTotal-relayClientsFailed.length}}/{{relayClientsTotal}}
+                      <span ng-if="relaysFailed.length != 0" class="data" ng-class="{'text-danger': relaysFailed.length == relaysTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{relaysFailed.join('<br>\n')}}">
+                          {{relaysTotal-relaysFailed.length}}/{{relaysTotal}}
                         </span>
                       </span>
                     </td>

gui/syncthing/core/syncthingController.js

@@ -378,24 +378,25 @@ angular.module('syncthing.core')
                 $scope.myID = data.myID;
                 $scope.system = data;
 
-                $scope.announceServersTotal = data.extAnnounceOK ? Object.keys(data.extAnnounceOK).length : 0;
-                var failedAnnounce = [];
-                for (var server in data.extAnnounceOK) {
-                    if (!data.extAnnounceOK[server]) {
-                        failedAnnounce.push(server);
+                $scope.discoveryTotal = data.discoveryMethods;
+                var discoveryFailed = [];
+                for (var disco in data.discoveryErrors) {
+                    if (data.discoveryErrors[disco]) {
+                        discoveryFailed.push(disco + ": " + data.discoveryErrors[disco]);
                     }
                 }
-                $scope.announceServersFailed = failedAnnounce;
+                $scope.discoveryFailed = discoveryFailed;
 
-                $scope.relayClientsTotal = data.relayClientStatus ? Object.keys(data.relayClientStatus).length : 0;
-                var failedRelays = [];
+                var relaysFailed = [];
+                var relaysTotal = 0;
                 for (var relay in data.relayClientStatus) {
                     if (!data.relayClientStatus[relay]) {
-                        failedRelays.push(relay);
+                        relaysFailed.push(relay);
                     }
+                    relaysTotal++;
                 }
-                $scope.relayClientsFailed = failedRelays;
-
+                $scope.relaysFailed = relaysFailed;
+                $scope.relaysTotal = relaysTotal;
 
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);

gui/syncthing/device/editDeviceModalView.html

@@ -13,7 +13,7 @@
             <label translate for="deviceID">Device ID</label>
             <input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.deviceID" required valid-deviceid list="discovery-list" />
             <datalist id="discovery-list" ng-if="!editingExisting">
-              <option ng-repeat="(id,address) in discovery" value="{{ id }}" />
+              <option ng-repeat="(id, data) in discovery" value="{{id}}" />
             </datalist>
             <div ng-if="editingExisting" class="well well-sm text-monospace">{{currentDevice.deviceID}}</div>
             <p class="help-block">

lib/beacon/beacon.go

@@ -6,7 +6,12 @@
 
 package beacon
 
-import "net"
+import (
+	"net"
+	stdsync "sync"
+
+	"github.com/thejerf/suture"
+)
 
 type recv struct {
 	data []byte
@@ -14,34 +19,30 @@ type recv struct {
 }
 
 type Interface interface {
+	suture.Service
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
+	Error() error
 }
 
 type readerFrom interface {
 	ReadFrom([]byte) (int, net.Addr, error)
 }
 
-func genericReader(conn readerFrom, outbox chan<- recv) {
-	bs := make([]byte, 65536)
-	for {
-		n, addr, err := conn.ReadFrom(bs)
-		if err != nil {
-			l.Warnln("multicast read:", err)
-			return
-		}
-		if debug {
-			l.Debugf("recv %d bytes from %s", n, addr)
-		}
-
-		c := make([]byte, n)
-		copy(c, bs)
-		select {
-		case outbox <- recv{c, addr}:
-		default:
-			if debug {
-				l.Debugln("dropping message")
-			}
-		}
-	}
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
+
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
 }

lib/beacon/broadcast.go

@@ -19,6 +19,8 @@ type Broadcast struct {
 	port   int
 	inbox  chan []byte
 	outbox chan recv
+	br     *broadcastReader
+	bw     *broadcastWriter
 }
 
 func NewBroadcast(port int) *Broadcast {
@@ -41,14 +43,16 @@ func NewBroadcast(port int) *Broadcast {
 		outbox: make(chan recv, 16),
 	}
 
-	b.Add(&broadcastReader{
+	b.br = &broadcastReader{
 		port:   port,
 		outbox: b.outbox,
-	})
-	b.Add(&broadcastWriter{
+	}
+	b.Add(b.br)
+	b.bw = &broadcastWriter{
 		port:  port,
 		inbox: b.inbox,
-	})
+	}
+	b.Add(b.bw)
 
 	return b
 }
@@ -62,11 +66,18 @@ func (b *Broadcast) Recv() ([]byte, net.Addr) {
 	return recv.data, recv.src
 }
 
+func (b *Broadcast) Error() error {
+	if err := b.br.Error(); err != nil {
+		return err
+	}
+	return b.bw.Error()
+}
+
 type broadcastWriter struct {
-	port   int
-	inbox  chan []byte
-	conn   *net.UDPConn
-	failed bool // Have we already logged a failure reason?
+	port  int
+	inbox chan []byte
+	conn  *net.UDPConn
+	errorHolder
 }
 
 func (w *broadcastWriter) Serve() {
@@ -78,22 +89,21 @@ func (w *broadcastWriter) Serve() {
 	var err error
 	w.conn, err = net.ListenUDP("udp4", nil)
 	if err != nil {
-		if !w.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			w.failed = true
+		if debug {
+			l.Debugln(err)
 		}
+		w.setError(err)
 		return
 	}
 	defer w.conn.Close()
 
-	w.failed = false
-
 	for bs := range w.inbox {
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
 			if debug {
-				l.Debugln("Local discovery (broadcast writer):", err)
+				l.Debugln(err)
 			}
+			w.setError(err)
 			continue
 		}
 
@@ -117,13 +127,16 @@ func (w *broadcastWriter) Serve() {
 		for _, ip := range dsts {
 			dst := &net.UDPAddr{IP: ip, Port: w.port}
 
-			w.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
+			w.conn.SetWriteDeadline(time.Now().Add(time.Second))
 			_, err := w.conn.WriteTo(bs, dst)
+			w.conn.SetWriteDeadline(time.Time{})
 			if err, ok := err.(net.Error); ok && err.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
+				if debug {
+					l.Debugln(err)
+				}
+				w.setError(err)
 				return
 			} else if err, ok := err.(net.Error); ok && err.Temporary() {
 				// A transient error. Lets hope for better luck in the future.
@@ -133,11 +146,14 @@ func (w *broadcastWriter) Serve() {
 				continue
 			} else if err != nil {
 				// Some other error that we don't expect. Bail and retry.
-				l.Infoln("Local discovery (broadcast writer):", err)
-				w.failed = true
+				if debug {
+					l.Debugln(err)
+				}
+				w.setError(err)
 				return
 			} else if debug {
 				l.Debugf("sent %d bytes to %s", len(bs), dst)
+				w.setError(nil)
 			}
 		}
 	}
@@ -155,7 +171,7 @@ type broadcastReader struct {
 	port   int
 	outbox chan recv
 	conn   *net.UDPConn
-	failed bool
+	errorHolder
 }
 
 func (r *broadcastReader) Serve() {
@@ -167,10 +183,10 @@ func (r *broadcastReader) Serve() {
 	var err error
 	r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
 	if err != nil {
-		if !r.failed {
-			l.Warnln("Local discovery over IPv4 unavailable:", err)
-			r.failed = true
+		if debug {
+			l.Debugln(err)
 		}
+		r.setError(err)
 		return
 	}
 	defer r.conn.Close()
@@ -179,14 +195,14 @@ func (r *broadcastReader) Serve() {
 	for {
 		n, addr, err := r.conn.ReadFrom(bs)
 		if err != nil {
-			if !r.failed {
-				l.Infoln("Local discovery (broadcast reader):", err)
-				r.failed = true
+			if debug {
+				l.Debugln(err)
 			}
+			r.setError(err)
 			return
 		}
 
-		r.failed = false
+		r.setError(nil)
 
 		if debug {
 			l.Debugf("recv %d bytes from %s", n, addr)

lib/beacon/multicast.go

@@ -8,39 +8,200 @@ package beacon
 
 import (
 	"errors"
+	"fmt"
 	"net"
+	"time"
 
+	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
 )
 
 type Multicast struct {
-	conn   *ipv6.PacketConn
+	*suture.Supervisor
 	addr   *net.UDPAddr
 	inbox  chan []byte
 	outbox chan recv
-	intfs  []net.Interface
+	mr     *multicastReader
+	mw     *multicastWriter
 }
 
-func NewMulticast(addr string) (*Multicast, error) {
-	gaddr, err := net.ResolveUDPAddr("udp6", addr)
+func NewMulticast(addr string) *Multicast {
+	m := &Multicast{
+		Supervisor: suture.New("multicastBeacon", suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				if debug {
+					l.Debugln(line)
+				}
+			},
+		}),
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	m.mr = &multicastReader{
+		addr:   addr,
+		outbox: m.outbox,
+		stop:   make(chan struct{}),
+	}
+	m.Add(m.mr)
+
+	m.mw = &multicastWriter{
+		addr:  addr,
+		inbox: m.inbox,
+		stop:  make(chan struct{}),
+	}
+	m.Add(m.mw)
+
+	return m
+}
+
+func (m *Multicast) Send(data []byte) {
+	m.inbox <- data
+}
+
+func (m *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-m.outbox
+	return recv.data, recv.src
+}
+
+func (m *Multicast) Error() error {
+	if err := m.mr.Error(); err != nil {
+		return err
+	}
+	return m.mw.Error()
+}
+
+type multicastWriter struct {
+	addr  string
+	inbox <-chan []byte
+	errorHolder
+	stop chan struct{}
+}
+
+func (w *multicastWriter) Serve() {
+	if debug {
+		l.Debugln(w, "starting")
+		defer l.Debugln(w, "stopping")
+	}
+
+	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
+	if err != nil {
+		if debug {
+			l.Debugln(err)
+		}
+		w.setError(err)
+		return
+	}
+
+	conn, err := net.ListenPacket("udp6", ":0")
+	if err != nil {
+		if debug {
+			l.Debugln(err)
+		}
+		w.setError(err)
+		return
+	}
+
+	pconn := ipv6.NewPacketConn(conn)
+
+	wcm := &ipv6.ControlMessage{
+		HopLimit: 1,
+	}
+
+	for bs := range w.inbox {
+		intfs, err := net.Interfaces()
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			w.setError(err)
+			return
+		}
+
+		var success int
+
+		for _, intf := range intfs {
+			wcm.IfIndex = intf.Index
+			pconn.SetWriteDeadline(time.Now().Add(time.Second))
+			_, err = pconn.WriteTo(bs, wcm, gaddr)
+			pconn.SetWriteDeadline(time.Time{})
+			if err != nil && debug {
+				l.Debugln(err, "on write to", gaddr, intf.Name)
+			} else if debug {
+				l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)
+				success++
+			}
+		}
+
+		if success > 0 {
+			w.setError(nil)
+		} else {
+			if debug {
+				l.Debugln(err)
+			}
+			w.setError(err)
+		}
+	}
+}
+
+func (w *multicastWriter) Stop() {
+	close(w.stop)
+}
+
+func (w *multicastWriter) String() string {
+	return fmt.Sprintf("multicastWriter@%p", w)
+}
+
+type multicastReader struct {
+	addr   string
+	outbox chan<- recv
+	errorHolder
+	stop chan struct{}
+}
+
+func (r *multicastReader) Serve() {
+	if debug {
+		l.Debugln(r, "starting")
+		defer l.Debugln(r, "stopping")
+	}
+
+	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
 	if err != nil {
-		return nil, err
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
 	}
 
-	conn, err := net.ListenPacket("udp6", addr)
+	conn, err := net.ListenPacket("udp6", r.addr)
 	if err != nil {
-		return nil, err
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
 	}
 
 	intfs, err := net.Interfaces()
 	if err != nil {
-		return nil, err
+		if debug {
+			l.Debugln(err)
+		}
+		r.setError(err)
+		return
 	}
 
-	p := ipv6.NewPacketConn(conn)
+	pconn := ipv6.NewPacketConn(conn)
 	joined := 0
 	for _, intf := range intfs {
-		err := p.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
+		err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
 		if debug {
 			if err != nil {
 				l.Debugln("IPv6 join", intf.Name, "failed:", err)
@@ -52,57 +213,43 @@ func NewMulticast(addr string) (*Multicast, error) {
 	}
 
 	if joined == 0 {
-		return nil, errors.New("no multicast interfaces available")
-	}
-
-	b := &Multicast{
-		conn:   p,
-		addr:   gaddr,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-		intfs:  intfs,
+		if debug {
+			l.Debugln("no multicast interfaces available")
+		}
+		r.setError(errors.New("no multicast interfaces available"))
+		return
 	}
 
-	go genericReader(ipv6ReaderAdapter{b.conn}, b.outbox)
-	go b.writer()
-
-	return b, nil
-}
-
-func (b *Multicast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Multicast) writer() {
-	wcm := &ipv6.ControlMessage{
-		HopLimit: 1,
-	}
+	bs := make([]byte, 65536)
+	for {
+		n, _, addr, err := pconn.ReadFrom(bs)
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			r.setError(err)
+			continue
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
 
-	for bs := range b.inbox {
-		for _, intf := range b.intfs {
-			wcm.IfIndex = intf.Index
-			_, err := b.conn.WriteTo(bs, wcm, b.addr)
-			if err != nil && debug {
-				l.Debugln(err, "on write to", b.addr)
-			} else if debug {
-				l.Debugf("sent %d bytes to %v on %s", len(bs), b.addr, intf.Name)
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case r.outbox <- recv{c, addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
 			}
 		}
 	}
 }
 
-// This makes ReadFrom on an *ipv6.PacketConn behave like ReadFrom on a
-// net.PacketConn.
-type ipv6ReaderAdapter struct {
-	c *ipv6.PacketConn
+func (r *multicastReader) Stop() {
+	close(r.stop)
 }
 
-func (i ipv6ReaderAdapter) ReadFrom(bs []byte) (int, net.Addr, error) {
-	n, _, src, err := i.c.ReadFrom(bs)
-	return n, src, err
+func (r *multicastReader) String() string {
+	return fmt.Sprintf("multicastReader@%p", r)
 }

lib/config/config.go

@@ -31,6 +31,21 @@ const (
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
+var (
+	// DefaultDiscoveryServers should be substituted when the configuration
+	// contains <globalAnnounceServer>default</globalAnnounceServer>. This is
+	// done by the "consumer" of the configuration, as we don't want these
+	// saved to the config.
+	DefaultDiscoveryServers = []string{
+		"https://v4-1.discover.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 194.126.249.5, Sweden
+		"https://v4-2.discover.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 45.55.230.38, USA
+		"https://v4-3.discover.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 128.199.95.124, Singapore
+		"https://v6-1.discover.syncthing.net/?id=SR7AARM-TCBUZ5O-VFAXY4D-CECGSDE-3Q6IZ4G-XG7AH75-OBIXJQV-QJ6NLQA", // 2001:470:28:4d6::5, Sweden
+		"https://v6-2.discover.syncthing.net/?id=AQEHEO2-XOS7QRA-X2COH5K-PO6OPVA-EWOSEGO-KZFMD32-XJ4ZV46-CUUVKAS", // 2604:a880:800:10::182:a001, USA
+		"https://v6-3.discover.syncthing.net/?id=7WT2BVR-FX62ZOW-TNVVW25-6AHFJGD-XEXQSBW-VO3MPL2-JBTLL4T-P4572Q4", // 2400:6180:0:d0::d9:d001, Singapore
+	}
+)
+
 type Configuration struct {
 	Version        int                   `xml:"version,attr" json:"version"`
 	Folders        []FolderConfiguration `xml:"folder" json:"folders"`
@@ -215,7 +230,7 @@ type FolderDeviceConfiguration struct {
 
 type OptionsConfiguration struct {
 	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"tcp://0.0.0.0:22000"`
-	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"udp4://announce.syncthing.net:22027, udp6://announce-v6.syncthing.net:22027"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
 	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
 	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
@@ -498,17 +513,21 @@ func convertV11V12(cfg *Configuration) {
 	}
 
 	// Use new discovery server
-	for i, addr := range cfg.Options.GlobalAnnServers {
+	var newDiscoServers []string
+	var useDefault bool
+	for _, addr := range cfg.Options.GlobalAnnServers {
 		if addr == "udp4://announce.syncthing.net:22026" {
-			cfg.Options.GlobalAnnServers[i] = "udp4://announce.syncthing.net:22027"
+			useDefault = true
 		} else if addr == "udp6://announce-v6.syncthing.net:22026" {
-			cfg.Options.GlobalAnnServers[i] = "udp6://announce-v6.syncthing.net:22027"
-		} else if addr == "udp4://194.126.249.5:22026" {
-			cfg.Options.GlobalAnnServers[i] = "udp4://194.126.249.5:22027"
-		} else if addr == "udp6://[2001:470:28:4d6::5]:22026" {
-			cfg.Options.GlobalAnnServers[i] = "udp6://[2001:470:28:4d6::5]:22027"
+			useDefault = true
+		} else {
+			newDiscoServers = append(newDiscoServers, addr)
 		}
 	}
+	if useDefault {
+		newDiscoServers = append(newDiscoServers, "default")
+	}
+	cfg.Options.GlobalAnnServers = newDiscoServers
 
 	// Use new multicast group
 	if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" {

lib/config/config_test.go

@@ -32,7 +32,7 @@ func init() {
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
 		ListenAddress:           []string{"tcp://0.0.0.0:22000"},
-		GlobalAnnServers:        []string{"udp4://announce.syncthing.net:22027", "udp6://announce-v6.syncthing.net:22027"},
+		GlobalAnnServers:        []string{"default"},
 		GlobalAnnEnabled:        true,
 		LocalAnnEnabled:         true,
 		LocalAnnPort:            21027,

lib/config/wrapper.go

@@ -317,3 +317,15 @@ func (w *Wrapper) Save() error {
 	events.Default.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
+
+func (w *Wrapper) GlobalDiscoveryServers() []string {
+	var servers []string
+	for _, srv := range w.cfg.Options.GlobalAnnServers {
+		if srv == "default" {
+			servers = append(servers, DefaultDiscoveryServers...)
+		} else {
+			servers = append(servers, srv)
+		}
+	}
+	return uniqueStrings(servers)
+}

lib/discover/cache.go

@@ -0,0 +1,192 @@
+package discover
+
+import (
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
+)
+
+// The CachingMux aggregates results from multiple Finders. Each Finder has
+// an associated cache time and negative cache time. The cache time sets how
+// long we cache and return successfull lookup results, the negative cache
+// time sets how long we refrain from asking about the same device ID after
+// receiving a negative answer. The value of zero disables caching (positive
+// or negative).
+type CachingMux struct {
+	*suture.Supervisor
+	finders []cachedFinder
+	caches  []*cache
+	mut     sync.Mutex
+}
+
+// A cachedFinder is a Finder with associated cache timeouts.
+type cachedFinder struct {
+	Finder
+	cacheTime    time.Duration
+	negCacheTime time.Duration
+}
+
+func NewCachingMux() *CachingMux {
+	return &CachingMux{
+		Supervisor: suture.NewSimple("discover.cachingMux"),
+		mut:        sync.NewMutex(),
+	}
+}
+
+// Add registers a new Finder, with associated cache timeouts.
+func (m *CachingMux) Add(finder Finder, cacheTime, negCacheTime time.Duration) {
+	m.mut.Lock()
+	m.finders = append(m.finders, cachedFinder{finder, cacheTime, negCacheTime})
+	m.caches = append(m.caches, newCache())
+	m.mut.Unlock()
+
+	if svc, ok := finder.(suture.Service); ok {
+		m.Supervisor.Add(svc)
+	}
+}
+
+// Lookup attempts to resolve the device ID using any of the added Finders,
+// while obeying the cache settings.
+func (m *CachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	m.mut.Lock()
+	for i, finder := range m.finders {
+		if cacheEntry, ok := m.caches[i].Get(deviceID); ok {
+			// We have a cache entry. Lets see what it says.
+
+			if cacheEntry.found && time.Since(cacheEntry.when) < finder.cacheTime {
+				// It's a positive, valid entry. Use it.
+				if debug {
+					l.Debugln("cached discovery entry for", deviceID, "at", finder.String())
+					l.Debugln("   ", cacheEntry)
+				}
+				direct = append(direct, cacheEntry.Direct...)
+				relays = append(relays, cacheEntry.Relays...)
+				continue
+			}
+
+			if !cacheEntry.found && time.Since(cacheEntry.when) < finder.negCacheTime {
+				// It's a negative, valid entry. We should not make another
+				// attempt right now.
+				if debug {
+					l.Debugln("negative cache entry for", deviceID, "at", finder.String())
+				}
+				continue
+			}
+
+			// It's expired. Ignore and continue.
+		}
+
+		// Perform the actual lookup and cache the result.
+		if td, tr, err := finder.Lookup(deviceID); err == nil {
+			if debug {
+				l.Debugln("lookup for", deviceID, "at", finder.String())
+				l.Debugln("   ", td)
+				l.Debugln("   ", tr)
+			}
+			direct = append(direct, td...)
+			relays = append(relays, tr...)
+			m.caches[i].Set(deviceID, CacheEntry{
+				Direct: td,
+				Relays: tr,
+				when:   time.Now(),
+				found:  len(td)+len(tr) > 0,
+			})
+		}
+	}
+	m.mut.Unlock()
+
+	if debug {
+		l.Debugln("lookup results for", deviceID)
+		l.Debugln("   ", direct)
+		l.Debugln("   ", relays)
+	}
+
+	return direct, relays, nil
+}
+
+func (m *CachingMux) String() string {
+	return "discovery cache"
+}
+
+func (m *CachingMux) Error() error {
+	return nil
+}
+
+func (m *CachingMux) ChildErrors() map[string]error {
+	m.mut.Lock()
+	children := make(map[string]error, len(m.finders))
+	for _, f := range m.finders {
+		children[f.String()] = f.Error()
+	}
+	m.mut.Unlock()
+	return children
+}
+
+func (m *CachingMux) Cache() map[protocol.DeviceID]CacheEntry {
+	// Res will be the "total" cache, i.e. the union of our cache and all our
+	// children's caches.
+	res := make(map[protocol.DeviceID]CacheEntry)
+
+	m.mut.Lock()
+	for i := range m.finders {
+		// Each finder[i] has a corresponding cache at cache[i]. Go through it
+		// and populate the total, if it's newer than what's already in there.
+		// We skip any negative cache entries.
+		for k, v := range m.caches[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+
+		// Then ask the finder itself for it's cache and do the same. If this
+		// finder is a global discovery client, it will have no cache. If it's
+		// a local discovery client, this will be it's current state.
+		for k, v := range m.finders[i].Cache() {
+			if v.found && v.when.After(res[k].when) {
+				res[k] = v
+			}
+		}
+	}
+	m.mut.Unlock()
+
+	return res
+}
+
+// A cache can be embedded wherever useful
+
+type cache struct {
+	entries map[protocol.DeviceID]CacheEntry
+	mut     stdsync.Mutex
+}
+
+func newCache() *cache {
+	return &cache{
+		entries: make(map[protocol.DeviceID]CacheEntry),
+	}
+}
+
+func (c *cache) Set(id protocol.DeviceID, ce CacheEntry) {
+	c.mut.Lock()
+	c.entries[id] = ce
+	c.mut.Unlock()
+}
+
+func (c *cache) Get(id protocol.DeviceID) (CacheEntry, bool) {
+	c.mut.Lock()
+	ce, ok := c.entries[id]
+	c.mut.Unlock()
+	return ce, ok
+}
+
+func (c *cache) Cache() map[protocol.DeviceID]CacheEntry {
+	c.mut.Lock()
+	m := make(map[protocol.DeviceID]CacheEntry, len(c.entries))
+	for k, v := range c.entries {
+		m[k] = v
+	}
+	c.mut.Unlock()
+	return m
+}

lib/discover/client.go

@@ -1,54 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net/url"
-	"time"
-
-	"github.com/syncthing/protocol"
-)
-
-type Announcer interface {
-	Announcement() Announce
-}
-
-type Factory func(*url.URL, Announcer) (Client, error)
-
-var (
-	factories                      = make(map[string]Factory)
-	DefaultErrorRetryInternval     = 60 * time.Second
-	DefaultGlobalBroadcastInterval = 1800 * time.Second
-)
-
-func Register(proto string, factory Factory) {
-	factories[proto] = factory
-}
-
-func New(addr string, announcer Announcer) (Client, error) {
-	uri, err := url.Parse(addr)
-	if err != nil {
-		return nil, err
-	}
-	factory, ok := factories[uri.Scheme]
-	if !ok {
-		return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
-	}
-	client, err := factory(uri, announcer)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-type Client interface {
-	Lookup(device protocol.DeviceID) (Announce, error)
-	StatusOK() bool
-	Address() string
-	Stop()
-}

lib/discover/client_test.go

@@ -1,239 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-var device protocol.DeviceID
-
-func init() {
-	device, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
-}
-
-type FakeAnnouncer struct {
-	pkt Announce
-}
-
-func (f *FakeAnnouncer) Announcement() Announce {
-	return f.pkt
-}
-
-func TestUDP4Success(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d", port)
-	pkt := Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]string{"tcp://123.123.123.123:1234"},
-			nil,
-		},
-	}
-	ann := &FakeAnnouncer{
-		pkt: pkt,
-	}
-
-	client, err := New(address, ann)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != DefaultErrorRetryInternval {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if udpclient.listenAddress.IP != nil || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, addr, err := conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Reply to it.
-	_, err = conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be ok
-	if !client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		pkt, err := client.Lookup(device)
-		if err == nil {
-			for _, addr := range pkt.This.Addresses {
-				addrs = append(addrs, addr)
-			}
-		}
-		wg.Done()
-	}()
-
-	// Receive the lookup and reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, addr, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
-
-	// Wait for the lookup to arrive, verify that the number of answers is correct
-	wg.Wait()
-
-	if len(addrs) != 1 || addrs[0] != "tcp://123.123.123.123:1234" {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}
-
-func TestUDP4Failure(t *testing.T) {
-	conn, err := net.ListenUDP("udp4", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-
-	address := fmt.Sprintf("udp4://127.0.0.1:%d/?listenaddress=127.0.0.1&retry=5", port)
-
-	pkt := Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			device[:],
-			[]string{"tcp://123.123.123.123:1234"},
-			nil,
-		},
-	}
-	ann := &FakeAnnouncer{
-		pkt: pkt,
-	}
-
-	client, err := New(address, ann)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	udpclient := client.(*UDPClient)
-	if udpclient.errorRetryInterval != time.Second*5 {
-		t.Fatal("Incorrect retry interval")
-	}
-
-	if !udpclient.listenAddress.IP.Equal(net.IPv4(127, 0, 0, 1)) || udpclient.listenAddress.Port != 0 {
-		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
-	}
-
-	if client.Address() != address {
-		t.Fatal("Incorrect address")
-	}
-
-	buf := make([]byte, 2048)
-
-	// First announcement
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Announcement verification
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Don't reply
-	// We should get nothing else
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, err = conn.Read(buf)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
-	// Status should be failure
-	if client.StatusOK() {
-		t.Fatal("Wrong status")
-	}
-
-	// Do a lookup in a separate routine
-	addrs := []string{}
-	wg := sync.NewWaitGroup()
-	wg.Add(1)
-	go func() {
-		pkt, err := client.Lookup(device)
-		if err == nil {
-			for _, addr := range pkt.This.Addresses {
-				addrs = append(addrs, addr)
-			}
-		}
-		wg.Done()
-	}()
-
-	// Receive the lookup and don't reply
-	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
-	_, _, err = conn.ReadFromUDP(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Wait for the lookup to timeout, verify that the number of answers is none
-	wg.Wait()
-
-	if len(addrs) != 0 {
-		t.Fatal("Wrong number of answers")
-	}
-
-	client.Stop()
-}

lib/discover/client_udp.go

@@ -1,261 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"encoding/hex"
-	"io"
-	"net"
-	"net/url"
-	"strconv"
-	"time"
-
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-func init() {
-	for _, proto := range []string{"udp", "udp4", "udp6"} {
-		Register(proto, func(uri *url.URL, announcer Announcer) (Client, error) {
-			c := &UDPClient{
-				announcer: announcer,
-				wg:        sync.NewWaitGroup(),
-				mut:       sync.NewRWMutex(),
-			}
-			err := c.Start(uri)
-			if err != nil {
-				return nil, err
-			}
-			return c, nil
-		})
-	}
-}
-
-type UDPClient struct {
-	url *url.URL
-
-	stop          chan struct{}
-	wg            sync.WaitGroup
-	listenAddress *net.UDPAddr
-
-	globalBroadcastInterval time.Duration
-	errorRetryInterval      time.Duration
-	announcer               Announcer
-
-	status bool
-	mut    sync.RWMutex
-}
-
-func (d *UDPClient) Start(uri *url.URL) error {
-	d.url = uri
-	d.stop = make(chan struct{})
-
-	params := uri.Query()
-	// The address must not have a port, as otherwise both announce and lookup
-	// sockets would try to bind to the same port.
-	addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
-	if err != nil {
-		return err
-	}
-	d.listenAddress = addr
-
-	broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
-	if err != nil {
-		d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
-	} else {
-		d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
-	}
-
-	retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
-	if err != nil {
-		d.errorRetryInterval = DefaultErrorRetryInternval
-	} else {
-		d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
-	}
-
-	d.wg.Add(1)
-	go d.broadcast()
-	return nil
-}
-
-func (d *UDPClient) broadcast() {
-	defer d.wg.Done()
-
-	conn, err := net.ListenUDP(d.url.Scheme, d.listenAddress)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast listen: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		conn, err = net.ListenUDP(d.url.Scheme, d.listenAddress)
-	}
-	defer conn.Close()
-
-	remote, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	for err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast resolve: %v; trying again in %v", d.url, err, d.errorRetryInterval)
-		}
-		select {
-		case <-d.stop:
-			return
-		case <-time.After(d.errorRetryInterval):
-		}
-		remote, err = net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	}
-
-	timer := time.NewTimer(0)
-
-	eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged)
-	defer events.Default.Unsubscribe(eventSub)
-
-	for {
-		select {
-		case <-d.stop:
-			return
-
-		case <-eventSub.C():
-			ok := d.sendAnnouncement(remote, conn)
-
-			d.mut.Lock()
-			d.status = ok
-			d.mut.Unlock()
-
-		case <-timer.C:
-			ok := d.sendAnnouncement(remote, conn)
-
-			d.mut.Lock()
-			d.status = ok
-			d.mut.Unlock()
-
-			if ok {
-				timer.Reset(d.globalBroadcastInterval)
-			} else {
-				timer.Reset(d.errorRetryInterval)
-			}
-		}
-	}
-}
-
-func (d *UDPClient) sendAnnouncement(remote net.Addr, conn *net.UDPConn) bool {
-	if debug {
-		l.Debugf("discover %s: broadcast: Sending self announcement to %v", d.url, remote)
-	}
-
-	ann := d.announcer.Announcement()
-	pkt, err := ann.MarshalXDR()
-	if err != nil {
-		return false
-	}
-
-	myID := protocol.DeviceIDFromBytes(ann.This.ID)
-
-	_, err = conn.WriteTo(pkt, remote)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: broadcast: Failed to send self announcement: %s", d.url, err)
-		}
-		return false
-	}
-
-	// Verify that the announce server responds positively for our device ID
-
-	time.Sleep(1 * time.Second)
-
-	ann, err = d.Lookup(myID)
-	if err != nil && debug {
-		l.Debugf("discover %s: broadcast: Self-lookup failed: %v", d.url, err)
-	} else if debug {
-		l.Debugf("discover %s: broadcast: Self-lookup returned: %v", d.url, ann.This.Addresses)
-	}
-	return len(ann.This.Addresses) > 0
-}
-
-func (d *UDPClient) Lookup(device protocol.DeviceID) (Announce, error) {
-	extIP, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return Announce{}, err
-	}
-
-	conn, err := net.DialUDP(d.url.Scheme, d.listenAddress, extIP)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return Announce{}, err
-	}
-	defer conn.Close()
-
-	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return Announce{}, err
-	}
-
-	buf := Query{QueryMagic, device[:]}.MustMarshalXDR()
-	_, err = conn.Write(buf)
-	if err != nil {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return Announce{}, err
-	}
-
-	buf = make([]byte, 2048)
-	n, err := conn.Read(buf)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			// Expected if the server doesn't know about requested device ID
-			return Announce{}, err
-		}
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
-		}
-		return Announce{}, err
-	}
-
-	var pkt Announce
-	err = pkt.UnmarshalXDR(buf[:n])
-	if err != nil && err != io.EOF {
-		if debug {
-			l.Debugf("discover %s: Lookup(%s): %s\n%s", d.url, device, err, hex.Dump(buf[:n]))
-		}
-		return Announce{}, err
-	}
-
-	if debug {
-		l.Debugf("discover %s: Lookup(%s) result: %v relays: %v", d.url, device, pkt.This.Addresses, pkt.This.Relays)
-	}
-	return pkt, nil
-}
-
-func (d *UDPClient) Stop() {
-	if d.stop != nil {
-		close(d.stop)
-		d.wg.Wait()
-	}
-}
-
-func (d *UDPClient) StatusOK() bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-	return d.status
-}
-
-func (d *UDPClient) Address() string {
-	return d.url.String()
-}

lib/discover/discover.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
@@ -7,539 +7,48 @@
 package discover
 
 import (
-	"bytes"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/url"
-	"sort"
 	"time"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/lib/beacon"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
 )
 
-type Discoverer struct {
-	myID                protocol.DeviceID
-	listenAddrs         []string
-	relayStatusProvider relayStatusProvider
-	localBcastIntv      time.Duration
-	localBcastStart     time.Time
-	cacheLifetime       time.Duration
-	negCacheCutoff      time.Duration
-	beacons             []beacon.Interface
-	extAddr             externalAddr
-	localBcastTick      <-chan time.Time
-	forcedBcastTick     chan time.Time
-
-	registryLock    sync.RWMutex
-	addressRegistry map[protocol.DeviceID][]CacheEntry
-	relayRegistry   map[protocol.DeviceID][]CacheEntry
-	lastLookup      map[protocol.DeviceID]time.Time
-
-	clients []Client
-	mut     sync.RWMutex
-}
-
-type relayStatusProvider interface {
-	ClientStatus() map[string]bool
-}
-
-type externalAddr interface {
-	ExternalAddresses() []string
+// A Finder provides lookup services of some kind.
+type Finder interface {
+	Lookup(deviceID protocol.DeviceID) (direct []string, relays []Relay, err error)
+	Error() error
+	String() string
+	Cache() map[protocol.DeviceID]CacheEntry
 }
 
 type CacheEntry struct {
-	Address string
-	Seen    time.Time
-}
-
-var (
-	ErrIncorrectMagic = errors.New("incorrect magic number")
-)
-
-func NewDiscoverer(id protocol.DeviceID, addresses []string, relayStatusProvider relayStatusProvider) *Discoverer {
-	return &Discoverer{
-		myID:                id,
-		listenAddrs:         addresses,
-		relayStatusProvider: relayStatusProvider,
-		localBcastIntv:      30 * time.Second,
-		cacheLifetime:       5 * time.Minute,
-		negCacheCutoff:      3 * time.Minute,
-		addressRegistry:     make(map[protocol.DeviceID][]CacheEntry),
-		relayRegistry:       make(map[protocol.DeviceID][]CacheEntry),
-		lastLookup:          make(map[protocol.DeviceID]time.Time),
-		registryLock:        sync.NewRWMutex(),
-		mut:                 sync.NewRWMutex(),
-	}
-}
-
-func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
-	if localPort > 0 {
-		d.startLocalIPv4Broadcasts(localPort)
-	}
-
-	if len(localMCAddr) > 0 {
-		d.startLocalIPv6Multicasts(localMCAddr)
-	}
-
-	if len(d.beacons) == 0 {
-		l.Warnln("Local discovery unavailable")
-		return
-	}
-
-	d.localBcastTick = time.Tick(d.localBcastIntv)
-	d.forcedBcastTick = make(chan time.Time)
-	d.localBcastStart = time.Now()
-	go d.sendLocalAnnouncements()
-}
-
-func (d *Discoverer) startLocalIPv4Broadcasts(localPort int) {
-	bb := beacon.NewBroadcast(localPort)
-	d.beacons = append(d.beacons, bb)
-	go d.recvAnnouncements(bb)
-	bb.ServeBackground()
-}
-
-func (d *Discoverer) startLocalIPv6Multicasts(localMCAddr string) {
-	mb, err := beacon.NewMulticast(localMCAddr)
-	if err != nil {
-		if debug {
-			l.Debugln("beacon.NewMulticast:", err)
-		}
-		l.Infoln("Local discovery over IPv6 unavailable")
-		return
-	}
-	d.beacons = append(d.beacons, mb)
-	go d.recvAnnouncements(mb)
-}
-
-func (d *Discoverer) StartGlobal(servers []string, extAddr externalAddr) {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-
-	if len(d.clients) > 0 {
-		d.stopGlobal()
-	}
-
-	d.extAddr = extAddr
-	wg := sync.NewWaitGroup()
-	clients := make(chan Client, len(servers))
-	for _, address := range servers {
-		wg.Add(1)
-		go func(addr string) {
-			defer wg.Done()
-			client, err := New(addr, d)
-			if err != nil {
-				l.Infoln("Error creating discovery client", addr, err)
-				return
-			}
-			clients <- client
-		}(address)
-	}
-
-	wg.Wait()
-	close(clients)
-
-	for client := range clients {
-		d.clients = append(d.clients, client)
-	}
-}
-
-func (d *Discoverer) StopGlobal() {
-	d.mut.Lock()
-	defer d.mut.Unlock()
-	d.stopGlobal()
-}
-
-func (d *Discoverer) stopGlobal() {
-	for _, client := range d.clients {
-		client.Stop()
-	}
-	d.clients = []Client{}
+	Direct []string  `json:"direct"`
+	Relays []Relay   `json:"relays"`
+	when   time.Time // When did we get the result
+	found  bool      // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
 }
 
-func (d *Discoverer) ExtAnnounceOK() map[string]bool {
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	ret := make(map[string]bool)
-	for _, client := range d.clients {
-		ret[client.Address()] = client.StatusOK()
-	}
-	return ret
-}
-
-// Lookup returns a list of addresses the device is available at, as well as
-// a list of relays the device is supposed to be available on sorted by the
-// sum of latencies between this device, and the device in question.
-func (d *Discoverer) Lookup(device protocol.DeviceID) ([]string, []string) {
-	d.registryLock.RLock()
-	cachedAddresses := d.filterCached(d.addressRegistry[device])
-	cachedRelays := d.filterCached(d.relayRegistry[device])
-	lastLookup := d.lastLookup[device]
-	d.registryLock.RUnlock()
-
-	d.mut.RLock()
-	defer d.mut.RUnlock()
-
-	relays := make([]string, len(cachedRelays))
-	for i := range cachedRelays {
-		relays[i] = cachedRelays[i].Address
-	}
-
-	if len(cachedAddresses) > 0 {
-		// There are cached address entries.
-		addrs := make([]string, len(cachedAddresses))
-		for i := range cachedAddresses {
-			addrs[i] = cachedAddresses[i].Address
-		}
-		return addrs, relays
-	}
-
-	if time.Since(lastLookup) < d.negCacheCutoff {
-		// We have recently tried to lookup this address and failed. Lets
-		// chill for a while.
-		return nil, relays
-	}
-
-	if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
-		// Only perform external lookups if we have at least one external
-		// server client and one local announcement interval has passed. This is
-		// to avoid finding local peers on their remote address at startup.
-		results := make(chan Announce, len(d.clients))
-		wg := sync.NewWaitGroup()
-		for _, client := range d.clients {
-			wg.Add(1)
-			go func(c Client) {
-				defer wg.Done()
-				ann, err := c.Lookup(device)
-				if err == nil {
-					results <- ann
-				}
-
-			}(client)
-		}
-
-		wg.Wait()
-		close(results)
-
-		cachedAddresses := []CacheEntry{}
-		availableRelays := []Relay{}
-		seenAddresses := make(map[string]struct{})
-		seenRelays := make(map[string]struct{})
-		now := time.Now()
-
-		var addrs []string
-		for result := range results {
-			for _, addr := range result.This.Addresses {
-				_, ok := seenAddresses[addr]
-				if !ok {
-					cachedAddresses = append(cachedAddresses, CacheEntry{
-						Address: addr,
-						Seen:    now,
-					})
-					seenAddresses[addr] = struct{}{}
-					addrs = append(addrs, addr)
-				}
-			}
-
-			for _, relay := range result.This.Relays {
-				_, ok := seenRelays[relay.Address]
-				if !ok {
-					availableRelays = append(availableRelays, relay)
-					seenRelays[relay.Address] = struct{}{}
-				}
-			}
-		}
-
-		relays = RelayAddressesSortedByLatency(availableRelays)
-		cachedRelays := make([]CacheEntry, len(relays))
-		for i := range relays {
-			cachedRelays[i] = CacheEntry{
-				Address: relays[i],
-				Seen:    now,
-			}
-		}
-
-		d.registryLock.Lock()
-		d.addressRegistry[device] = cachedAddresses
-		d.relayRegistry[device] = cachedRelays
-		d.lastLookup[device] = time.Now()
-		d.registryLock.Unlock()
-
-		return addrs, relays
-	}
-
-	return nil, relays
+// A FinderService is a Finder that has background activity and must be run as
+// a suture.Service.
+type FinderService interface {
+	Finder
+	suture.Service
 }
 
-func (d *Discoverer) Hint(device string, addrs []string) {
-	resAddrs := resolveAddrs(addrs)
-	var id protocol.DeviceID
-	id.UnmarshalText([]byte(device))
-	d.registerDevice(nil, Device{
-		Addresses: resAddrs,
-		ID:        id[:],
-	})
+type FinderMux interface {
+	Finder
+	ChildStatus() map[string]error
 }
 
-func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
-	d.registryLock.RLock()
-	devices := make(map[protocol.DeviceID][]CacheEntry, len(d.addressRegistry))
-	for device, addrs := range d.addressRegistry {
-		addrsCopy := make([]CacheEntry, len(addrs))
-		copy(addrsCopy, addrs)
-		devices[device] = addrsCopy
-	}
-	d.registryLock.RUnlock()
-	return devices
+// The RelayStatusProvider answers questions about current relay status.
+type RelayStatusProvider interface {
+	Relays() []string
+	RelayStatus(uri string) (time.Duration, bool)
 }
 
-func (d *Discoverer) Announcement() Announce {
-	return d.announcementPkt(true)
-}
-
-func (d *Discoverer) announcementPkt(allowExternal bool) Announce {
-	var addrs []string
-	if allowExternal && d.extAddr != nil {
-		addrs = d.extAddr.ExternalAddresses()
-	} else {
-		addrs = resolveAddrs(d.listenAddrs)
-	}
-
-	var relayAddrs []string
-	if d.relayStatusProvider != nil {
-		status := d.relayStatusProvider.ClientStatus()
-		for uri, ok := range status {
-			if ok {
-				relayAddrs = append(relayAddrs, uri)
-			}
-		}
-	}
-
-	return Announce{
-		Magic: AnnouncementMagic,
-		This:  Device{d.myID[:], addrs, measureLatency(relayAddrs)},
-	}
-}
-
-func (d *Discoverer) sendLocalAnnouncements() {
-	var pkt = d.announcementPkt(false)
-	msg := pkt.MustMarshalXDR()
-
-	for {
-		for _, b := range d.beacons {
-			b.Send(msg)
-		}
-
-		select {
-		case <-d.localBcastTick:
-		case <-d.forcedBcastTick:
-		}
-	}
-}
-
-func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
-	for {
-		buf, addr := b.Recv()
-
-		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
-		if err != nil && err != io.EOF {
-			if debug {
-				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
-			}
-			continue
-		}
-
-		if debug {
-			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
-		}
-
-		var newDevice bool
-		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
-			newDevice = d.registerDevice(addr, pkt.This)
-		}
-
-		if newDevice {
-			select {
-			case d.forcedBcastTick <- time.Now():
-			}
-		}
-	}
-}
-
-func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
-	var id protocol.DeviceID
-	copy(id[:], device.ID)
-
-	d.registryLock.Lock()
-	defer d.registryLock.Unlock()
-
-	current := d.filterCached(d.addressRegistry[id])
-
-	orig := current
-
-	for _, deviceAddr := range device.Addresses {
-		uri, err := url.Parse(deviceAddr)
-		if err != nil {
-			if debug {
-				l.Debugf("discover: Failed to parse address %s: %s", deviceAddr, err)
-			}
-			continue
-		}
-
-		host, port, err := net.SplitHostPort(uri.Host)
-		if err != nil {
-			if debug {
-				l.Debugf("discover: Failed to split address host %s: %s", deviceAddr, err)
-			}
-			continue
-		}
-
-		if host == "" {
-			uri.Host = net.JoinHostPort(addr.(*net.UDPAddr).IP.String(), port)
-			deviceAddr = uri.String()
-		}
-
-		for i := range current {
-			if current[i].Address == deviceAddr {
-				current[i].Seen = time.Now()
-				goto done
-			}
-		}
-		current = append(current, CacheEntry{
-			Address: deviceAddr,
-			Seen:    time.Now(),
-		})
-	done:
-	}
-
-	if debug {
-		l.Debugf("discover: Caching %s addresses: %v", id, current)
-	}
-
-	d.addressRegistry[id] = current
-
-	if len(current) > len(orig) {
-		addrs := make([]string, len(current))
-		for i := range current {
-			addrs[i] = current[i].Address
-		}
-		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
-			"device": id.String(),
-			"addrs":  addrs,
-		})
-	}
-
-	return len(current) > len(orig)
-}
-
-func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
-	for i := 0; i < len(c); {
-		if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
-			if debug {
-				l.Debugf("discover: Removing cached entry %s - seen %v ago", c[i].Address, ago)
-			}
-			c[i] = c[len(c)-1]
-			c = c[:len(c)-1]
-		} else {
-			i++
-		}
-	}
-	return c
-}
-
-func addrToAddr(addr *net.TCPAddr) string {
-	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
-		return fmt.Sprintf(":%d", addr.Port)
-	} else if bs := addr.IP.To4(); bs != nil {
-		return fmt.Sprintf("%s:%d", bs.String(), addr.Port)
-	} else if bs := addr.IP.To16(); bs != nil {
-		return fmt.Sprintf("[%s]:%d", bs.String(), addr.Port)
-	}
-	return ""
-}
-
-func resolveAddrs(addrs []string) []string {
-	var raddrs []string
-	for _, addrStr := range addrs {
-		uri, err := url.Parse(addrStr)
-		if err != nil {
-			continue
-		}
-		addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
-		if err != nil {
-			continue
-		}
-		addr := addrToAddr(addrRes)
-		if len(addr) > 0 {
-			uri.Host = addr
-			raddrs = append(raddrs, uri.String())
-		}
-	}
-	return raddrs
-}
-
-func measureLatency(relayAdresses []string) []Relay {
-	relays := make([]Relay, 0, len(relayAdresses))
-	for i, addr := range relayAdresses {
-		relay := Relay{
-			Address: addr,
-			Latency: int32(time.Hour / time.Millisecond),
-		}
-		relays = append(relays, relay)
-
-		if latency, err := osutil.GetLatencyForURL(addr); err == nil {
-			if debug {
-				l.Debugf("Relay %s latency %s", addr, latency)
-			}
-			relays[i].Latency = int32(latency / time.Millisecond)
-		} else {
-			l.Debugf("Failed to get relay %s latency %s", addr, err)
-		}
-	}
-	return relays
-}
-
-// RelayAddressesSortedByLatency adds local latency to the relay, and sorts them
-// by sum latency, and returns the addresses.
-func RelayAddressesSortedByLatency(input []Relay) []string {
-	relays := make([]Relay, len(input))
-	copy(relays, input)
-	for i, relay := range relays {
-		if latency, err := osutil.GetLatencyForURL(relay.Address); err == nil {
-			relays[i].Latency += int32(latency / time.Millisecond)
-		} else {
-			relays[i].Latency += int32(time.Hour / time.Millisecond)
-		}
-	}
-
-	sort.Sort(relayList(relays))
-
-	addresses := make([]string, 0, len(relays))
-	for _, relay := range relays {
-		addresses = append(addresses, relay.Address)
-	}
-	return addresses
-}
-
-type relayList []Relay
-
-func (l relayList) Len() int {
-	return len(l)
-}
-
-func (l relayList) Less(a, b int) bool {
-	return l[a].Latency < l[b].Latency
-}
-
-func (l relayList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
+// The AddressLister answers questions about what addresses we are listening
+// on.
+type AddressLister interface {
+	ExternalAddresses() []string
+	AllAddresses() []string
 }

lib/discover/discover_test.go

@@ -1,163 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package discover
-
-import (
-	"net/url"
-	"time"
-
-	"testing"
-
-	"github.com/syncthing/protocol"
-)
-
-type DummyClient struct {
-	url          *url.URL
-	lookups      []protocol.DeviceID
-	lookupRet    Announce
-	stops        int
-	statusRet    bool
-	statusChecks int
-}
-
-func (c *DummyClient) Lookup(device protocol.DeviceID) (Announce, error) {
-	c.lookups = append(c.lookups, device)
-	return c.lookupRet, nil
-}
-
-func (c *DummyClient) StatusOK() bool {
-	c.statusChecks++
-	return c.statusRet
-}
-
-func (c *DummyClient) Stop() {
-	c.stops++
-}
-
-func (c *DummyClient) Address() string {
-	return c.url.String()
-}
-
-func TestGlobalDiscovery(t *testing.T) {
-	c1 := &DummyClient{
-		statusRet: false,
-		lookupRet: Announce{
-			Magic: AnnouncementMagic,
-			This: Device{
-				ID:        protocol.LocalDeviceID[:],
-				Addresses: []string{"test.com:1234"},
-				Relays:    nil,
-			},
-			Extra: nil,
-		},
-	}
-
-	c2 := &DummyClient{
-		statusRet: true,
-		lookupRet: Announce{
-			Magic: AnnouncementMagic,
-			This: Device{
-				ID:        protocol.LocalDeviceID[:],
-				Addresses: nil,
-				Relays:    nil,
-			},
-			Extra: nil,
-		},
-	}
-
-	c3 := &DummyClient{
-		statusRet: true,
-		lookupRet: Announce{
-			Magic: AnnouncementMagic,
-			This: Device{
-				ID:        protocol.LocalDeviceID[:],
-				Addresses: []string{"best.com:2345"},
-				Relays:    nil,
-			},
-			Extra: nil,
-		},
-	}
-
-	clients := []*DummyClient{c1, c2}
-
-	Register("test1", func(uri *url.URL, ann Announcer) (Client, error) {
-		c := clients[0]
-		clients = clients[1:]
-		c.url = uri
-		return c, nil
-	})
-
-	Register("test2", func(uri *url.URL, ann Announcer) (Client, error) {
-		c3.url = uri
-		return c3, nil
-	})
-
-	d := NewDiscoverer(device, []string{}, nil)
-	d.localBcastStart = time.Time{}
-	servers := []string{
-		"test1://123.123.123.123:1234",
-		"test1://23.23.23.23:234",
-		"test2://234.234.234.234.2345",
-	}
-	d.StartGlobal(servers, nil)
-
-	if len(d.clients) != 3 {
-		t.Fatal("Wrong number of clients")
-	}
-
-	status := d.ExtAnnounceOK()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if status[c.url.String()] != c.statusRet || c.statusChecks != 1 {
-			t.Fatal("Wrong status")
-		}
-	}
-
-	addrs, _ := d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	for _, addr := range []string{"test.com:1234", "best.com:2345"} {
-		found := false
-		for _, laddr := range addrs {
-			if laddr == addr {
-				found = true
-				break
-			}
-		}
-		if !found {
-			t.Fatal("Couldn't find", addr)
-		}
-	}
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	addrs, _ = d.Lookup(device)
-	if len(addrs) != 2 {
-		t.Fatal("Wrong number of addresses", addrs)
-	}
-
-	// Answer should be cached, so number of lookups should have not increased
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if len(c.lookups) != 1 || c.lookups[0] != device {
-			t.Fatal("Wrong lookups")
-		}
-	}
-
-	d.StopGlobal()
-
-	for _, c := range []*DummyClient{c1, c2, c3} {
-		if c.stops != 1 {
-			t.Fatal("Wrong number of stops")
-		}
-	}
-}

lib/discover/doc.go

@@ -1,8 +1,75 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2015 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// Package discover implements the device discovery protocol.
+/*
+Package discover implements the local and global device discovery protocols.
+
+Global Discovery
+================
+
+Announcements
+-------------
+
+A device should announce itself at startup. It does this by an HTTPS POST to
+the announce server URL (with the path usually being "/", but this is of
+course up to the discovery server). The POST has a JSON payload listing direct
+connection addresses (if any) and relay addresses (if any).
+
+	{
+		direct: ["tcp://192.0.2.45:22000", "tcp://:22202"],
+		relays: [{"url": "relay://192.0.2.99:22028", "latency": 142}]
+	}
+
+It's OK for either of the "direct" or "relays" fields to be either the empty
+list ([]), null, or missing entirely. An announcment with both fields missing
+or empty is however not useful...
+
+Any empty or unspecified IP addresses (i.e. addresses like tcp://:22000,
+tcp://0.0.0.0:22000, tcp://[::]:22000) are interpreted as referring to the
+source IP address of the announcement.
+
+The device ID of the announcing device is not part of the announcement.
+Instead, the server requires that the client perform certificate
+authentication. The device ID is deduced from the presented certificate.
+
+The server response is empty, with code 200 (OK) on success. If no certificate
+was presented, status 403 (Forbidden) is returned. If the posted data doesn't
+conform to the expected format, 400 (Bad Request) is returned.
+
+In successfull responses, the server may return a "Reannounce-After" header
+containing the number of seconds after which the client should perform a new
+announcement.
+
+In error responses, the server may return a "Retry-After" header containing
+the number of seconds after which the client should retry.
+
+Performing announcements significantly more often than indicated by the
+Reannounce-After or Retry-After headers may result in the client being
+throttled. In such cases the server may respond with status code 429 (Too Many
+Requests).
+
+Queries
+=======
+
+Queries are performed as HTTPS GET requests to the announce server URL. The
+requested device ID is passed as the query parameter "device", in canonical
+string form, i.e. https://announce.syncthing.net/?device=ABC12345-....
+
+Successfull responses will have status code 200 (OK) and carry a JSON payload
+of the same format as the announcement above. The response will not contain
+empty or unspecified addresses.
+
+If the "device" query parameter is missing or malformed, the status code 400
+(Bad Request) is returned.
+
+If the device ID is of a valid format but not found in the registry, 404 (Not
+Found) is returned.
+
+If the client has exceeded a rate limit, the server may respond with 429 (Too
+Many Requests).
+
+*/
 package discover

lib/discover/global.go

@@ -0,0 +1,385 @@
+package discover
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/events"
+)
+
+type globalClient struct {
+	server         string
+	addrList       AddressLister
+	relayStat      RelayStatusProvider
+	announceClient httpClient
+	queryClient    httpClient
+	noAnnounce     bool
+	stop           chan struct{}
+	errorHolder
+}
+
+type httpClient interface {
+	Get(url string) (*http.Response, error)
+	Post(url, ctype string, data io.Reader) (*http.Response, error)
+}
+
+const (
+	defaultReannounceInterval  = 30 * time.Minute
+	announceErrorRetryInterval = 5 * time.Minute
+)
+
+type announcement struct {
+	Direct []string `json:"direct"`
+	Relays []Relay  `json:"relays"`
+}
+
+type serverOptions struct {
+	insecure   bool   // don't check certificate
+	noAnnounce bool   // don't announce
+	id         string // expected server device ID
+}
+
+func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	server, opts, err := parseOptions(server)
+	if err != nil {
+		return nil, err
+	}
+
+	var devID protocol.DeviceID
+	if opts.id != "" {
+		devID, err = protocol.DeviceIDFromString(opts.id)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// The http.Client used for announcements. It needs to have our
+	// certificate to prove our identity, and may or may not verify the server
+	// certificate depending on the insecure setting.
+	var announceClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+				Certificates:       []tls.Certificate{cert},
+			},
+		},
+	}
+	if opts.id != "" {
+		announceClient = newIDCheckingHTTPClient(announceClient, devID)
+	}
+
+	// The http.Client used for queries. We don't need to present our
+	// certificate here, so lets not include it. May be insecure if requested.
+	var queryClient httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: opts.insecure,
+			},
+		},
+	}
+	if opts.id != "" {
+		queryClient = newIDCheckingHTTPClient(queryClient, devID)
+	}
+
+	cl := &globalClient{
+		server:         server,
+		addrList:       addrList,
+		relayStat:      relayStat,
+		announceClient: announceClient,
+		queryClient:    queryClient,
+		noAnnounce:     opts.noAnnounce,
+		stop:           make(chan struct{}),
+	}
+	cl.setError(errors.New("not announced"))
+
+	return cl, nil
+}
+
+// Lookup returns the list of addresses where the given device is available;
+// direct, and via relays.
+func (c *globalClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	qURL, err := url.Parse(c.server)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	q := qURL.Query()
+	q.Set("device", device.String())
+	qURL.RawQuery = q.Encode()
+
+	resp, err := c.queryClient.Get(qURL.String())
+	if err != nil {
+		if debug {
+			l.Debugln("globalClient.Lookup", qURL.String(), err)
+		}
+		return nil, nil, err
+	}
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		if debug {
+			l.Debugln("globalClient.Lookup", qURL.String(), resp.Status)
+		}
+		return nil, nil, errors.New(resp.Status)
+	}
+
+	// TODO: Handle 429 and Retry-After?
+
+	var ann announcement
+	err = json.NewDecoder(resp.Body).Decode(&ann)
+	resp.Body.Close()
+	return ann.Direct, ann.Relays, err
+}
+
+func (c *globalClient) String() string {
+	return "global@" + c.server
+}
+
+func (c *globalClient) Serve() {
+	if c.noAnnounce {
+		// We're configured to not do announcements, only lookups. To maintain
+		// the same interface, we just pause here if Serve() is run.
+		<-c.stop
+		return
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+
+	eventSub := events.Default.Subscribe(events.ExternalPortMappingChanged | events.RelayStateChanged)
+	defer events.Default.Unsubscribe(eventSub)
+
+	for {
+		select {
+		case <-eventSub.C():
+			c.sendAnnouncement(timer)
+
+		case <-timer.C:
+			c.sendAnnouncement(timer)
+
+		case <-c.stop:
+			return
+		}
+	}
+}
+
+func (c *globalClient) sendAnnouncement(timer *time.Timer) {
+
+	var ann announcement
+	if c.addrList != nil {
+		ann.Direct = c.addrList.ExternalAddresses()
+	}
+
+	if c.relayStat != nil {
+		for _, relay := range c.relayStat.Relays() {
+			latency, ok := c.relayStat.RelayStatus(relay)
+			if ok {
+				ann.Relays = append(ann.Relays, Relay{
+					URL:     relay,
+					Latency: int32(latency / time.Millisecond),
+				})
+			}
+		}
+	}
+
+	if len(ann.Direct)+len(ann.Relays) == 0 {
+		c.setError(errors.New("nothing to announce"))
+		if debug {
+			l.Debugln("Nothing to announce")
+		}
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	// The marshal doesn't fail, I promise.
+	postData, _ := json.Marshal(ann)
+
+	if debug {
+		l.Debugf("Announcement: %s", postData)
+	}
+
+	resp, err := c.announceClient.Post(c.server, "application/json", bytes.NewReader(postData))
+	if err != nil {
+		if debug {
+			l.Debugln("announce POST:", err)
+		}
+		c.setError(err)
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+	if debug {
+		l.Debugln("announce POST:", resp.Status)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		if debug {
+			l.Debugln("announce POST:", resp.Status)
+		}
+		c.setError(errors.New(resp.Status))
+
+		if h := resp.Header.Get("Retry-After"); h != "" {
+			// The server has a recommendation on when we should
+			// retry. Follow it.
+			if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+				if debug {
+					l.Debugln("announce Retry-After:", secs, err)
+				}
+				timer.Reset(time.Duration(secs) * time.Second)
+				return
+			}
+		}
+
+		timer.Reset(announceErrorRetryInterval)
+		return
+	}
+
+	c.setError(nil)
+
+	if h := resp.Header.Get("Reannounce-After"); h != "" {
+		// The server has a recommendation on when we should
+		// reannounce. Follow it.
+		if secs, err := strconv.Atoi(h); err == nil && secs > 0 {
+			if debug {
+				l.Debugln("announce Reannounce-After:", secs, err)
+			}
+			timer.Reset(time.Duration(secs) * time.Second)
+			return
+		}
+	}
+
+	timer.Reset(defaultReannounceInterval)
+}
+
+func (c *globalClient) Stop() {
+	close(c.stop)
+}
+
+func (c *globalClient) Cache() map[protocol.DeviceID]CacheEntry {
+	// The globalClient doesn't do caching
+	return nil
+}
+
+// parseOptions parses and strips away any ?query=val options, setting the
+// corresponding field in the serverOptions struct. Unknown query options are
+// ignored and removed.
+func parseOptions(dsn string) (server string, opts serverOptions, err error) {
+	p, err := url.Parse(dsn)
+	if err != nil {
+		return "", serverOptions{}, err
+	}
+
+	// Grab known options from the query string
+	q := p.Query()
+	opts.id = q.Get("id")
+	opts.insecure = opts.id != "" || queryBool(q, "insecure")
+	opts.noAnnounce = queryBool(q, "noannounce")
+
+	// Check for disallowed combinations
+	if p.Scheme == "http" {
+		if !opts.insecure {
+			return "", serverOptions{}, errors.New("http without insecure not supported")
+		}
+		if !opts.noAnnounce {
+			return "", serverOptions{}, errors.New("http without noannounce not supported")
+		}
+	} else if p.Scheme != "https" {
+		return "", serverOptions{}, errors.New("unsupported scheme " + p.Scheme)
+	}
+
+	// Remove the query string
+	p.RawQuery = ""
+	server = p.String()
+
+	return
+}
+
+// queryBool returns the query parameter parsed as a boolean. An empty value
+// ("?foo") is considered true, as is any value string except false
+// ("?foo=false").
+func queryBool(q url.Values, key string) bool {
+	if _, ok := q[key]; !ok {
+		return false
+	}
+
+	return q.Get(key) != "false"
+}
+
+type idCheckingHTTPClient struct {
+	httpClient
+	id protocol.DeviceID
+}
+
+func newIDCheckingHTTPClient(client httpClient, id protocol.DeviceID) *idCheckingHTTPClient {
+	return &idCheckingHTTPClient{
+		httpClient: client,
+		id:         id,
+	}
+}
+
+func (c *idCheckingHTTPClient) check(resp *http.Response) error {
+	if resp.TLS == nil {
+		return errors.New("security: not TLS")
+	}
+
+	if len(resp.TLS.PeerCertificates) == 0 {
+		return errors.New("security: no certificates")
+	}
+
+	id := protocol.NewDeviceID(resp.TLS.PeerCertificates[0].Raw)
+	if !id.Equals(c.id) {
+		return errors.New("security: incorrect device id")
+	}
+
+	return nil
+}
+
+func (c *idCheckingHTTPClient) Get(url string) (*http.Response, error) {
+	resp, err := c.httpClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (c *idCheckingHTTPClient) Post(url, ctype string, data io.Reader) (*http.Response, error) {
+	resp, err := c.httpClient.Post(url, ctype, data)
+	if err != nil {
+		return nil, err
+	}
+	if err := c.check(resp); err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+type errorHolder struct {
+	err error
+	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+}
+
+func (e *errorHolder) setError(err error) {
+	e.mut.Lock()
+	e.err = err
+	e.mut.Unlock()
+}
+
+func (e *errorHolder) Error() error {
+	e.mut.Lock()
+	err := e.err
+	e.mut.Unlock()
+	return err
+}

lib/discover/global_test.go

@@ -0,0 +1,253 @@
+package discover
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+)
+
+func TestParseOptions(t *testing.T) {
+	testcases := []struct {
+		in   string
+		out  string
+		opts serverOptions
+	}{
+		{"https://example.com/", "https://example.com/", serverOptions{}},
+		{"https://example.com/?insecure", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=true", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=yes", "https://example.com/", serverOptions{insecure: true}},
+		{"https://example.com/?insecure=false&noannounce", "https://example.com/", serverOptions{noAnnounce: true}},
+		{"https://example.com/?id=abc", "https://example.com/", serverOptions{id: "abc", insecure: true}},
+	}
+
+	for _, tc := range testcases {
+		res, opts, err := parseOptions(tc.in)
+		if err != nil {
+			t.Errorf("Unexpected err %v for %v", err, tc.in)
+			continue
+		}
+		if res != tc.out {
+			t.Errorf("Incorrect server, %v!= %v for %v", res, tc.out, tc.in)
+		}
+		if opts != tc.opts {
+			t.Errorf("Incorrect options, %v!= %v for %v", opts, tc.opts, tc.in)
+		}
+	}
+}
+
+func TestGlobalOverHTTP(t *testing.T) {
+	// HTTP works for queries, but is obviously insecure and we can't do
+	// announces over it (as we don't present a certificate). As such, http://
+	// is only allowed in combination with the "insecure" and "noannounce"
+	// parameters.
+
+	if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure and noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without noannounce")
+	}
+
+	if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil, nil); err == nil {
+		t.Fatal("http is not allowed without insecure")
+	}
+
+	// Now lets check that lookups work over HTTP, given the correct options.
+
+	list, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	direct, relays, err := testLookup("http://" + list.Addr().String() + "?insecure&noannounce")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+		t.Errorf("incorrect direct list: %+v", direct)
+	}
+	if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+		t.Errorf("incorrect relays list: %+v", direct)
+	}
+}
+
+func TestGlobalOverHTTPS(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	// With default options the lookup code expects the server certificate to
+	// check out according to the usual CA chains etc. That won't be the case
+	// here so we expect the lookup to fail.
+
+	url := "https://" + list.Addr().String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error when we should have got a certificate error")
+	}
+
+	// With "insecure" set, whatever certificate is on the other side should
+	// be accepted.
+
+	url = "https://" + list.Addr().String() + "?insecure"
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+
+	// With "id" set to something incorrect, the checks should fail again.
+
+	url = "https://" + list.Addr().String() + "?id=" + protocol.LocalDeviceID.String()
+	if _, _, err := testLookup(url); err == nil {
+		t.Fatalf("unexpected nil error for incorrect discovery server ID")
+	}
+
+	// With the correct device ID, the check should pass and we should get a
+	// lookup response.
+
+	id := protocol.NewDeviceID(cert.Certificate[0])
+	url = "https://" + list.Addr().String() + "?id=" + id.String()
+	if direct, relays, err := testLookup(url); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	} else {
+		if len(direct) != 1 || direct[0] != "tcp://192.0.2.42::22000" {
+			t.Errorf("incorrect direct list: %+v", direct)
+		}
+		if len(relays) != 1 || relays[0] != (Relay{URL: "relay://192.0.2.43:443", Latency: 42}) {
+			t.Errorf("incorrect relays list: %+v", direct)
+		}
+	}
+}
+
+func TestGlobalAnnounce(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate a server certificate, using fewer bits than usual to hurry the
+	// process along a bit.
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := tls.Listen("tcp4", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer list.Close()
+
+	s := new(fakeDiscoveryServer)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", s.handler)
+	go http.Serve(list, mux)
+
+	url := "https://" + list.Addr().String() + "?insecure"
+	disco, err := NewGlobal(url, cert, new(fakeAddressLister), new(fakeRelayStatus))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go disco.Serve()
+	defer disco.Stop()
+
+	// The discovery thing should attempt an announcement immediately. We wait
+	// for it to succeed, a while.
+	t0 := time.Now()
+	for err := disco.Error(); err != nil; err = disco.Error() {
+		if time.Since(t0) > 10*time.Second {
+			t.Fatal("announce failed:", err)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	if !strings.Contains(string(s.announce), "tcp://0.0.0.0:22000") {
+		t.Errorf("announce missing direct address: %s", s.announce)
+	}
+	if !strings.Contains(string(s.announce), "relay://192.0.2.42:443") {
+		t.Errorf("announce missing relay address: %s", s.announce)
+	}
+}
+
+func testLookup(url string) ([]string, []Relay, error) {
+	disco, err := NewGlobal(url, tls.Certificate{}, nil, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	go disco.Serve()
+	defer disco.Stop()
+
+	return disco.Lookup(protocol.LocalDeviceID)
+}
+
+type fakeDiscoveryServer struct {
+	announce []byte
+}
+
+func (s *fakeDiscoveryServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "POST" {
+		s.announce, _ = ioutil.ReadAll(r.Body)
+		w.WriteHeader(204)
+	} else {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"direct":["tcp://192.0.2.42::22000"], "relays":[{"url": "relay://192.0.2.43:443", "latency": 42}]}`))
+	}
+}
+
+type fakeAddressLister struct{}
+
+func (f *fakeAddressLister) ExternalAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000"}
+}
+func (f *fakeAddressLister) AllAddresses() []string {
+	return []string{"tcp://0.0.0.0:22000", "tcp://192.168.0.1:22000"}
+}
+
+type fakeRelayStatus struct{}
+
+func (f *fakeRelayStatus) Relays() []string {
+	return []string{"relay://192.0.2.42:443"}
+}
+func (f *fakeRelayStatus) RelayStatus(uri string) (time.Duration, bool) {
+	return 42 * time.Millisecond, true
+}

lib/discover/local.go

@@ -0,0 +1,270 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/beacon"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/thejerf/suture"
+)
+
+type localClient struct {
+	*suture.Supervisor
+	myID      protocol.DeviceID
+	addrList  AddressLister
+	relayStat RelayStatusProvider
+	name      string
+
+	beacon          beacon.Interface
+	localBcastStart time.Time
+	localBcastTick  <-chan time.Time
+	forcedBcastTick chan time.Time
+
+	*cache
+}
+
+const (
+	BroadcastInterval = 30 * time.Second
+	CacheLifeTime     = 3 * BroadcastInterval
+)
+
+var (
+	ErrIncorrectMagic = errors.New("incorrect magic number")
+)
+
+func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, relayStat RelayStatusProvider) (FinderService, error) {
+	c := &localClient{
+		Supervisor:      suture.NewSimple("local"),
+		myID:            id,
+		addrList:        addrList,
+		relayStat:       relayStat,
+		localBcastTick:  time.Tick(BroadcastInterval),
+		forcedBcastTick: make(chan time.Time),
+		localBcastStart: time.Now(),
+		cache:           newCache(),
+	}
+
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(host) == 0 {
+		// A broadcast client
+		c.name = "IPv4 local"
+		bcPort, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, err
+		}
+		c.startLocalIPv4Broadcasts(bcPort)
+	} else {
+		// A multicast client
+		c.name = "IPv6 local"
+		c.startLocalIPv6Multicasts(addr)
+	}
+
+	go c.sendLocalAnnouncements()
+
+	return c, nil
+}
+
+func (c *localClient) startLocalIPv4Broadcasts(localPort int) {
+	c.beacon = beacon.NewBroadcast(localPort)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+func (c *localClient) startLocalIPv6Multicasts(localMCAddr string) {
+	c.beacon = beacon.NewMulticast(localMCAddr)
+	c.Add(c.beacon)
+	go c.recvAnnouncements(c.beacon)
+}
+
+// Lookup returns a list of addresses the device is available at. Local
+// discovery never returns relays.
+func (c *localClient) Lookup(device protocol.DeviceID) (direct []string, relays []Relay, err error) {
+	if cache, ok := c.Get(device); ok {
+		if time.Since(cache.when) < CacheLifeTime {
+			direct = cache.Direct
+			relays = cache.Relays
+		}
+	}
+
+	return
+}
+
+func (c *localClient) String() string {
+	return c.name
+}
+
+func (c *localClient) Error() error {
+	return c.beacon.Error()
+}
+
+func (c *localClient) announcementPkt() Announce {
+	addrs := c.addrList.AllAddresses()
+
+	var relays []Relay
+	for _, relay := range c.relayStat.Relays() {
+		latency, ok := c.relayStat.RelayStatus(relay)
+		if ok {
+			relays = append(relays, Relay{
+				URL:     relay,
+				Latency: int32(latency / time.Millisecond),
+			})
+		}
+	}
+
+	return Announce{
+		Magic: AnnouncementMagic,
+		This: Device{
+			ID:        c.myID[:],
+			Addresses: addrs,
+			Relays:    relays,
+		},
+	}
+}
+
+func (c *localClient) sendLocalAnnouncements() {
+	var pkt = c.announcementPkt()
+	msg := pkt.MustMarshalXDR()
+
+	for {
+		c.beacon.Send(msg)
+
+		select {
+		case <-c.localBcastTick:
+		case <-c.forcedBcastTick:
+		}
+	}
+}
+
+func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	for {
+		buf, addr := b.Recv()
+
+		var pkt Announce
+		err := pkt.UnmarshalXDR(buf)
+		if err != nil && err != io.EOF {
+			if debug {
+				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
+			}
+			continue
+		}
+
+		if debug {
+			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		}
+
+		var newDevice bool
+		if bytes.Compare(pkt.This.ID, c.myID[:]) != 0 {
+			newDevice = c.registerDevice(addr, pkt.This)
+		}
+
+		if newDevice {
+			select {
+			case c.forcedBcastTick <- time.Now():
+			}
+		}
+	}
+}
+
+func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+	var id protocol.DeviceID
+	copy(id[:], device.ID)
+
+	// Remember whether we already had a valid cache entry for this device.
+
+	ce, existsAlready := c.Get(id)
+	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
+
+	// Any empty or unspecified addresses should be set to the source address
+	// of the announcement. We also skip any addresses we can't parse.
+
+	var validAddresses []string
+	for _, addr := range device.Addresses {
+		u, err := url.Parse(addr)
+		if err != nil {
+			continue
+		}
+
+		tcpAddr, err := net.ResolveTCPAddr("tcp", u.Host)
+		if err != nil {
+			continue
+		}
+
+		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			host, _, err := net.SplitHostPort(src.String())
+			if err != nil {
+				continue
+			}
+			u.Host = fmt.Sprintf("%s:%d", host, tcpAddr.Port)
+			validAddresses = append(validAddresses, u.String())
+		} else {
+			validAddresses = append(validAddresses, addr)
+		}
+	}
+
+	c.Set(id, CacheEntry{
+		Direct: validAddresses,
+		Relays: device.Relays,
+		when:   time.Now(),
+		found:  true,
+	})
+
+	if isNewDevice {
+		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
+			"device": id.String(),
+			"addrs":  device.Addresses,
+			"relays": device.Relays,
+		})
+	}
+
+	return isNewDevice
+}
+
+func addrToAddr(addr *net.TCPAddr) string {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return fmt.Sprintf(":%c", addr.Port)
+	} else if bs := addr.IP.To4(); bs != nil {
+		return fmt.Sprintf("%s:%c", bs.String(), addr.Port)
+	} else if bs := addr.IP.To16(); bs != nil {
+		return fmt.Sprintf("[%s]:%c", bs.String(), addr.Port)
+	}
+	return ""
+}
+
+func resolveAddrs(addrs []string) []string {
+	var raddrs []string
+	for _, addrStr := range addrs {
+		uri, err := url.Parse(addrStr)
+		if err != nil {
+			continue
+		}
+		addrRes, err := net.ResolveTCPAddr("tcp", uri.Host)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr) > 0 {
+			uri.Host = addr
+			raddrs = append(raddrs, uri.String())
+		}
+	}
+	return raddrs
+}

lib/discover/packets.go → lib/discover/localpackets.go

@@ -5,7 +5,7 @@
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o packets_xdr.go packets.go
+//go:generate genxdr -o localpackets_xdr.go localpackets.go
 
 package discover
 
@@ -26,8 +26,8 @@ type Announce struct {
 }
 
 type Relay struct {
-	Address string // max:256
-	Latency int32
+	URL     string `json:"url"` // max:2083
+	Latency int32  `json:"latency"`
 }
 
 type Device struct {

lib/discover/packets_xdr.go → lib/discover/localpackets_xdr.go

@@ -192,10 +192,10 @@ Relay Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                       Length of Address                       |
+|                         Length of URL                         |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 /                                                               /
-\                   Address (variable length)                   \
+\                     URL (variable length)                     \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                            Latency                            |
@@ -203,7 +203,7 @@ Relay Structure:
 
 
 struct Relay {
-	string Address<256>;
+	string URL<256>;
 	int Latency;
 }
 
@@ -234,10 +234,10 @@ func (o Relay) AppendXDR(bs []byte) ([]byte, error) {
 }
 
 func (o Relay) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	if l := len(o.Address); l > 256 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Address", l, 256)
+	if l := len(o.URL); l > 256 {
+		return xw.Tot(), xdr.ElementSizeExceeded("URL", l, 256)
 	}
-	xw.WriteString(o.Address)
+	xw.WriteString(o.URL)
 	xw.WriteUint32(uint32(o.Latency))
 	return xw.Tot(), xw.Error()
 }
@@ -254,7 +254,7 @@ func (o *Relay) UnmarshalXDR(bs []byte) error {
 }
 
 func (o *Relay) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.Address = xr.ReadStringMax(256)
+	o.URL = xr.ReadStringMax(256)
 	o.Latency = int32(xr.ReadUint32())
 	return xr.Error()
 }

lib/events/events.go

@@ -40,6 +40,7 @@ const (
 	FolderErrors
 	FolderScanProgress
 	ExternalPortMappingChanged
+	RelayStateChanged
 
 	AllEvents = (1 << iota) - 1
 )
@@ -90,6 +91,8 @@ func (t EventType) String() string {
 		return "FolderScanProgress"
 	case ExternalPortMappingChanged:
 		return "ExternalPortMappingChanged"
+	case RelayStateChanged:
+		return "RelayStateChanged"
 	default:
 		return "Unknown"
 	}

lib/ignore/ignore.go

@@ -12,6 +12,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -236,6 +237,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
 			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
 			includes, err := loadIgnoreFile(includeFile, seen)
 			if err != nil {
+				log.Println(err)
 				return err
 			}
 			patterns = append(patterns, includes...)

lib/relay/relay.go

@@ -12,18 +12,23 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"time"
 
 	"github.com/syncthing/relaysrv/client"
 	"github.com/syncthing/relaysrv/protocol"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/sync"
 
 	"github.com/thejerf/suture"
 )
 
+const (
+	eventBroadcasterCheckInterval = 10 * time.Second
+)
+
 type Svc struct {
 	*suture.Supervisor
 	cfg    *config.Wrapper
@@ -71,7 +76,12 @@ func NewSvc(cfg *config.Wrapper, tlsCfg *tls.Config) *Svc {
 		stop:        make(chan struct{}),
 	}
 
+	eventBc := &eventBroadcaster{
+		svc: svc,
+	}
+
 	svc.Add(receiver)
+	svc.Add(eventBc)
 
 	return svc
 }
@@ -132,7 +142,7 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
 			continue
 		}
 
-		dynRelays := make([]discover.Relay, 0, len(ann.Relays))
+		var dynRelayAddrs []string
 		for _, relayAnn := range ann.Relays {
 			ruri, err := url.Parse(relayAnn.URL)
 			if err != nil {
@@ -144,13 +154,11 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
 			if debug {
 				l.Debugln("Found", ruri, "via", uri)
 			}
-			dynRelays = append(dynRelays, discover.Relay{
-				Address: ruri.String(),
-			})
+			dynRelayAddrs = append(dynRelayAddrs, ruri.String())
 		}
 
-		dynRelayAddrs := discover.RelayAddressesSortedByLatency(dynRelays)
 		if len(dynRelayAddrs) > 0 {
+			dynRelayAddrs = relayAddressesSortedByLatency(dynRelayAddrs)
 			closestRelay := dynRelayAddrs[0]
 			if debug {
 				l.Debugln("Picking", closestRelay, "as closest dynamic relay from", uri)
@@ -193,7 +201,14 @@ func (s *Svc) CommitConfiguration(from, to config.Configuration) bool {
 	return true
 }
 
-func (s *Svc) ClientStatus() map[string]bool {
+type Status struct {
+	URL     string
+	OK      bool
+	Latency int
+}
+
+// Relays return the list of relays that currently have an OK status.
+func (s *Svc) Relays() []string {
 	if s == nil {
 		// A nil client does not have a status, really. Yet we may be called
 		// this way, for raisins...
@@ -201,12 +216,34 @@ func (s *Svc) ClientStatus() map[string]bool {
 	}
 
 	s.mut.RLock()
-	status := make(map[string]bool, len(s.clients))
-	for uri, client := range s.clients {
-		status[uri] = client.StatusOK()
+	relays := make([]string, 0, len(s.clients))
+	for uri := range s.clients {
+		relays = append(relays, uri)
+	}
+	s.mut.RUnlock()
+
+	sort.Strings(relays)
+
+	return relays
+}
+
+// RelayStatus returns the latency and OK status for a given relay.
+func (s *Svc) RelayStatus(uri string) (time.Duration, bool) {
+	if s == nil {
+		// A nil client does not have a status, really. Yet we may be called
+		// this way, for raisins...
+		return time.Hour, false
 	}
+
+	s.mut.RLock()
+	client, ok := s.clients[uri]
 	s.mut.RUnlock()
-	return status
+
+	if !ok || !client.StatusOK() {
+		return time.Hour, false
+	}
+
+	return client.Latency(), true
 }
 
 // Accept returns a new *tls.Conn. The connection is already handshaken.
@@ -266,6 +303,55 @@ func (r *invitationReceiver) Stop() {
 	close(r.stop)
 }
 
+// The eventBroadcaster sends a RelayStateChanged event when the relay status
+// changes. We need this somewhat ugly polling mechanism as there's currently
+// no way to get the event feed directly from the relay lib. This may be
+// somethign to revisit later, possibly.
+type eventBroadcaster struct {
+	svc  *Svc
+	stop chan struct{}
+}
+
+func (e *eventBroadcaster) Serve() {
+	timer := time.NewTicker(eventBroadcasterCheckInterval)
+	defer timer.Stop()
+
+	var prevOKRelays []string
+
+	for {
+		select {
+		case <-timer.C:
+			curOKRelays := e.svc.Relays()
+
+			changed := len(curOKRelays) != len(prevOKRelays)
+			if !changed {
+				for i := range curOKRelays {
+					if curOKRelays[i] != prevOKRelays[i] {
+						changed = true
+						break
+					}
+				}
+			}
+
+			if changed {
+				events.Default.Log(events.RelayStateChanged, map[string][]string{
+					"old": prevOKRelays,
+					"new": curOKRelays,
+				})
+			}
+
+			prevOKRelays = curOKRelays
+
+		case <-e.stop:
+			return
+		}
+	}
+}
+
+func (e *eventBroadcaster) Stop() {
+	close(e.stop)
+}
+
 // This is the announcement recieved from the relay server;
 // {"relays": [{"url": "relay://10.20.30.40:5060"}, ...]}
 type dynamicAnnouncement struct {
@@ -273,3 +359,43 @@ type dynamicAnnouncement struct {
 		URL string
 	}
 }
+
+// relayAddressesSortedByLatency adds local latency to the relay, and sorts them
+// by sum latency, and returns the addresses.
+func relayAddressesSortedByLatency(input []string) []string {
+	relays := make(relayList, len(input))
+	for i, relay := range input {
+		if latency, err := osutil.GetLatencyForURL(relay); err == nil {
+			relays[i] = relayWithLatency{relay, int(latency / time.Millisecond)}
+		} else {
+			relays[i] = relayWithLatency{relay, int(time.Hour / time.Millisecond)}
+		}
+	}
+
+	sort.Sort(relays)
+
+	addresses := make([]string, len(relays))
+	for i, relay := range relays {
+		addresses[i] = relay.relay
+	}
+	return addresses
+}
+
+type relayWithLatency struct {
+	relay   string
+	latency int
+}
+
+type relayList []relayWithLatency
+
+func (l relayList) Len() int {
+	return len(l)
+}
+
+func (l relayList) Less(a, b int) bool {
+	return l[a].latency < l[b].latency
+}
+
+func (l relayList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}

test/h1/config.xml

@@ -46,8 +46,7 @@
     </gui>
     <options>
         <listenAddress>tcp://127.0.0.1:22001</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>

test/h2/config.xml

@@ -53,8 +53,7 @@
     </gui>
     <options>
         <listenAddress>tcp://127.0.0.1:22002</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>true</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>

test/h3/config.xml

@@ -39,8 +39,7 @@
     </gui>
     <options>
         <listenAddress>tcp://127.0.0.1:22003</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>

test/h4/config.xml

@@ -18,8 +18,7 @@
     </gui>
     <options>
         <listenAddress>tcp://127.0.0.1:22004</listenAddress>
-        <globalAnnounceServer>udp4://announce.syncthing.net:22027</globalAnnounceServer>
-        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22027</globalAnnounceServer>
+        <globalAnnounceServer>default</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>