syncthing/lib/ur/usage_report.go

// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package ur

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/tls"
	"encoding/json"
	"net"
	"net/http"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/syncthing/syncthing/lib/build"
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/connections"
	"github.com/syncthing/syncthing/lib/dialer"
	"github.com/syncthing/syncthing/lib/model"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/scanner"
	"github.com/syncthing/syncthing/lib/upgrade"
	"github.com/syncthing/syncthing/lib/util"

	"github.com/thejerf/suture"
)

// Current version number of the usage report, for acceptance purposes. If
// fields are added or changed this integer must be incremented so that users
// are prompted for acceptance of the new report.
const Version = 3

var StartTime = time.Now()

type Service struct {
	suture.Service
	cfg                config.Wrapper
	model              model.Model
	connectionsService connections.Service
	noUpgrade          bool
	forceRun           chan struct{}
}

func New(cfg config.Wrapper, m model.Model, connectionsService connections.Service, noUpgrade bool) *Service {
	svc := &Service{
		cfg:                cfg,
		model:              m,
		connectionsService: connectionsService,
		noUpgrade:          noUpgrade,
		forceRun:           make(chan struct{}, 1), // Buffered to prevent locking
	}
	svc.Service = util.AsService(svc.serve, svc.String())
	return svc
}

// ReportData returns the data to be sent in a usage report with the currently
// configured usage reporting version.
func (s *Service) ReportData(ctx context.Context) map[string]interface{} {
	urVersion := s.cfg.Options().URAccepted
	return s.reportData(ctx, urVersion, false)
}

// ReportDataPreview returns a preview of the data to be sent in a usage report
// with the given version.
func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) map[string]interface{} {
	return s.reportData(ctx, urVersion, true)
}

func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) map[string]interface{} {
	opts := s.cfg.Options()
	res := make(map[string]interface{})
	res["urVersion"] = urVersion
	res["uniqueID"] = opts.URUniqueID
	res["version"] = build.Version
	res["longVersion"] = build.LongVersion
	res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
	res["numFolders"] = len(s.cfg.Folders())
	res["numDevices"] = len(s.cfg.Devices())

	var totFiles, maxFiles int
	var totBytes, maxBytes int64
	for folderID := range s.cfg.Folders() {
		snap, err := s.model.DBSnapshot(folderID)
		if err != nil {
			continue
		}
		global := snap.GlobalSize()
		snap.Release()
		totFiles += int(global.Files)
		totBytes += global.Bytes
		if int(global.Files) > maxFiles {
			maxFiles = int(global.Files)
		}
		if global.Bytes > maxBytes {
			maxBytes = global.Bytes
		}
	}

	res["totFiles"] = totFiles
	res["folderMaxFiles"] = maxFiles
	res["totMiB"] = totBytes / 1024 / 1024
	res["folderMaxMiB"] = maxBytes / 1024 / 1024

	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
	res["sha256Perf"] = CpuBench(ctx, 5, 125*time.Millisecond, false)
	res["hashPerf"] = CpuBench(ctx, 5, 125*time.Millisecond, true)

	bytes, err := memorySize()
	if err == nil {
		res["memorySize"] = bytes / 1024 / 1024
	}
	res["numCPU"] = runtime.NumCPU()

	var rescanIntvs []int
	folderUses := map[string]int{
		"sendonly":            0,
		"sendreceive":         0,
		"receiveonly":         0,
		"ignorePerms":         0,
		"ignoreDelete":        0,
		"autoNormalize":       0,
		"simpleVersioning":    0,
		"externalVersioning":  0,
		"staggeredVersioning": 0,
		"trashcanVersioning":  0,
	}
	for _, cfg := range s.cfg.Folders() {
		rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)

		switch cfg.Type {
		case config.FolderTypeSendOnly:
			folderUses["sendonly"]++
		case config.FolderTypeSendReceive:
			folderUses["sendreceive"]++
		case config.FolderTypeReceiveOnly:
			folderUses["receiveonly"]++
		}
		if cfg.IgnorePerms {
			folderUses["ignorePerms"]++
		}
		if cfg.IgnoreDelete {
			folderUses["ignoreDelete"]++
		}
		if cfg.AutoNormalize {
			folderUses["autoNormalize"]++
		}
		if cfg.Versioning.Type != "" {
			folderUses[cfg.Versioning.Type+"Versioning"]++
		}
	}
	sort.Ints(rescanIntvs)
	res["rescanIntvs"] = rescanIntvs
	res["folderUses"] = folderUses

	deviceUses := map[string]int{
		"introducer":       0,
		"customCertName":   0,
		"compressAlways":   0,
		"compressMetadata": 0,
		"compressNever":    0,
		"dynamicAddr":      0,
		"staticAddr":       0,
	}
	for _, cfg := range s.cfg.Devices() {
		if cfg.Introducer {
			deviceUses["introducer"]++
		}
		if cfg.CertName != "" && cfg.CertName != "syncthing" {
			deviceUses["customCertName"]++
		}
		if cfg.Compression == protocol.CompressAlways {
			deviceUses["compressAlways"]++
		} else if cfg.Compression == protocol.CompressMetadata {
			deviceUses["compressMetadata"]++
		} else if cfg.Compression == protocol.CompressNever {
			deviceUses["compressNever"]++
		}
		for _, addr := range cfg.Addresses {
			if addr == "dynamic" {
				deviceUses["dynamicAddr"]++
			} else {
				deviceUses["staticAddr"]++
			}
		}
	}
	res["deviceUses"] = deviceUses

	defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
	for _, addr := range opts.RawGlobalAnnServers {
		if addr == "default" || addr == "default-v4" || addr == "default-v6" {
			defaultAnnounceServersDNS++
		} else {
			otherAnnounceServers++
		}
	}
	res["announce"] = map[string]interface{}{
		"globalEnabled":     opts.GlobalAnnEnabled,
		"localEnabled":      opts.LocalAnnEnabled,
		"defaultServersDNS": defaultAnnounceServersDNS,
		"defaultServersIP":  defaultAnnounceServersIP,
		"otherServers":      otherAnnounceServers,
	}

	defaultRelayServers, otherRelayServers := 0, 0
	for _, addr := range s.cfg.Options().ListenAddresses() {
		switch {
		case addr == "dynamic+https://relays.syncthing.net/endpoint":
			defaultRelayServers++
		case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"):
			otherRelayServers++
		}
	}
	res["relays"] = map[string]interface{}{
		"enabled":        defaultRelayServers+otherAnnounceServers > 0,
		"defaultServers": defaultRelayServers,
		"otherServers":   otherRelayServers,
	}

	res["usesRateLimit"] = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0

	res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || s.noUpgrade)
	res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0
	res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases

	if urVersion >= 3 {
		res["uptime"] = s.UptimeS()
		res["natType"] = s.connectionsService.NATType()
		res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0
		res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles
		res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
		res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
		res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
		res["customTrafficClass"] = opts.TrafficClass != 0
		res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
		res["temporariesDisabled"] = opts.KeepTemporariesH == 0
		res["temporariesCustom"] = opts.KeepTemporariesH != 24
		res["limitBandwidthInLan"] = opts.LimitBandwidthInLan
		res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json"
		res["restartOnWakeup"] = opts.RestartOnWakeup

		folderUsesV3 := map[string]int{
			"scanProgressDisabled":    0,
			"conflictsDisabled":       0,
			"conflictsUnlimited":      0,
			"conflictsOther":          0,
			"disableSparseFiles":      0,
			"disableTempIndexes":      0,
			"alwaysWeakHash":          0,
			"customWeakHashThreshold": 0,
			"fsWatcherEnabled":        0,
		}
		pullOrder := make(map[string]int)
		filesystemType := make(map[string]int)
		var fsWatcherDelays []int
		for _, cfg := range s.cfg.Folders() {
			if cfg.ScanProgressIntervalS < 0 {
				folderUsesV3["scanProgressDisabled"]++
			}
			if cfg.MaxConflicts == 0 {
				folderUsesV3["conflictsDisabled"]++
			} else if cfg.MaxConflicts < 0 {
				folderUsesV3["conflictsUnlimited"]++
			} else {
				folderUsesV3["conflictsOther"]++
			}
			if cfg.DisableSparseFiles {
				folderUsesV3["disableSparseFiles"]++
			}
			if cfg.DisableTempIndexes {
				folderUsesV3["disableTempIndexes"]++
			}
			if cfg.WeakHashThresholdPct < 0 {
				folderUsesV3["alwaysWeakHash"]++
			} else if cfg.WeakHashThresholdPct != 25 {
				folderUsesV3["customWeakHashThreshold"]++
			}
			if cfg.FSWatcherEnabled {
				folderUsesV3["fsWatcherEnabled"]++
			}
			pullOrder[cfg.Order.String()]++
			filesystemType[cfg.FilesystemType.String()]++
			fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS)
		}
		sort.Ints(fsWatcherDelays)
		folderUsesV3Interface := map[string]interface{}{
			"pullOrder":       pullOrder,
			"filesystemType":  filesystemType,
			"fsWatcherDelays": fsWatcherDelays,
		}
		for key, value := range folderUsesV3 {
			folderUsesV3Interface[key] = value
		}
		res["folderUsesV3"] = folderUsesV3Interface

		guiCfg := s.cfg.GUI()
		// Anticipate multiple GUI configs in the future, hence store counts.
		guiStats := map[string]int{
			"enabled":                   0,
			"useTLS":                    0,
			"useAuth":                   0,
			"insecureAdminAccess":       0,
			"debugging":                 0,
			"insecureSkipHostCheck":     0,
			"insecureAllowFrameLoading": 0,
			"listenLocal":               0,
			"listenUnspecified":         0,
		}
		theme := make(map[string]int)
		if guiCfg.Enabled {
			guiStats["enabled"]++
			if guiCfg.UseTLS() {
				guiStats["useTLS"]++
			}
			if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
				guiStats["useAuth"]++
			}
			if guiCfg.InsecureAdminAccess {
				guiStats["insecureAdminAccess"]++
			}
			if guiCfg.Debugging {
				guiStats["debugging"]++
			}
			if guiCfg.InsecureSkipHostCheck {
				guiStats["insecureSkipHostCheck"]++
			}
			if guiCfg.InsecureAllowFrameLoading {
				guiStats["insecureAllowFrameLoading"]++
			}

			addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address())
			if err == nil {
				if addr.IP.IsLoopback() {
					guiStats["listenLocal"]++
				} else if addr.IP.IsUnspecified() {
					guiStats["listenUnspecified"]++
				}
			}

			theme[guiCfg.Theme]++
		}
		guiStatsInterface := map[string]interface{}{
			"theme": theme,
		}
		for key, value := range guiStats {
			guiStatsInterface[key] = value
		}
		res["guiStats"] = guiStatsInterface
	}

	for key, value := range s.model.UsageReportingStats(urVersion, preview) {
		res[key] = value
	}

	return res
}

func (s *Service) UptimeS() int {
	return int(time.Since(StartTime).Seconds())
}

func (s *Service) sendUsageReport(ctx context.Context) error {
	d := s.ReportData(ctx)
	var b bytes.Buffer
	if err := json.NewEncoder(&b).Encode(d); err != nil {
		return err
	}

	client := &http.Client{
		Transport: &http.Transport{
			DialContext: dialer.DialContext,
			Proxy:       http.ProxyFromEnvironment,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: s.cfg.Options().URPostInsecurely,
			},
		},
	}
	req, err := http.NewRequest("POST", s.cfg.Options().URURL, &b)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Cancel = ctx.Done()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	resp.Body.Close()
	return nil
}

func (s *Service) serve(ctx context.Context) {
	s.cfg.Subscribe(s)
	defer s.cfg.Unsubscribe(s)

	t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
	for {
		select {
		case <-ctx.Done():
			return
		case <-s.forceRun:
			t.Reset(0)
		case <-t.C:
			if s.cfg.Options().URAccepted >= 2 {
				err := s.sendUsageReport(ctx)
				if err != nil {
					l.Infoln("Usage report:", err)
				} else {
					l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
				}
			}
			t.Reset(24 * time.Hour) // next report tomorrow
		}
	}
}

func (s *Service) VerifyConfiguration(from, to config.Configuration) error {
	return nil
}

func (s *Service) CommitConfiguration(from, to config.Configuration) bool {
	if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
		select {
		case s.forceRun <- struct{}{}:
		default:
			// s.forceRun is one buffered, so even though nothing
			// was sent, a run will still happen after this point.
		}
	}
	return true
}

func (*Service) String() string {
	return "ur.Service"
}

var (
	blocksResult    []protocol.BlockInfo // so the result is not optimized away
	blocksResultMut sync.Mutex
)

// CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 {
	blocksResultMut.Lock()
	defer blocksResultMut.Unlock()

	dataSize := 16 * protocol.MinBlockSize
	bs := make([]byte, dataSize)
	rand.Reader.Read(bs)

	var perf float64
	for i := 0; i < iterations; i++ {
		if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf {
			perf = v
		}
	}
	blocksResult = nil
	return perf
}

func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 {
	t0 := time.Now()
	b := 0
	var err error
	for time.Since(t0) < duration {
		r := bytes.NewReader(bs)
		blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
		if err != nil {
			return 0 // Context done
		}
		b += len(bs)
	}
	d := time.Since(t0)
	return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
}