ipn/localapi, cmd/tailscale: add CPU & memory profile support, debug command

This was already possible on Linux if you ran tailscaled with --debug
(which runs net/http/pprof), but it requires the user have the Go
toolchain around.

Also, it wasn't possible on macOS, as there's no way to run the IPNExtension
with a debug server (it doesn't run tailscaled).

And on Windows it's super tedious: beyond what users want to do or
what we want to explain.

Instead, put it in "tailscale debug" so it works and works the same on
all platforms. Then we can ask users to run it when we're debugging something
and they can email us the output files.

Signed-off-by: Brad Fitzpatrick <[email protected]>

client/tailscale/tailscale.go

@@ -154,6 +154,18 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// Profile returns a pprof profile of the Tailscale daemon.
+func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
+	var secArg string
+	if sec < 0 || sec > 300 {
+		return nil, errors.New("duration out of range")
+	}
+	if sec != 0 || pprofType == "profile" {
+		secArg = fmt.Sprint(sec)
+	}
+	return get200(ctx, fmt.Sprintf("/localapi/v0/profile?name=%s&seconds=%v", url.QueryEscape(pprofType), secArg))
+}
+
 // BugReport logs and returns a log marker that can be shared by the user with support.
 func BugReport(ctx context.Context, note string) (string, error) {
 	body, err := send(ctx, "POST", "/localapi/v0/bugreport?note="+url.QueryEscape(note), 200, nil)

cmd/tailscale/cli/debug.go

@@ -36,6 +36,9 @@ var debugCmd = &ffcli.Command{
 		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
 		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
+		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file")
+		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file")
+		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
 }
@@ -49,6 +52,9 @@ var debugArgs struct {
 	file       string
 	prefs      bool
 	pretty     bool
+	cpuSec     int
+	cpuFile    string
+	memFile    string
 }
 
 func runDebug(ctx context.Context, args []string) error {
@@ -68,6 +74,28 @@ func runDebug(ctx context.Context, args []string) error {
 		fmt.Printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
 		return nil
 	}
+	if out := debugArgs.cpuFile; out != "" {
+		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
+		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
+			return err
+		} else {
+			if err := os.WriteFile(out, v, 0600); err != nil {
+				return err
+			}
+			log.Printf("CPU profile written to %s", out)
+		}
+	}
+	if out := debugArgs.memFile; out != "" {
+		log.Printf("Capturing memory profile ...")
+		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
+			return err
+		} else {
+			if err := os.WriteFile(out, v, 0600); err != nil {
+				return err
+			}
+			log.Printf("Memory profile written to %s", out)
+		}
+	}
 	if debugArgs.prefs {
 		prefs, err := tailscale.GetPrefs(ctx)
 		if err != nil {

cmd/tailscaled/depaware.txt

@@ -279,7 +279,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         net/http/httptrace                                           from github.com/tcnksm/go-httpstat+
         net/http/httputil                                            from tailscale.com/ipn/localapi
         net/http/internal                                            from net/http+
-        net/http/pprof                                               from tailscale.com/cmd/tailscaled
+        net/http/pprof                                               from tailscale.com/cmd/tailscaled+
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
         os                                                           from crypto/rand+

ipn/localapi/localapi.go

@@ -94,6 +94,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveWhoIs(w, r)
 	case "/localapi/v0/goroutines":
 		h.serveGoroutines(w, r)
+	case "/localapi/v0/profile":
+		h.serveProfile(w, r)
 	case "/localapi/v0/status":
 		h.serveStatus(w, r)
 	case "/localapi/v0/logout":
@@ -181,6 +183,24 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+// serveProfileFunc is the implementation of Handler.serveProfile, after auth,
+// for platforms where we want to link it in.
+var serveProfileFunc func(http.ResponseWriter, *http.Request)
+
+func (h *Handler) serveProfile(w http.ResponseWriter, r *http.Request) {
+	// Require write access out of paranoia that the profile dump
+	// might contain something sensitive.
+	if !h.PermitWrite {
+		http.Error(w, "profile access denied", http.StatusForbidden)
+		return
+	}
+	if serveProfileFunc == nil {
+		http.Error(w, "not implemented on this platform", http.StatusServiceUnavailable)
+		return
+	}
+	serveProfileFunc(w, r)
+}
+
 func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitRead {
 		http.Error(w, "IP forwarding check access denied", http.StatusForbidden)

ipn/localapi/profile.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !ios && !android
+// +build !ios,!android
+
+// We don't include it on mobile where we're more memory constrained and
+// there's no CLI to get at the results anyway.
+
+package localapi
+
+import (
+	"net/http"
+	"net/http/pprof"
+)
+
+func init() {
+	serveProfileFunc = serveProfile
+}
+
+func serveProfile(w http.ResponseWriter, r *http.Request) {
+	name := r.FormValue("name")
+	switch name {
+	case "profile":
+		pprof.Profile(w, r)
+	default:
+		pprof.Handler(name).ServeHTTP(w, r)
+	}
+}