cmd/k8s-operator: send operator logs to tailscale (#17110)

This commit modifies the k8s operator to wrap its logger using the logtail
logger provided via the tsnet server. This causes any logs written by
the operator to make their way to Tailscale in the same fashion as
wireguard logs to be used by support.

This functionality can also be opted-out of entirely using the
"TS_NO_LOGS_NO_SUPPORT" environment variable.

Updates https://github.com/tailscale/corp/issues/32037

Signed-off-by: David Bond <[email protected]>

cmd/k8s-operator/logger.go

@@ -0,0 +1,26 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"io"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+// wrapZapCore returns a zapcore.Core implementation that splits the core chain using zapcore.NewTee. This causes
+// logs to be simultaneously written to both the original core and the provided io.Writer implementation.
+func wrapZapCore(core zapcore.Core, writer io.Writer) zapcore.Core {
+	encoder := &kzap.KubeAwareEncoder{
+		Encoder: zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
+	}
+
+	// We use a tee logger here so that logs are written to stdout/stderr normally while at the same time being
+	// sent upstream.
+	return zapcore.NewTee(core, zapcore.NewCore(encoder, zapcore.AddSync(writer), zap.DebugLevel))
+}

cmd/k8s-operator/operator.go

@@ -44,6 +44,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/envknob"
 
 	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
@@ -133,6 +134,14 @@ func main() {
 			}
 		}()
 	}
+
+	// Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable.
+	if !envknob.NoLogsNoSupport() {
+		zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
+			return wrapZapCore(core, s.LogtailWriter())
+		}))
+	}
+
 	rOpts := reconcilerOpts{
 		log:                           zlog,
 		tsServer:                      s,

cmd/k8s-operator/sts.go

@@ -439,12 +439,12 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
 		}
 
 		if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) {
-			logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+			logger.With("config", sanitizeConfig(latestConfig)).Debugf("patching the existing proxy Secret")
 			if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
 				return nil, err
 			}
 		} else {
-			logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+			logger.With("config", sanitizeConfig(latestConfig)).Debugf("creating a new Secret for the proxy")
 			if err = a.Create(ctx, secret); err != nil {
 				return nil, err
 			}
@@ -494,17 +494,16 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
 	return secretNames, nil
 }
 
-// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
-// auth key.
-func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
+// sanitizeConfig returns an ipn.ConfigVAlpha with sensitive fields redacted. Since we pump everything
+// into JSON-encoded logs it's easier to read this with a .With method than converting it to a string.
+func sanitizeConfig(c ipn.ConfigVAlpha) ipn.ConfigVAlpha {
+	// Explicitly redact AuthKey because we never want it appearing in logs. Never populate this with the
+	// actual auth key.
 	if c.AuthKey != nil {
 		c.AuthKey = ptr.To("**redacted**")
 	}
-	sanitizedBytes, err := json.Marshal(c)
-	if err != nil {
-		return "invalid config"
-	}
-	return string(sanitizedBytes)
+
+	return c
 }
 
 // DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy.

tsnet/tsnet.go

@@ -495,14 +495,14 @@ func (s *Server) TailscaleIPs() (ip4, ip6 netip.Addr) {
 	return ip4, ip6
 }
 
-// Logtailf returns a [logger.Logf] that outputs to Tailscale's logging service and will be only visible to Tailscale's
+// LogtailWriter returns an [io.Writer] that writes to Tailscale's logging service and will be only visible to Tailscale's
 // support team. Logs written there cannot be retrieved by the user. This method always returns a non-nil value.
-func (s *Server) Logtailf() logger.Logf {
+func (s *Server) LogtailWriter() io.Writer {
 	if s.logtail == nil {
-		return logger.Discard
+		return io.Discard
 	}
 
-	return s.logtail.Logf
+	return s.logtail
 }
 
 func (s *Server) getAuthKey() string {