cmd/k8s-operator: put Tailscale IPs in Service ingress status

Updates #502

Signed-off-by: Mike Beaumont <[email protected]>

cmd/containerboot/kube.go

@@ -7,9 +7,11 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
+	"net/netip"
 	"os"
 
 	"tailscale.com/kube"
@@ -32,7 +34,7 @@ func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error)
 
 // storeDeviceInfo writes deviceID into the "device_id" data field of the kube
 // secret secretName.
-func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string) error {
+func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
 	// First check if the secret exists at all. Even if running on
 	// kubernetes, we do not necessarily store state in a k8s secret.
 	if _, err := kc.GetSecret(ctx, secretName); err != nil {
@@ -46,10 +48,20 @@ func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.St
 		return err
 	}
 
+	var ips []string
+	for _, addr := range addresses {
+		ips = append(ips, addr.Addr().String())
+	}
+	deviceIPs, err := json.Marshal(ips)
+	if err != nil {
+		return err
+	}
+
 	m := &kube.Secret{
 		Data: map[string][]byte{
 			"device_id":   []byte(deviceID),
 			"device_fqdn": []byte(fqdn),
+			"device_ips":  deviceIPs,
 		},
 	}
 	return kc.StrategicMergePatchSecret(ctx, secretName, m, "tailscale-container")

cmd/containerboot/main.go

@@ -314,7 +314,7 @@ authLoop:
 			}
 			deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()}
 			if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
-				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()); err != nil {
+				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
 					log.Fatalf("storing device ID in kube secret: %v", err)
 				}
 			}

cmd/containerboot/main_test.go

@@ -113,10 +113,10 @@ func TestContainerBoot(t *testing.T) {
 		State: ptr.To(ipn.Running),
 		NetMap: &netmap.NetworkMap{
 			SelfNode: (&tailcfg.Node{
-				StableID: tailcfg.StableNodeID("myID"),
-				Name:     "test-node.test.ts.net",
+				StableID:  tailcfg.StableNodeID("myID"),
+				Name:      "test-node.test.ts.net",
+				Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
 			}).View(),
-			Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
 		},
 	}
 	tests := []struct {
@@ -359,6 +359,7 @@ func TestContainerBoot(t *testing.T) {
 						"authkey":     "tskey-key",
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},
@@ -447,6 +448,7 @@ func TestContainerBoot(t *testing.T) {
 					WantKubeSecret: map[string]string{
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},
@@ -476,6 +478,7 @@ func TestContainerBoot(t *testing.T) {
 						"authkey":     "tskey-key",
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 				{
@@ -483,16 +486,17 @@ func TestContainerBoot(t *testing.T) {
 						State: ptr.To(ipn.Running),
 						NetMap: &netmap.NetworkMap{
 							SelfNode: (&tailcfg.Node{
-								StableID: tailcfg.StableNodeID("newID"),
-								Name:     "new-name.test.ts.net",
+								StableID:  tailcfg.StableNodeID("newID"),
+								Name:      "new-name.test.ts.net",
+								Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
 							}).View(),
-							Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
 						},
 					},
 					WantKubeSecret: map[string]string{
 						"authkey":     "tskey-key",
 						"device_fqdn": "new-name.test.ts.net",
 						"device_id":   "newID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},

cmd/k8s-operator/ingress.go

@@ -194,7 +194,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to provision: %w", err)
 	}
 
-	_, tsHost, err := a.ssr.DeviceInfo(ctx, crl)
+	_, tsHost, _, err := a.ssr.DeviceInfo(ctx, crl)
 	if err != nil {
 		return fmt.Errorf("failed to get device ID: %w", err)
 	}

cmd/k8s-operator/operator_test.go

@@ -80,6 +80,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -104,6 +105,12 @@ func TestLoadBalancerClass(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
+					{
+						IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf",
+					},
 				},
 			},
 		},
@@ -306,6 +313,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -364,6 +372,12 @@ func TestAnnotationIntoLB(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
+					{
+						IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf",
+					},
 				},
 			},
 		},
@@ -425,6 +439,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -449,6 +464,12 @@ func TestLBIntoAnnotation(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
+					{
+						IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf",
+					},
 				},
 			},
 		},

cmd/k8s-operator/sts.go

@@ -122,7 +122,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
 		return false, nil
 	}
 
-	id, _, err := a.DeviceInfo(ctx, labels)
+	id, _, _, err := a.DeviceInfo(ctx, labels)
 	if err != nil {
 		return false, fmt.Errorf("getting device info: %w", err)
 	}
@@ -232,25 +232,31 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 
 // DeviceInfo returns the device ID and hostname for the Tailscale device
 // associated with the given labels.
-func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, err error) {
+func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
 	if err != nil {
-		return "", "", err
+		return "", "", nil, err
 	}
 	if sec == nil {
-		return "", "", nil
+		return "", "", nil, nil
 	}
 	id = tailcfg.StableNodeID(sec.Data["device_id"])
 	if id == "" {
-		return "", "", nil
+		return "", "", nil, nil
 	}
 	// Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have
 	// to remove it.
 	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
 	if hostname == "" {
-		return "", "", nil
+		return "", "", nil, nil
 	}
-	return id, hostname, nil
+	if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
+		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
+			return "", "", nil, err
+		}
+	}
+
+	return id, hostname, ips, nil
 }
 
 func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {

cmd/k8s-operator/svc.go

@@ -139,7 +139,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	_, tsHost, err := a.ssr.DeviceInfo(ctx, crl)
+	_, tsHost, tsIPs, err := a.ssr.DeviceInfo(ctx, crl)
 	if err != nil {
 		return fmt.Errorf("failed to get device ID: %w", err)
 	}
@@ -153,12 +153,14 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	logger.Debugf("setting ingress hostname to %q", tsHost)
-	svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{
-		{
-			Hostname: tsHost,
-		},
+	logger.Debugf("setting ingress to %q, %s", tsHost, strings.Join(tsIPs, ", "))
+	ingress := []corev1.LoadBalancerIngress{
+		{Hostname: tsHost},
 	}
+	for _, ip := range tsIPs {
+		ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip})
+	}
+	svc.Status.LoadBalancer.Ingress = ingress
 	if err := a.Status().Update(ctx, svc); err != nil {
 		return fmt.Errorf("failed to update service status: %w", err)
 	}