cmd/k8s-operator, k8s-operator: support Static Endpoints on ProxyGroups (#16115)

updates: #14674

Signed-off-by: chaosinthecrd <[email protected]>

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -16,6 +16,9 @@ kind: ClusterRole
 metadata:
   name: tailscale-operator
 rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list", "watch"]
 - apiGroups: [""]
   resources: ["events", "services", "services/status"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2203,6 +2203,51 @@ spec:
                                   won't make it *more* imbalanced.
                                   It's a required field.
                                 type: string
+                staticEndpoints:
+                  description: |-
+                    Configuration for 'static endpoints' on proxies in order to facilitate
+                    direct connections from other devices on the tailnet.
+                    See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
+                  type: object
+                  required:
+                    - nodePort
+                  properties:
+                    nodePort:
+                      description: The configuration for static endpoints using NodePort Services.
+                      type: object
+                      required:
+                        - ports
+                      properties:
+                        ports:
+                          description: |-
+                            The port ranges from which the operator will select NodePorts for the Services.
+                            You must ensure that firewall rules allow UDP ingress traffic for these ports
+                            to the node's external IPs.
+                            The ports must be in the range of service node ports for the cluster (default `30000-32767`).
+                            See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
+                          type: array
+                          minItems: 1
+                          items:
+                            type: object
+                            required:
+                              - port
+                            properties:
+                              endPort:
+                                description: |-
+                                  endPort indicates that the range of ports from port to endPort if set, inclusive,
+                                  should be used. This field cannot be defined if the port field is not defined.
+                                  The endPort must be either unset, or equal or greater than port.
+                                type: integer
+                              port:
+                                description: port represents a port selected to be used. This is a required field.
+                                type: integer
+                        selector:
+                          description: |-
+                            A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
+                            by the ProxyGroup as Static Endpoints.
+                          type: object
+                          additionalProperties:
+                            type: string
                 tailscale:
                   description: |-
                     TailscaleConfig contains options to configure the tailscale-specific

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -196,6 +196,11 @@ spec:
                           If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                           node.
                         type: string
+                      staticEndpoints:
+                        description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
+                        type: array
+                        items:
+                          type: string
                       tailnetIPs:
                         description: |-
                           TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2679,6 +2679,51 @@ spec:
                                                 type: array
                                         type: object
                                 type: object
+                            staticEndpoints:
+                                description: |-
+                                    Configuration for 'static endpoints' on proxies in order to facilitate
+                                    direct connections from other devices on the tailnet.
+                                    See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
+                                properties:
+                                    nodePort:
+                                        description: The configuration for static endpoints using NodePort Services.
+                                        properties:
+                                            ports:
+                                                description: |-
+                                                    The port ranges from which the operator will select NodePorts for the Services.
+                                                    You must ensure that firewall rules allow UDP ingress traffic for these ports
+                                                    to the node's external IPs.
+                                                    The ports must be in the range of service node ports for the cluster (default `30000-32767`).
+                                                    See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
+                                                items:
+                                                    properties:
+                                                        endPort:
+                                                            description: |-
+                                                                endPort indicates that the range of ports from port to endPort if set, inclusive,
+                                                                should be used. This field cannot be defined if the port field is not defined.
+                                                                The endPort must be either unset, or equal or greater than port.
+                                                            type: integer
+                                                        port:
+                                                            description: port represents a port selected to be used. This is a required field.
+                                                            type: integer
+                                                    required:
+                                                        - port
+                                                    type: object
+                                                minItems: 1
+                                                type: array
+                                            selector:
+                                                additionalProperties:
+                                                    type: string
+                                                description: |-
+                                                    A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
+                                                    by the ProxyGroup as Static Endpoints.
+                                                type: object
+                                        required:
+                                            - ports
+                                        type: object
+                                required:
+                                    - nodePort
+                                type: object
                             tailscale:
                                 description: |-
                                     TailscaleConfig contains options to configure the tailscale-specific
@@ -2976,6 +3021,11 @@ spec:
                                                 If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
                                                 node.
                                             type: string
+                                        staticEndpoints:
+                                            description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
+                                            items:
+                                                type: string
+                                            type: array
                                         tailnetIPs:
                                             description: |-
                                                 TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
@@ -4791,6 +4841,14 @@ kind: ClusterRole
 metadata:
     name: tailscale-operator
 rules:
+    - apiGroups:
+        - ""
+      resources:
+        - nodes
+      verbs:
+        - get
+        - list
+        - watch
     - apiGroups:
         - ""
       resources:

cmd/k8s-operator/nodeport-service-ports.go

@@ -0,0 +1,203 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"math/rand/v2"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	k8soperator "tailscale.com/k8s-operator"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
+)
+
+const (
+	tailscaledPortMax = 65535
+	tailscaledPortMin = 1024
+	testSvcName       = "test-node-port-range"
+
+	invalidSvcNodePort = 777777
+)
+
+// getServicesNodePortRange is a hacky function that attempts to determine Service NodePort range by
+// creating a deliberately invalid Service with a NodePort that is too large and parsing the returned
+// validation error. Returns nil if unable to determine port range.
+// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+func getServicesNodePortRange(ctx context.Context, c client.Client, tsNamespace string, logger *zap.SugaredLogger) *tsapi.PortRange {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      testSvcName,
+			Namespace: tsNamespace,
+			Labels: map[string]string{
+				kubetypes.LabelManaged: "true",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeNodePort,
+			Ports: []corev1.ServicePort{
+				{
+					Name:       testSvcName,
+					Port:       8080,
+					TargetPort: intstr.FromInt32(8080),
+					Protocol:   corev1.ProtocolUDP,
+					NodePort:   invalidSvcNodePort,
+				},
+			},
+		},
+	}
+
+	// NOTE(ChaosInTheCRD): ideally this would be a server side dry-run but could not get it working
+	err := c.Create(ctx, svc)
+	if err == nil {
+		return nil
+	}
+
+	if validPorts := getServicesNodePortRangeFromErr(err.Error()); validPorts != "" {
+		pr, err := parseServicesNodePortRange(validPorts)
+		if err != nil {
+			logger.Debugf("failed to parse NodePort range set for Kubernetes Cluster: %w", err)
+			return nil
+		}
+
+		return pr
+	}
+
+	return nil
+}
+
+func getServicesNodePortRangeFromErr(err string) string {
+	reg := regexp.MustCompile(`\d{1,5}-\d{1,5}`)
+	matches := reg.FindAllString(err, -1)
+	if len(matches) != 1 {
+		return ""
+	}
+
+	return matches[0]
+}
+
+// parseServicesNodePortRange converts the `ValidPorts` string field in the Kubernetes PortAllocator error and converts it to
+// PortRange
+func parseServicesNodePortRange(p string) (*tsapi.PortRange, error) {
+	parts := strings.Split(p, "-")
+	s, err := strconv.ParseUint(parts[0], 10, 16)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse string as uint16: %w", err)
+	}
+
+	var e uint64
+	switch len(parts) {
+	case 1:
+		e = uint64(s)
+	case 2:
+		e, err = strconv.ParseUint(parts[1], 10, 16)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse string as uint16: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("failed to parse port range %q", p)
+	}
+
+	portRange := &tsapi.PortRange{Port: uint16(s), EndPort: uint16(e)}
+	if !portRange.IsValid() {
+		return nil, fmt.Errorf("port range %q is not valid", portRange.String())
+	}
+
+	return portRange, nil
+}
+
+// validateNodePortRanges checks that the port range specified is valid. It also ensures that the specified ranges
+// lie within the NodePort Service port range specified for the Kubernetes API Server.
+func validateNodePortRanges(ctx context.Context, c client.Client, kubeRange *tsapi.PortRange, pc *tsapi.ProxyClass) error {
+	if pc.Spec.StaticEndpoints == nil {
+		return nil
+	}
+
+	portRanges := pc.Spec.StaticEndpoints.NodePort.Ports
+
+	if kubeRange != nil {
+		for _, pr := range portRanges {
+			if !kubeRange.Contains(pr.Port) || (pr.EndPort != 0 && !kubeRange.Contains(pr.EndPort)) {
+				return fmt.Errorf("range %q is not within Cluster configured range %q", pr.String(), kubeRange.String())
+			}
+		}
+	}
+
+	for _, r := range portRanges {
+		if !r.IsValid() {
+			return fmt.Errorf("port range %q is invalid", r.String())
+		}
+	}
+
+	// TODO(ChaosInTheCRD): if a ProxyClass that made another invalid (due to port range clash) is deleted,
+	// the invalid ProxyClass doesn't get reconciled on, and therefore will not go valid. We should fix this.
+	proxyClassRanges, err := getPortsForProxyClasses(ctx, c)
+	if err != nil {
+		return fmt.Errorf("failed to get port ranges for ProxyClasses: %w", err)
+	}
+
+	for _, r := range portRanges {
+		for pcName, pcr := range proxyClassRanges {
+			if pcName == pc.Name {
+				continue
+			}
+			if pcr.ClashesWith(r) {
+				return fmt.Errorf("port ranges for ProxyClass %q clash with existing ProxyClass %q", pc.Name, pcName)
+			}
+		}
+	}
+
+	if len(portRanges) == 1 {
+		return nil
+	}
+
+	sort.Slice(portRanges, func(i, j int) bool {
+		return portRanges[i].Port < portRanges[j].Port
+	})
+
+	for i := 1; i < len(portRanges); i++ {
+		prev := portRanges[i-1]
+		curr := portRanges[i]
+		if curr.Port <= prev.Port || curr.Port <= prev.EndPort {
+			return fmt.Errorf("overlapping ranges: %q and %q", prev.String(), curr.String())
+		}
+	}
+
+	return nil
+}
+
+// getPortsForProxyClasses gets the port ranges for all the other existing ProxyClasses
+func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]tsapi.PortRanges, error) {
+	pcs := new(tsapi.ProxyClassList)
+
+	err := c.List(ctx, pcs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list ProxyClasses: %w", err)
+	}
+
+	portRanges := make(map[string]tsapi.PortRanges)
+	for _, i := range pcs.Items {
+		if !k8soperator.ProxyClassIsReady(&i) {
+			continue
+		}
+		if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil {
+			portRanges[i.Name] = se.NodePort.Ports
+		}
+	}
+
+	return portRanges, nil
+}
+
+func getRandomPort() uint16 {
+	return uint16(rand.IntN(tailscaledPortMax-tailscaledPortMin+1) + tailscaledPortMin)
+}

cmd/k8s-operator/nodeport-services-ports_test.go

@@ -0,0 +1,277 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+)
+
+func TestGetServicesNodePortRangeFromErr(t *testing.T) {
+	tests := []struct {
+		name   string
+		errStr string
+		want   string
+	}{
+		{
+			name:   "valid_error_string",
+			errStr: "NodePort 777777 is not in the allowed range 30000-32767",
+			want:   "30000-32767",
+		},
+		{
+			name:   "error_string_with_different_message",
+			errStr: "some other error without a port range",
+			want:   "",
+		},
+		{
+			name:   "error_string_with_multiple_port_ranges",
+			errStr: "range 1000-2000 and another range 3000-4000",
+			want:   "",
+		},
+		{
+			name:   "empty_error_string",
+			errStr: "",
+			want:   "",
+		},
+		{
+			name:   "error_string_with_range_at_start",
+			errStr: "30000-32767 is the range",
+			want:   "30000-32767",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getServicesNodePortRangeFromErr(tt.errStr); got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseServicesNodePortRange(t *testing.T) {
+	tests := []struct {
+		name    string
+		p       string
+		want    *tsapi.PortRange
+		wantErr bool
+	}{
+		{
+			name:    "valid_range",
+			p:       "30000-32767",
+			want:    &tsapi.PortRange{Port: 30000, EndPort: 32767},
+			wantErr: false,
+		},
+		{
+			name:    "single_port_range",
+			p:       "30000",
+			want:    &tsapi.PortRange{Port: 30000, EndPort: 30000},
+			wantErr: false,
+		},
+		{
+			name:    "invalid_format_non_numeric_end",
+			p:       "30000-abc",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "invalid_format_non_numeric_start",
+			p:       "abc-32767",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "empty_string",
+			p:       "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "too_many_parts",
+			p:       "1-2-3",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "port_too_large_start",
+			p:       "65536-65537",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "port_too_large_end",
+			p:       "30000-65536",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "inverted_range",
+			p:       "32767-30000",
+			want:    nil,
+			wantErr: true, // IsValid() will fail
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			portRange, err := parseServicesNodePortRange(tt.p)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if tt.wantErr {
+				return
+			}
+
+			if portRange == nil {
+				t.Fatalf("got nil port range, expected %v", tt.want)
+			}
+
+			if portRange.Port != tt.want.Port || portRange.EndPort != tt.want.EndPort {
+				t.Errorf("got = %v, want %v", portRange, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateNodePortRanges(t *testing.T) {
+	tests := []struct {
+		name       string
+		portRanges []tsapi.PortRange
+		wantErr    bool
+	}{
+		{
+			name: "valid_ranges_with_unknown_kube_range",
+			portRanges: []tsapi.PortRange{
+				{Port: 30003, EndPort: 30005},
+				{Port: 30006, EndPort: 30007},
+			},
+			wantErr: false,
+		},
+		{
+			name: "overlapping_ranges",
+			portRanges: []tsapi.PortRange{
+				{Port: 30000, EndPort: 30010},
+				{Port: 30005, EndPort: 30015},
+			},
+			wantErr: true,
+		},
+		{
+			name: "adjacent_ranges_no_overlap",
+			portRanges: []tsapi.PortRange{
+				{Port: 30010, EndPort: 30020},
+				{Port: 30021, EndPort: 30022},
+			},
+			wantErr: false,
+		},
+		{
+			name: "identical_ranges_are_overlapping",
+			portRanges: []tsapi.PortRange{
+				{Port: 30005, EndPort: 30010},
+				{Port: 30005, EndPort: 30010},
+			},
+			wantErr: true,
+		},
+		{
+			name: "range_clashes_with_existing_proxyclass",
+			portRanges: []tsapi.PortRange{
+				{Port: 31005, EndPort: 32070},
+			},
+			wantErr: true,
+		},
+	}
+
+	// as part of this test, we want to create an adjacent ProxyClass in order to ensure that if it clashes with the one created in this test
+	// that we get an error
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	opc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "other-pc",
+		},
+		Spec: tsapi.ProxyClassSpec{
+			StatefulSet: &tsapi.StatefulSet{
+				Annotations: defaultProxyClassAnnotations,
+			},
+			StaticEndpoints: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 31000}, {Port: 32000},
+					},
+					Selector: map[string]string{
+						"foo/bar": "baz",
+					},
+				},
+			},
+		},
+		Status: tsapi.ProxyClassStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyClassReady),
+				Status:             metav1.ConditionTrue,
+				Reason:             reasonProxyClassValid,
+				Message:            reasonProxyClassValid,
+				LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			}},
+		},
+	}
+
+	fc := fake.NewClientBuilder().
+		WithObjects(opc).
+		WithStatusSubresource(opc).
+		WithScheme(tsapi.GlobalScheme).
+		Build()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pc := &tsapi.ProxyClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pc",
+				},
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Annotations: defaultProxyClassAnnotations,
+					},
+					StaticEndpoints: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: tt.portRanges,
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+				},
+				Status: tsapi.ProxyClassStatus{
+					Conditions: []metav1.Condition{{
+						Type:               string(tsapi.ProxyClassReady),
+						Status:             metav1.ConditionTrue,
+						Reason:             reasonProxyClassValid,
+						Message:            reasonProxyClassValid,
+						LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+					}},
+				},
+			}
+			err := validateNodePortRanges(context.Background(), fc, &tsapi.PortRange{Port: 30000, EndPort: 32767}, pc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestGetRandomPort(t *testing.T) {
+	for range 100 {
+		port := getRandomPort()
+		if port < tailscaledPortMin || port > tailscaledPortMax {
+			t.Errorf("generated port %d which is out of range [%d, %d]", port, tailscaledPortMin, tailscaledPortMax)
+		}
+	}
+}

cmd/k8s-operator/operator.go

@@ -26,7 +26,9 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	klabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	toolscache "k8s.io/client-go/tools/cache"
@@ -39,6 +41,7 @@ import (
 	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
@@ -228,6 +231,17 @@ waitOnline:
 	return s, tsc
 }
 
+// predicate function for filtering to ensure we *don't* reconcile on tailscale managed Kubernetes Services
+func serviceManagedResourceFilterPredicate() predicate.Predicate {
+	return predicate.NewPredicateFuncs(func(object client.Object) bool {
+		if svc, ok := object.(*corev1.Service); !ok {
+			return false
+		} else {
+			return !isManagedResource(svc)
+		}
+	})
+}
+
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
 func runReconcilers(opts reconcilerOpts) {
@@ -374,7 +388,7 @@ func runReconcilers(opts reconcilerOpts) {
 	ingressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(ingressSvcFromEps(mgr.GetClient(), opts.log.Named("service-pg-reconciler")))
 	err = builder.
 		ControllerManagedBy(mgr).
-		For(&corev1.Service{}).
+		For(&corev1.Service{}, builder.WithPredicates(serviceManagedResourceFilterPredicate())).
 		Named("service-pg-reconciler").
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))).
 		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
@@ -519,16 +533,19 @@ func runReconcilers(opts reconcilerOpts) {
 	// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
 	// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
 	// reconciled if the CRD is applied at a later point.
+	kPortRange := getServicesNodePortRange(context.Background(), mgr.GetClient(), opts.tailscaleNamespace, startlog)
 	serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyClass{}).
 		Named("proxyclass-reconciler").
 		Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
 		Complete(&ProxyClassReconciler{
-			Client:   mgr.GetClient(),
-			recorder: eventRecorder,
-			logger:   opts.log.Named("proxyclass-reconciler"),
-			clock:    tstime.DefaultClock{},
+			Client:        mgr.GetClient(),
+			nodePortRange: kPortRange,
+			recorder:      eventRecorder,
+			tsNamespace:   opts.tailscaleNamespace,
+			logger:        opts.log.Named("proxyclass-reconciler"),
+			clock:         tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatal("could not create proxyclass reconciler: %v", err)
@@ -587,9 +604,11 @@ func runReconcilers(opts reconcilerOpts) {
 	// ProxyGroup reconciler.
 	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
+	nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyGroup{}).
 		Named("proxygroup-reconciler").
+		Watches(&corev1.Service{}, ownedByProxyGroupFilter).
 		Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
@@ -597,6 +616,7 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
 		Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
+		Watches(&corev1.Node{}, nodeFilterForProxyGroup).
 		Complete(&ProxyGroupReconciler{
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
@@ -840,6 +860,64 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
+// nodeHandlerForProxyGroup returns a handler that, for a given Node, returns a
+// list of reconcile requests for ProxyGroups that should be reconciled for the
+// Node event. ProxyGroups need to be reconciled for Node events if they are
+// configured to expose tailscaled static endpoints to tailnet using NodePort
+// Services.
+func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		pgList := new(tsapi.ProxyGroupList)
+		if err := cl.List(ctx, pgList); err != nil {
+			logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err)
+			return nil
+		}
+
+		reqs := make([]reconcile.Request, 0)
+		for _, pg := range pgList.Items {
+			if pg.Spec.ProxyClass == "" && defaultProxyClass == "" {
+				continue
+			}
+
+			pc := defaultProxyClass
+			if pc == "" {
+				pc = pg.Spec.ProxyClass
+			}
+
+			proxyClass := &tsapi.ProxyClass{}
+			if err := cl.Get(ctx, types.NamespacedName{Name: pc}, proxyClass); err != nil {
+				logger.Debugf("error getting ProxyClass %q: %v", pg.Spec.ProxyClass, err)
+				return nil
+			}
+
+			stat := proxyClass.Spec.StaticEndpoints
+			if stat == nil {
+				continue
+			}
+
+			// If the selector is empty, all nodes match.
+			// TODO(ChaosInTheCRD): think about how this must be handled if we want to limit the number of nodes used
+			if len(stat.NodePort.Selector) == 0 {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
+				continue
+			}
+
+			selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+				MatchLabels: stat.NodePort.Selector,
+			})
+			if err != nil {
+				logger.Debugf("error converting `spec.staticEndpoints.nodePort.selector` to Selector: %v", err)
+				return nil
+			}
+
+			if selector.Matches(klabels.Set(o.GetLabels())) {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
+			}
+		}
+		return reqs
+	}
+}
+
 // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass,
 // returns a list of reconcile requests for all Connectors that have
 // .spec.proxyClass set.

cmd/k8s-operator/proxyclass.go

@@ -44,22 +44,24 @@ const (
 type ProxyClassReconciler struct {
 	client.Client
 
-	recorder record.EventRecorder
-	logger   *zap.SugaredLogger
-	clock    tstime.Clock
+	recorder    record.EventRecorder
+	logger      *zap.SugaredLogger
+	clock       tstime.Clock
+	tsNamespace string
 
 	mu sync.Mutex // protects following
 
 	// managedProxyClasses is a set of all ProxyClass resources that we're currently
 	// managing. This is only used for metrics.
 	managedProxyClasses set.Slice[types.UID]
+	// nodePortRange is the NodePort range set for the Kubernetes Cluster. This is used
+	// when validating port ranges configured by users for spec.StaticEndpoints
+	nodePortRange *tsapi.PortRange
 }
 
-var (
-	// gaugeProxyClassResources tracks the number of ProxyClass resources
-	// that we're currently managing.
-	gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
-)
+// gaugeProxyClassResources tracks the number of ProxyClass resources
+// that we're currently managing.
+var gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
 
 func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
 	logger := pcr.logger.With("ProxyClass", req.Name)
@@ -96,7 +98,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	pcr.mu.Unlock()
 
 	oldPCStatus := pc.Status.DeepCopy()
-	if errs := pcr.validate(ctx, pc); errs != nil {
+	if errs := pcr.validate(ctx, pc, logger); errs != nil {
 		msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
 		pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
 		tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
@@ -112,7 +114,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 	return reconcile.Result{}, nil
 }
 
-func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
+func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) (violations field.ErrorList) {
 	if sts := pc.Spec.StatefulSet; sts != nil {
 		if len(sts.Labels) > 0 {
 			if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil {
@@ -183,6 +185,17 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
 			violations = append(violations, errs...)
 		}
 	}
+
+	if stat := pc.Spec.StaticEndpoints; stat != nil {
+		if err := validateNodePortRanges(ctx, pcr.Client, pcr.nodePortRange, pc); err != nil {
+			var prs tsapi.PortRanges = stat.NodePort.Ports
+			violations = append(violations, field.TypeInvalid(field.NewPath("spec", "staticEndpoints", "nodePort", "ports"), prs.String(), err.Error()))
+		}
+
+		if len(stat.NodePort.Selector) < 1 {
+			logger.Debug("no Selectors specified on `spec.staticEndpoints.nodePort.selectors` field")
+		}
+	}
 	// We do not validate embedded fields (security context, resource
 	// requirements etc) as we inherit upstream validation for those fields.
 	// Invalid values would get rejected by upstream validations at apply

cmd/k8s-operator/proxyclass_test.go

@@ -131,9 +131,11 @@ func TestProxyClass(t *testing.T) {
 		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image
 		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}}
 	})
-	expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
+	expectedEvents := []string{
+		"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
 		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
-		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."}
+		"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
+	}
 	expectReconciled(t, pcr, "", "test")
 	expectEvents(t, fr, expectedEvents)
 
@@ -176,6 +178,110 @@ func TestProxyClass(t *testing.T) {
 	expectEqual(t, fc, pc)
 }
 
+func TestValidateProxyClassStaticEndpoints(t *testing.T) {
+	for name, tc := range map[string]struct {
+		staticEndpointConfig *tsapi.StaticEndpointsConfig
+		valid                bool
+	}{
+		"no_static_endpoints": {
+			staticEndpointConfig: nil,
+			valid:                true,
+		},
+		"valid_specific_ports": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 3001},
+						{Port: 3005},
+					},
+					Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
+				},
+			},
+			valid: true,
+		},
+		"valid_port_ranges": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 3000, EndPort: 3002},
+						{Port: 3005, EndPort: 3007},
+					},
+					Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
+				},
+			},
+			valid: true,
+		},
+		"overlapping_port_ranges": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 1000, EndPort: 2000},
+						{Port: 1500, EndPort: 1800},
+					},
+					Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
+				},
+			},
+			valid: false,
+		},
+		"clashing_port_and_range": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 3005},
+						{Port: 3001, EndPort: 3010},
+					},
+					Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
+				},
+			},
+			valid: false,
+		},
+		"malformed_port_range": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports: []tsapi.PortRange{
+						{Port: 3001, EndPort: 3000},
+					},
+					Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
+				},
+			},
+			valid: false,
+		},
+		"empty_selector": {
+			staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+				NodePort: &tsapi.NodePortConfig{
+					Ports:    []tsapi.PortRange{{Port: 3000}},
+					Selector: map[string]string{},
+				},
+			},
+			valid: true,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			fc := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme).
+				Build()
+			zl, _ := zap.NewDevelopment()
+			pcr := &ProxyClassReconciler{
+				logger: zl.Sugar(),
+				Client: fc,
+			}
+
+			pc := &tsapi.ProxyClass{
+				Spec: tsapi.ProxyClassSpec{
+					StaticEndpoints: tc.staticEndpointConfig,
+				},
+			}
+
+			logger := pcr.logger.With("ProxyClass", pc)
+			err := pcr.validate(context.Background(), pc, logger)
+			valid := err == nil
+			if valid != tc.valid {
+				t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
+			}
+		})
+	}
+}
+
 func TestValidateProxyClass(t *testing.T) {
 	for name, tc := range map[string]struct {
 		pc    *tsapi.ProxyClass
@@ -219,8 +325,12 @@ func TestValidateProxyClass(t *testing.T) {
 		},
 	} {
 		t.Run(name, func(t *testing.T) {
-			pcr := &ProxyClassReconciler{}
-			err := pcr.validate(context.Background(), tc.pc)
+			zl, _ := zap.NewDevelopment()
+			pcr := &ProxyClassReconciler{
+				logger: zl.Sugar(),
+			}
+			logger := pcr.logger.With("ProxyClass", tc.pc)
+			err := pcr.validate(context.Background(), tc.pc, logger)
 			valid := err == nil
 			if valid != tc.valid {
 				t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)

cmd/k8s-operator/proxygroup.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"net/netip"
 	"slices"
 	"strings"
 	"sync"
@@ -24,6 +25,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -48,7 +50,8 @@ const (
 	reasonProxyGroupInvalid        = "ProxyGroupInvalid"
 
 	// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
-	optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
+	optimisticLockErrorMsg  = "the object has been modified; please apply your changes to the latest version and try again"
+	staticEndpointsMaxAddrs = 2
 )
 
 var (
@@ -174,7 +177,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		}
 	}
 
-	if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
+	isProvisioned, err := r.maybeProvision(ctx, pg, proxyClass)
+	if err != nil {
 		reason := reasonProxyGroupCreationFailed
 		msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err)
 		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
@@ -185,9 +189,20 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		} else {
 			r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
 		}
+
 		return setStatusReady(pg, metav1.ConditionFalse, reason, msg)
 	}
 
+	if !isProvisioned {
+		if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
+				return reconcile.Result{}, errors.Join(err, updateErr)
+			}
+		}
+		return
+	}
+
 	desiredReplicas := int(pgReplicas(pg))
 	if len(pg.Status.Devices) < desiredReplicas {
 		message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
@@ -230,15 +245,42 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc
 	}
 }
 
-func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
+func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (isProvisioned bool, err error) {
 	logger := r.logger(pg.Name)
 	r.mu.Lock()
 	r.ensureAddedToGaugeForProxyGroup(pg)
 	r.mu.Unlock()
 
-	if err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass); err != nil {
-		return fmt.Errorf("error provisioning config Secrets: %w", err)
+	svcToNodePorts := make(map[string]uint16)
+	var tailscaledPort *uint16
+	if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil {
+		svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass)
+		if err != nil {
+			wrappedErr := fmt.Errorf("error provisioning NodePort Services for static endpoints: %w", err)
+			var allocatePortErr *allocatePortsErr
+			if errors.As(err, &allocatePortErr) {
+				reason := reasonProxyGroupCreationFailed
+				msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr)
+				r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger)
+				return false, nil
+			}
+			return false, wrappedErr
+		}
 	}
+
+	staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts)
+	if err != nil {
+		wrappedErr := fmt.Errorf("error provisioning config Secrets: %w", err)
+		var selectorErr *FindStaticEndpointErr
+		if errors.As(err, &selectorErr) {
+			reason := reasonProxyGroupCreationFailed
+			msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr)
+			r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger)
+			return false, nil
+		}
+		return false, wrappedErr
+	}
+
 	// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
 	stateSecrets := pgStateSecrets(pg, r.tsNamespace)
 	for _, sec := range stateSecrets {
@@ -247,7 +289,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
 			s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
 		}); err != nil {
-			return fmt.Errorf("error provisioning state Secrets: %w", err)
+			return false, fmt.Errorf("error provisioning state Secrets: %w", err)
 		}
 	}
 	sa := pgServiceAccount(pg, r.tsNamespace)
@@ -256,7 +298,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
 		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
 	}); err != nil {
-		return fmt.Errorf("error provisioning ServiceAccount: %w", err)
+		return false, fmt.Errorf("error provisioning ServiceAccount: %w", err)
 	}
 	role := pgRole(pg, r.tsNamespace)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
@@ -265,7 +307,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
 		r.Rules = role.Rules
 	}); err != nil {
-		return fmt.Errorf("error provisioning Role: %w", err)
+		return false, fmt.Errorf("error provisioning Role: %w", err)
 	}
 	roleBinding := pgRoleBinding(pg, r.tsNamespace)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
@@ -275,7 +317,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		r.RoleRef = roleBinding.RoleRef
 		r.Subjects = roleBinding.Subjects
 	}); err != nil {
-		return fmt.Errorf("error provisioning RoleBinding: %w", err)
+		return false, fmt.Errorf("error provisioning RoleBinding: %w", err)
 	}
 	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
 		cm, hp := pgEgressCM(pg, r.tsNamespace)
@@ -284,7 +326,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
 			mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp)
 		}); err != nil {
-			return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
+			return false, fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
 		}
 	}
 	if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
@@ -293,12 +335,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
 			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
 		}); err != nil {
-			return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
+			return false, fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
 		}
 	}
-	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass)
+	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass)
 	if err != nil {
-		return fmt.Errorf("error generating StatefulSet spec: %w", err)
+		return false, fmt.Errorf("error generating StatefulSet spec: %w", err)
 	}
 	cfg := &tailscaleSTSConfig{
 		proxyType: string(pg.Spec.Type),
@@ -306,7 +348,6 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger)
 
 	updateSS := func(s *appsv1.StatefulSet) {
-
 		s.Spec = ss.Spec
 
 		s.ObjectMeta.Labels = ss.ObjectMeta.Labels
@@ -314,7 +355,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
 	}
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil {
-		return fmt.Errorf("error provisioning StatefulSet: %w", err)
+		return false, fmt.Errorf("error provisioning StatefulSet: %w", err)
 	}
 	mo := &metricsOpts{
 		tsNamespace:  r.tsNamespace,
@@ -323,26 +364,150 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 		proxyType:    "proxygroup",
 	}
 	if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
-		return fmt.Errorf("error reconciling metrics resources: %w", err)
+		return false, fmt.Errorf("error reconciling metrics resources: %w", err)
 	}
 
-	if err := r.cleanupDanglingResources(ctx, pg); err != nil {
-		return fmt.Errorf("error cleaning up dangling resources: %w", err)
+	if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil {
+		return false, fmt.Errorf("error cleaning up dangling resources: %w", err)
 	}
 
-	devices, err := r.getDeviceInfo(ctx, pg)
+	devices, err := r.getDeviceInfo(ctx, staticEndpoints, pg)
 	if err != nil {
-		return fmt.Errorf("failed to get device info: %w", err)
+		return false, fmt.Errorf("failed to get device info: %w", err)
 	}
 
 	pg.Status.Devices = devices
 
-	return nil
+	return true, nil
+}
+
+// getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts,
+// and a set of all allocated NodePorts for quick occupancy checking.
+func getServicePortsForProxyGroups(ctx context.Context, c client.Client, namespace string, portRanges tsapi.PortRanges) (map[string]uint16, set.Set[uint16], error) {
+	svcs := new(corev1.ServiceList)
+	matchingLabels := client.MatchingLabels(map[string]string{
+		LabelParentType: "proxygroup",
+	})
+
+	err := c.List(ctx, svcs, matchingLabels, client.InNamespace(namespace))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to list ProxyGroup Services: %w", err)
+	}
+
+	svcToNodePorts := map[string]uint16{}
+	usedPorts := set.Set[uint16]{}
+	for _, svc := range svcs.Items {
+		if len(svc.Spec.Ports) == 1 && svc.Spec.Ports[0].NodePort != 0 {
+			p := uint16(svc.Spec.Ports[0].NodePort)
+			if portRanges.Contains(p) {
+				svcToNodePorts[svc.Name] = p
+				usedPorts.Add(p)
+			}
+		}
+	}
+
+	return svcToNodePorts, usedPorts, nil
+}
+
+type allocatePortsErr struct {
+	msg string
+}
+
+func (e *allocatePortsErr) Error() string {
+	return e.msg
+}
+
+func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) {
+	replicaCount := int(pgReplicas(pg))
+	svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges)
+	if err != nil {
+		return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())}
+	}
+
+	replicasAllocated := 0
+	for i := range pgReplicas(pg) {
+		if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok {
+			svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0
+		} else {
+			replicasAllocated++
+		}
+	}
+
+	for replica, port := range svcToNodePorts {
+		if port == 0 {
+			for p := range portRanges.All() {
+				if !usedPorts.Contains(p) {
+					svcToNodePorts[replica] = p
+					usedPorts.Add(p)
+					replicasAllocated++
+					break
+				}
+			}
+		}
+	}
+
+	if replicasAllocated < replicaCount {
+		return nil, &allocatePortsErr{msg: fmt.Sprintf("not enough available ports to allocate all replicas (needed %d, got %d). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass %q must have bigger range allocated", replicaCount, usedPorts.Len(), proxyClassName)}
+	}
+
+	return svcToNodePorts, nil
+}
+
+func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) {
+	// NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup
+	tailscaledPort := getRandomPort()
+	svcs := []*corev1.Service{}
+	for i := range pgReplicas(pg) {
+		replicaName := pgNodePortServiceName(pg.Name, i)
+
+		svc := &corev1.Service{}
+		err := r.Get(ctx, types.NamespacedName{Name: replicaName, Namespace: r.tsNamespace}, svc)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", replicaName, err)
+		}
+		if apierrors.IsNotFound(err) {
+			svcs = append(svcs, pgNodePortService(pg, replicaName, r.tsNamespace))
+		} else {
+			// NOTE: if we can we want to recover the random port used for tailscaled,
+			// as well as the NodePort previously used for that Service
+			if len(svc.Spec.Ports) == 1 {
+				if svc.Spec.Ports[0].Port != 0 {
+					tailscaledPort = uint16(svc.Spec.Ports[0].Port)
+				}
+			}
+			svcs = append(svcs, svc)
+		}
+	}
+
+	svcToNodePorts, err := r.allocatePorts(ctx, pg, pc.Name, pc.Spec.StaticEndpoints.NodePort.Ports)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to allocate NodePorts to ProxyGroup Services: %w", err)
+	}
+
+	for _, svc := range svcs {
+		// NOTE: we know that every service is going to have 1 port here
+		svc.Spec.Ports[0].Port = int32(tailscaledPort)
+		svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(tailscaledPort))
+		svc.Spec.Ports[0].NodePort = int32(svcToNodePorts[svc.Name])
+
+		_, err = createOrUpdate(ctx, r.Client, r.tsNamespace, svc, func(s *corev1.Service) {
+			s.ObjectMeta.Labels = svc.ObjectMeta.Labels
+			s.ObjectMeta.Annotations = svc.ObjectMeta.Annotations
+			s.ObjectMeta.OwnerReferences = svc.ObjectMeta.OwnerReferences
+			s.Spec.Selector = svc.Spec.Selector
+			s.Spec.Ports = svc.Spec.Ports
+		})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error creating/updating Kubernetes NodePort Service %q: %w", svc.Name, err)
+		}
+	}
+
+	return svcToNodePorts, ptr.To(tailscaledPort), nil
 }
 
 // cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
 // tailnet devices when the number of replicas specified is reduced.
-func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error {
+func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error {
 	logger := r.logger(pg.Name)
 	metadata, err := r.getNodeMetadata(ctx, pg)
 	if err != nil {
@@ -371,6 +536,30 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg
 				return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
 			}
 		}
+		// NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough
+		svc := &corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-nodeport", m.stateSecret.Name),
+				Namespace: m.stateSecret.Namespace,
+			},
+		}
+		if err := r.Delete(ctx, svc); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return fmt.Errorf("error deleting static endpoints Kubernetes Service %q: %w", svc.Name, err)
+			}
+		}
+	}
+
+	// If the ProxyClass has its StaticEndpoints config removed, we want to remove all of the NodePort Services
+	if pc != nil && pc.Spec.StaticEndpoints == nil {
+		labels := map[string]string{
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        proxyTypeProxyGroup,
+			LabelParentName:        pg.Name,
+		}
+		if err := r.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+			return fmt.Errorf("error deleting Kubernetes Services for static endpoints: %w", err)
+		}
 	}
 
 	return nil
@@ -396,7 +585,8 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
 	mo := &metricsOpts{
 		proxyLabels: pgLabels(pg.Name, nil),
 		tsNamespace: r.tsNamespace,
-		proxyType:   "proxygroup"}
+		proxyType:   "proxygroup",
+	}
 	if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil {
 		return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
 	}
@@ -424,8 +614,9 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
 	return nil
 }
 
-func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (err error) {
+func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) {
 	logger := r.logger(pg.Name)
+	endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg))
 	for i := range pgReplicas(pg) {
 		cfgSecret := &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
@@ -441,7 +632,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
 			existingCfgSecret = cfgSecret.DeepCopy()
 		} else if !apierrors.IsNotFound(err) {
-			return err
+			return nil, err
 		}
 
 		var authKey string
@@ -453,19 +644,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			}
 			authKey, err = newAuthKey(ctx, r.tsClient, tags)
 			if err != nil {
-				return err
+				return nil, err
+			}
+		}
+
+		replicaName := pgNodePortServiceName(pg.Name, i)
+		if len(svcToNodePorts) > 0 {
+			port, ok := svcToNodePorts[replicaName]
+			if !ok {
+				return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName)
+			}
+
+			endpoints[replicaName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger)
+			if err != nil {
+				return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err)
 			}
 		}
 
-		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret)
+		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret, endpoints[replicaName])
 		if err != nil {
-			return fmt.Errorf("error creating tailscaled config: %w", err)
+			return nil, fmt.Errorf("error creating tailscaled config: %w", err)
 		}
 
 		for cap, cfg := range configs {
 			cfgJSON, err := json.Marshal(cfg)
 			if err != nil {
-				return fmt.Errorf("error marshalling tailscaled config: %w", err)
+				return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
 			}
 			mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 		}
@@ -474,18 +678,111 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) {
 				logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name)
 				if err := r.Update(ctx, cfgSecret); err != nil {
-					return err
+					return nil, err
 				}
 			}
 		} else {
 			logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
 			if err := r.Create(ctx, cfgSecret); err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
 
-	return nil
+	return endpoints, nil
+}
+
+type FindStaticEndpointErr struct {
+	msg string
+}
+
+func (e *FindStaticEndpointErr) Error() string {
+	return e.msg
+}
+
+// findStaticEndpoints returns up to two `netip.AddrPort` entries, derived from the ExternalIPs of Nodes that
+// match the `proxyClass`'s selector within the StaticEndpoints configuration. The port is set to the replica's NodePort Service Port.
+func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existingCfgSecret *corev1.Secret, proxyClass *tsapi.ProxyClass, port uint16, logger *zap.SugaredLogger) ([]netip.AddrPort, error) {
+	var currAddrs []netip.AddrPort
+	if existingCfgSecret != nil {
+		oldConfB := existingCfgSecret.Data[tsoperator.TailscaledConfigFileName(106)]
+		if len(oldConfB) > 0 {
+			var oldConf ipn.ConfigVAlpha
+			if err := json.Unmarshal(oldConfB, &oldConf); err == nil {
+				currAddrs = oldConf.StaticEndpoints
+			} else {
+				logger.Debugf("failed to unmarshal tailscaled config from secret %q: %v", existingCfgSecret.Name, err)
+			}
+		} else {
+			logger.Debugf("failed to get tailscaled config from secret %q: empty data", existingCfgSecret.Name)
+		}
+	}
+
+	nodes := new(corev1.NodeList)
+	selectors := client.MatchingLabels(proxyClass.Spec.StaticEndpoints.NodePort.Selector)
+
+	err := r.List(ctx, nodes, selectors)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list nodes: %w", err)
+	}
+
+	if len(nodes.Items) == 0 {
+		return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass %q", proxyClass.Name)}
+	}
+
+	endpoints := []netip.AddrPort{}
+
+	// NOTE(ChaosInTheCRD): Setting a hard limit of two static endpoints.
+	newAddrs := []netip.AddrPort{}
+	for _, n := range nodes.Items {
+		for _, a := range n.Status.Addresses {
+			if a.Type == corev1.NodeExternalIP {
+				addr := getStaticEndpointAddress(&a, port)
+				if addr == nil {
+					logger.Debugf("failed to parse %q address on node %q: %q", corev1.NodeExternalIP, n.Name, a.Address)
+					continue
+				}
+
+				// we want to add the currently used IPs first before
+				// adding new ones.
+				if currAddrs != nil && slices.Contains(currAddrs, *addr) {
+					endpoints = append(endpoints, *addr)
+				} else {
+					newAddrs = append(newAddrs, *addr)
+				}
+			}
+
+			if len(endpoints) == 2 {
+				break
+			}
+		}
+	}
+
+	// if the 2 endpoints limit hasn't been reached, we
+	// can start adding newIPs.
+	if len(endpoints) < 2 {
+		for _, a := range newAddrs {
+			endpoints = append(endpoints, a)
+			if len(endpoints) == 2 {
+				break
+			}
+		}
+	}
+
+	if len(endpoints) == 0 {
+		return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)}
+	}
+
+	return endpoints, nil
+}
+
+func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort {
+	addr, err := netip.ParseAddr(a.Address)
+	if err != nil {
+		return nil
+	}
+
+	return ptr.To(netip.AddrPortFrom(addr, port))
 }
 
 // ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup
@@ -514,7 +811,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
 }
 
-func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
+func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:      "alpha0",
 		AcceptDNS:    "false",
@@ -531,6 +828,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
 		conf.AcceptRoutes = "true"
 	}
 
+	if len(staticEndpoints) > 0 {
+		conf.StaticEndpoints = staticEndpoints
+	}
+
 	deviceAuthed := false
 	for _, d := range pg.Status.Devices {
 		if d.Hostname == *conf.Hostname {
@@ -624,7 +925,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
 	return metadata, nil
 }
 
-func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
+func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoints map[string][]netip.AddrPort, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
 	metadata, err := r.getNodeMetadata(ctx, pg)
 	if err != nil {
 		return nil, err
@@ -638,10 +939,21 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox
 		if !ok {
 			continue
 		}
-		devices = append(devices, tsapi.TailnetDevice{
+
+		dev := tsapi.TailnetDevice{
 			Hostname:   device.Hostname,
 			TailnetIPs: device.TailnetIPs,
-		})
+		}
+
+		if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 {
+			eps := make([]string, 0, len(ep))
+			for _, e := range ep {
+				eps = append(eps, e.String())
+			}
+			dev.StaticEndpoints = eps
+		}
+
+		devices = append(devices, dev)
 	}
 
 	return devices, nil
@@ -655,3 +967,8 @@ type nodeMetadata struct {
 	tsID    tailcfg.StableNodeID
 	dnsName string
 }
+
+func (pr *ProxyGroupReconciler) setStatusReady(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason string, msg string, logger *zap.SugaredLogger) {
+	pr.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
+	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, msg, pg.Generation, pr.clock, logger)
+}

cmd/k8s-operator/proxygroup_specs.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"slices"
 	"strconv"
+	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -23,12 +24,43 @@ import (
 	"tailscale.com/types/ptr"
 )
 
-// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
-const deletionGracePeriodSeconds int64 = 360
+const (
+	// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
+	deletionGracePeriodSeconds int64 = 360
+	staticEndpointPortName           = "static-endpoint-port"
+)
+
+func pgNodePortServiceName(proxyGroupName string, replica int32) string {
+	return fmt.Sprintf("%s-%d-nodeport", proxyGroupName, replica)
+}
+
+func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeNodePort,
+			Ports: []corev1.ServicePort{
+				// NOTE(ChaosInTheCRD): we set the ports once we've iterated over every svc and found any old configuration we want to persist.
+				{
+					Name:     staticEndpointPortName,
+					Protocol: corev1.ProtocolUDP,
+				},
+			},
+			Selector: map[string]string{
+				appsv1.StatefulSetPodNameLabel: strings.TrimSuffix(name, "-nodeport"),
+			},
+		},
+	}
+}
 
 // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
 // applied over the top after.
-func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
+func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
 	ss := new(appsv1.StatefulSet)
 	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
@@ -144,6 +176,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 			},
 		}
 
+		if port != nil {
+			envs = append(envs, corev1.EnvVar{
+				Name:  "PORT",
+				Value: strconv.Itoa(int(*port)),
+			})
+		}
+
 		if tsFirewallMode != "" {
 			envs = append(envs, corev1.EnvVar{
 				Name:  "TS_DEBUG_FIREWALL_MODE",

cmd/k8s-operator/proxygroup_test.go

@@ -9,6 +9,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/netip"
+	"slices"
 	"testing"
 	"time"
 
@@ -18,6 +20,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
@@ -32,14 +35,772 @@ import (
 	"tailscale.com/types/ptr"
 )
 
-const testProxyImage = "tailscale/tailscale:test"
+const (
+	testProxyImage = "tailscale/tailscale:test"
+	initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
+)
+
+var (
+	defaultProxyClassAnnotations = map[string]string{
+		"some-annotation": "from-the-proxy-class",
+	}
+
+	defaultReplicas             = ptr.To(int32(2))
+	defaultStaticEndpointConfig = &tsapi.StaticEndpointsConfig{
+		NodePort: &tsapi.NodePortConfig{
+			Ports: []tsapi.PortRange{
+				{Port: 30001}, {Port: 30002},
+			},
+			Selector: map[string]string{
+				"foo/bar": "baz",
+			},
+		},
+	}
+)
+
+func TestProxyGroupWithStaticEndpoints(t *testing.T) {
+	type testNodeAddr struct {
+		ip       string
+		addrType corev1.NodeAddressType
+	}
 
-var defaultProxyClassAnnotations = map[string]string{
-	"some-annotation": "from-the-proxy-class",
+	type testNode struct {
+		name      string
+		addresses []testNodeAddr
+		labels    map[string]string
+	}
+
+	type reconcile struct {
+		staticEndpointConfig *tsapi.StaticEndpointsConfig
+		replicas             *int32
+		nodes                []testNode
+		expectedIPs          []netip.Addr
+		expectedEvents       []string
+		expectedErr          string
+		expectStatefulSet    bool
+	}
+
+	testCases := []struct {
+		name        string
+		description string
+		reconciles  []reconcile
+	}{
+		{
+			// the reconciler should manage to create static endpoints when Nodes have IPv6 addresses.
+			name: "IPv6",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3001},
+								{Port: 3005},
+								{Port: 3007},
+								{Port: 3009},
+							},
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+					replicas: ptr.To(int32(4)),
+					nodes: []testNode{
+						{
+							name:      "foobar",
+							addresses: []testNodeAddr{{ip: "2001:0db8::1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbaz",
+							addresses: []testNodeAddr{{ip: "2001:0db8::2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbazz",
+							addresses: []testNodeAddr{{ip: "2001:0db8::3", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("2001:0db8::1"), netip.MustParseAddr("2001:0db8::2"), netip.MustParseAddr("2001:0db8::3")},
+					expectedEvents:    []string{},
+					expectedErr:       "",
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// declaring specific ports (with no `endPort`s) in the `spec.staticEndpoints.nodePort` should work.
+			name: "SpecificPorts",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3001},
+								{Port: 3005},
+								{Port: 3007},
+								{Port: 3009},
+							},
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+					replicas: ptr.To(int32(4)),
+					nodes: []testNode{
+						{
+							name:      "foobar",
+							addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbaz",
+							addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbazz",
+							addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("192.168.0.2"), netip.MustParseAddr("192.168.0.3")},
+					expectedEvents:    []string{},
+					expectedErr:       "",
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// if too narrow a range of `spec.staticEndpoints.nodePort.Ports` on the proxyClass should result in no StatefulSet being created.
+			name: "NotEnoughPorts",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3001},
+								{Port: 3005},
+								{Port: 3007},
+							},
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+					replicas: ptr.To(int32(4)),
+					nodes: []testNode{
+						{
+							name:      "foobar",
+							addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbaz",
+							addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbazz",
+							addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{},
+					expectedEvents:    []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"},
+					expectedErr:       "",
+					expectStatefulSet: false,
+				},
+			},
+		},
+		{
+			// when supplying a variety of ranges that are not clashing, the reconciler should manage to create a StatefulSet.
+			name: "NonClashingRanges",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3000, EndPort: 3002},
+								{Port: 3003, EndPort: 3005},
+								{Port: 3006},
+							},
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+					replicas: ptr.To(int32(3)),
+					nodes: []testNode{
+						{name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
+						{name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
+						{name: "node3", addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.3")},
+					expectedEvents:    []string{},
+					expectedErr:       "",
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// when there isn't a node that matches the selector, the ProxyGroup enters a failed state as there are no valid Static Endpoints.
+			// while it does create an event on the resource, It does not return an error
+			name: "NoMatchingNodes",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3000, EndPort: 3005},
+							},
+							Selector: map[string]string{
+								"zone": "us-west",
+							},
+						},
+					},
+					replicas: defaultReplicas,
+					nodes: []testNode{
+						{name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"zone": "eu-central"}},
+						{name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}},
+					},
+					expectedIPs:       []netip.Addr{},
+					expectedEvents:    []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""},
+					expectedErr:       "",
+					expectStatefulSet: false,
+				},
+			},
+		},
+		{
+			// when all the nodes have only have addresses of type InternalIP populated in their status, the ProxyGroup enters a failed state as there are no valid Static Endpoints.
+			// while it does create an event on the resource, It does not return an error
+			name: "AllInternalIPAddresses",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: &tsapi.StaticEndpointsConfig{
+						NodePort: &tsapi.NodePortConfig{
+							Ports: []tsapi.PortRange{
+								{Port: 3001},
+								{Port: 3005},
+								{Port: 3007},
+								{Port: 3009},
+							},
+							Selector: map[string]string{
+								"foo/bar": "baz",
+							},
+						},
+					},
+					replicas: ptr.To(int32(4)),
+					nodes: []testNode{
+						{
+							name:      "foobar",
+							addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeInternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbaz",
+							addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeInternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "foobarbazz",
+							addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeInternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{},
+					expectedEvents:    []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""},
+					expectedErr:       "",
+					expectStatefulSet: false,
+				},
+			},
+		},
+		{
+			// When the node's (and some of their addresses) change between reconciles, the reconciler should first pick addresses that
+			// have been used previously (provided that they are still populated on a node that matches the selector)
+			name: "NodeIPChangesAndPersists",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node3",
+							addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.10", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node3",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectStatefulSet: true,
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+				},
+			},
+		},
+		{
+			// given a new node being created with a new IP, and a node previously used for Static Endpoints being removed, the Static Endpoints should be updated
+			// correctly
+			name: "NodeIPChangesWithNewNode",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node3",
+							addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.3")},
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// when all the node IPs change, they should all update
+			name: "AllNodeIPsChange",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.100", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.200", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.100"), netip.MustParseAddr("10.0.0.200")},
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// if there are less ExternalIPs after changes to the nodes between reconciles, the reconciler should complete without issues
+			name: "LessExternalIPsAfterChange",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1")},
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// if node address parsing fails (given an invalid address), the reconciler should continue without failure and find other
+			// valid addresses
+			name: "NodeAddressParsingFails",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+			},
+		},
+		{
+			// if the node's become unlabeled, the ProxyGroup should enter a ProxyGroupInvalid state, but the reconciler should not fail
+			name: "NodesBecomeUnlabeled",
+			reconciles: []reconcile{
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node1",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+						{
+							name:      "node2",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{"foo/bar": "baz"},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectStatefulSet: true,
+				},
+				{
+					staticEndpointConfig: defaultStaticEndpointConfig,
+					replicas:             defaultReplicas,
+					nodes: []testNode{
+						{
+							name:      "node3",
+							addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{},
+						},
+						{
+							name:      "node4",
+							addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
+							labels:    map[string]string{},
+						},
+					},
+					expectedIPs:       []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
+					expectedEvents:    []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""},
+					expectStatefulSet: true,
+				},
+			},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			tsClient := &fakeTSClient{}
+			zl, _ := zap.NewDevelopment()
+			fr := record.NewFakeRecorder(10)
+			cl := tstest.NewClock(tstest.ClockOpts{})
+
+			pc := &tsapi.ProxyClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "default-pc",
+				},
+				Spec: tsapi.ProxyClassSpec{
+					StatefulSet: &tsapi.StatefulSet{
+						Annotations: defaultProxyClassAnnotations,
+					},
+				},
+				Status: tsapi.ProxyClassStatus{
+					Conditions: []metav1.Condition{{
+						Type:               string(tsapi.ProxyClassReady),
+						Status:             metav1.ConditionTrue,
+						Reason:             reasonProxyClassValid,
+						Message:            reasonProxyClassValid,
+						LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+					}},
+				},
+			}
+
+			pg := &tsapi.ProxyGroup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       "test",
+					Finalizers: []string{"tailscale.com/finalizer"},
+				},
+				Spec: tsapi.ProxyGroupSpec{
+					Type:       tsapi.ProxyGroupTypeEgress,
+					ProxyClass: pc.Name,
+				},
+			}
+
+			fc := fake.NewClientBuilder().
+				WithObjects(pc, pg).
+				WithStatusSubresource(pc, pg).
+				WithScheme(tsapi.GlobalScheme).
+				Build()
+
+			reconciler := &ProxyGroupReconciler{
+				tsNamespace:       tsNamespace,
+				proxyImage:        testProxyImage,
+				defaultTags:       []string{"tag:test-tag"},
+				tsFirewallMode:    "auto",
+				defaultProxyClass: "default-pc",
+
+				Client:   fc,
+				tsClient: tsClient,
+				recorder: fr,
+				clock:    cl,
+			}
+
+			for i, r := range tt.reconciles {
+				createdNodes := []corev1.Node{}
+				t.Run(tt.name, func(t *testing.T) {
+					for _, n := range r.nodes {
+						no := &corev1.Node{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:   n.name,
+								Labels: n.labels,
+							},
+							Status: corev1.NodeStatus{
+								Addresses: []corev1.NodeAddress{},
+							},
+						}
+						for _, addr := range n.addresses {
+							no.Status.Addresses = append(no.Status.Addresses, corev1.NodeAddress{
+								Type:    addr.addrType,
+								Address: addr.ip,
+							})
+						}
+						if err := fc.Create(context.Background(), no); err != nil {
+							t.Fatalf("failed to create node %q: %v", n.name, err)
+						}
+						createdNodes = append(createdNodes, *no)
+						t.Logf("created node %q with data", n.name)
+					}
+
+					reconciler.l = zl.Sugar().With("TestName", tt.name).With("Reconcile", i)
+					pg.Spec.Replicas = r.replicas
+					pc.Spec.StaticEndpoints = r.staticEndpointConfig
+
+					createOrUpdate(context.Background(), fc, "", pg, func(o *tsapi.ProxyGroup) {
+						o.Spec.Replicas = pg.Spec.Replicas
+					})
+
+					createOrUpdate(context.Background(), fc, "", pc, func(o *tsapi.ProxyClass) {
+						o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints
+					})
+
+					if r.expectedErr != "" {
+						expectError(t, reconciler, "", pg.Name)
+					} else {
+						expectReconciled(t, reconciler, "", pg.Name)
+					}
+					expectEvents(t, fr, r.expectedEvents)
+
+					sts := &appsv1.StatefulSet{}
+					err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts)
+					if r.expectStatefulSet {
+						if err != nil {
+							t.Fatalf("failed to get StatefulSet: %v", err)
+						}
+
+						for j := range 2 {
+							sec := &corev1.Secret{}
+							if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil {
+								t.Fatalf("failed to get state Secret for replica %d: %v", j, err)
+							}
+
+							config := &ipn.ConfigVAlpha{}
+							foundConfig := false
+							for _, d := range sec.Data {
+								if err := json.Unmarshal(d, config); err == nil {
+									foundConfig = true
+									break
+								}
+							}
+							if !foundConfig {
+								t.Fatalf("could not unmarshal config from secret data for replica %d", j)
+							}
+
+							if len(config.StaticEndpoints) > staticEndpointsMaxAddrs {
+								t.Fatalf("expected %d StaticEndpoints in config Secret, but got %d for replica %d. Found Static Endpoints: %v", staticEndpointsMaxAddrs, len(config.StaticEndpoints), j, config.StaticEndpoints)
+							}
+
+							for _, e := range config.StaticEndpoints {
+								if !slices.Contains(r.expectedIPs, e.Addr()) {
+									t.Fatalf("found unexpected static endpoint IP %q for replica %d. Expected one of %v", e.Addr().String(), j, r.expectedIPs)
+								}
+								if c := r.staticEndpointConfig; c != nil && c.NodePort.Ports != nil {
+									var ports tsapi.PortRanges = c.NodePort.Ports
+									found := false
+									for port := range ports.All() {
+										if port == e.Port() {
+											found = true
+											break
+										}
+									}
+
+									if !found {
+										t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected one of %v .", e.Port(), j, ports.All())
+									}
+								} else {
+									if e.Port() != 3001 && e.Port() != 3002 {
+										t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected 3001 or 3002.", e.Port(), j)
+									}
+								}
+							}
+						}
+
+						pgroup := &tsapi.ProxyGroup{}
+						err = fc.Get(context.Background(), client.ObjectKey{Name: pg.Name}, pgroup)
+						if err != nil {
+							t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err)
+						}
+
+						t.Logf("getting proxygroup after reconcile")
+						for _, d := range pgroup.Status.Devices {
+							t.Logf("found device %q", d.Hostname)
+							for _, e := range d.StaticEndpoints {
+								t.Logf("found static endpoint %q", e)
+							}
+						}
+					} else {
+						if err == nil {
+							t.Fatal("expected error when getting Statefulset")
+						}
+					}
+				})
+
+				// node cleanup between reconciles
+				// we created a new set of nodes for each
+				for _, n := range createdNodes {
+					err := fc.Delete(context.Background(), &n)
+					if err != nil && !apierrors.IsNotFound(err) {
+						t.Fatalf("failed to delete node: %v", err)
+					}
+				}
+			}
+
+			t.Run("delete_and_cleanup", func(t *testing.T) {
+				reconciler := &ProxyGroupReconciler{
+					tsNamespace:       tsNamespace,
+					proxyImage:        testProxyImage,
+					defaultTags:       []string{"tag:test-tag"},
+					tsFirewallMode:    "auto",
+					defaultProxyClass: "default-pc",
+
+					Client:   fc,
+					tsClient: tsClient,
+					recorder: fr,
+					l:        zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"),
+					clock:    cl,
+				}
+
+				if err := fc.Delete(context.Background(), pg); err != nil {
+					t.Fatalf("error deleting ProxyGroup: %v", err)
+				}
+
+				expectReconciled(t, reconciler, "", pg.Name)
+				expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
+
+				if err := fc.Delete(context.Background(), pc); err != nil {
+					t.Fatalf("error deleting ProxyClass: %v", err)
+				}
+				expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name)
+			})
+		})
+	}
 }
 
 func TestProxyGroup(t *testing.T) {
-
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "default-pc",
@@ -598,7 +1359,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass)
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass)
 	if err != nil {
 		t.Fatal(err)
 	}

k8s-operator/api.md

@@ -425,6 +425,23 @@ _Appears in:_
 | `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. |  |  |
 
 
+#### NodePortConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [StaticEndpointsConfig](#staticendpointsconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `ports` _[PortRange](#portrange) array_ | The port ranges from which the operator will select NodePorts for the Services.<br />You must ensure that firewall rules allow UDP ingress traffic for these ports<br />to the node's external IPs.<br />The ports must be in the range of service node ports for the cluster (default `30000-32767`).<br />See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. |  | MinItems: 1 <br /> |
+| `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised<br />by the ProxyGroup as Static Endpoints. |  |  |
+
+
 #### Pod
 
 
@@ -451,6 +468,26 @@ _Appears in:_
 | `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#topologyspreadconstraint-v1-core) array_ | Proxy Pod's topology spread constraints.<br />By default Tailscale Kubernetes operator does not apply any topology spread constraints.<br />https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ |  |  |
 
 
+#### PortRange
+
+
+
+
+
+
+
+_Appears in:_
+- [NodePortConfig](#nodeportconfig)
+- [PortRanges](#portranges)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `port` _integer_ | port represents a port selected to be used. This is a required field. |  |  |
+| `endPort` _integer_ | endPort indicates that the range of ports from port to endPort if set, inclusive,<br />should be used. This field cannot be defined if the port field is not defined.<br />The endPort must be either unset, or equal or greater than port. |  |  |
+
+
+
+
 #### ProxyClass
 
 
@@ -518,6 +555,7 @@ _Appears in:_
 | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported<br />for egress proxies and for Ingress proxies that have been configured<br />with tailscale.com/experimental-forward-cluster-traffic-via-ingress<br />annotation. Note that the metrics are currently considered unstable<br />and will likely change in breaking ways in the future - we only<br />recommend that you use those for debugging purposes. |  |  |
 | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific<br />parameters of proxies. |  |  |
 | `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS<br />certificates for any HTTPS endpoints exposed to the tailnet from<br />LetsEncrypt's staging environment.<br />https://letsencrypt.org/docs/staging-environment/<br />This setting only affects Tailscale Ingress resources.<br />By default Ingress TLS certificates are issued from LetsEncrypt's<br />production environment.<br />Changing this setting true -> false, will result in any<br />existing certs being re-issued from the production environment.<br />Changing this setting false (default) -> true, when certs have already<br />been provisioned from production environment will NOT result in certs<br />being re-issued from the staging environment before they need to be<br />renewed. |  |  |
+| `staticEndpoints` _[StaticEndpointsConfig](#staticendpointsconfig)_ | Configuration for 'static endpoints' on proxies in order to facilitate<br />direct connections from other devices on the tailnet.<br />See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. |  |  |
 
 
 #### ProxyClassStatus
@@ -935,6 +973,22 @@ _Appears in:_
 | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. |  |  |
 
 
+#### StaticEndpointsConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ProxyClassSpec](#proxyclassspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `nodePort` _[NodePortConfig](#nodeportconfig)_ | The configuration for static endpoints using NodePort Services. |  |  |
+
+
 #### Storage
 
 
@@ -1015,6 +1069,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `hostname` _string_ | Hostname is the fully qualified domain name of the device.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
 | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the device. |  |  |
+| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. |  |  |
 
 
 #### TailscaleConfig

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -6,6 +6,10 @@
 package v1alpha1
 
 import (
+	"fmt"
+	"iter"
+	"strings"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -82,6 +86,124 @@ type ProxyClassSpec struct {
 	// renewed.
 	// +optional
 	UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"`
+	// Configuration for 'static endpoints' on proxies in order to facilitate
+	// direct connections from other devices on the tailnet.
+	// See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
+	// +optional
+	StaticEndpoints *StaticEndpointsConfig `json:"staticEndpoints,omitempty"`
+}
+
+type StaticEndpointsConfig struct {
+	// The configuration for static endpoints using NodePort Services.
+	NodePort *NodePortConfig `json:"nodePort"`
+}
+
+type NodePortConfig struct {
+	// The port ranges from which the operator will select NodePorts for the Services.
+	// You must ensure that firewall rules allow UDP ingress traffic for these ports
+	// to the node's external IPs.
+	// The ports must be in the range of service node ports for the cluster (default `30000-32767`).
+	// See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
+	// +kubebuilder:validation:MinItems=1
+	Ports []PortRange `json:"ports"`
+	// A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
+	// by the ProxyGroup as Static Endpoints.
+	Selector map[string]string `json:"selector,omitempty"`
+}
+
+// PortRanges is a list of PortRange(s)
+type PortRanges []PortRange
+
+func (prs PortRanges) String() string {
+	var prStrings []string
+
+	for _, pr := range prs {
+		prStrings = append(prStrings, pr.String())
+	}
+
+	return strings.Join(prStrings, ", ")
+}
+
+// All allows us to iterate over all the ports in the PortRanges
+func (prs PortRanges) All() iter.Seq[uint16] {
+	return func(yield func(uint16) bool) {
+		for _, pr := range prs {
+			end := pr.EndPort
+			if end == 0 {
+				end = pr.Port
+			}
+
+			for port := pr.Port; port <= end; port++ {
+				if !yield(port) {
+					return
+				}
+			}
+		}
+	}
+}
+
+// Contains reports whether port is in any of the PortRanges.
+func (prs PortRanges) Contains(port uint16) bool {
+	for _, r := range prs {
+		if r.Contains(port) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ClashesWith reports whether the supplied PortRange clashes with any of the PortRanges.
+func (prs PortRanges) ClashesWith(pr PortRange) bool {
+	for p := range prs.All() {
+		if pr.Contains(p) {
+			return true
+		}
+	}
+
+	return false
+}
+
+type PortRange struct {
+	// port represents a port selected to be used. This is a required field.
+	Port uint16 `json:"port"`
+
+	// endPort indicates that the range of ports from port to endPort if set, inclusive,
+	// should be used. This field cannot be defined if the port field is not defined.
+	// The endPort must be either unset, or equal or greater than port.
+	// +optional
+	EndPort uint16 `json:"endPort,omitempty"`
+}
+
+// Contains reports whether port is in pr.
+func (pr PortRange) Contains(port uint16) bool {
+	switch pr.EndPort {
+	case 0:
+		return port == pr.Port
+	default:
+		return port >= pr.Port && port <= pr.EndPort
+	}
+}
+
+// String returns the PortRange in a string form.
+func (pr PortRange) String() string {
+	if pr.EndPort == 0 {
+		return fmt.Sprintf("%d", pr.Port)
+	}
+
+	return fmt.Sprintf("%d-%d", pr.Port, pr.EndPort)
+}
+
+// IsValid reports whether the port range is valid.
+func (pr PortRange) IsValid() bool {
+	if pr.Port == 0 {
+		return false
+	}
+	if pr.EndPort == 0 {
+		return true
+	}
+
+	return pr.Port <= pr.EndPort
 }
 
 type TailscaleConfig struct {

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -111,6 +111,10 @@ type TailnetDevice struct {
 	// assigned to the device.
 	// +optional
 	TailnetIPs []string `json:"tailnetIPs,omitempty"`
+
+	// StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
+	// +optional
+	StaticEndpoints []string `json:"staticEndpoints,omitempty"`
 }
 
 // +kubebuilder:validation:Type=string

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -407,6 +407,33 @@ func (in *NameserverStatus) DeepCopy() *NameserverStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) {
+	*out = *in
+	if in.Ports != nil {
+		in, out := &in.Ports, &out.Ports
+		*out = make([]PortRange, len(*in))
+		copy(*out, *in)
+	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig.
+func (in *NodePortConfig) DeepCopy() *NodePortConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(NodePortConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pod) DeepCopyInto(out *Pod) {
 	*out = *in
@@ -482,6 +509,40 @@ func (in *Pod) DeepCopy() *Pod {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PortRange) DeepCopyInto(out *PortRange) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange.
+func (in *PortRange) DeepCopy() *PortRange {
+	if in == nil {
+		return nil
+	}
+	out := new(PortRange)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in PortRanges) DeepCopyInto(out *PortRanges) {
+	{
+		in := &in
+		*out = make(PortRanges, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRanges.
+func (in PortRanges) DeepCopy() PortRanges {
+	if in == nil {
+		return nil
+	}
+	out := new(PortRanges)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
 	*out = *in
@@ -559,6 +620,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
 		*out = new(TailscaleConfig)
 		**out = **in
 	}
+	if in.StaticEndpoints != nil {
+		in, out := &in.StaticEndpoints, &out.StaticEndpoints
+		*out = new(StaticEndpointsConfig)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
@@ -1096,6 +1162,26 @@ func (in *StatefulSet) DeepCopy() *StatefulSet {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StaticEndpointsConfig) DeepCopyInto(out *StaticEndpointsConfig) {
+	*out = *in
+	if in.NodePort != nil {
+		in, out := &in.NodePort, &out.NodePort
+		*out = new(NodePortConfig)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticEndpointsConfig.
+func (in *StaticEndpointsConfig) DeepCopy() *StaticEndpointsConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(StaticEndpointsConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Storage) DeepCopyInto(out *Storage) {
 	*out = *in
@@ -1163,6 +1249,11 @@ func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.StaticEndpoints != nil {
+		in, out := &in.StaticEndpoints, &out.StaticEndpoints
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetDevice.