net/interfaces, net/netns: add node attributes to control default interface getting and binding

With #6566 we started to more aggressively bind to the default interface
on Darwin. We are seeing some reports of the wrong cellular interface
being chosen on iOS. To help with the investigation, this adds to knobs
to control the behavior changes:
- CapabilityDebugDisableAlternateDefaultRouteInterface disables the
  alternate function that we use to get the default interface on macOS
  and iOS (implemented in tailscale/corp#8201). We still log what it
  would have returned so we can see if it gets things wrong.
- CapabilityDebugDisableBindConnToInterface is a bigger hammer that
  disables binding of connections to the default interface altogether.

Updates #7184
Updates #7188

Signed-off-by: Mihai Parparita <[email protected]>

ipn/ipnlocal/local.go

@@ -3840,6 +3840,8 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 
 	// See the netns package for documentation on what this capability does.
 	netns.SetBindToInterfaceByRoute(hasCapability(nm, tailcfg.CapabilityBindToInterfaceByRoute))
+	interfaces.SetDisableAlternateDefaultRouteInterface(hasCapability(nm, tailcfg.CapabilityDebugDisableAlternateDefaultRouteInterface))
+	netns.SetDisableBindConnToInterface(hasCapability(nm, tailcfg.CapabilityDebugDisableBindConnToInterface))
 
 	b.setTCPPortsInterceptedFromNetmapAndPrefsLocked(b.pm.CurrentPrefs())
 	if nm == nil {

net/interfaces/interfaces.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync/atomic"
 
 	"tailscale.com/hostinfo"
 	"tailscale.com/net/netaddr"
@@ -756,3 +757,14 @@ func HasCGNATInterface() (bool, error) {
 	}
 	return hasCGNATInterface, nil
 }
+
+var disableAlternateDefaultRouteInterface atomic.Bool
+
+// SetDisableAlternateDefaultRouteInterface disables the optional external/
+// alternate mechanism for getting the default route network interface.
+//
+// Currently, this only changes the behaviour on BSD-like sytems (FreeBSD and
+// Darwin).
+func SetDisableAlternateDefaultRouteInterface(v bool) {
+	disableAlternateDefaultRouteInterface.Store(v)
+}

net/interfaces/interfaces_bsd.go

@@ -40,9 +40,15 @@ func defaultRoute() (d DefaultRouteDetails, err error) {
 // owns the default route. It returns the first IPv4 or IPv6 default route it
 // finds (it does not prefer one or the other).
 func DefaultRouteInterfaceIndex() (int, error) {
+	disabledAlternateDefaultRouteInterface := false
 	if f := defaultRouteInterfaceIndexFunc.Load(); f != nil {
 		if ifIndex := f(); ifIndex != 0 {
-			return ifIndex, nil
+			if !disableAlternateDefaultRouteInterface.Load() {
+				return ifIndex, nil
+			} else {
+				disabledAlternateDefaultRouteInterface = true
+				log.Printf("interfaces_bsd: alternate default route interface function disabled, would have returned interface %d", ifIndex)
+			}
 		}
 		// Fallthrough if we can't use the alternate implementation.
 	}
@@ -75,6 +81,9 @@ func DefaultRouteInterfaceIndex() (int, error) {
 			continue
 		}
 		if isDefaultGateway(rm) {
+			if disabledAlternateDefaultRouteInterface {
+				log.Printf("interfaces_bsd: alternate default route interface function disabled, default implementation returned %d", rm.Index)
+			}
 			return rm.Index, nil
 		}
 	}

net/netns/netns.go

@@ -42,6 +42,16 @@ func SetBindToInterfaceByRoute(v bool) {
 	bindToInterfaceByRoute.Store(v)
 }
 
+var disableBindConnToInterface atomic.Bool
+
+// SetDisableBindConnToInterface disables the (normal) behavior of binding
+// connections to the default network interface.
+//
+// Currently, this only has an effect on Darwin.
+func SetDisableBindConnToInterface(v bool) {
+	disableBindConnToInterface.Store(v)
+}
+
 // Listener returns a new net.Listener with its Control hook func
 // initialized as necessary to run in logical network namespace that
 // doesn't route back into Tailscale.

net/netns/netns_darwin.go

@@ -42,6 +42,11 @@ func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) e
 		return nil
 	}
 
+	if disableBindConnToInterface.Load() {
+		logf("netns_darwin: binding connection to interfaces disabled")
+		return nil
+	}
+
 	idx, err := getInterfaceIndex(logf, address)
 	if err != nil {
 		// callee logged

tailcfg/tailcfg.go

@@ -1765,6 +1765,17 @@ const (
 	// details on the behaviour of this capability.
 	CapabilityBindToInterfaceByRoute = "https://tailscale.com/cap/bind-to-interface-by-route"
 
+	// CapabilityDebugDisableAlternateDefaultRouteInterface changes how Darwin
+	// nodes get the default interface. There is an optional hook (used by the
+	// macOS and iOS clients) to override the default interface, this capability
+	// disables that and uses the default behavior (of parsing the routing
+	// table).
+	CapabilityDebugDisableAlternateDefaultRouteInterface = "https://tailscale.com/cap/debug-disable-alternate-default-route-interface"
+
+	// CapabilityDebugDisableBindConnToInterface disables the automatic binding
+	// of connections to the default network interface on Darwin nodes.
+	CapabilityDebugDisableBindConnToInterface = "https://tailscale.com/cap/debug-disable-bind-conn-to-interface"
+
 	// CapabilityTailnetLockAlpha indicates the node is in the tailnet lock alpha,
 	// and initialization of tailnet lock may proceed.
 	//