Fix some flaky tests related to pick port (#985)

* Fix some tests to PickPort correctly

* Add retry logic for two socks tests

Two socks tests listen for both TCP and UDP with same port number, in some cases the port is not available
Add retry logic for server core, start server core and client core separately
Extract a common method

* Add retry logic for two dokodemo tests

Two dokodemo tests listen for 5 ports, in some cases they are not available
Add retry logic for client core, start server and client separately

core/xray_test.go

@@ -58,7 +58,7 @@ func TestXrayClose(t *testing.T) {
 					Address: net.NewIPOrDomain(net.LocalHostIP),
 					Port:    uint32(0),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_TCP},
 					},
 				}),
 			},

testing/scenarios/common.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"sync"
 	"syscall"
+	"testing"
 	"time"
 
 	"github.com/golang/protobuf/proto"
@@ -235,3 +236,20 @@ func testTCPConn2(conn net.Conn, payloadSize int, timeout time.Duration) func()
 		return nil
 	}
 }
+
+func WaitConnAvailableWithTest(t *testing.T, testFunc func() error) bool {
+	for i := 1; ; i++ {
+		if i > 10 {
+			t.Log("All attempts failed to test tcp conn")
+			return false
+		}
+		time.Sleep(time.Millisecond * 10)
+		if err := testFunc(); err != nil {
+			t.Log("err ", err)
+		} else {
+			t.Log("success with", i, "attempts")
+			break
+		}
+	}
+	return true
+}

testing/scenarios/dokodemo_test.go

@@ -64,55 +64,68 @@ func TestDokodemoTCP(t *testing.T) {
 			},
 		},
 	}
+	server, err := InitializeServerConfig(serverConfig)
+	common.Must(err)
+	defer CloseServer(server)
 
-	clientPort := uint32(tcp.PickPort())
 	clientPortRange := uint32(5)
-	clientConfig := &core.Config{
-		App: []*serial.TypedMessage{
-			serial.ToTypedMessage(&log.Config{
-				ErrorLogLevel: clog.Severity_Debug,
-				ErrorLogType:  log.LogType_Console,
-			}),
-		},
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
-					Address: net.NewIPOrDomain(dest.Address),
-					Port:    uint32(dest.Port),
-					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP},
-					},
+	retry := 1
+	clientPort := uint32(tcp.PickPort())
+	for {
+		clientConfig := &core.Config{
+			App: []*serial.TypedMessage{
+				serial.ToTypedMessage(&log.Config{
+					ErrorLogLevel: clog.Severity_Debug,
+					ErrorLogType:  log.LogType_Console,
 				}),
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&outbound.Config{
-					Receiver: []*protocol.ServerEndpoint{
-						{
-							Address: net.NewIPOrDomain(net.LocalHostIP),
-							Port:    uint32(serverPort),
-							User: []*protocol.User{
-								{
-									Account: serial.ToTypedMessage(&vmess.Account{
-										Id: userID.String(),
-									}),
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_TCP},
+						},
+					}),
+				},
+			},
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&outbound.Config{
+						Receiver: []*protocol.ServerEndpoint{
+							{
+								Address: net.NewIPOrDomain(net.LocalHostIP),
+								Port:    uint32(serverPort),
+								User: []*protocol.User{
+									{
+										Account: serial.ToTypedMessage(&vmess.Account{
+											Id: userID.String(),
+										}),
+									},
 								},
 							},
 						},
-					},
-				}),
+					}),
+				},
 			},
-		},
-	}
+		}
 
-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
-	common.Must(err)
-	defer CloseAllServers(servers)
+		server, _ := InitializeServerConfig(clientConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testTCPConn(net.Port(clientPort), 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start client")
+		}
+		clientPort = uint32(tcp.PickPort())
+	}
 
 	for port := clientPort; port <= clientPort+clientPortRange; port++ {
 		if err := testTCPConn(net.Port(port), 1024, time.Second*2)(); err != nil {
@@ -155,49 +168,62 @@ func TestDokodemoUDP(t *testing.T) {
 			},
 		},
 	}
+	server, err := InitializeServerConfig(serverConfig)
+	common.Must(err)
+	defer CloseServer(server)
 
-	clientPort := uint32(udp.PickPort())
 	clientPortRange := uint32(5)
-	clientConfig := &core.Config{
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
-					Address: net.NewIPOrDomain(dest.Address),
-					Port:    uint32(dest.Port),
-					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_UDP},
-					},
-				}),
+	retry := 1
+	clientPort := uint32(udp.PickPort())
+	for {
+		clientConfig := &core.Config{
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
+				},
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&outbound.Config{
-					Receiver: []*protocol.ServerEndpoint{
-						{
-							Address: net.NewIPOrDomain(net.LocalHostIP),
-							Port:    uint32(serverPort),
-							User: []*protocol.User{
-								{
-									Account: serial.ToTypedMessage(&vmess.Account{
-										Id: userID.String(),
-									}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&outbound.Config{
+						Receiver: []*protocol.ServerEndpoint{
+							{
+								Address: net.NewIPOrDomain(net.LocalHostIP),
+								Port:    uint32(serverPort),
+								User: []*protocol.User{
+									{
+										Account: serial.ToTypedMessage(&vmess.Account{
+											Id: userID.String(),
+										}),
+									},
 								},
 							},
 						},
-					},
-				}),
+					}),
+				},
 			},
-		},
-	}
+		}
 
-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
-	common.Must(err)
-	defer CloseAllServers(servers)
+		server, _ := InitializeServerConfig(clientConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(net.Port(clientPort), 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start client")
+		}
+		clientPort = uint32(udp.PickPort())
+	}
 
 	var errg errgroup.Group
 	for port := clientPort; port <= clientPort+clientPortRange; port++ {

testing/scenarios/feature_test.go

@@ -234,7 +234,7 @@ func TestProxyOverKCP(t *testing.T) {
 	defer tcpServer.Close()
 
 	serverUserID := protocol.NewID(uuid.New())
-	serverPort := tcp.PickPort()
+	serverPort := udp.PickPort()
 	serverConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{

testing/scenarios/socks_test.go

@@ -111,32 +111,59 @@ func TestSocksBridageUDP(t *testing.T) {
 	common.Must(err)
 	defer udpServer.Close()
 
+	retry := 1
 	serverPort := tcp.PickPort()
-	serverConfig := &core.Config{
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
-					AuthType: socks.AuthType_PASSWORD,
-					Accounts: map[string]string{
-						"Test Account": "Test Password",
-					},
-					Address:    net.NewIPOrDomain(net.LocalHostIP),
-					UdpEnabled: true,
-				}),
+	for {
+		serverConfig := &core.Config{
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
+						AuthType: socks.AuthType_PASSWORD,
+						Accounts: map[string]string{
+							"Test Account": "Test Password",
+						},
+						Address:    net.NewIPOrDomain(net.LocalHostIP),
+						UdpEnabled: true,
+					}),
+				},
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort + 1)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
+				},
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+				},
 			},
-		},
+		}
+
+		server, _ := InitializeServerConfig(serverConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(serverPort+1, 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start server")
+		}
+		serverPort = tcp.PickPort()
 	}
 
-	clientPort := tcp.PickPort()
+	clientPort := udp.PickPort()
 	clientConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{
@@ -148,7 +175,7 @@ func TestSocksBridageUDP(t *testing.T) {
 					Address: net.NewIPOrDomain(dest.Address),
 					Port:    uint32(dest.Port),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_UDP},
 					},
 				}),
 			},
@@ -175,12 +202,12 @@ func TestSocksBridageUDP(t *testing.T) {
 		},
 	}
 
-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
+	server, err := InitializeServerConfig(clientConfig)
 	common.Must(err)
-	defer CloseAllServers(servers)
+	defer CloseServer(server)
 
-	if err := testUDPConn(clientPort, 1024, time.Second*5)(); err != nil {
-		t.Error(err)
+	if !WaitConnAvailableWithTest(t, testUDPConn(clientPort, 1024, time.Second*2)) {
+		t.Fail()
 	}
 }
 
@@ -192,46 +219,74 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 	common.Must(err)
 	defer udpServer.Close()
 
+	retry := 1
 	serverPort := tcp.PickPort()
-	serverConfig := &core.Config{
-		App: []*serial.TypedMessage{
-			serial.ToTypedMessage(&router.Config{
-				Rule: []*router.RoutingRule{
-					{
-						TargetTag: &router.RoutingRule_Tag{
-							Tag: "out",
+	for {
+		serverConfig := &core.Config{
+			App: []*serial.TypedMessage{
+				serial.ToTypedMessage(&router.Config{
+					Rule: []*router.RoutingRule{
+						{
+							TargetTag: &router.RoutingRule_Tag{
+								Tag: "out",
+							},
+							InboundTag: []string{"socks", "dokodemo"},
 						},
-						InboundTag: []string{"socks"},
 					},
-				},
-			}),
-		},
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				Tag: "socks",
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
-					AuthType:   socks.AuthType_NO_AUTH,
-					Address:    net.NewIPOrDomain(net.LocalHostIP),
-					UdpEnabled: true,
 				}),
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&blackhole.Config{}),
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					Tag: "socks",
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
+						AuthType:   socks.AuthType_NO_AUTH,
+						Address:    net.NewIPOrDomain(net.LocalHostIP),
+						UdpEnabled: true,
+					}),
+				},
+				{
+					Tag: "dokodemo",
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort + 1)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
+				},
 			},
-			{
-				Tag:           "out",
-				ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&blackhole.Config{}),
+				},
+				{
+					Tag:           "out",
+					ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+				},
 			},
-		},
+		}
+
+		server, _ := InitializeServerConfig(serverConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(serverPort+1, 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start server")
+		}
+		serverPort = tcp.PickPort()
 	}
 
-	clientPort := tcp.PickPort()
+	clientPort := udp.PickPort()
 	clientConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{
@@ -243,7 +298,7 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 					Address: net.NewIPOrDomain(dest.Address),
 					Port:    uint32(dest.Port),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_UDP},
 					},
 				}),
 			},
@@ -262,12 +317,12 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 		},
 	}
 
-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
+	server, err := InitializeServerConfig(clientConfig)
 	common.Must(err)
-	defer CloseAllServers(servers)
+	defer CloseServer(server)
 
-	if err := testUDPConn(clientPort, 1024, time.Second*5)(); err != nil {
-		t.Error(err)
+	if !WaitConnAvailableWithTest(t, testUDPConn(clientPort, 1024, time.Second*2)) {
+		t.Fail()
 	}
 }
 

testing/scenarios/vmess_test.go

@@ -108,7 +108,7 @@ func TestVMessDynamicPort(t *testing.T) {
 		}
 
 		server, _ := InitializeServerConfig(serverConfig)
-		if server != nil && tcpConnAvailableAtPort(t, serverPort+100) {
+		if server != nil && WaitConnAvailableWithTest(t, testTCPConn(serverPort+100, 1024, time.Second*2)) {
 			defer CloseServer(server)
 			break
 		}
@@ -167,28 +167,11 @@ func TestVMessDynamicPort(t *testing.T) {
 	common.Must(err)
 	defer CloseServer(server)
 
-	if !tcpConnAvailableAtPort(t, clientPort) {
+	if !WaitConnAvailableWithTest(t, testTCPConn(clientPort, 1024, time.Second*2)) {
 		t.Fail()
 	}
 }
 
-func tcpConnAvailableAtPort(t *testing.T, port net.Port) bool {
-	for i := 1; ; i++ {
-		if i > 10 {
-			t.Log("All attempts failed to test tcp conn")
-			return false
-		}
-		time.Sleep(time.Millisecond * 10)
-		if err := testTCPConn(port, 1024, time.Second*2)(); err != nil {
-			t.Log("err ", err)
-		} else {
-			t.Log("success with", i, "attempts")
-			break
-		}
-	}
-	return true
-}
-
 func TestVMessGCM(t *testing.T) {
 	tcpServer := tcp.Server{
 		MsgProcessor: xor,