Rework e2e tests into suite

Signed-off-by: Christopher Crone <[email protected]>

Makefile

@@ -44,10 +44,10 @@ cli: ## Compile the cli
 	--target cli
 
 e2e-local: ## Run End to end local tests
-	go run ./tests/e2e/e2e.go
+	go test -v ./tests/e2e
 
 e2e-aci: ## Run End to end ACI tests (requires azure login)
-	go run ./tests/aci-e2e/e2e-aci.go
+	go test -v ./tests/aci-e2e
 
 cross: ## Compile the CLI for linux, darwin and windows
 	@docker build . \

builder.Makefile

@@ -55,7 +55,7 @@ cross:
 	@GOOS=windows GOARCH=amd64 $(GO_BUILD) -o $(BINARY)-windows-amd64.exe ./cli
 
 test:
-	@go test ./...
+	@go test -cover $(shell go list ./... | grep -vE 'e2e')
 
 lint:
 	golangci-lint run --timeout 10m0s ./...

tests/aci-e2e/e2e-aci.go → tests/aci-e2e/e2e-aci_test.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net/url"
 	"strings"
+	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -14,6 +15,7 @@ import (
 	"github.com/Azure/azure-storage-file-go/azfile"
 
 	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/suite"
 
 	"github.com/docker/api/azure"
 	"github.com/docker/api/context/store"
@@ -29,51 +31,60 @@ const (
 	testContainerName = "testcontainername"
 )
 
-func main() {
-	SetupTest()
+var (
+	subscriptionID string
+)
+
+type E2eACISuite struct {
+	Suite
+}
 
+func (s *E2eACISuite) TestContextHelp() {
 	It("ensures context command includes azure-login and aci-create", func() {
-		output := NewDockerCommand("context", "create", "--help").ExecOrDie()
+		output := s.NewDockerCommand("context", "create", "--help").ExecOrDie()
 		Expect(output).To(ContainSubstring("docker context create CONTEXT BACKEND [OPTIONS] [flags]"))
 		Expect(output).To(ContainSubstring("--aci-location"))
 		Expect(output).To(ContainSubstring("--aci-subscription-id"))
 		Expect(output).To(ContainSubstring("--aci-resource-group"))
 	})
+}
 
+func (s *E2eACISuite) TestContextDefault() {
 	It("should be initialized with default context", func() {
-		_, err := NewCommand("docker", "context", "rm", "-f", contextName).Exec()
+		_, err := s.NewCommand("docker", "context", "rm", "-f", contextName).Exec()
 		if err == nil {
 			log.Println("Cleaning existing test context")
 		}
 
-		NewCommand("docker", "context", "use", "default").ExecOrDie()
-		output := NewCommand("docker", "context", "ls").ExecOrDie()
+		s.NewCommand("docker", "context", "use", "default").ExecOrDie()
+		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
 		Expect(output).To(Not(ContainSubstring(contextName)))
 		Expect(output).To(ContainSubstring("default *"))
 	})
+}
 
-	var subscriptionID string
+func (s *E2eACISuite) TestACIBackend() {
 	It("creates a new aci context for tests", func() {
 		setupTestResourceGroup(resourceGroupName)
 		var err error
 		subscriptionID, err = azure.GetSubscriptionID(context.TODO())
 		Expect(err).To(BeNil())
 
-		NewDockerCommand("context", "create", contextName, "aci", "--aci-subscription-id", subscriptionID, "--aci-resource-group", resourceGroupName, "--aci-location", location).ExecOrDie()
+		s.NewDockerCommand("context", "create", contextName, "aci", "--aci-subscription-id", subscriptionID, "--aci-resource-group", resourceGroupName, "--aci-location", location).ExecOrDie()
 		// Expect(output).To(ContainSubstring("ACI context acitest created"))
 	})
 
 	defer deleteResourceGroup(resourceGroupName)
 
 	It("uses the aci context", func() {
-		currentContext := NewCommand("docker", "context", "use", contextName).ExecOrDie()
+		currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie()
 		Expect(currentContext).To(ContainSubstring(contextName))
-		output := NewCommand("docker", "context", "ls").ExecOrDie()
+		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
 		Expect(output).To(ContainSubstring("acitest *"))
 	})
 
 	It("ensures no container is running initially", func() {
-		output := NewDockerCommand("ps").ExecOrDie()
+		output := s.NewDockerCommand("ps").ExecOrDie()
 		Expect(len(Lines(output))).To(Equal(1))
 	})
 
@@ -91,13 +102,13 @@ func main() {
 		uploadFile(credential, u.String(), testFileName, testFileContent)
 
 		mountTarget := "/usr/share/nginx/html"
-		output := NewDockerCommand("run", "nginx",
+		output := s.NewDockerCommand("run", "nginx",
 			"-v", fmt.Sprintf("%s:%s@%s:%s",
 				testStorageAccountName, firstKey, testShareName, mountTarget),
 			"-p", "80:80",
 			"--name", testContainerName).ExecOrDie()
 		Expect(output).To(Equal(testContainerName + "\n"))
-		output = NewDockerCommand("ps").ExecOrDie()
+		output = s.NewDockerCommand("ps").ExecOrDie()
 		lines := Lines(output)
 		Expect(len(lines)).To(Equal(2))
 
@@ -108,19 +119,19 @@ func main() {
 		Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
 
 		publishedURL := strings.ReplaceAll(exposedIP, "->80/tcp", "")
-		output = NewCommand("curl", publishedURL).ExecOrDie()
+		output = s.NewCommand("curl", publishedURL).ExecOrDie()
 		Expect(output).To(ContainSubstring(testFileContent))
 	})
 
 	It("removes container nginx", func() {
-		output := NewDockerCommand("rm", testContainerName).ExecOrDie()
+		output := s.NewDockerCommand("rm", testContainerName).ExecOrDie()
 		Expect(Lines(output)[0]).To(Equal(testContainerName))
 	})
 
 	It("deploys a compose app", func() {
-		NewDockerCommand("compose", "up", "-f", "./tests/composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
+		s.NewDockerCommand("compose", "up", "-f", "../composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
 		// Expect(output).To(ContainSubstring("Successfully deployed"))
-		output := NewDockerCommand("ps").ExecOrDie()
+		output := s.NewDockerCommand("ps").ExecOrDie()
 		Lines := Lines(output)
 		Expect(len(Lines)).To(Equal(4))
 		webChecked := false
@@ -134,9 +145,9 @@ func main() {
 				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
 
 				url := strings.ReplaceAll(exposedIP, "->80/tcp", "")
-				output = NewCommand("curl", url).ExecOrDie()
+				output = s.NewCommand("curl", url).ExecOrDie()
 				Expect(output).To(ContainSubstring("Docker Compose demo"))
-				output = NewCommand("curl", url+"/words/noun").ExecOrDie()
+				output = s.NewCommand("curl", url+"/words/noun").ExecOrDie()
 				Expect(output).To(ContainSubstring("\"word\":"))
 			}
 		}
@@ -145,20 +156,20 @@ func main() {
 	})
 
 	It("get logs from web service", func() {
-		output := NewDockerCommand("logs", "acidemo_web").ExecOrDie()
+		output := s.NewDockerCommand("logs", "acidemo_web").ExecOrDie()
 		Expect(output).To(ContainSubstring("Listening on port 80"))
 	})
 
 	It("shutdown compose app", func() {
-		NewDockerCommand("compose", "down", "-f", "./tests/composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
+		s.NewDockerCommand("compose", "down", "-f", "../composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
 	})
 	It("switches back to default context", func() {
-		output := NewCommand("docker", "context", "use", "default").ExecOrDie()
+		output := s.NewCommand("docker", "context", "use", "default").ExecOrDie()
 		Expect(output).To(ContainSubstring("default"))
 	})
 
 	It("deletes test context", func() {
-		output := NewCommand("docker", "context", "rm", contextName).ExecOrDie()
+		output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie()
 		Expect(output).To(ContainSubstring(contextName))
 	})
 }
@@ -212,6 +223,10 @@ func uploadFile(credential azfile.SharedKeyCredential, baseURL, fileName, fileCo
 	Expect(err).To(BeNil())
 }
 
+func TestE2eACI(t *testing.T) {
+	suite.Run(t, new(E2eACISuite))
+}
+
 func setupTestResourceGroup(groupName string) {
 	log.Println("Creating resource group " + resourceGroupName)
 	ctx := context.TODO()

tests/e2e/e2e.go → tests/e2e/e2e_test.go

@@ -1,81 +1,85 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
+	"testing"
 	"time"
 
 	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/suite"
 
 	. "github.com/docker/api/tests/framework"
 )
 
-func main() {
-	SetupTest()
+type E2eSuite struct {
+	Suite
+}
 
+func (s *E2eSuite) TestContextHelp() {
 	It("ensures context command includes azure-login and aci-create", func() {
-		output := NewDockerCommand("context", "create", "--help").ExecOrDie()
+		output := s.NewDockerCommand("context", "create", "--help").ExecOrDie()
 		Expect(output).To(ContainSubstring("docker context create CONTEXT BACKEND [OPTIONS] [flags]"))
 		Expect(output).To(ContainSubstring("--aci-location"))
 		Expect(output).To(ContainSubstring("--aci-subscription-id"))
 		Expect(output).To(ContainSubstring("--aci-resource-group"))
 	})
+}
 
+func (s *E2eSuite) TestContextDefault() {
 	It("should be initialized with default context", func() {
-		NewDockerCommand("context", "use", "default").ExecOrDie()
-		output := NewDockerCommand("context", "show").ExecOrDie()
+		s.NewDockerCommand("context", "use", "default").ExecOrDie()
+		output := s.NewDockerCommand("context", "show").ExecOrDie()
 		Expect(output).To(ContainSubstring("default"))
-		output = NewCommand("docker", "context", "ls").ExecOrDie()
+		output = s.NewCommand("docker", "context", "ls").ExecOrDie()
 		Expect(output).To(Not(ContainSubstring("test-example")))
 		Expect(output).To(ContainSubstring("default *"))
 	})
+}
 
+func (s *E2eSuite) TestLegacy() {
 	It("should list all legacy commands", func() {
-		output := NewDockerCommand("--help").ExecOrDie()
+		output := s.NewDockerCommand("--help").ExecOrDie()
 		Expect(output).To(ContainSubstring("swarm"))
 	})
 
 	It("should execute legacy commands", func() {
-		output, _ := NewDockerCommand("swarm", "join").Exec()
+		output, _ := s.NewDockerCommand("swarm", "join").Exec()
 		Expect(output).To(ContainSubstring("\"docker swarm join\" requires exactly 1 argument."))
 	})
 
-	It("should run local container in less than 5 secs", func() {
-		NewDockerCommand("pull", "hello-world").ExecOrDie()
-		output := NewDockerCommand("run", "hello-world").WithTimeout(time.NewTimer(5 * time.Second).C).ExecOrDie()
+	It("should run local container in less than 10 secs", func() {
+		s.NewDockerCommand("pull", "hello-world").ExecOrDie()
+		output := s.NewDockerCommand("run", "--rm", "hello-world").WithTimeout(time.NewTimer(10 * time.Second).C).ExecOrDie()
 		Expect(output).To(ContainSubstring("Hello from Docker!"))
 	})
+}
 
-	It("should list local container", func() {
-		output := NewDockerCommand("ps", "-a").ExecOrDie()
-		Expect(output).To(ContainSubstring("hello-world"))
-	})
-
+func (s *E2eSuite) TestMockBackend() {
 	It("creates a new test context to hardcoded example backend", func() {
-		NewDockerCommand("context", "create", "test-example", "example").ExecOrDie()
+		s.NewDockerCommand("context", "create", "test-example", "example").ExecOrDie()
 		// Expect(output).To(ContainSubstring("test-example context acitest created"))
 	})
-	defer NewDockerCommand("context", "rm", "test-example").ExecOrDie()
-	defer NewDockerCommand("context", "use", "default").ExecOrDie()
 
 	It("uses the test context", func() {
-		currentContext := NewDockerCommand("context", "use", "test-example").ExecOrDie()
+		currentContext := s.NewDockerCommand("context", "use", "test-example").ExecOrDie()
 		Expect(currentContext).To(ContainSubstring("test-example"))
-		output := NewDockerCommand("context", "ls").ExecOrDie()
+		output := s.NewDockerCommand("context", "ls").ExecOrDie()
 		Expect(output).To(ContainSubstring("test-example *"))
-		output = NewDockerCommand("context", "show").ExecOrDie()
+		output = s.NewDockerCommand("context", "show").ExecOrDie()
 		Expect(output).To(ContainSubstring("test-example"))
 	})
 
 	It("can run ps command", func() {
-		output := NewDockerCommand("ps").ExecOrDie()
+		output := s.NewDockerCommand("ps").ExecOrDie()
 		lines := Lines(output)
 		Expect(len(lines)).To(Equal(3))
 		Expect(lines[2]).To(ContainSubstring("1234                alpine"))
 	})
 
 	It("can run quiet ps command", func() {
-		output := NewDockerCommand("ps", "-q").ExecOrDie()
+		output := s.NewDockerCommand("ps", "-q").ExecOrDie()
 		lines := Lines(output)
 		Expect(len(lines)).To(Equal(2))
 		Expect(lines[0]).To(Equal("id"))
@@ -83,7 +87,7 @@ func main() {
 	})
 
 	It("can run ps command with all ", func() {
-		output := NewDockerCommand("ps", "-q", "--all").ExecOrDie()
+		output := s.NewDockerCommand("ps", "-q", "--all").ExecOrDie()
 		lines := Lines(output)
 		Expect(len(lines)).To(Equal(3))
 		Expect(lines[0]).To(Equal("id"))
@@ -92,28 +96,42 @@ func main() {
 	})
 
 	It("can run 'run' command", func() {
-		output := NewDockerCommand("run", "nginx", "-p", "80:80").ExecOrDie()
+		output := s.NewDockerCommand("run", "nginx", "-p", "80:80").ExecOrDie()
 		Expect(output).To(ContainSubstring("Running container \"nginx\" with name"))
 	})
+}
 
+func (s *E2eSuite) TestAPIServer() {
+	_, err := exec.LookPath("yarn")
+	if err != nil || os.Getenv("SKIP_NODE") != "" {
+		s.T().Skip("skipping, yarn not installed")
+	}
 	It("can run 'serve' command", func() {
-		server, err := startCliServer()
+		cName := "test-example"
+		s.NewDockerCommand("context", "create", cName, "example").ExecOrDie()
+
+		sPath := fmt.Sprintf("unix:///%s/docker.sock", s.ConfigDir)
+		server, err := serveAPI(s.ConfigDir, sPath)
 		Expect(err).To(BeNil())
-		defer killCliServer(server)
+		defer killProcess(server)
 
-		NewCommand("yarn", "install").WithinDirectory("tests/node-client").ExecOrDie()
-		output := NewCommand("yarn", "run", "start", "test-example").WithinDirectory("tests/node-client").ExecOrDie()
+		s.NewCommand("yarn", "install").WithinDirectory("../node-client").ExecOrDie()
+		output := s.NewCommand("yarn", "run", "start", cName, sPath).WithinDirectory("../node-client").ExecOrDie()
 		Expect(output).To(ContainSubstring("nginx"))
 	})
 }
 
-func killCliServer(process *os.Process) {
+func TestE2e(t *testing.T) {
+	suite.Run(t, new(E2eSuite))
+}
+
+func killProcess(process *os.Process) {
 	err := process.Kill()
 	Expect(err).To(BeNil())
 }
 
-func startCliServer() (*os.Process, error) {
-	cmd := exec.Command("./bin/docker", "serve", "--address", "unix:///tmp/backend.sock")
+func serveAPI(configDir string, address string) (*os.Process, error) {
+	cmd := exec.Command("../../bin/docker", "--config", configDir, "serve", "--address", address)
 	err := cmd.Start()
 	if err != nil {
 		return nil, err

tests/framework/exec.go

@@ -4,13 +4,18 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
 	"github.com/onsi/gomega"
 	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
 )
 
 func (b CmdContext) makeCmd() *exec.Cmd {
@@ -35,25 +40,79 @@ type RetriesContext struct {
 	interval time.Duration
 }
 
+// Suite is used to store context information for e2e tests
+type Suite struct {
+	suite.Suite
+	ConfigDir string
+	BinDir    string
+}
+
+// SetupSuite is run before running any tests
+func (s *Suite) SetupSuite() {
+	d, _ := ioutil.TempDir("", "")
+	s.BinDir = d
+	gomega.RegisterFailHandler(func(message string, callerSkip ...int) {
+		log.Error(message)
+		cp := filepath.Join(s.ConfigDir, "config.json")
+		d, _ := ioutil.ReadFile(cp)
+		fmt.Printf("Contents of %s:\n%s\n\nContents of config dir:\n", cp, string(d))
+		out, _ := s.NewCommand("find", s.ConfigDir).Exec()
+		fmt.Println(out)
+		s.T().Fail()
+	})
+	s.linkClassicDocker()
+}
+
+// TearDownSuite is run after all tests
+func (s *Suite) TearDownSuite() {
+	_ = os.RemoveAll(s.BinDir)
+}
+
+func (s *Suite) linkClassicDocker() {
+	p, err := exec.LookPath("docker")
+	gomega.Expect(err).To(gomega.BeNil())
+	err = os.Symlink(p, filepath.Join(s.BinDir, "docker-classic"))
+	gomega.Expect(err).To(gomega.BeNil())
+	err = os.Setenv("PATH", fmt.Sprintf("%s:%s", s.BinDir, os.Getenv("PATH")))
+	gomega.Expect(err).To(gomega.BeNil())
+}
+
+// BeforeTest is run before each test
+func (s *Suite) BeforeTest(suite, test string) {
+	d, _ := ioutil.TempDir("", "")
+	s.ConfigDir = d
+}
+
+// AfterTest is run after each test
+func (s *Suite) AfterTest(suite, test string) {
+	err := os.RemoveAll(s.ConfigDir)
+	require.NoError(s.T(), err)
+}
+
 // NewCommand creates a command context.
-func NewCommand(command string, args ...string) *CmdContext {
+func (s *Suite) NewCommand(command string, args ...string) *CmdContext {
+	var envs []string
+	if s.ConfigDir != "" {
+		envs = append(os.Environ(), fmt.Sprintf("DOCKER_CONFIG=%s", s.ConfigDir))
+	}
 	return &CmdContext{
 		command: command,
 		args:    args,
+		envs:    envs,
 		retries: RetriesContext{interval: time.Second},
 	}
 }
 
 func dockerExecutable() string {
 	if runtime.GOOS == "windows" {
-		return "./bin/docker.exe"
+		return "../../bin/docker.exe"
 	}
-	return "./bin/docker"
+	return "../../bin/docker"
 }
 
 // NewDockerCommand creates a docker builder.
-func NewDockerCommand(args ...string) *CmdContext {
-	return NewCommand(dockerExecutable(), args...)
+func (s *Suite) NewDockerCommand(args ...string) *CmdContext {
+	return s.NewCommand(dockerExecutable(), args...)
 }
 
 // WithinDirectory tells Docker the cwd.

tests/framework/helper.go

@@ -1,14 +1,10 @@
 package framework
 
 import (
-	"fmt"
 	"log"
-	"os"
 	"strings"
 
 	"github.com/robpike/filter"
-
-	"github.com/onsi/gomega"
 )
 
 func nonEmptyString(s string) bool {
@@ -30,28 +26,3 @@ func It(description string, test func()) {
 	test()
 	log.Print("Passed: ", description)
 }
-
-func gomegaFailHandler(message string, callerSkip ...int) {
-	log.Fatal(message)
-}
-
-//SetupTest Init gomega fail handler
-func SetupTest() {
-	gomega.RegisterFailHandler(gomegaFailHandler)
-
-	linkClassicDocker()
-}
-
-func linkClassicDocker() {
-	dockerOriginal := strings.TrimSuffix(NewCommand("which", "docker").ExecOrDie(), "\n")
-	_, err := NewCommand("rm", "-r", "./bin/tests").Exec()
-	if err == nil {
-		fmt.Println("Removing existing /bin/tests folder before running tests")
-	}
-	_, err = NewCommand("mkdir", "-p", "./bin/tests").Exec()
-	gomega.Expect(err).To(gomega.BeNil())
-	NewCommand("ln", "-s", dockerOriginal, "./bin/tests/docker-classic").ExecOrDie()
-	newPath := "./bin/tests:" + os.Getenv("PATH")
-	err = os.Setenv("PATH", newPath)
-	gomega.Expect(err).To(gomega.BeNil())
-}

tests/node-client/index.ts

@@ -3,8 +3,10 @@ import * as continersPb from "./grpc/containers_grpc_pb";
 import { IContainersClient } from './grpc/containers_grpc_pb';
 import { ListRequest, ListResponse } from "./grpc/containers_pb";
 
+let address = process.argv[3] || "unix:///tmp/backend.sock";
+
 const ContainersServiceClient = grpc.makeClientConstructor(continersPb["com.docker.api.containers.v1.Containers"], "ContainersClient");
-const client = new ContainersServiceClient("unix:///tmp/backend.sock", grpc.credentials.createInsecure()) as unknown as IContainersClient;
+const client = new ContainersServiceClient(address, grpc.credentials.createInsecure()) as unknown as IContainersClient;
 
 let backend = process.argv[2] || "moby";
 const meta = new grpc.Metadata();