Merge pull request #537 from docker/e2e_better_logs

E2e improvements

local/e2e/backend_test.go

@@ -47,24 +47,20 @@ func TestLocalBackend(t *testing.T) {
 	c.RunDockerCmd("context", "use", "test-context").Assert(t, icmd.Success)
 
 	t.Run("run", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("run", "-d", "nginx")
-		res.Assert(t, icmd.Success)
 		containerName := strings.TrimSpace(res.Combined())
 		t.Cleanup(func() {
-			_ = c.RunDockerCmd("rm", "-f", containerName)
+			_ = c.RunDockerOrExitError("rm", "-f", containerName)
 		})
 		res = c.RunDockerCmd("inspect", containerName)
 		res.Assert(t, icmd.Expected{Out: `"Status": "running"`})
 	})
 
 	t.Run("run with ports", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("run", "-d", "-p", "8080:80", "nginx")
-		res.Assert(t, icmd.Success)
 		containerName := strings.TrimSpace(res.Combined())
 		t.Cleanup(func() {
-			_ = c.RunDockerCmd("rm", "-f", containerName)
+			_ = c.RunDockerOrExitError("rm", "-f", containerName)
 		})
 		res = c.RunDockerCmd("inspect", containerName)
 		res.Assert(t, icmd.Expected{Out: `"Status": "running"`})
@@ -73,8 +69,7 @@ func TestLocalBackend(t *testing.T) {
 	})
 
 	t.Run("inspect not found", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("inspect", "nonexistentcontainer")
+		res := c.RunDockerOrExitError("inspect", "nonexistentcontainer")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "Error: No such container: nonexistentcontainer",

tests/aci-e2e/e2e-aci_test.go

@@ -87,9 +87,8 @@ func TestLoginLogout(t *testing.T) {
 			_ = deleteResourceGroup(rg)
 		})
 
-		res := c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rg, "--location", location)
-		res.Assert(t, icmd.Success)
-		res = c.RunDockerCmd("context", "use", contextName)
+		c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rg, "--location", location)
+		res := c.RunDockerCmd("context", "use", contextName)
 		res.Assert(t, icmd.Expected{Out: contextName})
 		res = c.RunDockerCmd("context", "ls")
 		res.Assert(t, icmd.Expected{Out: contextName + " *"})
@@ -117,7 +116,7 @@ func TestLoginLogout(t *testing.T) {
 	})
 
 	t.Run("create context fail", func(t *testing.T) {
-		res := c.RunDockerCmd("context", "create", "aci", "fail-context")
+		res := c.RunDockerOrExitError("context", "create", "aci", "fail-context")
 		res.Assert(t, icmd.Expected{
 			ExitCode: errdefs.ExitCodeLoginRequired,
 			Err:      `not logged in to azure, you need to run "docker login azure" first`,
@@ -127,7 +126,7 @@ func TestLoginLogout(t *testing.T) {
 
 func TestContainerRun(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	sID, rg := setupTestResourceGroup(t, c, "run")
+	sID, rg := setupTestResourceGroup(t, c)
 
 	const (
 		testShareName   = "dockertestshare"
@@ -169,14 +168,11 @@ func TestContainerRun(t *testing.T) {
 			"-p", "80:80",
 			"nginx",
 		)
-		res.Assert(t, icmd.Success)
 		container = getContainerName(res.Stdout())
-		t.Logf("Container name: %q", container)
 	})
 
 	t.Run("inspect", func(t *testing.T) {
 		res := c.RunDockerCmd("inspect", container)
-		res.Assert(t, icmd.Success)
 
 		containerInspect, err := ParseContainerInspect(res.Stdout())
 		assert.NilError(t, err)
@@ -187,12 +183,10 @@ func TestContainerRun(t *testing.T) {
 		assert.Assert(t, is.Len(containerInspect.Ports, 1))
 		hostIP = containerInspect.Ports[0].HostIP
 		endpoint = fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
-		t.Logf("Endpoint: %s", endpoint)
 	})
 
 	t.Run("ps", func(t *testing.T) {
 		res := c.RunDockerCmd("ps")
-		res.Assert(t, icmd.Success)
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		l := out[len(out)-1]
 		assert.Assert(t, strings.Contains(l, container), "Looking for %q in line: %s", container, l)
@@ -219,7 +213,7 @@ func TestContainerRun(t *testing.T) {
 		res := c.RunDockerCmd("exec", container, "pwd")
 		res.Assert(t, icmd.Expected{Out: "/"})
 
-		res = c.RunDockerCmd("exec", container, "echo", "fail_with_argument")
+		res = c.RunDockerOrExitError("exec", container, "echo", "fail_with_argument")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "ACI exec command does not accept arguments to the command. Only the binary should be specified",
@@ -266,7 +260,7 @@ func TestContainerRun(t *testing.T) {
 	})
 
 	t.Run("rm a running container", func(t *testing.T) {
-		res := c.RunDockerCmd("rm", container)
+		res := c.RunDockerOrExitError("rm", container)
 		res.Assert(t, icmd.Expected{
 			Err:      fmt.Sprintf("Error: you cannot remove a running container %s. Stop the container before attempting removal or force remove", container),
 			ExitCode: 1,
@@ -275,13 +269,10 @@ func TestContainerRun(t *testing.T) {
 
 	t.Run("force rm", func(t *testing.T) {
 		res := c.RunDockerCmd("rm", "-f", container)
-		res.Assert(t, icmd.Expected{
-			Out:      container,
-			ExitCode: 0,
-		})
+		res.Assert(t, icmd.Expected{Out: container})
 
 		checkStopped := func(t poll.LogT) poll.Result {
-			res := c.RunDockerCmd("inspect", container)
+			res := c.RunDockerOrExitError("inspect", container)
 			if res.ExitCode == 1 {
 				return poll.Success()
 			}
@@ -293,7 +284,7 @@ func TestContainerRun(t *testing.T) {
 
 func TestContainerRunAttached(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	_, _ = setupTestResourceGroup(t, c, "runA")
+	_, _ = setupTestResourceGroup(t, c)
 
 	// Used in subtests
 	var (
@@ -315,7 +306,7 @@ func TestContainerRunAttached(t *testing.T) {
 		runRes := icmd.StartCmd(cmd)
 
 		checkRunning := func(t poll.LogT) poll.Result {
-			res := c.RunDockerCmd("inspect", container)
+			res := c.RunDockerOrExitError("inspect", container)
 			if res.ExitCode == 0 {
 				return poll.Success()
 			}
@@ -324,7 +315,6 @@ func TestContainerRunAttached(t *testing.T) {
 		poll.WaitOn(t, checkRunning, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
 
 		inspectRes := c.RunDockerCmd("inspect", container)
-		inspectRes.Assert(t, icmd.Success)
 
 		containerInspect, err := ParseContainerInspect(inspectRes.Stdout())
 		assert.NilError(t, err)
@@ -339,7 +329,6 @@ func TestContainerRunAttached(t *testing.T) {
 		assert.Equal(t, port.ContainerPort, uint32(80))
 		assert.Equal(t, port.HostPort, uint32(80))
 		endpoint = fmt.Sprintf("http://%s:%d", port.HostIP, port.HostPort)
-		t.Logf("Endpoint: %s", endpoint)
 
 		assert.Assert(t, !strings.Contains(runRes.Stdout(), "/test"))
 		checkRequest := func(t poll.LogT) poll.Result {
@@ -361,7 +350,7 @@ func TestContainerRunAttached(t *testing.T) {
 	})
 
 	t.Run("stop wrong container", func(t *testing.T) {
-		res := c.RunDockerCmd("stop", "unknown-container")
+		res := c.RunDockerOrExitError("stop", "unknown-container")
 		res.Assert(t, icmd.Expected{
 			Err:      "Error: container unknown-container not found",
 			ExitCode: 1,
@@ -375,12 +364,10 @@ func TestContainerRunAttached(t *testing.T) {
 
 	t.Run("ps stopped container with --all", func(t *testing.T) {
 		res := c.RunDockerCmd("ps", container)
-		res.Assert(t, icmd.Success)
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		assert.Assert(t, is.Len(out, 1))
 
 		res = c.RunDockerCmd("ps", "--all", container)
-		res.Assert(t, icmd.Success)
 		out = strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		assert.Assert(t, is.Len(out, 2))
 	})
@@ -403,7 +390,7 @@ func TestContainerRunAttached(t *testing.T) {
 
 func TestCompose(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	_, _ = setupTestResourceGroup(t, c, "compose")
+	_, _ = setupTestResourceGroup(t, c)
 
 	const (
 		composeFile              = "../composefiles/aci-demo/aci_demo_port.yaml"
@@ -415,11 +402,8 @@ func TestCompose(t *testing.T) {
 
 	t.Run("compose up", func(t *testing.T) {
 		// Name of Compose project is taken from current folder "acie2e"
-		res := c.RunDockerCmd("compose", "up", "-f", composeFile)
-		res.Assert(t, icmd.Success)
-
-		res = c.RunDockerCmd("ps")
-		res.Assert(t, icmd.Success)
+		c.RunDockerCmd("compose", "up", "-f", composeFile)
+		res := c.RunDockerCmd("ps")
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		// Check three containers are running
 		assert.Assert(t, is.Len(out, 4))
@@ -433,13 +417,11 @@ func TestCompose(t *testing.T) {
 		assert.Assert(t, webRunning, "web container not running")
 
 		res = c.RunDockerCmd("inspect", serverContainer)
-		res.Assert(t, icmd.Success)
 
 		containerInspect, err := ParseContainerInspect(res.Stdout())
 		assert.NilError(t, err)
 		assert.Assert(t, is.Len(containerInspect.Ports, 1))
 		endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
-		t.Logf("Endpoint: %s", endpoint)
 
 		r, err := http.Get(endpoint + "/words/noun")
 		assert.NilError(t, err)
@@ -455,24 +437,19 @@ func TestCompose(t *testing.T) {
 	})
 
 	t.Run("update", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName)
-		res.Assert(t, icmd.Success)
-
-		res = c.RunDockerCmd("ps")
-		res.Assert(t, icmd.Success)
+		c.RunDockerCmd("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName)
+		res := c.RunDockerCmd("ps")
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		// Check three containers are running
 		assert.Assert(t, is.Len(out, 4))
 
 		for _, cName := range []string{serverContainer, wordsContainer} {
 			res = c.RunDockerCmd("inspect", cName)
-			res.Assert(t, icmd.Success)
 
 			containerInspect, err := ParseContainerInspect(res.Stdout())
 			assert.NilError(t, err)
 			assert.Assert(t, is.Len(containerInspect.Ports, 1))
 			endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
-			t.Logf("Endpoint: %s", endpoint)
 			var route string
 			switch cName {
 			case serverContainer:
@@ -502,11 +479,8 @@ func TestCompose(t *testing.T) {
 	})
 
 	t.Run("down", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "down", "--project-name", composeProjectName)
-		res.Assert(t, icmd.Success)
-
-		res = c.RunDockerCmd("ps")
-		res.Assert(t, icmd.Success)
+		c.RunDockerCmd("compose", "down", "--project-name", composeProjectName)
+		res := c.RunDockerCmd("ps")
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		assert.Equal(t, len(out), 1)
 	})
@@ -514,7 +488,7 @@ func TestCompose(t *testing.T) {
 
 func TestRunEnvVars(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	_, _ = setupTestResourceGroup(t, c, "runEnv")
+	_, _ = setupTestResourceGroup(t, c)
 
 	t.Run("run", func(t *testing.T) {
 		cmd := c.NewDockerCmd(
@@ -530,17 +504,15 @@ func TestRunEnvVars(t *testing.T) {
 		res.Assert(t, icmd.Success)
 		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
 		container := strings.TrimSpace(out[len(out)-1])
-		t.Logf("Container name: %q", container)
 
 		res = c.RunDockerCmd("inspect", container)
-		res.Assert(t, icmd.Success)
 
 		containerInspect, err := ParseContainerInspect(res.Stdout())
 		assert.NilError(t, err)
 		assert.Equal(t, containerInspect.Image, "mysql:5.7")
 
 		check := func(t poll.LogT) poll.Result {
-			res := c.RunDockerCmd("logs", container)
+			res := c.RunDockerOrExitError("logs", container)
 			if strings.Contains(res.Stdout(), "Giving user user1 access to schema mytestdb") {
 				return poll.Success()
 			}
@@ -550,12 +522,11 @@ func TestRunEnvVars(t *testing.T) {
 	})
 }
 
-func setupTestResourceGroup(t *testing.T, c *E2eCLI, tName string) (string, string) {
+func setupTestResourceGroup(t *testing.T, c *E2eCLI) (string, string) {
 	startTime := strconv.Itoa(int(time.Now().Unix()))
-	rg := "E2E-" + tName + "-" + startTime
+	rg := "E2E-" + t.Name() + "-" + startTime
 	azureLogin(t, c)
 	sID := getSubscriptionID(t)
-	t.Logf("Create resource group %q", rg)
 	err := createResourceGroup(sID, rg)
 	assert.Check(t, is.Nil(err))
 	t.Cleanup(func() {
@@ -566,7 +537,6 @@ func setupTestResourceGroup(t *testing.T, c *E2eCLI, tName string) (string, stri
 	createAciContextAndUseIt(t, c, sID, rg)
 	// Check nothing is running
 	res := c.RunDockerCmd("ps")
-	res.Assert(t, icmd.Success)
 	assert.Assert(t, is.Len(strings.Split(strings.TrimSpace(res.Stdout()), "\n"), 1))
 	return sID, rg
 }
@@ -585,7 +555,6 @@ func deleteResourceGroup(rgName string) error {
 }
 
 func azureLogin(t *testing.T, c *E2eCLI) {
-	t.Log("Log in to Azure")
 	// in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth`
 	clientID := os.Getenv("AZURE_CLIENT_ID")
 	clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
@@ -593,8 +562,7 @@ func azureLogin(t *testing.T, c *E2eCLI) {
 	assert.Check(t, clientID != "", "AZURE_CLIENT_ID must not be empty")
 	assert.Check(t, clientSecret != "", "AZURE_CLIENT_SECRET must not be empty")
 	assert.Check(t, tenantID != "", "AZURE_TENANT_ID must not be empty")
-	res := c.RunDockerCmd("login", "azure", "--client-id", clientID, "--client-secret", clientSecret, "--tenant-id", tenantID)
-	res.Assert(t, icmd.Success)
+	c.RunDockerCmd("login", "azure", "--client-id", clientID, "--client-secret", clientSecret, "--tenant-id", tenantID)
 }
 
 func getSubscriptionID(t *testing.T) string {
@@ -613,17 +581,14 @@ func createResourceGroup(sID, rgName string) error {
 }
 
 func createAciContextAndUseIt(t *testing.T, c *E2eCLI, sID, rgName string) {
-	t.Log("Create ACI context")
-	res := c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rgName, "--location", location)
-	res.Assert(t, icmd.Success)
-	res = c.RunDockerCmd("context", "use", contextName)
+	c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rgName, "--location", location)
+	res := c.RunDockerCmd("context", "use", contextName)
 	res.Assert(t, icmd.Expected{Out: contextName})
 	res = c.RunDockerCmd("context", "ls")
 	res.Assert(t, icmd.Expected{Out: contextName + " *"})
 }
 
 func createStorageAccount(t *testing.T, aciContext store.AciContext, name string) (azure_storage.Account, func() error) {
-	t.Logf("Create storage account %q", name)
 	account, err := storage.CreateStorageAccount(context.TODO(), aciContext, name)
 	assert.Check(t, is.Nil(err))
 	assert.Check(t, is.Equal(*(account.Name), name))

tests/e2e/e2e_test.go

@@ -51,7 +51,7 @@ func TestComposeNotImplemented(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 	res := c.RunDockerCmd("context", "show")
 	res.Assert(t, icmd.Expected{Out: "default"})
-	res = c.RunDockerCmd("compose", "up")
+	res = c.RunDockerOrExitError("compose", "up")
 	res.Assert(t, icmd.Expected{
 		ExitCode: 1,
 		Err:      `Command "compose up" not available in current context (default)`,
@@ -62,26 +62,21 @@ func TestContextDefault(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("show", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "show")
 		res.Assert(t, icmd.Expected{Out: "default"})
 	})
 
 	t.Run("ls", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "ls")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), GoldenFile("ls-out-default"))
 	})
 
 	t.Run("inspect", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "inspect", "default")
 		res.Assert(t, icmd.Expected{Out: `"Name": "default"`})
 	})
 
 	t.Run("inspect current", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "inspect")
 		res.Assert(t, icmd.Expected{Out: `"Name": "default"`})
 	})
@@ -93,20 +88,16 @@ func TestContextCreateDocker(t *testing.T) {
 	res.Assert(t, icmd.Expected{Out: "test-docker"})
 
 	t.Run("ls", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "ls")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), GoldenFile("ls-out-test-docker"))
 	})
 
 	t.Run("ls quiet", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "ls", "-q")
 		golden.Assert(t, res.Stdout(), "ls-out-test-docker-quiet.golden")
 	})
 
 	t.Run("ls format", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "ls", "--format", "{{ json . }}")
 		res.Assert(t, icmd.Expected{Out: `"Name":"default"`})
 	})
@@ -131,7 +122,6 @@ func TestContextHelpACI(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("help", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "create", "aci", "--help")
 		// Can't use golden here as the help prints the config directory which changes
 		res.Assert(t, icmd.Expected{Out: "docker context create aci CONTEXT [flags]"})
@@ -141,8 +131,7 @@ func TestContextHelpACI(t *testing.T) {
 	})
 
 	t.Run("check exec", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("context", "create", "aci", "--subscription-id", "invalid-id")
+		res := c.RunDockerOrExitError("context", "create", "aci", "--subscription-id", "invalid-id")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "accepts 1 arg(s), received 0",
@@ -154,8 +143,8 @@ func TestContextHelpACI(t *testing.T) {
 func TestContextDuplicateACI(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
-	c.RunDockerCmd("context", "create", "mycontext", "--from", "default").Assert(t, icmd.Success)
-	res := c.RunDockerCmd("context", "create", "aci", "mycontext")
+	c.RunDockerCmd("context", "create", "mycontext", "--from", "default")
+	res := c.RunDockerOrExitError("context", "create", "aci", "mycontext")
 	res.Assert(t, icmd.Expected{
 		ExitCode: 1,
 		Err:      "context mycontext: already exists",
@@ -167,10 +156,10 @@ func TestContextRemove(t *testing.T) {
 	t.Run("remove current", func(t *testing.T) {
 		c := NewParallelE2eCLI(t, binDir)
 
-		c.RunDockerCmd("context", "create", "test-context-rm", "--from", "default").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "create", "test-context-rm", "--from", "default")
 		res := c.RunDockerCmd("context", "use", "test-context-rm")
 		res.Assert(t, icmd.Expected{Out: "test-context-rm"})
-		res = c.RunDockerCmd("context", "rm", "test-context-rm")
+		res = c.RunDockerOrExitError("context", "rm", "test-context-rm")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "cannot delete current context",
@@ -180,8 +169,8 @@ func TestContextRemove(t *testing.T) {
 	t.Run("force remove current", func(t *testing.T) {
 		c := NewParallelE2eCLI(t, binDir)
 
-		c.RunDockerCmd("context", "create", "test-context-rmf").Assert(t, icmd.Success)
-		c.RunDockerCmd("context", "use", "test-context-rmf").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "create", "test-context-rmf")
+		c.RunDockerCmd("context", "use", "test-context-rmf")
 		res := c.RunDockerCmd("context", "rm", "-f", "test-context-rmf")
 		res.Assert(t, icmd.Expected{Out: "test-context-rmf"})
 		res = c.RunDockerCmd("context", "ls")
@@ -195,8 +184,7 @@ func TestLoginCommandDelegation(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("default context", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("login", "-u", "nouser", "-p", "wrongpasword")
+		res := c.RunDockerOrExitError("login", "-u", "nouser", "-p", "wrongpasword")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "unauthorized: incorrect username or password",
@@ -204,8 +192,7 @@ func TestLoginCommandDelegation(t *testing.T) {
 	})
 
 	t.Run("interactive", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("login", "someregistry.docker.io")
+		res := c.RunDockerOrExitError("login", "someregistry.docker.io")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "Cannot perform an interactive login from a non TTY device",
@@ -213,16 +200,14 @@ func TestLoginCommandDelegation(t *testing.T) {
 	})
 
 	t.Run("logout", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("logout", "someregistry.docker.io")
 		res.Assert(t, icmd.Expected{Out: "someregistry.docker.io"})
 	})
 
 	t.Run("existing context", func(t *testing.T) {
-		c := NewParallelE2eCLI(t, binDir)
-		c.RunDockerCmd("context", "create", "local", "local").Assert(t, icmd.Success)
-		c.RunDockerCmd("context", "use", "local").Assert(t, icmd.Success)
-		res := c.RunDockerCmd("login", "-u", "nouser", "-p", "wrongpasword")
+		c.RunDockerCmd("context", "create", "local", "local")
+		c.RunDockerCmd("context", "use", "local")
+		res := c.RunDockerOrExitError("login", "-u", "nouser", "-p", "wrongpasword")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "unauthorized: incorrect username or password",
@@ -234,8 +219,7 @@ func TestCloudLogin(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("unknown backend", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("login", "mycloudbackend")
+		res := c.RunDockerOrExitError("login", "mycloudbackend")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "unknown backend type for cloud login: mycloudbackend",
@@ -282,14 +266,12 @@ func TestLegacy(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("help", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("--help")
 		res.Assert(t, icmd.Expected{Out: "swarm"})
 	})
 
 	t.Run("swarm", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("swarm", "join")
+		res := c.RunDockerOrExitError("swarm", "join")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      `"docker swarm join" requires exactly 1 argument.`,
@@ -297,7 +279,6 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("local run", func(t *testing.T) {
-		t.Parallel()
 		cmd := c.NewDockerCmd("run", "--rm", "hello-world")
 		cmd.Timeout = 40 * time.Second
 		res := icmd.RunCmd(cmd)
@@ -305,8 +286,7 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("error messages", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("foo")
+		res := c.RunDockerOrExitError("foo")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "docker: 'foo' is not a docker command.",
@@ -314,12 +294,11 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("host flag", func(t *testing.T) {
-		t.Parallel()
 		stderr := "Cannot connect to the Docker daemon at tcp://localhost:123"
 		if runtime.GOOS == "windows" {
 			stderr = "error during connect: Get http://localhost:123"
 		}
-		res := c.RunDockerCmd("-H", "tcp://localhost:123", "version")
+		res := c.RunDockerOrExitError("-H", "tcp://localhost:123", "version")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      stderr,
@@ -327,10 +306,9 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("existing contexts delegate", func(t *testing.T) {
-		c := NewParallelE2eCLI(t, binDir)
-		c.RunDockerCmd("context", "create", "moby-ctx", "--from=default").Assert(t, icmd.Success)
-		c.RunDockerCmd("context", "use", "moby-ctx").Assert(t, icmd.Success)
-		res := c.RunDockerCmd("swarm", "join")
+		c.RunDockerCmd("context", "create", "moby-ctx", "--from=default")
+		c.RunDockerCmd("context", "use", "moby-ctx")
+		res := c.RunDockerOrExitError("swarm", "join")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      `"docker swarm join" requires exactly 1 argument.`,
@@ -338,18 +316,16 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("host flag overrides context", func(t *testing.T) {
-		c := NewParallelE2eCLI(t, binDir)
-		c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
-		c.RunDockerCmd("context", "use", "test-example").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "create", "example", "test-example")
+		c.RunDockerCmd("context", "use", "test-example")
 		endpoint := "unix:///var/run/docker.sock"
 		if runtime.GOOS == "windows" {
 			endpoint = "npipe:////./pipe/docker_engine"
 		}
 		res := c.RunDockerCmd("-H", endpoint, "ps")
-		res.Assert(t, icmd.Success)
 		// Example backend's ps output includes these strings
-		assert.Assert(t, !strings.Contains(res.Stdout(), "id"))
-		assert.Assert(t, !strings.Contains(res.Stdout(), "1234"))
+		assert.Assert(t, !strings.Contains(res.Stdout(), "id"), "%q does not contains %q", res.Stdout(), "id")
+		assert.Assert(t, !strings.Contains(res.Stdout(), "1234"), "%q does not contains %q", res.Stdout(), "1234")
 	})
 }
 
@@ -357,8 +333,7 @@ func TestLegacyLogin(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("host flag login", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("-H", "tcp://localhost:123", "login", "-u", "nouser", "-p", "wrongpasword")
+		res := c.RunDockerOrExitError("-H", "tcp://localhost:123", "login", "-u", "nouser", "-p", "wrongpasword")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "WARNING! Using --password via the CLI is insecure. Use --password-stdin.",
@@ -366,8 +341,7 @@ func TestLegacyLogin(t *testing.T) {
 	})
 
 	t.Run("log level flag login", func(t *testing.T) {
-		t.Parallel()
-		res := c.RunDockerCmd("--log-level", "debug", "login", "-u", "nouser", "-p", "wrongpasword")
+		res := c.RunDockerOrExitError("--log-level", "debug", "login", "-u", "nouser", "-p", "wrongpasword")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      "WARNING! Using --password via the CLI is insecure",
@@ -375,9 +349,7 @@ func TestLegacyLogin(t *testing.T) {
 	})
 
 	t.Run("login help global flags", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("login", "--help")
-		res.Assert(t, icmd.Success)
 		assert.Assert(t, !strings.Contains(res.Combined(), "--log-level"))
 	})
 }
@@ -385,9 +357,8 @@ func TestLegacyLogin(t *testing.T) {
 func TestUnsupportedCommand(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
-	res := c.RunDockerCmd("context", "create", "example", "test-example")
-	res.Assert(t, icmd.Success)
-	res = c.RunDockerCmd("--context", "test-example", "images")
+	c.RunDockerCmd("context", "create", "example", "test-example")
+	res := c.RunDockerOrExitError("--context", "test-example", "images")
 	res.Assert(t, icmd.Expected{
 		ExitCode: 1,
 		Err:      `Command "images" not available in current context (test-example), you can use the "default" context to run this command`,
@@ -398,13 +369,11 @@ func TestVersion(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	t.Run("azure version", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("version")
 		res.Assert(t, icmd.Expected{Out: "Azure integration"})
 	})
 
 	t.Run("format", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("version", "-f", "{{ json . }}")
 		res.Assert(t, icmd.Expected{Out: `"Client":`})
 		res = c.RunDockerCmd("version", "--format", "{{ json . }}")
@@ -412,9 +381,8 @@ func TestVersion(t *testing.T) {
 	})
 
 	t.Run("delegate version flag", func(t *testing.T) {
-		c := NewParallelE2eCLI(t, binDir)
-		c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
-		c.RunDockerCmd("context", "use", "test-example").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "create", "example", "test-example")
+		c.RunDockerCmd("context", "use", "test-example")
 		res := c.RunDockerCmd("-v")
 		res.Assert(t, icmd.Expected{Out: "Docker version"})
 	})
@@ -422,12 +390,11 @@ func TestVersion(t *testing.T) {
 
 func TestMockBackend(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
+	c.RunDockerCmd("context", "create", "example", "test-example")
 	res := c.RunDockerCmd("context", "use", "test-example")
 	res.Assert(t, icmd.Expected{Out: "test-example"})
 
 	t.Run("use", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("context", "show")
 		res.Assert(t, icmd.Expected{Out: "test-example"})
 		res = c.RunDockerCmd("context", "ls")
@@ -435,35 +402,26 @@ func TestMockBackend(t *testing.T) {
 	})
 
 	t.Run("ps", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("ps")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), "ps-out-example.golden")
 	})
 
 	t.Run("ps quiet", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("ps", "-q")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), "ps-quiet-out-example.golden")
 	})
 
 	t.Run("ps quiet all", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("ps", "-q", "--all")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), "ps-quiet-all-out-example.golden")
 	})
 
 	t.Run("inspect", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("inspect", "id")
-		res.Assert(t, icmd.Success)
 		golden.Assert(t, res.Stdout(), "inspect-id.golden")
 	})
 
 	t.Run("run", func(t *testing.T) {
-		t.Parallel()
 		res := c.RunDockerCmd("run", "-d", "nginx", "-p", "80:80")
 		res.Assert(t, icmd.Expected{
 			Out: `Running container "nginx" with name`,

tests/ecs-e2e/e2e-ecs_test.go

@@ -49,35 +49,30 @@ func TestMain(m *testing.M) {
 }
 
 func TestSecrets(t *testing.T) {
-	c, testID := setupTest(t)
+	cmd, testID := setupTest(t)
 	secretName := "secret" + testID
 	description := "description " + testID
 
 	t.Run("create secret", func(t *testing.T) {
-		res := c.RunDockerCmd("secret", "create", secretName, "-u", "user1", "-p", "pass1", "-d", description)
-		res.Assert(t, icmd.Success)
+		res := cmd.RunDockerCmd("secret", "create", secretName, "-u", "user1", "-p", "pass1", "-d", description)
 		assert.Check(t, strings.Contains(res.Stdout(), "secret:"+secretName))
 	})
 
 	t.Run("list secrets", func(t *testing.T) {
-		res := c.RunDockerCmd("secret", "list")
-		res.Assert(t, icmd.Success)
+		res := cmd.RunDockerCmd("secret", "list")
 		assert.Check(t, strings.Contains(res.Stdout(), secretName))
 		assert.Check(t, strings.Contains(res.Stdout(), description))
 	})
 
 	t.Run("inspect secret", func(t *testing.T) {
-		res := c.RunDockerCmd("secret", "inspect", secretName)
-		res.Assert(t, icmd.Success)
+		res := cmd.RunDockerCmd("secret", "inspect", secretName)
 		assert.Check(t, strings.Contains(res.Stdout(), `"Name": "`+secretName+`"`))
 		assert.Check(t, strings.Contains(res.Stdout(), `"Description": "`+description+`"`))
 	})
 
 	t.Run("rm secret", func(t *testing.T) {
-		res := c.RunDockerCmd("secret", "rm", secretName)
-		res.Assert(t, icmd.Success)
-		res = c.RunDockerCmd("secret", "list")
-		res.Assert(t, icmd.Success)
+		cmd.RunDockerCmd("secret", "rm", secretName)
+		res := cmd.RunDockerCmd("secret", "list")
 		assert.Check(t, !strings.Contains(res.Stdout(), secretName))
 	})
 }
@@ -86,14 +81,12 @@ func TestCompose(t *testing.T) {
 	c, stack := setupTest(t)
 
 	t.Run("compose up", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
-		res.Assert(t, icmd.Success)
+		c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
 	})
 
 	var url string
 	t.Run("compose ps", func(t *testing.T) {
 		res := c.RunDockerCmd("compose", "ps", "--project-name", stack)
-		res.Assert(t, icmd.Success)
 		lines := strings.Split(res.Stdout(), "\n")
 
 		assert.Equal(t, 3, len(lines))
@@ -124,8 +117,7 @@ func TestCompose(t *testing.T) {
 	})
 
 	t.Run("compose down", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "down", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
-		res.Assert(t, icmd.Success)
+		c.RunDockerCmd("compose", "down", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
 	})
 }
 
@@ -140,8 +132,7 @@ func setupTest(t *testing.T) (*E2eCLI, string) {
 		if localTestProfile != "" {
 			region := os.Getenv("TEST_AWS_REGION")
 			assert.Check(t, region != "")
-			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile, "--region", region)
-			res.Assert(t, icmd.Success)
+			c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile, "--region", region)
 		} else {
 			profile := contextName
 			region := os.Getenv("AWS_DEFAULT_REGION")
@@ -150,8 +141,7 @@ func setupTest(t *testing.T) (*E2eCLI, string) {
 			assert.Check(t, keyID != "")
 			assert.Check(t, secretKey != "")
 			assert.Check(t, region != "")
-			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region, "--secret-key", secretKey, "--key-id", keyID)
-			res.Assert(t, icmd.Success)
+			c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region, "--secret-key", secretKey, "--key-id", keyID)
 		}
 		res = c.RunDockerCmd("context", "use", contextName)
 		res.Assert(t, icmd.Expected{Out: contextName})

tests/framework/e2e.go

@@ -20,11 +20,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
@@ -51,6 +53,7 @@ func init() {
 type E2eCLI struct {
 	BinDir    string
 	ConfigDir string
+	test      *testing.T
 }
 
 // NewParallelE2eCLI returns a configured TestE2eCLI with t.Parallel() set
@@ -80,7 +83,7 @@ func newE2eCLI(t *testing.T, binDir string) *E2eCLI {
 		_ = os.RemoveAll(d)
 	})
 
-	return &E2eCLI{binDir, d}
+	return &E2eCLI{binDir, d, t}
 }
 
 func dirContents(dir string) []string {
@@ -158,11 +161,19 @@ func (c *E2eCLI) NewDockerCmd(args ...string) icmd.Cmd {
 	return c.NewCmd(filepath.Join(c.BinDir, DockerExecutableName), args...)
 }
 
-// RunDockerCmd runs a docker command and returns a result
-func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
+// RunDockerOrExitError runs a docker command and returns a result
+func (c *E2eCLI) RunDockerOrExitError(args ...string) *icmd.Result {
+	fmt.Printf("	[%s] docker %s\n", c.test.Name(), strings.Join(args, " "))
 	return icmd.RunCmd(c.NewDockerCmd(args...))
 }
 
+// RunDockerCmd runs a docker command, expects no error and returns a result
+func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
+	res := c.RunDockerOrExitError(args...)
+	res.Assert(c.test, icmd.Success)
+	return res
+}
+
 // GoldenFile golden file specific to platform
 func GoldenFile(name string) string {
 	if runtime.GOOS == "windows" {