docker-compose/tests/aci-e2e/e2e-aci_test.go

/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/icmd"
	"gotest.tools/v3/poll"

	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
	azure_storage "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/storage/mgmt/storage"
	"github.com/Azure/azure-storage-file-go/azfile"
	"github.com/Azure/go-autorest/autorest/to"

	"github.com/docker/api/aci"
	"github.com/docker/api/aci/login"
	"github.com/docker/api/containers"
	"github.com/docker/api/context/store"
	"github.com/docker/api/errdefs"
	"github.com/docker/api/tests/aci-e2e/storage"
	. "github.com/docker/api/tests/framework"
)

const (
	contextName = "aci-test"
	location    = "eastus2"
)

var binDir string

func TestMain(m *testing.M) {
	p, cleanup, err := SetupExistingCLI()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	binDir = p
	exitCode := m.Run()
	cleanup()
	os.Exit(exitCode)
}

// Cannot be parallelized as login/logout is global.
func TestLoginLogout(t *testing.T) {
	startTime := strconv.Itoa(int(time.Now().UnixNano()))
	c := NewE2eCLI(t, binDir)
	rg := "E2E-" + startTime

	t.Run("login", func(t *testing.T) {
		azureLogin(t, c)
	})

	t.Run("create context", func(t *testing.T) {
		sID := getSubscriptionID(t)
		err := createResourceGroup(sID, rg)
		assert.Check(t, is.Nil(err))
		t.Cleanup(func() {
			_ = deleteResourceGroup(rg)
		})

		c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rg, "--location", location)
		res := c.RunDockerCmd("context", "use", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
		res = c.RunDockerCmd("context", "ls")
		res.Assert(t, icmd.Expected{Out: contextName + " *"})
	})

	t.Run("delete context", func(t *testing.T) {
		res := c.RunDockerCmd("context", "use", "default")
		res.Assert(t, icmd.Expected{Out: "default"})

		res = c.RunDockerCmd("context", "rm", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
	})

	t.Run("logout", func(t *testing.T) {
		_, err := os.Stat(login.GetTokenStorePath())
		assert.NilError(t, err)
		res := c.RunDockerCmd("logout", "azure")
		res.Assert(t, icmd.Expected{Out: "Removing login credentials for Azure"})
		_, err = os.Stat(login.GetTokenStorePath())
		errMsg := "no such file or directory"
		if runtime.GOOS == "windows" {
			errMsg = "The system cannot find the file specified"
		}
		assert.ErrorContains(t, err, errMsg)
	})

	t.Run("create context fail", func(t *testing.T) {
		res := c.RunDockerOrExitError("context", "create", "aci", "fail-context")
		res.Assert(t, icmd.Expected{
			ExitCode: errdefs.ExitCodeLoginRequired,
			Err:      `not logged in to azure, you need to run "docker login azure" first`,
		})
	})
}

func TestContainerRun(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)
	sID, rg := setupTestResourceGroup(t, c)

	const (
		testShareName   = "dockertestshare"
		testFileContent = "Volume mounted successfully!"
		testFileName    = "index.html"
	)

	// Bootstrap volume
	aciContext := store.AciContext{
		SubscriptionID: sID,
		Location:       location,
		ResourceGroup:  rg,
	}
	saName := "e2e" + strconv.Itoa(int(time.Now().UnixNano()))
	_, cleanupSa := createStorageAccount(t, aciContext, saName)
	t.Cleanup(func() {
		if err := cleanupSa(); err != nil {
			t.Error(err)
		}
	})
	keys := getStorageKeys(t, aciContext, saName)
	assert.Assert(t, len(keys) > 0)
	k := *keys[0].Value
	cred, u := createFileShare(t, k, testShareName, saName)
	uploadFile(t, *cred, u.String(), testFileName, testFileContent)

	// Used in subtests
	var (
		container string
		hostIP    string
		endpoint  string
	)

	t.Run("run", func(t *testing.T) {
		mountTarget := "/usr/share/nginx/html"
		res := c.RunDockerCmd(
			"run", "-d",
			"-v", fmt.Sprintf("%s@%s:%s", saName, testShareName, mountTarget),
			"-p", "80:80",
			"nginx",
		)
		container = getContainerName(res.Stdout())
	})

	t.Run("inspect", func(t *testing.T) {
		res := c.RunDockerCmd("inspect", container)

		containerInspect, err := ParseContainerInspect(res.Stdout())
		assert.NilError(t, err)
		assert.Equal(t, containerInspect.Platform, "Linux")
		assert.Equal(t, containerInspect.CPULimit, 1.0)
		assert.Equal(t, containerInspect.RestartPolicyCondition, containers.RestartPolicyNone)

		assert.Assert(t, is.Len(containerInspect.Ports, 1))
		hostIP = containerInspect.Ports[0].HostIP
		endpoint = fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
	})

	t.Run("ps", func(t *testing.T) {
		res := c.RunDockerCmd("ps")
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		l := out[len(out)-1]
		assert.Assert(t, strings.Contains(l, container), "Looking for %q in line: %s", container, l)
		assert.Assert(t, strings.Contains(l, "nginx"))
		assert.Assert(t, strings.Contains(l, "Running"))
		assert.Assert(t, strings.Contains(l, hostIP+":80->80/tcp"))
	})

	t.Run("http get", func(t *testing.T) {
		r, err := http.Get(endpoint)
		assert.NilError(t, err)
		assert.Equal(t, r.StatusCode, http.StatusOK)
		b, err := ioutil.ReadAll(r.Body)
		assert.NilError(t, err)
		assert.Assert(t, strings.Contains(string(b), testFileContent), "Actual content: "+string(b))
	})

	t.Run("logs", func(t *testing.T) {
		res := c.RunDockerCmd("logs", container)
		res.Assert(t, icmd.Expected{Out: "GET"})
	})

	t.Run("exec", func(t *testing.T) {
		res := c.RunDockerCmd("exec", container, "pwd")
		res.Assert(t, icmd.Expected{Out: "/"})

		res = c.RunDockerOrExitError("exec", container, "echo", "fail_with_argument")
		res.Assert(t, icmd.Expected{
			ExitCode: 1,
			Err:      "ACI exec command does not accept arguments to the command. Only the binary should be specified",
		})
	})

	t.Run("logs follow", func(t *testing.T) {
		cmd := c.NewDockerCmd("logs", "--follow", container)
		res := icmd.StartCmd(cmd)

		checkUp := func(t poll.LogT) poll.Result {
			r, _ := http.Get(endpoint + "/is_up")
			if r != nil && r.StatusCode == http.StatusNotFound {
				return poll.Success()
			}
			return poll.Continue("waiting for container to serve request")
		}
		poll.WaitOn(t, checkUp, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))

		assert.Assert(t, !strings.Contains(res.Stdout(), "/test"))

		checkLogs := func(t poll.LogT) poll.Result {
			if strings.Contains(res.Stdout(), "/test") {
				return poll.Success()
			}
			return poll.Continue("waiting for logs to contain /test")
		}

		// Do request on /test
		go func() {
			time.Sleep(3 * time.Second)
			_, _ = http.Get(endpoint + "/test")
		}()

		poll.WaitOn(t, checkLogs, poll.WithDelay(3*time.Second), poll.WithTimeout(20*time.Second))

		if runtime.GOOS == "windows" {
			err := res.Cmd.Process.Kill()
			assert.NilError(t, err)
		} else {
			err := res.Cmd.Process.Signal(syscall.SIGTERM)
			assert.NilError(t, err)
		}
	})

	t.Run("rm a running container", func(t *testing.T) {
		res := c.RunDockerOrExitError("rm", container)
		res.Assert(t, icmd.Expected{
			Err:      fmt.Sprintf("Error: you cannot remove a running container %s. Stop the container before attempting removal or force remove", container),
			ExitCode: 1,
		})
	})

	t.Run("force rm", func(t *testing.T) {
		res := c.RunDockerCmd("rm", "-f", container)
		res.Assert(t, icmd.Expected{Out: container})

		checkStopped := func(t poll.LogT) poll.Result {
			res := c.RunDockerOrExitError("inspect", container)
			if res.ExitCode == 1 {
				return poll.Success()
			}
			return poll.Continue("waiting for container to stop")
		}
		poll.WaitOn(t, checkStopped, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
	})
}

func TestContainerRunAttached(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)
	_, _ = setupTestResourceGroup(t, c)

	// Used in subtests
	var (
		container string
		endpoint  string
	)

	container = "test-container"

	t.Run("run attached limits", func(t *testing.T) {
		cmd := c.NewDockerCmd(
			"run",
			"--name", container,
			"--restart", "on-failure",
			"--memory", "0.1G", "--cpus", "0.1",
			"-p", "80:80",
			"nginx",
		)
		runRes := icmd.StartCmd(cmd)

		checkRunning := func(t poll.LogT) poll.Result {
			res := c.RunDockerOrExitError("inspect", container)
			if res.ExitCode == 0 {
				return poll.Success()
			}
			return poll.Continue("waiting for container to be running")
		}
		poll.WaitOn(t, checkRunning, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))

		inspectRes := c.RunDockerCmd("inspect", container)

		containerInspect, err := ParseContainerInspect(inspectRes.Stdout())
		assert.NilError(t, err)
		assert.Equal(t, containerInspect.Platform, "Linux")
		assert.Equal(t, containerInspect.CPULimit, 0.1)
		assert.Equal(t, containerInspect.MemoryLimit, uint64(107374182))
		assert.Equal(t, containerInspect.RestartPolicyCondition, containers.RestartPolicyOnFailure)

		assert.Assert(t, is.Len(containerInspect.Ports, 1))
		port := containerInspect.Ports[0]
		assert.Assert(t, len(port.HostIP) > 0)
		assert.Equal(t, port.ContainerPort, uint32(80))
		assert.Equal(t, port.HostPort, uint32(80))
		endpoint = fmt.Sprintf("http://%s:%d", port.HostIP, port.HostPort)

		assert.Assert(t, !strings.Contains(runRes.Stdout(), "/test"))
		checkRequest := func(t poll.LogT) poll.Result {
			r, _ := http.Get(endpoint + "/test")
			if r != nil && r.StatusCode == http.StatusNotFound {
				return poll.Success()
			}
			return poll.Continue("waiting for container to serve request")
		}
		poll.WaitOn(t, checkRequest, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))

		checkLog := func(t poll.LogT) poll.Result {
			if strings.Contains(runRes.Stdout(), "/test") {
				return poll.Success()
			}
			return poll.Continue("waiting for logs to contain /test")
		}
		poll.WaitOn(t, checkLog, poll.WithDelay(1*time.Second), poll.WithTimeout(20*time.Second))
	})

	t.Run("stop wrong container", func(t *testing.T) {
		res := c.RunDockerOrExitError("stop", "unknown-container")
		res.Assert(t, icmd.Expected{
			Err:      "Error: container unknown-container not found",
			ExitCode: 1,
		})
	})

	t.Run("stop container", func(t *testing.T) {
		res := c.RunDockerCmd("stop", container)
		res.Assert(t, icmd.Expected{Out: container})
	})

	t.Run("ps stopped container with --all", func(t *testing.T) {
		res := c.RunDockerCmd("ps", container)
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		assert.Assert(t, is.Len(out, 1))

		res = c.RunDockerCmd("ps", "--all", container)
		out = strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		assert.Assert(t, is.Len(out, 2))
	})

	t.Run("start container", func(t *testing.T) {
		res := c.RunDockerCmd("start", container)
		res.Assert(t, icmd.Expected{Out: container})
		waitForStatus(t, c, container, "Running")
	})

	t.Run("rm stopped container", func(t *testing.T) {
		res := c.RunDockerCmd("stop", container)
		res.Assert(t, icmd.Expected{Out: container})
		waitForStatus(t, c, container, "Terminated", "Node Stopped")

		res = c.RunDockerCmd("rm", container)
		res.Assert(t, icmd.Expected{Out: container})
	})
}

func TestCompose(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)
	_, _ = setupTestResourceGroup(t, c)

	const (
		composeFile              = "../composefiles/aci-demo/aci_demo_port.yaml"
		composeFileMultiplePorts = "../composefiles/aci-demo/aci_demo_multi_port.yaml"
		composeProjectName       = "acidemo"
		serverContainer          = composeProjectName + "_web"
		wordsContainer           = composeProjectName + "_words"
	)

	t.Run("compose up", func(t *testing.T) {
		// Name of Compose project is taken from current folder "acie2e"
		c.RunDockerCmd("compose", "up", "-f", composeFile)
		res := c.RunDockerCmd("ps")
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		// Check three containers are running
		assert.Assert(t, is.Len(out, 4))
		webRunning := false
		for _, l := range out {
			if strings.Contains(l, serverContainer) {
				webRunning = true
				strings.Contains(l, ":80->80/tcp")
			}
		}
		assert.Assert(t, webRunning, "web container not running")

		res = c.RunDockerCmd("inspect", serverContainer)

		containerInspect, err := ParseContainerInspect(res.Stdout())
		assert.NilError(t, err)
		assert.Assert(t, is.Len(containerInspect.Ports, 1))
		endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)

		r, err := http.Get(endpoint + "/words/noun")
		assert.NilError(t, err)
		assert.Equal(t, r.StatusCode, http.StatusOK)
		b, err := ioutil.ReadAll(r.Body)
		assert.NilError(t, err)
		assert.Assert(t, strings.Contains(string(b), `"word":`))
	})

	t.Run("logs web", func(t *testing.T) {
		res := c.RunDockerCmd("logs", serverContainer)
		res.Assert(t, icmd.Expected{Out: "Listening on port 80"})
	})

	t.Run("update", func(t *testing.T) {
		c.RunDockerCmd("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName)
		res := c.RunDockerCmd("ps")
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		// Check three containers are running
		assert.Assert(t, is.Len(out, 4))

		for _, cName := range []string{serverContainer, wordsContainer} {
			res = c.RunDockerCmd("inspect", cName)

			containerInspect, err := ParseContainerInspect(res.Stdout())
			assert.NilError(t, err)
			assert.Assert(t, is.Len(containerInspect.Ports, 1))
			endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
			var route string
			switch cName {
			case serverContainer:
				route = "/words/noun"
				assert.Equal(t, containerInspect.Ports[0].HostPort, uint32(80))
				assert.Equal(t, containerInspect.Ports[0].ContainerPort, uint32(80))
			case wordsContainer:
				route = "/noun"
				assert.Equal(t, containerInspect.Ports[0].HostPort, uint32(8080))
				assert.Equal(t, containerInspect.Ports[0].ContainerPort, uint32(8080))
			}
			checkUp := func(t poll.LogT) poll.Result {
				r, _ := http.Get(endpoint + route)
				if r != nil && r.StatusCode == http.StatusOK {
					return poll.Success()
				}
				return poll.Continue("Waiting for container to serve request")
			}
			poll.WaitOn(t, checkUp, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))

			res = c.RunDockerCmd("ps")
			p := containerInspect.Ports[0]
			res.Assert(t, icmd.Expected{
				Out: fmt.Sprintf("%s:%d->%d/tcp", p.HostIP, p.HostPort, p.ContainerPort),
			})
		}
	})

	t.Run("down", func(t *testing.T) {
		c.RunDockerCmd("compose", "down", "--project-name", composeProjectName)
		res := c.RunDockerCmd("ps")
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		assert.Equal(t, len(out), 1)
	})
}

func TestRunEnvVars(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)
	_, _ = setupTestResourceGroup(t, c)

	t.Run("run", func(t *testing.T) {
		cmd := c.NewDockerCmd(
			"run", "-d",
			"-e", "MYSQL_ROOT_PASSWORD=rootpwd",
			"-e", "MYSQL_DATABASE=mytestdb",
			"-e", "MYSQL_USER",
			"-e", "MYSQL_PASSWORD=userpwd",
			"mysql:5.7",
		)
		cmd.Env = append(cmd.Env, "MYSQL_USER=user1")
		res := icmd.RunCmd(cmd)
		res.Assert(t, icmd.Success)
		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
		container := strings.TrimSpace(out[len(out)-1])

		res = c.RunDockerCmd("inspect", container)

		containerInspect, err := ParseContainerInspect(res.Stdout())
		assert.NilError(t, err)
		assert.Equal(t, containerInspect.Image, "mysql:5.7")

		check := func(t poll.LogT) poll.Result {
			res := c.RunDockerOrExitError("logs", container)
			if strings.Contains(res.Stdout(), "Giving user user1 access to schema mytestdb") {
				return poll.Success()
			}
			return poll.Continue("waiting for DB container to be up")
		}
		poll.WaitOn(t, check, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
	})
}

func setupTestResourceGroup(t *testing.T, c *E2eCLI) (string, string) {
	startTime := strconv.Itoa(int(time.Now().Unix()))
	rg := "E2E-" + t.Name() + "-" + startTime
	azureLogin(t, c)
	sID := getSubscriptionID(t)
	err := createResourceGroup(sID, rg)
	assert.Check(t, is.Nil(err))
	t.Cleanup(func() {
		if err := deleteResourceGroup(rg); err != nil {
			t.Error(err)
		}
	})
	createAciContextAndUseIt(t, c, sID, rg)
	// Check nothing is running
	res := c.RunDockerCmd("ps")
	assert.Assert(t, is.Len(strings.Split(strings.TrimSpace(res.Stdout()), "\n"), 1))
	return sID, rg
}

func deleteResourceGroup(rgName string) error {
	ctx := context.TODO()
	helper := aci.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	if err != nil {
		return err
	}
	if len(models) == 0 {
		return errors.New("unable to delete resource group: no models")
	}
	return helper.DeleteAsync(ctx, *models[0].SubscriptionID, rgName)
}

func azureLogin(t *testing.T, c *E2eCLI) {
	// in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth`
	clientID := os.Getenv("AZURE_CLIENT_ID")
	clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
	tenantID := os.Getenv("AZURE_TENANT_ID")
	assert.Check(t, clientID != "", "AZURE_CLIENT_ID must not be empty")
	assert.Check(t, clientSecret != "", "AZURE_CLIENT_SECRET must not be empty")
	assert.Check(t, tenantID != "", "AZURE_TENANT_ID must not be empty")
	c.RunDockerCmd("login", "azure", "--client-id", clientID, "--client-secret", clientSecret, "--tenant-id", tenantID)
}

func getSubscriptionID(t *testing.T) string {
	ctx := context.TODO()
	helper := aci.NewACIResourceGroupHelper()
	models, err := helper.GetSubscriptionIDs(ctx)
	assert.Check(t, is.Nil(err))
	assert.Check(t, len(models) == 1)
	return *models[0].SubscriptionID
}

func createResourceGroup(sID, rgName string) error {
	helper := aci.NewACIResourceGroupHelper()
	_, err := helper.CreateOrUpdate(context.TODO(), sID, rgName, resources.Group{Location: to.StringPtr(location)})
	return err
}

func createAciContextAndUseIt(t *testing.T, c *E2eCLI, sID, rgName string) {
	c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rgName, "--location", location)
	res := c.RunDockerCmd("context", "use", contextName)
	res.Assert(t, icmd.Expected{Out: contextName})
	res = c.RunDockerCmd("context", "ls")
	res.Assert(t, icmd.Expected{Out: contextName + " *"})
}

func createStorageAccount(t *testing.T, aciContext store.AciContext, name string) (azure_storage.Account, func() error) {
	account, err := storage.CreateStorageAccount(context.TODO(), aciContext, name)
	assert.Check(t, is.Nil(err))
	assert.Check(t, is.Equal(*(account.Name), name))
	return account, func() error { return deleteStorageAccount(aciContext, name) }
}

func deleteStorageAccount(aciContext store.AciContext, name string) error {
	_, err := storage.DeleteStorageAccount(context.TODO(), aciContext, name)
	return err
}

func getStorageKeys(t *testing.T, aciContext store.AciContext, saName string) []azure_storage.AccountKey {
	l, err := storage.ListKeys(context.TODO(), aciContext, saName)
	assert.NilError(t, err)
	assert.Assert(t, l.Keys != nil)
	return *l.Keys
}

func createFileShare(t *testing.T, key, share, storageAccount string) (*azfile.SharedKeyCredential, *url.URL) {
	// Create a ShareURL object that wraps a soon-to-be-created share's URL and a default pipeline.
	u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", storageAccount, share))
	cred, err := azfile.NewSharedKeyCredential(storageAccount, key)
	assert.NilError(t, err)

	shareURL := azfile.NewShareURL(*u, azfile.NewPipeline(cred, azfile.PipelineOptions{}))
	_, err = shareURL.Create(context.TODO(), azfile.Metadata{}, 0)
	assert.NilError(t, err)
	return cred, u
}

func uploadFile(t *testing.T, cred azfile.SharedKeyCredential, baseURL, fileName, content string) {
	fURL, err := url.Parse(baseURL + "/" + fileName)
	assert.NilError(t, err)
	fileURL := azfile.NewFileURL(*fURL, azfile.NewPipeline(&cred, azfile.PipelineOptions{}))
	err = azfile.UploadBufferToAzureFile(context.TODO(), []byte(content), fileURL, azfile.UploadToAzureFileOptions{})
	assert.NilError(t, err)
}

func getContainerName(stdout string) string {
	out := strings.Split(strings.TrimSpace(stdout), "\n")
	return strings.TrimSpace(out[len(out)-1])
}

func waitForStatus(t *testing.T, c *E2eCLI, containerID string, statuses ...string) {
	checkStopped := func(logt poll.LogT) poll.Result {
		res := c.RunDockerCmd("inspect", containerID)
		containerInspect, err := ParseContainerInspect(res.Stdout())
		assert.NilError(t, err)
		for _, status := range statuses {
			if containerInspect.Status == status {
				return poll.Success()
			}
		}
		return poll.Continue("Status %s != %s (expected) for container %s", containerInspect.Status, statuses, containerID)
	}

	poll.WaitOn(t, checkStopped, poll.WithDelay(5*time.Second), poll.WithTimeout(90*time.Second))
}