Move compose e2e tests into pkg

Signed-off-by: Nicolas De Loof <[email protected]>

.github/workflows/ci.yml

@@ -102,4 +102,4 @@ jobs:
         run: make -f builder.Makefile cli compose-plugin
 
       - name: E2E Test
-        run: make e2e-local
+        run: make e2e-compose

Makefile

@@ -51,8 +51,11 @@ compose-plugin: ## Compile the compose cli-plugin
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--output ./bin
 
+e2e-compose: ## Run End to end local tests. Set E2E_TEST=TestName to run a single test
+	gotestsum $(TEST_FLAGS) ./pkg/e2e -- -count=1
+
 e2e-local: ## Run End to end local tests. Set E2E_TEST=TestName to run a single test
-	gotestsum $(TEST_FLAGS) ./local/e2e/compose ./local/e2e/container ./local/e2e/cli-only -- -count=1
+	gotestsum $(TEST_FLAGS) ./local/e2e/container ./local/e2e/cli-only -- -count=1
 
 e2e-win-ci: ## Run end to end local tests on Windows CI, no Docker for Linux containers available ATM. Set E2E_TEST=TestName to run a single test
 	go test -count=1 -v $(TEST_FLAGS) ./local/e2e/cli-only

local/e2e/compose/metrics_test.go

@@ -1,91 +0,0 @@
-/*
-   Copyright 2020 Docker Compose CLI authors
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package e2e
-
-import (
-	"fmt"
-	"runtime"
-	"testing"
-	"time"
-
-	"gotest.tools/v3/assert"
-	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
-)
-
-func TestComposeMetrics(t *testing.T) {
-	c := NewParallelE2eCLI(t, binDir)
-	s := NewMetricsServer(c.MetricsSocket())
-	s.Start()
-	defer s.Stop()
-
-	started := false
-	for i := 0; i < 30; i++ {
-		c.RunDockerCmd("help", "ps")
-		if len(s.GetUsage()) > 0 {
-			started = true
-			fmt.Printf("	[%s] Server up in %d ms\n", t.Name(), i*100)
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	assert.Assert(t, started, "Metrics mock server not available after 3 secs")
-
-	t.Run("catch specific failure metrics", func(t *testing.T) {
-		s.ResetUsage()
-
-		res := c.RunDockerOrExitError("compose", "-f", "../compose/fixtures/does-not-exist/compose.yaml", "build")
-		expectedErr := "compose/fixtures/does-not-exist/compose.yaml: no such file or directory"
-		if runtime.GOOS == "windows" {
-			expectedErr = "does-not-exist\\compose.yaml: The system cannot find the path specified"
-		}
-		res.Assert(t, icmd.Expected{ExitCode: 14, Err: expectedErr})
-		res = c.RunDockerOrExitError("compose", "-f", "../compose/fixtures/wrong-composefile/compose.yaml", "up", "-d")
-		res.Assert(t, icmd.Expected{ExitCode: 15, Err: "services.simple Additional property wrongField is not allowed"})
-		res = c.RunDockerOrExitError("compose", "up")
-		res.Assert(t, icmd.Expected{ExitCode: 14, Err: "can't find a suitable configuration file in this directory or any parent: not found"})
-		res = c.RunDockerOrExitError("compose", "up", "-f", "../compose/fixtures/wrong-composefile/compose.yaml")
-		res.Assert(t, icmd.Expected{ExitCode: 16, Err: "unknown shorthand flag: 'f' in -f"})
-		res = c.RunDockerOrExitError("compose", "up", "--file", "../compose/fixtures/wrong-composefile/compose.yaml")
-		res.Assert(t, icmd.Expected{ExitCode: 16, Err: "unknown flag: --file"})
-		res = c.RunDockerOrExitError("compose", "donw", "--file", "../compose/fixtures/wrong-composefile/compose.yaml")
-		res.Assert(t, icmd.Expected{ExitCode: 16, Err: `unknown docker command: "compose donw"`})
-		res = c.RunDockerOrExitError("compose", "--file", "../compose/fixtures/wrong-composefile/build-error.yml", "build")
-		res.Assert(t, icmd.Expected{ExitCode: 17, Err: `line 17: unknown instruction: WRONG`})
-		res = c.RunDockerOrExitError("compose", "--file", "../compose/fixtures/wrong-composefile/build-error.yml", "up")
-		res.Assert(t, icmd.Expected{ExitCode: 17, Err: `line 17: unknown instruction: WRONG`})
-		res = c.RunDockerOrExitError("compose", "--file", "../compose/fixtures/wrong-composefile/unknown-image.yml", "pull")
-		res.Assert(t, icmd.Expected{ExitCode: 18, Err: `pull access denied for unknownimage, repository does not exist or may require 'docker login'`})
-		res = c.RunDockerOrExitError("compose", "--file", "../compose/fixtures/wrong-composefile/unknown-image.yml", "up")
-		res.Assert(t, icmd.Expected{ExitCode: 18, Err: `pull access denied for unknownimage, repository does not exist or may require 'docker login'`})
-
-		usage := s.GetUsage()
-		assert.DeepEqual(t, []string{
-			`{"command":"compose build","context":"moby","source":"cli","status":"failure-file-not-found"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-compose-parse"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-file-not-found"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-cmd-syntax"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-cmd-syntax"}`,
-			`{"command":"compose","context":"moby","source":"cli","status":"failure-cmd-syntax"}`,
-			`{"command":"compose build","context":"moby","source":"cli","status":"failure-build"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-build"}`,
-			`{"command":"compose pull","context":"moby","source":"cli","status":"failure-pull"}`,
-			`{"command":"compose up","context":"moby","source":"cli","status":"failure-pull"}`,
-		}, usage)
-	})
-}

local/e2e/compose/scan_message_test.go

@@ -1,147 +0,0 @@
-/*
-   Copyright 2020 Docker Compose CLI authors
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package e2e
-
-import (
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"testing"
-
-	"gotest.tools/v3/assert"
-	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
-)
-
-func TestDisplayScanMessageAfterBuild(t *testing.T) {
-
-	c := NewParallelE2eCLI(t, binDir)
-	setupScanPlugin(t, c)
-
-	res := c.RunDockerCmd("info")
-	res.Assert(t, icmd.Expected{Out: "scan: Docker Scan"})
-
-	t.Run("display when docker build", func(t *testing.T) {
-		res := c.RunDockerCmd("build", "-t", "test-image-scan-msg", "./fixtures/simple-build-test/nginx-build")
-		defer c.RunDockerOrExitError("rmi", "-f", "test-image-scan-msg")
-		res.Assert(t, icmd.Expected{Err: "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them"})
-	})
-
-	t.Run("do not display with docker build and quiet flag", func(t *testing.T) {
-		res := c.RunDockerCmd("build", "-t", "test-image-scan-msg-quiet", "--quiet", "./fixtures/simple-build-test/nginx-build")
-		defer c.RunDockerOrExitError("rmi", "-f", "test-image-scan-msg-quiet")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"))
-
-		res = c.RunDockerCmd("build", "-t", "test-image-scan-msg-q", "-q", "./fixtures/simple-build-test/nginx-build")
-		defer c.RunDockerOrExitError("rmi", "-f", "test-image-scan-msg-q")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"))
-	})
-
-	t.Run("do not display if envvar DOCKER_SCAN_SUGGEST=false", func(t *testing.T) {
-		cmd := c.NewDockerCmd("build", "-t", "test-image-scan-msg", "./fixtures/build-test/nginx-build")
-		defer c.RunDockerOrExitError("rmi", "-f", "test-image-scan-msg")
-		cmd.Env = append(cmd.Env, "DOCKER_SCAN_SUGGEST=false")
-		res := icmd.StartCmd(cmd)
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
-	})
-
-	t.Run("display on compose build", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-compose-build", "build")
-		defer c.RunDockerOrExitError("rmi", "-f", "scan-msg-test-compose-build_nginx")
-		res.Assert(t, icmd.Expected{Err: "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them"})
-	})
-
-	t.Run("do not display on compose build with quiet flag", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-quiet", "build", "--quiet")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
-		res = c.RunDockerCmd("rmi", "-f", "scan-msg-test-quiet_nginx")
-		assert.Assert(t, !strings.Contains(res.Combined(), "No such image"))
-
-		res = c.RunDockerCmd("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-q", "build", "-q")
-		defer c.RunDockerOrExitError("rmi", "-f", "scan-msg-test-q_nginx")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
-	})
-
-	_ = c.RunDockerOrExitError("rmi", "scan-msg-test_nginx")
-
-	t.Run("display on compose up if image is built", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "up", "-d")
-		defer c.RunDockerOrExitError("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "down")
-		res.Assert(t, icmd.Expected{Err: "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them"})
-	})
-
-	t.Run("do not display on compose up if no image built", func(t *testing.T) { // re-run the same Compose aproject
-		res := c.RunDockerCmd("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "up", "-d")
-		defer c.RunDockerOrExitError("compose", "-f", "./fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "down", "--rmi", "all")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
-	})
-
-	t.Run("do not display if scan already invoked", func(t *testing.T) {
-		_ = os.MkdirAll(filepath.Join(c.ConfigDir, "scan"), 0755)
-		scanConfigFile := filepath.Join(c.ConfigDir, "scan", "config.json")
-		err := ioutil.WriteFile(scanConfigFile, []byte(`{"optin":true}`), 0644)
-		assert.NilError(t, err)
-
-		res := c.RunDockerCmd("build", "-t", "test-image-scan-msg", "./fixtures/simple-build-test/nginx-build")
-		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
-	})
-}
-
-func setupScanPlugin(t *testing.T, c *E2eCLI) {
-	_ = os.MkdirAll(filepath.Join(c.ConfigDir, "cli-plugins"), 0755)
-
-	scanPluginFile := "docker-scan"
-	scanPluginURL := "https://github.com/docker/scan-cli-plugin/releases/download/v0.7.0/docker-scan_linux_amd64"
-	if runtime.GOOS == "windows" {
-		scanPluginFile += ".exe"
-		scanPluginURL = "https://github.com/docker/scan-cli-plugin/releases/download/v0.7.0/docker-scan_windows_amd64.exe"
-	}
-	if runtime.GOOS == "darwin" {
-		scanPluginURL = "https://github.com/docker/scan-cli-plugin/releases/download/v0.7.0/docker-scan_darwin_amd64"
-	}
-
-	localScanBinary := filepath.Join("..", "..", "..", "bin", scanPluginFile)
-	if _, err := os.Stat(localScanBinary); os.IsNotExist(err) {
-		out, err := os.Create(localScanBinary)
-		assert.NilError(t, err)
-		defer out.Close() //nolint:errcheck
-		resp, err := http.Get(scanPluginURL)
-		assert.NilError(t, err)
-		defer resp.Body.Close() //nolint:errcheck
-		_, err = io.Copy(out, resp.Body)
-		assert.NilError(t, err)
-	}
-
-	finalScanBinaryFile := filepath.Join(c.ConfigDir, "cli-plugins", scanPluginFile)
-
-	out, err := os.Create(finalScanBinaryFile)
-	assert.NilError(t, err)
-	defer out.Close() //nolint:errcheck
-	in, err := os.Open(localScanBinary)
-	assert.NilError(t, err)
-	defer in.Close() //nolint:errcheck
-	_, err = io.Copy(out, in)
-	assert.NilError(t, err)
-
-	err = os.Chmod(finalScanBinaryFile, 7777)
-	assert.NilError(t, err)
-}

local/e2e/compose/cancel_test.go → pkg/e2e/cancel_test.go

@@ -29,34 +29,14 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestComposeCancel(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
-	s := NewMetricsServer(c.MetricsSocket())
-	s.Start()
-	defer s.Stop()
-
-	started := false
-
-	for i := 0; i < 30; i++ {
-		c.RunDockerCmd("help", "ps")
-		if len(s.GetUsage()) > 0 {
-			started = true
-			fmt.Printf("    [%s] Server up in %d ms\n", t.Name(), i*100)
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	assert.Assert(t, started, "Metrics mock server not available after 3 secs")
 
 	t.Run("metrics on cancel Compose build", func(t *testing.T) {
-		s.ResetUsage()
-
 		c.RunDockerCmd("compose", "ls")
-		buildProjectPath := "../compose/fixtures/build-infinite/compose.yaml"
+		buildProjectPath := "fixtures/build-infinite/compose.yaml"
 
 		// require a separate groupID from the process running tests, in order to simulate ctrl+C from a terminal.
 		// sending kill signal
@@ -77,12 +57,6 @@ func TestComposeCancel(t *testing.T) {
 			errors := stderr.String()
 			return strings.Contains(out, "CANCELED"), fmt.Sprintf("'CANCELED' not found in : \n%s\nStderr: \n%s\n", out, errors)
 		}, 10*time.Second, 1*time.Second)
-
-		usage := s.GetUsage()
-		assert.DeepEqual(t, []string{
-			`{"command":"compose ls","context":"moby","source":"cli","status":"success"}`,
-			`{"command":"compose build","context":"moby","source":"cli","status":"canceled"}`,
-		}, usage)
 	})
 }
 

local/e2e/compose/cascade_stop_test.go → pkg/e2e/cascade_stop_test.go

@@ -20,8 +20,6 @@ import (
 	"testing"
 
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestCascadeStop(t *testing.T) {

local/e2e/compose/compose_build_test.go → pkg/e2e/compose_build_test.go

@@ -27,8 +27,6 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestLocalComposeBuild(t *testing.T) {

local/e2e/compose/compose_exec_test.go → pkg/e2e/compose_exec_test.go

@@ -22,8 +22,6 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestLocalComposeExec(t *testing.T) {

local/e2e/compose/compose_run_test.go → pkg/e2e/compose_run_test.go

@@ -23,8 +23,6 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestLocalComposeRun(t *testing.T) {

local/e2e/compose/compose_test.go → pkg/e2e/compose_test.go

@@ -22,7 +22,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -30,21 +29,12 @@ import (
 	testify "github.com/stretchr/testify/assert"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 var binDir string
 
 func TestMain(m *testing.M) {
-	p, cleanup, err := SetupExistingCLI()
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	binDir = p
 	exitCode := m.Run()
-	cleanup()
 	os.Exit(exitCode)
 }
 
@@ -133,36 +123,6 @@ func TestLocalComposeUp(t *testing.T) {
 	})
 }
 
-func binExt() string {
-	binaryExt := ""
-	if runtime.GOOS == "windows" {
-		binaryExt = ".exe"
-	}
-	return binaryExt
-}
-func TestComposeUsingCliPlugin(t *testing.T) {
-	c := NewParallelE2eCLI(t, binDir)
-
-	err := os.Remove(filepath.Join(c.ConfigDir, "cli-plugins", "docker-compose"+binExt()))
-	assert.NilError(t, err)
-	res := c.RunDockerOrExitError("compose", "ls")
-	res.Assert(t, icmd.Expected{Err: "'compose' is not a docker command", ExitCode: 1})
-}
-
-func TestComposeCliPluginWithoutCloudIntegration(t *testing.T) {
-	newBinFolder, cleanup, err := SetupExistingCLI() // do not share bin folder with other tests
-	assert.NilError(t, err)
-	defer cleanup()
-	c := NewParallelE2eCLI(t, newBinFolder)
-
-	err = os.Remove(filepath.Join(newBinFolder, "docker"+binExt()))
-	assert.NilError(t, err)
-	err = os.Rename(filepath.Join(newBinFolder, "com.docker.cli"+binExt()), filepath.Join(newBinFolder, "docker"+binExt()))
-	assert.NilError(t, err)
-	res := c.RunDockerOrExitError("compose", "ls")
-	res.Assert(t, icmd.Expected{Out: "NAME                STATUS", ExitCode: 0})
-}
-
 func TestComposePull(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 

local/e2e/compose/cp_test.go → pkg/e2e/cp_test.go

@@ -23,8 +23,6 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestCopy(t *testing.T) {

pkg/e2e/framework.go

@@ -0,0 +1,267 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/icmd"
+	"gotest.tools/v3/poll"
+)
+
+var (
+	// DockerExecutableName is the OS dependent Docker CLI binary name
+	DockerExecutableName = "docker"
+)
+
+func init() {
+	if runtime.GOOS == "windows" {
+		DockerExecutableName = DockerExecutableName + ".exe"
+	}
+}
+
+// E2eCLI is used to wrap the CLI for end to end testing
+// nolint stutter
+type E2eCLI struct {
+	BinDir    string
+	ConfigDir string
+	test      *testing.T
+}
+
+// NewParallelE2eCLI returns a configured TestE2eCLI with t.Parallel() set
+func NewParallelE2eCLI(t *testing.T, binDir string) *E2eCLI {
+	t.Parallel()
+	return newE2eCLI(t, binDir)
+}
+
+func newE2eCLI(t *testing.T, binDir string) *E2eCLI {
+	d, err := ioutil.TempDir("", "")
+	assert.Check(t, is.Nil(err))
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			conf, _ := ioutil.ReadFile(filepath.Join(d, "config.json"))
+			t.Errorf("Config: %s\n", string(conf))
+			t.Error("Contents of config dir:")
+			for _, p := range dirContents(d) {
+				t.Errorf(p)
+			}
+		}
+		_ = os.RemoveAll(d)
+	})
+
+	_ = os.MkdirAll(filepath.Join(d, "cli-plugins"), 0755)
+	composePluginFile := "docker-compose"
+	scanPluginFile := "docker-scan"
+	if runtime.GOOS == "windows" {
+		composePluginFile += ".exe"
+		scanPluginFile += ".exe"
+	}
+	composePlugin, err := findExecutable(composePluginFile, []string{"../../bin", "../../../bin"})
+	if os.IsNotExist(err) {
+		fmt.Println("WARNING: docker-compose cli-plugin not found")
+	}
+	if err == nil {
+		err = CopyFile(composePlugin, filepath.Join(d, "cli-plugins", composePluginFile))
+		if err != nil {
+			panic(err)
+		}
+		// We don't need a functional scan plugin, but a valid plugin binary
+		err = CopyFile(composePlugin, filepath.Join(d, "cli-plugins", scanPluginFile))
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return &E2eCLI{binDir, d, t}
+}
+
+func dirContents(dir string) []string {
+	res := []string{}
+	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		res = append(res, filepath.Join(dir, path))
+		return nil
+	})
+	return res
+}
+
+func findExecutable(executableName string, paths []string) (string, error) {
+	for _, p := range paths {
+		bin, err := filepath.Abs(path.Join(p, executableName))
+		if err != nil {
+			return "", err
+		}
+
+		if _, err := os.Stat(bin); os.IsNotExist(err) {
+			continue
+		}
+
+		return bin, nil
+	}
+
+	return "", errors.Wrap(os.ErrNotExist, "executable not found")
+}
+
+// CopyFile copies a file from a sourceFile to a destinationFile setting permissions to 0755
+func CopyFile(sourceFile string, destinationFile string) error {
+	src, err := os.Open(sourceFile)
+	if err != nil {
+		return err
+	}
+	// nolint: errcheck
+	defer src.Close()
+
+	dst, err := os.OpenFile(destinationFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		return err
+	}
+	// nolint: errcheck
+	defer dst.Close()
+
+	if _, err = io.Copy(dst, src); err != nil {
+		return err
+	}
+
+	return err
+}
+
+// NewCmd creates a cmd object configured with the test environment set
+func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
+	env := append(os.Environ(),
+		"DOCKER_CONFIG="+c.ConfigDir,
+		"KUBECONFIG=invalid",
+	)
+	return icmd.Cmd{
+		Command: append([]string{command}, args...),
+		Env:     env,
+	}
+}
+
+// MetricsSocket get the path where test metrics will be sent
+func (c *E2eCLI) MetricsSocket() string {
+	return filepath.Join(c.ConfigDir, "./docker-cli.sock")
+}
+
+// NewDockerCmd creates a docker cmd without running it
+func (c *E2eCLI) NewDockerCmd(args ...string) icmd.Cmd {
+	return c.NewCmd(DockerExecutableName, args...)
+}
+
+// RunDockerOrExitError runs a docker command and returns a result
+func (c *E2eCLI) RunDockerOrExitError(args ...string) *icmd.Result {
+	fmt.Printf("\t[%s] docker %s\n", c.test.Name(), strings.Join(args, " "))
+	return icmd.RunCmd(c.NewDockerCmd(args...))
+}
+
+// RunCmd runs a command, expects no error and returns a result
+func (c *E2eCLI) RunCmd(args ...string) *icmd.Result {
+	fmt.Printf("\t[%s] %s\n", c.test.Name(), strings.Join(args, " "))
+	assert.Assert(c.test, len(args) >= 1, "require at least one command in parameters")
+	res := icmd.RunCmd(c.NewCmd(args[0], args[1:]...))
+	res.Assert(c.test, icmd.Success)
+	return res
+}
+
+// RunDockerCmd runs a docker command, expects no error and returns a result
+func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
+	res := c.RunDockerOrExitError(args...)
+	res.Assert(c.test, icmd.Success)
+	return res
+}
+
+// StdoutContains returns a predicate on command result expecting a string in stdout
+func StdoutContains(expected string) func(*icmd.Result) bool {
+	return func(res *icmd.Result) bool {
+		return strings.Contains(res.Stdout(), expected)
+	}
+}
+
+// WaitForCmdResult try to execute a cmd until resulting output matches given predicate
+func (c *E2eCLI) WaitForCmdResult(command icmd.Cmd, predicate func(*icmd.Result) bool, timeout time.Duration, delay time.Duration) {
+	assert.Assert(c.test, timeout.Nanoseconds() > delay.Nanoseconds(), "timeout must be greater than delay")
+	var res *icmd.Result
+	checkStopped := func(logt poll.LogT) poll.Result {
+		fmt.Printf("\t[%s] %s\n", c.test.Name(), strings.Join(command.Command, " "))
+		res = icmd.RunCmd(command)
+		if !predicate(res) {
+			return poll.Continue("Cmd output did not match requirement: %q", res.Combined())
+		}
+		return poll.Success()
+	}
+	poll.WaitOn(c.test, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
+}
+
+// WaitForCondition wait for predicate to execute to true
+func (c *E2eCLI) WaitForCondition(predicate func() (bool, string), timeout time.Duration, delay time.Duration) {
+	checkStopped := func(logt poll.LogT) poll.Result {
+		pass, description := predicate()
+		if !pass {
+			return poll.Continue("Condition not met: %q", description)
+		}
+		return poll.Success()
+	}
+	poll.WaitOn(c.test, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
+}
+
+//Lines split output into lines
+func Lines(output string) []string {
+	return strings.Split(strings.TrimSpace(output), "\n")
+}
+
+// HTTPGetWithRetry performs an HTTP GET on an `endpoint`, using retryDelay also as a request timeout.
+// In the case of an error or the response status is not the expeted one, it retries the same request,
+// returning the response body as a string (empty if we could not reach it)
+func HTTPGetWithRetry(t *testing.T, endpoint string, expectedStatus int, retryDelay time.Duration, timeout time.Duration) string {
+	var (
+		r   *http.Response
+		err error
+	)
+	client := &http.Client{
+		Timeout: retryDelay,
+	}
+	fmt.Printf("\t[%s] GET %s\n", t.Name(), endpoint)
+	checkUp := func(t poll.LogT) poll.Result {
+		r, err = client.Get(endpoint)
+		if err != nil {
+			return poll.Continue("reaching %q: Error %s", endpoint, err.Error())
+		}
+		if r.StatusCode == expectedStatus {
+			return poll.Success()
+		}
+		return poll.Continue("reaching %q: %d != %d", endpoint, r.StatusCode, expectedStatus)
+	}
+	poll.WaitOn(t, checkUp, poll.WithDelay(retryDelay), poll.WithTimeout(timeout))
+	if r != nil {
+		b, err := ioutil.ReadAll(r.Body)
+		assert.NilError(t, err)
+		return string(b)
+	}
+	return ""
+}

local/e2e/compose/ipc_test.go → pkg/e2e/ipc_test.go

@@ -22,8 +22,6 @@ import (
 	"testing"
 
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestIPC(t *testing.T) {

local/e2e/compose/logs_test.go → pkg/e2e/logs_test.go

@@ -23,8 +23,6 @@ import (
 	"gotest.tools/v3/assert"
 
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestLocalComposeLogs(t *testing.T) {

pkg/e2e/metrics_test.go

@@ -0,0 +1,55 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"runtime"
+	"testing"
+
+	"gotest.tools/v3/icmd"
+)
+
+func TestComposeMetrics(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("catch specific failure metrics", func(t *testing.T) {
+		res := c.RunDockerOrExitError("compose", "-f", "fixtures/does-not-exist/compose.yaml", "build")
+		expectedErr := "fixtures/does-not-exist/compose.yaml: no such file or directory"
+		if runtime.GOOS == "windows" {
+			expectedErr = "does-not-exist\\compose.yaml: The system cannot find the path specified"
+		}
+		res.Assert(t, icmd.Expected{ExitCode: 14, Err: expectedErr})
+		res = c.RunDockerOrExitError("compose", "-f", "fixtures/wrong-composefile/compose.yaml", "up", "-d")
+		res.Assert(t, icmd.Expected{ExitCode: 15, Err: "services.simple Additional property wrongField is not allowed"})
+		res = c.RunDockerOrExitError("compose", "up")
+		res.Assert(t, icmd.Expected{ExitCode: 14, Err: "can't find a suitable configuration file in this directory or any parent: not found"})
+		res = c.RunDockerOrExitError("compose", "up", "-f", "fixtures/wrong-composefile/compose.yaml")
+		res.Assert(t, icmd.Expected{ExitCode: 16, Err: "unknown shorthand flag: 'f' in -f"})
+		res = c.RunDockerOrExitError("compose", "up", "--file", "fixtures/wrong-composefile/compose.yaml")
+		res.Assert(t, icmd.Expected{ExitCode: 16, Err: "unknown flag: --file"})
+		res = c.RunDockerOrExitError("compose", "donw", "--file", "fixtures/wrong-composefile/compose.yaml")
+		res.Assert(t, icmd.Expected{ExitCode: 16, Err: `unknown docker command: "compose donw"`})
+		res = c.RunDockerOrExitError("compose", "--file", "fixtures/wrong-composefile/build-error.yml", "build")
+		res.Assert(t, icmd.Expected{ExitCode: 17, Err: `line 17: unknown instruction: WRONG`})
+		res = c.RunDockerOrExitError("compose", "--file", "fixtures/wrong-composefile/build-error.yml", "up")
+		res.Assert(t, icmd.Expected{ExitCode: 17, Err: `line 17: unknown instruction: WRONG`})
+		res = c.RunDockerOrExitError("compose", "--file", "fixtures/wrong-composefile/unknown-image.yml", "pull")
+		res.Assert(t, icmd.Expected{ExitCode: 18, Err: `pull access denied for unknownimage, repository does not exist or may require 'docker login'`})
+		res = c.RunDockerOrExitError("compose", "--file", "fixtures/wrong-composefile/unknown-image.yml", "up")
+		res.Assert(t, icmd.Expected{ExitCode: 18, Err: `pull access denied for unknownimage, repository does not exist or may require 'docker login'`})
+	})
+}

local/e2e/compose/networks_test.go → pkg/e2e/networks_test.go

@@ -24,8 +24,6 @@ import (
 
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestNetworks(t *testing.T) {

local/e2e/compose/restart_test.go → pkg/e2e/restart_test.go

@@ -24,8 +24,6 @@ import (
 
 	testify "github.com/stretchr/testify/assert"
 	"gotest.tools/v3/assert"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestRestart(t *testing.T) {

pkg/e2e/scan_message_test.go

@@ -0,0 +1,78 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package e2e
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/docker/compose-cli/pkg/utils"
+
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/icmd"
+)
+
+func TestDisplayScanMessageAfterBuild(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	// assert docker scan plugin is available
+	c.RunDockerOrExitError("scan", "--help")
+
+	t.Run("display on compose build", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-compose-build", "build")
+		defer c.RunDockerOrExitError("rmi", "-f", "scan-msg-test-compose-build_nginx")
+		res.Assert(t, icmd.Expected{Err: utils.ScanSuggestMsg})
+	})
+
+	t.Run("do not display on compose build with quiet flag", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-quiet", "build", "--quiet")
+		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
+		res = c.RunDockerCmd("rmi", "-f", "scan-msg-test-quiet_nginx")
+		assert.Assert(t, !strings.Contains(res.Combined(), "No such image"))
+
+		res = c.RunDockerCmd("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test-q", "build", "-q")
+		defer c.RunDockerOrExitError("rmi", "-f", "scan-msg-test-q_nginx")
+		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
+	})
+
+	_ = c.RunDockerOrExitError("rmi", "scan-msg-test_nginx")
+
+	t.Run("display on compose up if image is built", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "up", "-d")
+		defer c.RunDockerOrExitError("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "down")
+		res.Assert(t, icmd.Expected{Err: utils.ScanSuggestMsg})
+	})
+
+	t.Run("do not display on compose up if no image built", func(t *testing.T) { // re-run the same Compose aproject
+		res := c.RunDockerCmd("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "up", "-d")
+		defer c.RunDockerOrExitError("compose", "-f", "fixtures/simple-build-test/compose.yaml", "-p", "scan-msg-test", "down", "--rmi", "all")
+		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
+	})
+
+	t.Run("do not display if scan already invoked", func(t *testing.T) {
+		_ = os.MkdirAll(filepath.Join(c.ConfigDir, "scan"), 0755)
+		scanConfigFile := filepath.Join(c.ConfigDir, "scan", "config.json")
+		err := ioutil.WriteFile(scanConfigFile, []byte(`{"optin":true}`), 0644)
+		assert.NilError(t, err)
+
+		res := c.RunDockerCmd("build", "-t", "test-image-scan-msg", "fixtures/simple-build-test/nginx-build")
+		assert.Assert(t, !strings.Contains(res.Combined(), "docker scan"), res.Combined())
+	})
+}

local/e2e/compose/start_stop_test.go → pkg/e2e/start_stop_test.go

@@ -23,8 +23,6 @@ import (
 
 	testify "github.com/stretchr/testify/assert"
 	"gotest.tools/v3/assert"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestStartStop(t *testing.T) {

local/e2e/compose/volumes_test.go → pkg/e2e/volumes_test.go

@@ -23,8 +23,6 @@ import (
 	"time"
 
 	"gotest.tools/v3/assert"
-
-	. "github.com/docker/compose-cli/utils/e2e"
 )
 
 func TestLocalComposeVolume(t *testing.T) {

pkg/utils/scan_suggest.go

@@ -28,6 +28,9 @@ import (
 	cliConfig "github.com/docker/cli/cli/config"
 )
 
+// ScanSuggestMsg display a message after a successful build to suggest use of `docker scan` command
+const ScanSuggestMsg = "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them"
+
 // DisplayScanSuggestMsg displlay a message suggesting users can scan new image
 func DisplayScanSuggestMsg() {
 	if os.Getenv("DOCKER_SCAN_SUGGEST") == "false" {
@@ -39,7 +42,7 @@ func DisplayScanSuggestMsg() {
 	if scanAlreadyInvoked() {
 		return
 	}
-	fmt.Fprintf(os.Stderr, "\nUse 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them\n")
+	fmt.Fprintf(os.Stderr, "\n"+ScanSuggestMsg+"\n")
 }
 
 func scanAlreadyInvoked() bool {