Merge pull request #713 from docker/metrics_e2e

E2e test on metrics

.github/workflows/ci.yml

@@ -55,12 +55,12 @@ jobs:
 
       - name: Test
         env:
-          BUILD_TAGS: example,local,ecs
+          BUILD_TAGS: example,local
         run: make -f builder.Makefile test
 
-      - name: Build
+      - name: Build for local E2E
         env:
-          BUILD_TAGS: example,local,ecs
+          BUILD_TAGS: example,local,e2e
         run: make -f builder.Makefile cli
 
       - name: E2E Test

.github/workflows/optional-ci.yml

@@ -130,7 +130,7 @@ jobs:
 
       - name: Build
         env:
-          BUILD_TAGS: example,local
+          BUILD_TAGS: example,local,e2e
         run: make -f builder.Makefile cli
 
       - name: E2E Test

Makefile

@@ -39,7 +39,7 @@ protos: ## Generate go code from .proto files
 cli: ## Compile the cli
 	@docker build . --target cli \
 	--platform local \
-	--build-arg BUILD_TAGS=example,local,ecs \
+	--build-arg BUILD_TAGS=example,local,e2e \
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--output ./bin
 
@@ -63,7 +63,7 @@ cross: ## Compile the CLI for linux, darwin and windows
 
 test: ## Run unit tests
 	@docker build . \
-	--build-arg BUILD_TAGS=example,local,ecs \
+	--build-arg BUILD_TAGS=example,local \
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--target test
 

go.mod

@@ -42,6 +42,8 @@ require (
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/joho/godotenv v1.3.0
+	github.com/labstack/echo v3.3.10+incompatible
+	github.com/labstack/gommon v0.3.0 // indirect
 	github.com/moby/term v0.0.0-20200611042045-63b9a826fb74
 	github.com/morikuni/aec v1.0.0
 	github.com/onsi/gomega v1.10.1 // indirect
@@ -55,6 +57,7 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
+	github.com/valyala/fasttemplate v1.2.1 // indirect
 	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43

go.sum

@@ -325,6 +325,10 @@ github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
+github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
+github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
@@ -334,6 +338,8 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljm
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw=
 github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -458,6 +464,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
@@ -600,6 +611,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

metrics/conn_e2e.go

@@ -0,0 +1,31 @@
+// +build e2e
+
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package metrics
+
+import (
+	"os"
+)
+
+
+func init() {
+	testSocket, defined := os.LookupEnv("TEST_METRICS_SOCKET")
+	if defined {
+		socket = testSocket
+	}
+}

metrics/conn_other.go

@@ -20,7 +20,7 @@ package metrics
 
 import "net"
 
-const (
+var (
 	socket = "/var/run/docker-cli.sock"
 )
 

metrics/conn_windows.go

@@ -20,16 +20,20 @@ package metrics
 
 import (
 	"net"
+	"strings"
 	"time"
 
 	"github.com/Microsoft/go-winio"
 )
 
-const (
+var (
 	socket = `\\.\pipe\docker_cli`
 )
 
 func conn() (net.Conn, error) {
-	timeout := 200 * time.Millisecond
-	return winio.DialPipe(socket, &timeout)
+	if strings.HasPrefix(socket, `\\.\pipe\`) {
+		timeout := 200 * time.Millisecond
+		return winio.DialPipe(socket, &timeout)
+	}
+	return net.Dial("unix", socket)
 }

tests/aci-e2e/e2e-aci_test.go

@@ -498,7 +498,8 @@ func TestUpUpdate(t *testing.T) {
 	composeAccountName = strings.ToLower(composeAccountName)
 
 	dstDir := filepath.Join(os.TempDir(), "e2e-aci-volume-"+composeAccountName)
-	err := fileutil.CopyDirs("../composefiles/aci-demo/", dstDir)
+	srcDir := filepath.Join("..", "composefiles", "aci-demo")
+	err := fileutil.CopyDirs(srcDir, dstDir)
 	assert.NilError(t, err)
 	t.Cleanup(func() {
 		assert.NilError(t, os.RemoveAll(dstDir))

tests/e2e/e2e_test.go

@@ -158,6 +158,49 @@ func TestContextHelpACI(t *testing.T) {
 	})
 }
 
+func TestContextMetrics(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	s := NewMetricsServer(c.MetricsSocket())
+	s.Start()
+	defer s.Stop()
+
+	t.Run("metrics on default context", func(t *testing.T) {
+		s.ResetUsage()
+
+		c.RunDockerCmd("ps")
+		c.RunDockerCmd("version")
+		c.RunDockerOrExitError("version", "--xxx")
+
+		usage := s.GetUsage()
+		assert.Equal(t, 3, len(usage))
+		assert.Equal(t, `{"command":"ps","context":"moby","source":"cli","status":"success"}`, usage[0])
+		assert.Equal(t, `{"command":"version","context":"moby","source":"cli","status":"success"}`, usage[1])
+		assert.Equal(t, `{"command":"version","context":"moby","source":"cli","status":"failure"}`, usage[2])
+	})
+
+	t.Run("metrics on other context type", func(t *testing.T) {
+		s.ResetUsage()
+
+		c.RunDockerCmd("context", "create", "example", "test-example")
+		c.RunDockerCmd("ps")
+		c.RunDockerCmd("context", "use", "test-example")
+		c.RunDockerCmd("ps")
+		c.RunDockerOrExitError("error")
+		c.RunDockerCmd("context", "use", "default")
+		c.RunDockerCmd("--context", "test-example", "ps")
+
+		usage := s.GetUsage()
+		assert.Equal(t, 7, len(usage))
+		assert.Equal(t, `{"command":"context create","context":"moby","source":"cli","status":"success"}`, usage[0])
+		assert.Equal(t, `{"command":"ps","context":"moby","source":"cli","status":"success"}`, usage[1])
+		assert.Equal(t, `{"command":"context use","context":"moby","source":"cli","status":"success"}`, usage[2])
+		assert.Equal(t, `{"command":"ps","context":"example","source":"cli","status":"success"}`, usage[3])
+		assert.Equal(t, `{"command":"error","context":"example","source":"cli","status":"failure"}`, usage[4])
+		assert.Equal(t, `{"command":"context use","context":"example","source":"cli","status":"success"}`, usage[5])
+		assert.Equal(t, `{"command":"ps","context":"example","source":"cli","status":"success"}`, usage[6])
+	})
+}
+
 func TestContextDuplicateACI(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 

tests/framework/e2e.go

@@ -147,6 +147,7 @@ func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
 	env := append(os.Environ(),
 		"DOCKER_CONFIG="+c.ConfigDir,
 		"KUBECONFIG=invalid",
+		"TEST_METRICS_SOCKET="+c.MetricsSocket(),
 		"PATH="+c.PathEnvVar(),
 	)
 	return icmd.Cmd{
@@ -155,6 +156,11 @@ func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
 	}
 }
 
+// MetricsSocket get the path where test metrics will be sent
+func (c *E2eCLI) MetricsSocket() string {
+	return filepath.Join(c.ConfigDir, "./docker-cli.sock")
+}
+
 // NewDockerCmd creates a docker cmd without running it
 func (c *E2eCLI) NewDockerCmd(args ...string) icmd.Cmd {
 	return c.NewCmd(filepath.Join(c.BinDir, DockerExecutableName), args...)

tests/framework/mockmetrics.go

@@ -0,0 +1,81 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package framework
+
+import (
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/labstack/echo"
+)
+
+// MockMetricsServer a mock registring all metrics POST invocations
+type MockMetricsServer struct {
+	socket string
+	usage  []string
+	e      *echo.Echo
+}
+
+// NewMetricsServer instaniate a new MockMetricsServer
+func NewMetricsServer(socket string) *MockMetricsServer {
+	return &MockMetricsServer{
+		socket: socket,
+		e:      echo.New(),
+	}
+}
+
+// Handler
+func (s *MockMetricsServer) hello(c echo.Context) error {
+	body, error := ioutil.ReadAll(c.Request().Body)
+	if error != nil {
+		return error
+	}
+	cliUsage := string(body)
+	s.usage = append(s.usage, cliUsage)
+	return c.String(http.StatusOK, "")
+}
+
+// GetUsage get usage
+func (s *MockMetricsServer) GetUsage() []string {
+	return s.usage
+}
+
+// ResetUsage reset usage
+func (s *MockMetricsServer) ResetUsage() {
+	s.usage = []string{}
+}
+
+// Stop stop the mock server
+func (s *MockMetricsServer) Stop() {
+	_ = s.e.Close()
+}
+
+// Start start the mock server
+func (s *MockMetricsServer) Start() {
+	go func() {
+		listener, err := net.Listen("unix", strings.TrimPrefix(s.socket, "unix://"))
+		if err != nil {
+			log.Fatal(err)
+		}
+		s.e.Listener = listener
+		s.e.POST("/usage", s.hello)
+		_ = s.e.Start(":1323")
+	}()
+}