Build and pull in parallel with BuildKit

Signed-off-by: Nicolas De Loof <[email protected]>

go.sum

@@ -34,8 +34,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AkihiroSuda/containerd-fuse-overlayfs v0.0.0-20200220082720-bb896865146c/go.mod h1:K4kx7xAA5JimeQCnN+dbeLlfaBxzZLaLiDD8lusFI8w=
-github.com/AlecAivazis/survey/v2 v2.1.1 h1:LEMbHE0pLj75faaVEKClEX1TM4AJmmnOh9eimREzLWI=
-github.com/AlecAivazis/survey/v2 v2.1.1/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk=
+github.com/AlecAivazis/survey/v2 v2.2.3 h1:utJR2X4Ibp2fBxdjalQUiMFf3zfQNjA15YE8+ftlKEs=
+github.com/AlecAivazis/survey/v2 v2.2.3/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -86,7 +86,10 @@ github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab h1:9pygWVFqbY9lPxM0peffumuVDyMuIMzNLyO9uFjJuQo=
 github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-
+github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc=
+github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
 github.com/Microsoft/hcsshim v0.8.10 h1:k5wTrpnVU2/xv8ZuzGkbXVd3js5zJ8RnumPo5RxiIxU=
 github.com/Microsoft/hcsshim v0.8.10/go.mod h1:g5uw8EV2mAlzqe94tfNBNdr89fnbD/n3HV0OhsddkmM=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -145,10 +148,21 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-
+github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/compose-spec/compose-go v0.0.0-20201122180821-0f87549cda1e h1:EO6seAKT1UtMjk1RGmuXS1tFkSkWr6FPjQru/mC+r/U=
+github.com/compose-spec/compose-go v0.0.0-20201122180821-0f87549cda1e/go.mod h1:Pm8gjIoib7x9av1XGkijXddjyV4aTHEq9EwVOvWH624=
+github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
+github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d/go.mod h1:CStdkl05lBnJej94BPFoJ7vB8cELKXwViS+dgfW0/M8=
+github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk=
 github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-
+github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/console v0.0.0-20191219165238-8375c3424e4d/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/console v1.0.1 h1:u7SFAJyRqWcG6ogaMAx3KjSTy1e3hT9QxqX7Jco7dRc=
+github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
+github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55 h1:FGO0nwSBESgoGCakj+w3OQXyrMLsz2omdo9b2UfG/BQ=
+github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20181001140422-bd77b46c8352/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
 github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a h1:jEIoR0aA5GogXZ8pP3DUzE+zrhaF6/1rYZy+7KkYEWM=
@@ -416,7 +430,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U=
 github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -506,7 +521,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
-
+github.com/moby/buildkit v0.7.0 h1:5T41l8i0f+OqHvRThh2NKVXiLpclXYmgO8QePoyIJrw=
+github.com/moby/buildkit v0.7.0/go.mod h1:zOhLO1TiQepuSfzoNDQ542IGJQy0CHr79T4MOJvaewY=
 github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf h1:Un6PNx5oMK6CCwO3QTUyPiK2mtZnPrpDl5UnZ64eCkw=
 github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -527,7 +543,7 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -537,7 +553,10 @@ github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=

local/build.go

@@ -1,3 +1,5 @@
+// +build local
+
 /*
    Copyright 2020 Docker Compose CLI authors
 
@@ -18,13 +20,75 @@ package local
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path"
+	"strings"
 
 	"github.com/compose-spec/compose-go/types"
 	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/driver"
 	_ "github.com/docker/buildx/driver/docker" // required to get default driver registered
+	"github.com/docker/buildx/util/progress"
 )
 
+func (s *local) ensureImagesExists(ctx context.Context, project *types.Project) error {
+	opts := map[string]build.Options{}
+	for _, service := range project.Services {
+		if service.Image == "" && service.Build == nil {
+			return fmt.Errorf("invalid service %q. Must specify either image or build", service.Name)
+		}
+
+		// TODO build vs pull should be controlled by pull policy, see https://github.com/compose-spec/compose-spec/issues/26
+		if service.Image != "" {
+			needPull, err := s.needPull(ctx, service)
+			if err != nil {
+				return err
+			}
+			if !needPull {
+				continue
+			}
+		}
+		if service.Build != nil {
+			opts[service.Name] = s.buildImage(ctx, service, project.WorkingDir)
+			continue
+		}
+
+		// Buildx has no command to "just pull", see
+		// so we bake a temporary dockerfile that will just pull and export pulled image
+		opts[service.Name] = build.Options{
+			Inputs: build.Inputs{
+				ContextPath:    ".",
+				DockerfilePath: "-",
+				InStream:       strings.NewReader("FROM " + service.Image),
+			},
+			Tags: []string{service.Image},
+			Pull: true,
+		}
+
+	}
+
+	return s.build(ctx, project, opts)
+}
+
+func (s *local) build(ctx context.Context, project *types.Project, opts map[string]build.Options) error {
+	const drivername = "default"
+	d, err := driver.GetDriver(ctx, drivername, nil, s.containerService.apiClient, nil, nil, "", nil, project.WorkingDir)
+	if err != nil {
+		return err
+	}
+	driverInfo := []build.DriverInfo{
+		{
+			Name:   "default",
+			Driver: d,
+		},
+	}
+	// We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here
+	w := progress.NewPrinter(ctx, os.Stdout, "auto")
+	_, err = build.Build(ctx, driverInfo, opts, nil, nil, w)
+	return err
+}
+
 func (s *local) buildImage(ctx context.Context, service types.ServiceConfig, contextPath string) build.Options {
 	var tags []string
 	if service.Image != "" {

local/compose.go

@@ -23,7 +23,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -31,9 +30,6 @@ import (
 	"sync"
 
 	"github.com/compose-spec/compose-go/types"
-	"github.com/docker/buildx/build"
-	"github.com/docker/buildx/driver"
-	px "github.com/docker/buildx/util/progress"
 	moby "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
@@ -41,7 +37,6 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/go-connections/nat"
 	"github.com/pkg/errors"
 	"github.com/sanathkr/go-yaml"
@@ -76,14 +71,12 @@ func (s *local) Up(ctx context.Context, project *types.Project, detach bool) err
 		}
 	}
 
-	for _, service := range project.Services {
-		err := s.applyPullPolicy(ctx, project, service)
-		if err != nil {
-			return err
-		}
+	err := s.ensureImagesExists(ctx, project)
+	if err != nil {
+		return err
 	}
 
-	err := inDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error {
+	err = inDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error {
 		return s.ensureService(c, project, service)
 	})
 	return err
@@ -99,94 +92,15 @@ func getContainerName(c moby.Container) string {
 	return c.Names[0][1:]
 }
 
-func (s *local) applyPullPolicy(ctx context.Context, project *types.Project, service types.ServiceConfig) error {
-	w := progress.ContextWriter(ctx)
-	// TODO build vs pull should be controlled by pull policy
-	opts := map[string]build.Options{}
-	if service.Build != nil {
-		opts[service.Name] = s.buildImage(ctx, service, project.WorkingDir)
-	}
-
-	if len(opts) > 0 {
-		w := px.NewPrinter(ctx, os.Stdout, "auto")
-		const drivername = "buildx_buildkit_default"
-		d, err := driver.GetDriver(ctx, drivername, nil, s.containerService.apiClient, nil, nil, "", nil, project.WorkingDir)
-		if err != nil {
-			return err
-		}
-		driverInfo := []build.DriverInfo{
-			{
-				Name:   "default",
-				Driver: d,
-			},
-		}
-		// We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here
-		// FIXME pass auth file from
-		_, err = build.Build(ctx, driverInfo, opts, nil, nil, w)
-		return err
-	}
-
-	if service.Image != "" {
-		_, _, err := s.containerService.apiClient.ImageInspectWithRaw(ctx, service.Image)
-		if err != nil {
-			if errdefs.IsNotFound(err) {
-				stream, err := s.containerService.apiClient.ImagePull(ctx, service.Image, moby.ImagePullOptions{})
-				if err != nil {
-					return err
-				}
-				dec := json.NewDecoder(stream)
-				for {
-					var jm jsonmessage.JSONMessage
-					if err := dec.Decode(&jm); err != nil {
-						if err == io.EOF {
-							break
-						}
-						return err
-					}
-					toProgressEvent(jm, w)
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func toProgressEvent(jm jsonmessage.JSONMessage, w progress.Writer) {
-	if jm.Progress != nil {
-		if jm.Progress.Total != 0 {
-			status := progress.Working
-			if jm.Status == "Pull complete" {
-				status = progress.Done
-			}
-			w.Event(progress.Event{
-				ID:         jm.ID,
-				Text:       jm.Status,
-				Status:     status,
-				StatusText: jm.Progress.String(),
-			})
-		} else {
-			if jm.Error != nil {
-				w.Event(progress.Event{
-					ID:         jm.ID,
-					Text:       jm.Status,
-					Status:     progress.Error,
-					StatusText: jm.Error.Message,
-				})
-			} else if jm.Status == "Pull complete" || jm.Status == "Already exists" {
-				w.Event(progress.Event{
-					ID:     jm.ID,
-					Text:   jm.Status,
-					Status: progress.Done,
-				})
-			} else {
-				w.Event(progress.Event{
-					ID:     jm.ID,
-					Text:   jm.Status,
-					Status: progress.Working,
-				})
-			}
+func (s *local) needPull(ctx context.Context, service types.ServiceConfig) (bool, error) {
+	_, _, err := s.containerService.apiClient.ImageInspectWithRaw(ctx, service.Image)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return true, nil
 		}
+		return false, err
 	}
+	return false, nil
 }
 
 func (s *local) Down(ctx context.Context, projectName string) error {