publish Compose application as compose.yaml + images

Signed-off-by: Nicolas De Loof <[email protected]>

cmd/compose/publish.go

@@ -34,6 +34,7 @@ type publishOptions struct {
 	ociVersion          string
 	withEnvironment     bool
 	assumeYes           bool
+	app                 bool
 }
 
 func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command {
@@ -53,6 +54,7 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic
 	flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI image/artifact specification version (automatically determined by default)")
 	flags.BoolVar(&opts.withEnvironment, "with-env", false, "Include environment variables in the published OCI artifact")
 	flags.BoolVarP(&opts.assumeYes, "yes", "y", false, `Assume "yes" as answer to all prompts`)
+	flags.BoolVar(&opts.app, "app", false, "Published compose application (includes referenced images)")
 	flags.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
 		// assumeYes was introduced by mistake as `--y`
 		if name == "y" {
@@ -76,7 +78,8 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service,
 	}
 
 	return backend.Publish(ctx, project, repository, api.PublishOptions{
-		ResolveImageDigests: opts.resolveImageDigests,
+		ResolveImageDigests: opts.resolveImageDigests || opts.app,
+		Application:         opts.app,
 		OCIVersion:          api.OCIVersion(opts.ociVersion),
 		WithEnvironment:     opts.withEnvironment,
 		AssumeYes:           opts.assumeYes,

docs/reference/compose_publish.md

@@ -7,6 +7,7 @@ Publish compose application
 
 | Name                      | Type     | Default | Description                                                                    |
 |:--------------------------|:---------|:--------|:-------------------------------------------------------------------------------|
+| `--app`                   | `bool`   |         | Published compose application (includes referenced images)                     |
 | `--dry-run`               | `bool`   |         | Execute command in dry run mode                                                |
 | `--oci-version`           | `string` |         | OCI image/artifact specification version (automatically determined by default) |
 | `--resolve-image-digests` | `bool`   |         | Pin image tags to digests                                                      |

docs/reference/docker_compose_alpha_publish.yaml

@@ -5,6 +5,16 @@ usage: docker compose alpha publish [OPTIONS] REPOSITORY[:TAG]
 pname: docker compose alpha
 plink: docker_compose_alpha.yaml
 options:
+    - option: app
+      value_type: bool
+      default_value: "false"
+      description: Published compose application (includes referenced images)
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
     - option: oci-version
       value_type: string
       description: |

docs/reference/docker_compose_publish.yaml

@@ -5,6 +5,16 @@ usage: docker compose publish [OPTIONS] REPOSITORY[:TAG]
 pname: docker compose
 plink: docker_compose.yaml
 options:
+    - option: app
+      value_type: bool
+      default_value: "false"
+      description: Published compose application (includes referenced images)
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
     - option: oci-version
       value_type: string
       description: |

internal/oci/push.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/containerd/containerd/v2/core/remotes"
 	pusherrors "github.com/containerd/containerd/v2/core/remotes/errors"
-	"github.com/containerd/errdefs"
 	"github.com/distribution/reference"
 	"github.com/docker/compose/v2/pkg/api"
 	"github.com/opencontainers/go-digest"
@@ -94,12 +93,12 @@ func DescriptorForEnvFile(path string, content []byte) v1.Descriptor {
 	}
 }
 
-func PushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error {
+func PushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) (v1.Descriptor, error) {
 	// Check if we need an extra empty layer for the manifest config
 	if ociVersion == api.OCIVersion1_1 || ociVersion == "" {
 		err := push(ctx, resolver, named, v1.DescriptorEmptyJSON)
 		if err != nil {
-			return err
+			return v1.Descriptor{}, err
 		}
 	}
 	// prepare to push the manifest by pushing the layers
@@ -107,7 +106,7 @@ func PushManifest(ctx context.Context, resolver remotes.Resolver, named referenc
 	for i := range layers {
 		layerDescriptors[i] = layers[i]
 		if err := push(ctx, resolver, named, layers[i]); err != nil {
-			return err
+			return v1.Descriptor{}, err
 		}
 	}
 
@@ -119,13 +118,13 @@ func PushManifest(ctx context.Context, resolver remotes.Resolver, named referenc
 	// try to push in the OCI 1.1 format but fallback to OCI 1.0 on 4xx errors
 	// (other than auth) since it's most likely the result of the registry not
 	// having support
-	err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1)
+	descriptor, err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1)
 	var pushErr pusherrors.ErrUnexpectedStatus
 	if errors.As(err, &pushErr) && isNonAuthClientError(pushErr.StatusCode) {
 		// TODO(milas): show a warning here (won't work with logrus)
 		return createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_0)
 	}
-	return err
+	return descriptor, err
 }
 
 func push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, descriptor v1.Descriptor) error {
@@ -134,37 +133,21 @@ func push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, d
 		return err
 	}
 
-	pusher, err := resolver.Pusher(ctx, fullRef.String())
-	if err != nil {
-		return err
-	}
-	push, err := pusher.Push(ctx, descriptor)
-	if errdefs.IsAlreadyExists(err) {
-		return nil
-	}
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = push.Close()
-	}()
-
-	_, err = push.Write(descriptor.Data)
-	return err
+	return Push(ctx, resolver, fullRef, descriptor)
 }
 
-func createAndPushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error {
-	toPush, err := generateManifest(layers, ociVersion)
+func createAndPushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) (v1.Descriptor, error) {
+	descriptor, toPush, err := generateManifest(layers, ociVersion)
 	if err != nil {
-		return err
+		return v1.Descriptor{}, err
 	}
 	for _, p := range toPush {
 		err = push(ctx, resolver, named, p)
 		if err != nil {
-			return err
+			return v1.Descriptor{}, err
 		}
 	}
-	return nil
+	return descriptor, nil
 }
 
 func isNonAuthClientError(statusCode int) bool {
@@ -175,7 +158,7 @@ func isNonAuthClientError(statusCode int) bool {
 	return !slices.Contains(clientAuthStatusCodes, statusCode)
 }
 
-func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.Descriptor, error) {
+func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) (v1.Descriptor, []v1.Descriptor, error) {
 	var toPush []v1.Descriptor
 	var config v1.Descriptor
 	var artifactType string
@@ -205,10 +188,9 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De
 	case api.OCIVersion1_1:
 		config = v1.DescriptorEmptyJSON
 		artifactType = ComposeProjectArtifactType
-		// N.B. the descriptor has the data embedded in it
 		toPush = append(toPush, config)
 	default:
-		return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat)
+		return v1.Descriptor{}, nil, fmt.Errorf("unsupported OCI version: %s", ociCompat)
 	}
 
 	manifest, err := json.Marshal(v1.Manifest{
@@ -222,7 +204,7 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De
 		},
 	})
 	if err != nil {
-		return nil, err
+		return v1.Descriptor{}, nil, err
 	}
 
 	manifestDescriptor := v1.Descriptor{
@@ -236,5 +218,5 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De
 		Data:         manifest,
 	}
 	toPush = append(toPush, manifestDescriptor)
-	return toPush, nil
+	return manifestDescriptor, toPush, nil
 }

internal/oci/resolver.go

@@ -19,12 +19,17 @@ package oci
 import (
 	"context"
 	"io"
+	"net/url"
+	"strings"
 
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/labels"
+	"github.com/containerd/errdefs"
 	"github.com/distribution/reference"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/compose/v2/internal/registry"
+	"github.com/moby/buildkit/util/contentutil"
 	spec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -70,3 +75,60 @@ func Get(ctx context.Context, resolver remotes.Resolver, ref reference.Named) (s
 	}
 	return descriptor, content, nil
 }
+
+func Copy(ctx context.Context, resolver remotes.Resolver, image reference.Named, named reference.Named) (spec.Descriptor, error) {
+	src, desc, err := resolver.Resolve(ctx, image.String())
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+	if desc.Annotations == nil {
+		desc.Annotations = make(map[string]string)
+	}
+	// set LabelDistributionSource so push will actually use a registry mount
+	refspec := reference.TrimNamed(image).String()
+	u, err := url.Parse("dummy://" + refspec)
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+	source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/")
+	desc.Annotations[labels.LabelDistributionSource+"."+source] = repo
+
+	p, err := resolver.Pusher(ctx, named.Name())
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+	f, err := resolver.Fetcher(ctx, src)
+	if err != nil {
+		return spec.Descriptor{}, err
+	}
+
+	err = contentutil.CopyChain(ctx,
+		contentutil.FromPusher(p),
+		contentutil.FromFetcher(f), desc)
+	return desc, err
+}
+
+func Push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, descriptor spec.Descriptor) error {
+	pusher, err := resolver.Pusher(ctx, ref.String())
+	if err != nil {
+		return err
+	}
+	ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeYAMLMediaType, "artifact-")
+	ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeEnvFileMediaType, "artifact-")
+	ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeEmptyConfigMediaType, "config-")
+	ctx = remotes.WithMediaTypeKeyPrefix(ctx, spec.MediaTypeEmptyJSON, "config-")
+
+	push, err := pusher.Push(ctx, descriptor)
+	if errdefs.IsAlreadyExists(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = push.Close()
+	}()
+
+	_, err = push.Write(descriptor.Data)
+	return err
+}

pkg/api/api.go

@@ -444,9 +444,10 @@ const (
 // PublishOptions group options of the Publish API
 type PublishOptions struct {
 	ResolveImageDigests bool
+	Application         bool
 	WithEnvironment     bool
-	AssumeYes           bool
 
+	AssumeYes  bool
 	OCIVersion OCIVersion
 }
 

pkg/compose/publish.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -36,6 +37,8 @@ import (
 	"github.com/docker/compose/v2/pkg/compose/transform"
 	"github.com/docker/compose/v2/pkg/progress"
 	"github.com/docker/compose/v2/pkg/prompt"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -45,6 +48,7 @@ func (s *composeService) Publish(ctx context.Context, project *types.Project, re
 	}, s.stdinfo(), "Publishing")
 }
 
+//nolint:gocyclo
 func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
 	accept, err := s.preChecks(project, options)
 	if err != nil {
@@ -106,7 +110,7 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 		Status: progress.Working,
 	})
 	if !s.dryRun {
-		err = oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
+		descriptor, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
 		if err != nil {
 			w.Event(progress.Event{
 				ID:     repository,
@@ -115,6 +119,50 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 			})
 			return err
 		}
+
+		if options.Application {
+			manifests := []v1.Descriptor{}
+			for _, service := range project.Services {
+				ref, err := reference.ParseDockerRef(service.Image)
+				if err != nil {
+					return err
+				}
+
+				manifest, err := oci.Copy(ctx, resolver, ref, named)
+				if err != nil {
+					return err
+				}
+				manifests = append(manifests, manifest)
+			}
+
+			descriptor.Data = nil
+			index, err := json.Marshal(v1.Index{
+				Versioned: specs.Versioned{SchemaVersion: 2},
+				MediaType: v1.MediaTypeImageIndex,
+				Manifests: manifests,
+				Subject:   &descriptor,
+				Annotations: map[string]string{
+					"com.docker.compose.version": api.ComposeVersion,
+				},
+			})
+			if err != nil {
+				return err
+			}
+			imagesDescriptor := v1.Descriptor{
+				MediaType:    v1.MediaTypeImageIndex,
+				ArtifactType: oci.ComposeProjectArtifactType,
+				Digest:       digest.FromString(string(index)),
+				Size:         int64(len(index)),
+				Annotations: map[string]string{
+					"com.docker.compose.version": api.ComposeVersion,
+				},
+				Data: index,
+			}
+			err = oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	w.Event(progress.Event{
 		ID:     repository,

pkg/remote/oci.go

@@ -26,11 +26,12 @@ import (
 	"strings"
 
 	"github.com/compose-spec/compose-go/v2/loader"
+	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/distribution/reference"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/compose/v2/internal/oci"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	spec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 const (
@@ -67,6 +68,7 @@ func (g ociRemoteLoader) Accept(path string) bool {
 	return strings.HasPrefix(path, OciPrefix)
 }
 
+//nolint:gocyclo
 func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error) {
 	enabled, err := ociRemoteLoaderEnabled()
 	if err != nil {
@@ -91,7 +93,7 @@ func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error)
 
 		descriptor, content, err := oci.Get(ctx, resolver, ref)
 		if err != nil {
-			return "", err
+			return "", fmt.Errorf("failed to pull OCI resource %q: %w", ref, err)
 		}
 
 		cache, err := cacheDir()
@@ -101,7 +103,35 @@ func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error)
 
 		local = filepath.Join(cache, descriptor.Digest.Hex())
 		if _, err = os.Stat(local); os.IsNotExist(err) {
-			var manifest v1.Manifest
+
+			// a Compose application bundle is published as image index
+			if images.IsIndexType(descriptor.MediaType) {
+				var index spec.Index
+				err = json.Unmarshal(content, &index)
+				if err != nil {
+					return "", err
+				}
+				found := false
+				for _, manifest := range index.Manifests {
+					if manifest.ArtifactType != oci.ComposeProjectArtifactType {
+						continue
+					}
+					found = true
+					digested, err := reference.WithDigest(ref, manifest.Digest)
+					if err != nil {
+						return "", err
+					}
+					descriptor, content, err = oci.Get(ctx, resolver, digested)
+					if err != nil {
+						return "", fmt.Errorf("failed to pull OCI resource %q: %w", ref, err)
+					}
+				}
+				if !found {
+					return "", fmt.Errorf("OCI index %s doesn't refer to compose artifacts", ref)
+				}
+			}
+
+			var manifest spec.Manifest
 			err = json.Unmarshal(content, &manifest)
 			if err != nil {
 				return "", err
@@ -123,7 +153,7 @@ func (g ociRemoteLoader) Dir(path string) string {
 	return g.known[path]
 }
 
-func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, manifest v1.Manifest, ref reference.Named, resolver remotes.Resolver) error { //nolint:gocyclo
+func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, manifest spec.Manifest, ref reference.Named, resolver remotes.Resolver) error { //nolint:gocyclo
 	err := os.MkdirAll(local, 0o700)
 	if err != nil {
 		return err
@@ -173,7 +203,7 @@ func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, man
 	return nil
 }
 
-func writeComposeFile(layer v1.Descriptor, i int, f *os.File, content []byte) error {
+func writeComposeFile(layer spec.Descriptor, i int, f *os.File, content []byte) error {
 	if _, ok := layer.Annotations["com.docker.compose.file"]; i > 0 && ok {
 		_, err := f.Write([]byte("\n---\n"))
 		if err != nil {
@@ -184,7 +214,7 @@ func writeComposeFile(layer v1.Descriptor, i int, f *os.File, content []byte) er
 	return err
 }
 
-func writeEnvFile(layer v1.Descriptor, local string, content []byte) error {
+func writeEnvFile(layer spec.Descriptor, local string, content []byte) error {
 	envfilePath, ok := layer.Annotations["com.docker.compose.envfile"]
 	if !ok {
 		return fmt.Errorf("missing annotation com.docker.compose.envfile in layer %q", layer.Digest)