Merge pull request #11503 from ndeloof/includes_metric

use listeners to collect include metrics

cmd/compose/compose.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/compose-spec/compose-go/v2/cli"
 	"github.com/compose-spec/compose-go/v2/dotenv"
+	"github.com/compose-spec/compose-go/v2/loader"
 	"github.com/compose-spec/compose-go/v2/types"
 	composegoutils "github.com/compose-spec/compose-go/v2/utils"
 	"github.com/docker/buildx/util/logutil"
@@ -147,7 +148,7 @@ func (o *ProjectOptions) WithServices(dockerCli command.Cli, fn ProjectServicesF
 			return err
 		}
 
-		ctx = context.WithValue(ctx, tracing.Metrics{}, metrics)
+		ctx = context.WithValue(ctx, tracing.MetricsKey{}, metrics)
 
 		return fn(ctx, project, args)
 	})
@@ -202,8 +203,10 @@ func (o *ProjectOptions) toProjectName(ctx context.Context, dockerCli command.Cl
 
 func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, services []string, po ...cli.ProjectOptionsFn) (*types.Project, tracing.Metrics, error) {
 	var metrics tracing.Metrics
-	if !o.Offline {
-		po = append(po, o.remoteLoaders(dockerCli)...)
+
+	remotes := o.remoteLoaders(dockerCli)
+	for _, r := range remotes {
+		po = append(po, cli.WithResourceLoader(r))
 	}
 
 	po = append(po, cli.WithContext(ctx))
@@ -214,10 +217,28 @@ func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, s
 	}
 
 	options.WithListeners(func(event string, metadata map[string]any) {
-		if event == "extends" {
+		switch event {
+		case "extends":
 			metrics.CountExtends++
+		case "include":
+			paths := metadata["path"].(types.StringList)
+			for _, path := range paths {
+				var isRemote bool
+				for _, r := range remotes {
+					if r.Accept(path) {
+						isRemote = true
+						break
+					}
+				}
+				if isRemote {
+					metrics.CountIncludesRemote++
+				} else {
+					metrics.CountIncludesLocal++
+				}
+			}
 		}
 	})
+
 	if o.Compatibility || utils.StringToBool(options.Environment[ComposeCompatibility]) {
 		api.Separator = "_"
 	}
@@ -257,10 +278,13 @@ func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, s
 	return project, metrics, err
 }
 
-func (o *ProjectOptions) remoteLoaders(dockerCli command.Cli) []cli.ProjectOptionsFn {
+func (o *ProjectOptions) remoteLoaders(dockerCli command.Cli) []loader.ResourceLoader {
+	if o.Offline {
+		return nil
+	}
 	git := remote.NewGitRemoteLoader(o.Offline)
 	oci := remote.NewOCIRemoteLoader(dockerCli, o.Offline)
-	return []cli.ProjectOptionsFn{cli.WithResourceLoader(git), cli.WithResourceLoader(oci)}
+	return []loader.ResourceLoader{git, oci}
 }
 
 func (o *ProjectOptions) toProjectOptions(po ...cli.ProjectOptionsFn) (*cli.ProjectOptions, error) {

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Microsoft/go-winio v0.6.1
 	github.com/buger/goterm v1.0.4
-	github.com/compose-spec/compose-go/v2 v2.0.0-rc.6
+	github.com/compose-spec/compose-go/v2 v2.0.0-rc.7
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.7.12
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -86,8 +86,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
-github.com/compose-spec/compose-go/v2 v2.0.0-rc.6 h1:sv9W3U0IEYqgGqTbSDpU2c8cttWQmlbJ0D6jdt//Dv8=
-github.com/compose-spec/compose-go/v2 v2.0.0-rc.6/go.mod h1:bEPizBkIojlQ20pi2vNluBa58tevvj0Y18oUSHPyfdc=
+github.com/compose-spec/compose-go/v2 v2.0.0-rc.7 h1:koFIK+JwplWu1m/DscSO6MJw7hodaEHOaKQZPUSL4OY=
+github.com/compose-spec/compose-go/v2 v2.0.0-rc.7/go.mod h1:bEPizBkIojlQ20pi2vNluBa58tevvj0Y18oUSHPyfdc=
 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
 github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=

internal/tracing/attributes.go

@@ -24,8 +24,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/compose/v2/pkg/utils"
-
 	"github.com/compose-spec/compose-go/v2/types"
 	moby "github.com/docker/docker/api/types"
 	"go.opentelemetry.io/otel/attribute"
@@ -35,8 +33,13 @@ import (
 // SpanOptions is a small helper type to make it easy to share the options helpers between
 // downstream functions that accept slices of trace.SpanStartOption and trace.EventOption.
 type SpanOptions []trace.SpanStartEventOption
+
+type MetricsKey struct{}
+
 type Metrics struct {
-	CountExtends int
+	CountExtends        int
+	CountIncludesLocal  int
+	CountIncludesRemote int
 }
 
 func (s SpanOptions) SpanStartOptions() []trace.SpanStartOption {
@@ -75,7 +78,6 @@ func ProjectOptions(ctx context.Context, proj *types.Project) SpanOptions {
 		attribute.StringSlice("project.secrets", proj.SecretNames()),
 		attribute.StringSlice("project.configs", proj.ConfigNames()),
 		attribute.StringSlice("project.extensions", keys(proj.Extensions)),
-		attribute.StringSlice("project.includes", flattenIncludeReferences(proj.IncludeReferences)),
 		attribute.StringSlice("project.services.active", proj.ServiceNames()),
 		attribute.StringSlice("project.services.disabled", proj.DisabledServiceNames()),
 		attribute.StringSlice("project.services.build", proj.ServicesWithBuild()),
@@ -84,8 +86,10 @@ func ProjectOptions(ctx context.Context, proj *types.Project) SpanOptions {
 		attribute.StringSlice("project.services.capabilities.gpu", gpu),
 		attribute.StringSlice("project.services.capabilities.tpu", tpu),
 	}
-	if metrics, ok := ctx.Value(Metrics{}).(Metrics); ok {
+	if metrics, ok := ctx.Value(MetricsKey{}).(Metrics); ok {
 		attrs = append(attrs, attribute.Int("project.services.extends", metrics.CountExtends))
+		attrs = append(attrs, attribute.Int("project.includes.local", metrics.CountIncludesLocal))
+		attrs = append(attrs, attribute.Int("project.includes.remote", metrics.CountIncludesRemote))
 	}
 
 	if projHash, ok := projectHash(proj); ok {
@@ -168,16 +172,6 @@ func unixTimeAttr(key string, value int64) attribute.KeyValue {
 	return timeAttr(key, time.Unix(value, 0).UTC())
 }
 
-func flattenIncludeReferences(includeRefs map[string][]types.IncludeConfig) []string {
-	ret := utils.NewSet[string]()
-	for _, included := range includeRefs {
-		for i := range included {
-			ret.AddAll(included[i].Path...)
-		}
-	}
-	return ret.Elements()
-}
-
 // projectHash returns a checksum from the JSON encoding of the project.
 func projectHash(p *types.Project) (string, bool) {
 	if p == nil {