Merge pull request #3058 from infosiftr/manifest-tool

Add an initial "manifest-tool" implementation

bashbrew/go/src/bashbrew/cmd-put-shared.go

@@ -4,11 +4,50 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"strings"
 
 	"github.com/codegangsta/cli"
+
+	"github.com/docker-library/go-dockerlibrary/architecture"
+	"github.com/docker-library/go-dockerlibrary/manifest"
 )
 
+func entriesToManifestToolYaml(r Repo, entries ...*manifest.Manifest2822Entry) (string, error) {
+	yaml := ""
+	entryIdentifiers := []string{}
+	for _, entry := range entries {
+		entryIdentifiers = append(entryIdentifiers, r.EntryIdentifier(*entry))
+
+		for _, arch := range entry.Architectures {
+			var ok bool
+
+			var ociArch architecture.OCIPlatform
+			if ociArch, ok = architecture.SupportedArches[arch]; !ok {
+				// skip unsupported arches
+				// TODO turn this into explicit validation checks at "parse" time instead (so that unsupported arches result in concrete user-facing errors long before this block of code)
+				continue
+			}
+
+			var archNamespace string
+			if archNamespace, ok = archNamespaces[arch]; !ok || archNamespace == "" {
+				fmt.Fprintf(os.Stderr, "warning: no arch-namespace specified for %q; skipping %q\n", arch, r.EntryIdentifier(*entry))
+				continue
+			}
+
+			yaml += fmt.Sprintf("  - image: %s/%s:%s\n    platform:\n", archNamespace, r.RepoName, entry.Tags[0])
+			yaml += fmt.Sprintf("      os: %s\n", ociArch.OS)
+			yaml += fmt.Sprintf("      architecture: %s\n", ociArch.Architecture)
+			if ociArch.Variant != "" {
+				yaml += fmt.Sprintf("      variant: %s\n", ociArch.Variant)
+			}
+		}
+	}
+	if yaml == "" {
+		return "", fmt.Errorf("failed gathering images for creating %q", entryIdentifiers)
+	}
+
+	return "manifests:\n" + yaml, nil
+}
+
 func cmdPutShared(c *cli.Context) error {
 	repos, err := repos(c.Bool("all"), c.Args()...)
 	if err != nil {
@@ -21,24 +60,55 @@ func cmdPutShared(c *cli.Context) error {
 		return fmt.Errorf(`"--namespace" is a required flag for "put-shared"`)
 	}
 
-	fmt.Fprintf(os.Stderr, "warning: this subcommand is still a big WIP -- it doesn't do anything yet!\n")
-
 	for _, repo := range repos {
 		r, err := fetch(repo)
 		if err != nil {
 			return cli.NewMultiError(fmt.Errorf(`failed fetching repo %q`, repo), err)
 		}
 
-		// TODO handle all multi-architecture tags first (regardless of whether they have SharedTags)
+		// handle all multi-architecture tags first (regardless of whether they have SharedTags)
+		for _, entry := range r.Entries() {
+			// "image:" will be added later so we don't have to regenerate the entire "manifests" section every time
+			yaml, err := entriesToManifestToolYaml(*r, &entry)
+			if err != nil {
+				return err
+			}
+
+			for _, tag := range r.Tags(namespace, false, entry) {
+				tagYaml := fmt.Sprintf("image: %s\n%s", tag, yaml)
+				fmt.Printf("Putting %s\n", tag)
+				if err := manifestToolPushFromSpec(tagYaml); err != nil {
+					return fmt.Errorf("failed pushing %q (%q)", tag, entry.TagsString())
+				}
+			}
+		}
+
+		// TODO do something better with r.TagName (ie, the user has done something crazy like "bashbrew put-shared single-repo:single-tag")
+		sharedTagGroups := r.Manifest.GetSharedTagGroups()
+		if len(sharedTagGroups) == 0 {
+			continue
+		}
+		if r.TagName != "" {
+			fmt.Fprintf(os.Stderr, "warning: a single tag was requested -- skipping SharedTags\n")
+			continue
+		}
 
 		targetRepo := path.Join(namespace, r.RepoName)
-		for _, group := range r.Manifest.GetSharedTagGroups() {
-			// TODO build up a YAML file
-			entryTags := []string{}
-			for _, entry := range group.Entries {
-				entryTags = append(entryTags, entry.Tags[0])
+		for _, group := range sharedTagGroups {
+			yaml, err := entriesToManifestToolYaml(*r, group.Entries...)
+			if err != nil {
+				return err
+			}
+
+			for _, tag := range group.SharedTags {
+				tag = targetRepo + ":" + tag
+
+				tagYaml := fmt.Sprintf("image: %s\n%s", tag, yaml)
+				fmt.Printf("Putting shared %s\n", tag)
+				if err := manifestToolPushFromSpec(tagYaml); err != nil {
+					return fmt.Errorf("failed pushing %s", tag)
+				}
 			}
-			fmt.Printf("Putting %s (tags %s) <= %s\n", targetRepo, strings.Join(group.SharedTags, ", "), strings.Join(entryTags, ", "))
 		}
 	}
 

bashbrew/go/src/bashbrew/config.go

@@ -18,8 +18,6 @@ type FlagsConfigEntry struct {
 
 	Commands []string `delim:"," strip:"\n\r\t "`
 
-	// TODO arch namespace mappings (for intermediate pushing before put-shared, and for put-shared to pull from to join together in one big happy family)
-
 	Library    string
 	Cache      string
 	Debug      string
@@ -32,6 +30,9 @@ type FlagsConfigEntry struct {
 	Constraints          []string `delim:"," strip:"\n\r\t "`
 	ExclusiveConstraints string
 	ApplyConstraints     string
+
+	// a list of "arch=namespace" mappings for pushing indexes (manifest lists)
+	ArchNamespaces []string `delim:"," strip:"\n\r\t "`
 }
 
 type FlagsConfig map[string]FlagsConfigEntry
@@ -70,6 +71,9 @@ func (dst *FlagsConfigEntry) Apply(src FlagsConfigEntry) {
 	if src.ApplyConstraints != "" {
 		dst.ApplyConstraints = src.ApplyConstraints
 	}
+	if len(src.ArchNamespaces) > 0 {
+		dst.ArchNamespaces = src.ArchNamespaces[:]
+	}
 }
 
 func (config FlagsConfigEntry) Vars() map[string]map[string]interface{} {
@@ -82,6 +86,8 @@ func (config FlagsConfigEntry) Vars() map[string]map[string]interface{} {
 			"arch":                  config.Arch,
 			"constraint":            config.Constraints,
 			"exclusive-constraints": config.ExclusiveConstraints,
+
+			"arch-namespace": config.ArchNamespaces,
 		},
 
 		"local": {

bashbrew/go/src/bashbrew/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/codegangsta/cli"
 
@@ -24,6 +25,8 @@ var (
 	constraints          []string
 	exclusiveConstraints bool
 
+	archNamespaces map[string]string
+
 	debugFlag  = false
 	noSortFlag = false
 
@@ -35,6 +38,9 @@ var (
 		"library": "BASHBREW_LIBRARY",
 		"cache":   "BASHBREW_CACHE",
 		"pull":    "BASHBREW_PULL",
+
+		"constraint":     "BASHBREW_CONSTRAINTS",
+		"arch-namespace": "BASHBREW_ARCH_NAMESPACES",
 	}
 )
 
@@ -84,14 +90,21 @@ func main() {
 			Usage:  "the current platform architecture",
 		},
 		cli.StringSliceFlag{
-			Name:  "constraint",
-			Usage: "build constraints (see Constraints in Manifest2822Entry)",
+			Name:   "constraint",
+			EnvVar: flagEnvVars["constraint"],
+			Usage:  "build constraints (see Constraints in Manifest2822Entry)",
 		},
 		cli.BoolFlag{
 			Name:  "exclusive-constraints",
 			Usage: "skip entries which do not have Constraints",
 		},
 
+		cli.StringSliceFlag{
+			Name:   "arch-namespace",
+			EnvVar: flagEnvVars["arch-namespace"],
+			Usage:  `architecture to push namespace mappings for creating indexes/manifest lists ("arch=namespace" ala "s390x=tianons390x")`,
+		},
+
 		cli.StringFlag{
 			Name:   "config",
 			Value:  initDefaultConfigPath(),
@@ -142,6 +155,13 @@ func main() {
 			constraints = c.GlobalStringSlice("constraint")
 			exclusiveConstraints = c.GlobalBool("exclusive-constraints")
 
+			archNamespaces = map[string]string{}
+			for _, archMapping := range c.GlobalStringSlice("arch-namespace") {
+				splitArchMapping := strings.SplitN(archMapping, "=", 2)
+				splitArch, splitNamespace := strings.TrimSpace(splitArchMapping[0]), strings.TrimSpace(splitArchMapping[1])
+				archNamespaces[splitArch] = splitNamespace
+			}
+
 			defaultLibrary, err = filepath.Abs(c.GlobalString("library"))
 			if err != nil {
 				return err

bashbrew/go/src/bashbrew/manifest-tool.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+func manifestToolPushFromSpec(yamlSpec string) error {
+	yamlFile, err := ioutil.TempFile("", "bashbrew-manifest-tool-yaml-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(yamlFile.Name())
+
+	if _, err := yamlFile.Write([]byte(yamlSpec)); err != nil {
+		return err
+	}
+	if err := yamlFile.Close(); err != nil {
+		return err
+	}
+
+	args := []string{"push", "from-spec", "--ignore-missing", yamlFile.Name()}
+	if debugFlag {
+		args = append([]string{"--debug"}, args...)
+		fmt.Printf("$ manifest-tool %q\n", args)
+	}
+	cmd := exec.Command("manifest-tool", args...)
+	cmd.Stderr = os.Stderr
+	if debugFlag {
+		cmd.Stdout = os.Stdout
+	}
+
+	return cmd.Run()
+}

bashbrew/go/src/bashbrew/repo.go

@@ -139,6 +139,7 @@ func (r Repo) Entries() []manifest.Manifest2822Entry {
 	if r.TagName == "" {
 		return r.Manifest.Entries
 	} else {
+		// TODO what if r.TagName isn't a single entry, but is a SharedTag ?
 		return []manifest.Manifest2822Entry{*r.Manifest.GetTag(r.TagName)}
 	}
 }

bashbrew/go/vendor/manifest

@@ -10,7 +10,7 @@
 		{
 			"importpath": "github.com/docker-library/go-dockerlibrary",
 			"repository": "https://github.com/docker-library/go-dockerlibrary",
-			"revision": "ce3ef0e05c16a5202b2c3dae35ef6a832eb18d7a",
+			"revision": "4fd80f3c84b66d3a62d907359f540c9417745f79",
 			"branch": "master"
 		},
 		{

bashbrew/go/vendor/src/github.com/docker-library/go-dockerlibrary/architecture/oci-platform.go

@@ -0,0 +1,25 @@
+package architecture
+
+// https://github.com/opencontainers/image-spec/blob/v1.0.0-rc6/image-index.md#image-index-property-descriptions
+// see "platform" (under "manifests")
+type OCIPlatform struct {
+	OS           string `json:"os"`
+	Architecture string `json:"architecture"`
+	Variant      string `json:"variant,omitempty"`
+
+	//OSVersion  string   `json:"os.version,omitempty"`
+	//OSFeatures []string `json:"os.features,omitempty"`
+}
+
+var SupportedArches = map[string]OCIPlatform{
+	"amd64":   {OS: "linux", Architecture: "amd64"},
+	"arm32v5": {OS: "linux", Architecture: "arm", Variant: "v5"},
+	"arm32v6": {OS: "linux", Architecture: "arm", Variant: "v6"},
+	"arm32v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
+	"arm64v8": {OS: "linux", Architecture: "arm64", Variant: "v8"},
+	"i386":    {OS: "linux", Architecture: "386"},
+	"ppc64le": {OS: "linux", Architecture: "ppc64le"},
+	"s390x":   {OS: "linux", Architecture: "s390x"},
+
+	"windows-amd64": {OS: "windows", Architecture: "amd64"},
+}