Merge pull request #1137 from jedevc/imagetools-multiple-repositories

Imagetools multiple repositories

build/build.go

@@ -925,7 +925,21 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
 
 							itpull := imagetools.New(imageopt)
 
-							dt, desc, err := itpull.Combine(ctx, names[0], descs)
+							ref, err := reference.ParseNormalizedNamed(names[0])
+							if err != nil {
+								return err
+							}
+							ref = reference.TagNameOnly(ref)
+
+							srcs := make([]*imagetools.Source, len(descs))
+							for i, desc := range descs {
+								srcs[i] = &imagetools.Source{
+									Desc: desc,
+									Ref:  ref,
+								}
+							}
+
+							dt, desc, err := itpull.Combine(ctx, srcs)
 							if err != nil {
 								return err
 							}

commands/imagetools/create.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -9,6 +10,7 @@ import (
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/distribution/reference"
 	"github.com/moby/buildkit/util/appcontext"
@@ -25,6 +27,7 @@ type createOptions struct {
 	tags         []string
 	dryrun       bool
 	actionAppend bool
+	progress     string
 }
 
 func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
@@ -78,18 +81,21 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	if len(repos) == 0 {
 		return errors.Errorf("no repositories specified, please set a reference in tag or source")
 	}
-	if len(repos) > 1 {
-		return errors.Errorf("multiple repositories currently not supported, found %v", repos)
-	}
 
-	var repo string
+	var defaultRepo *string
-	for r := range repos {
+	if len(repos) == 1 {
-		repo = r
+		for repo := range repos {
+			defaultRepo = &repo
+		}
 	}
 
 	for i, s := range srcs {
 		if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
-			n, err := reference.ParseNormalizedNamed(repo)
+			if defaultRepo == nil {
+				return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i])
+			}
+
+			n, err := reference.ParseNormalizedNamed(*defaultRepo)
 			if err != nil {
 				return err
 			}
@@ -143,7 +149,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 					if err != nil {
 						return err
 					}
-					srcs[i].Ref = nil
 					if srcs[i].Desc.Digest == "" {
 						srcs[i].Desc = desc
 					} else {
@@ -162,12 +167,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		}
 	}
 
-	descs := make([]ocispec.Descriptor, len(srcs))
+	dt, desc, err := r.Combine(ctx, srcs)
-	for i := range descs {
-		descs[i] = srcs[i].Desc
-	}
-
-	dt, desc, err := r.Combine(ctx, repo, descs)
 	if err != nil {
 		return err
 	}
@@ -180,23 +180,49 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	// new resolver cause need new auth
 	r = imagetools.New(imageopt)
 
+	ctx2, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+	printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
+
+	eg, _ := errgroup.WithContext(ctx)
+	pw := progress.WithPrefix(printer, "internal", true)
+
 	for _, t := range tags {
-		if err := r.Push(ctx, t, desc, dt); err != nil {
+		t := t
+		eg.Go(func() error {
+			return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
+				eg2, _ := errgroup.WithContext(ctx)
+				for _, s := range srcs {
+					if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
+						continue
+					}
+					s := s
+					eg2.Go(func() error {
+						sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String())))
+						return r.Copy(ctx, s, t)
+					})
+				}
+
+				if err := eg2.Wait(); err != nil {
 					return err
 				}
-		fmt.Println(t.String())
+				sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String())))
+				return r.Push(ctx, t, desc, dt)
+			})
+		})
 	}
 
-	return nil
+	err = eg.Wait()
+	err1 := printer.Wait()
+	if err == nil {
+		err = err1
 	}
 
-type src struct {
+	return err
-	Desc ocispec.Descriptor
-	Ref  reference.Named
 }
 
-func parseSources(in []string) ([]*src, error) {
+func parseSources(in []string) ([]*imagetools.Source, error) {
-	out := make([]*src, len(in))
+	out := make([]*imagetools.Source, len(in))
 	for i, in := range in {
 		s, err := parseSource(in)
 		if err != nil {
@@ -219,11 +245,11 @@ func parseRefs(in []string) ([]reference.Named, error) {
 	return refs, nil
 }
 
-func parseSource(in string) (*src, error) {
+func parseSource(in string) (*imagetools.Source, error) {
 	// source can be a digest, reference or a descriptor JSON
 	dgst, err := digest.Parse(in)
 	if err == nil {
-		return &src{
+		return &imagetools.Source{
 			Desc: ocispec.Descriptor{
 				Digest: dgst,
 			},
@@ -234,14 +260,14 @@ func parseSource(in string) (*src, error) {
 
 	ref, err := reference.ParseNormalizedNamed(in)
 	if err == nil {
-		return &src{
+		return &imagetools.Source{
 			Ref: ref,
 		}, nil
 	} else if !strings.HasPrefix(in, "{") {
 		return nil, err
 	}
 
-	var s src
+	var s imagetools.Source
 	if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -265,6 +291,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
 	flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
 	flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
 	flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
+	flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)
 
 	return cmd
 }

docs/reference/buildx_imagetools_create.md

@@ -15,6 +15,7 @@ Create a new image based on source images
 | [`--builder`](#builder) | `string` |  | Override the configured builder instance |
 | [`--dry-run`](#dry-run) |  |  | Show final image instead of pushing |
 | [`-f`](#file), [`--file`](#file) | `stringArray` |  | Read source descriptor from file |
+| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
 | [`-t`](#tag), [`--tag`](#tag) | `stringArray` |  | Set reference for new image |
 
 

util/imagetools/create.go

@@ -4,12 +4,15 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"net/url"
+	"strings"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/util/contentutil"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -17,46 +20,46 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) {
+type Source struct {
-	ref, err := parseRef(in)
+	Desc ocispec.Descriptor
-	if err != nil {
+	Ref  reference.Named
-		return nil, ocispec.Descriptor{}, err
 }
 
+func (r *Resolver) Combine(ctx context.Context, srcs []*Source) ([]byte, ocispec.Descriptor, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 
-	dts := make([][]byte, len(descs))
+	dts := make([][]byte, len(srcs))
 	for i := range dts {
 		func(i int) {
 			eg.Go(func() error {
-				dt, err := r.GetDescriptor(ctx, ref.String(), descs[i])
+				dt, err := r.GetDescriptor(ctx, srcs[i].Ref.String(), srcs[i].Desc)
 				if err != nil {
 					return err
 				}
 				dts[i] = dt
 
-				if descs[i].MediaType == "" {
+				if srcs[i].Desc.MediaType == "" {
 					mt, err := detectMediaType(dt)
 					if err != nil {
 						return err
 					}
-					descs[i].MediaType = mt
+					srcs[i].Desc.MediaType = mt
 				}
 
-				mt := descs[i].MediaType
+				mt := srcs[i].Desc.MediaType
 
 				switch mt {
 				case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
-					p := descs[i].Platform
+					p := srcs[i].Desc.Platform
-					if descs[i].Platform == nil {
+					if srcs[i].Desc.Platform == nil {
 						p = &ocispec.Platform{}
 					}
 					if p.OS == "" || p.Architecture == "" {
-						if err := r.loadPlatform(ctx, p, in, dt); err != nil {
+						if err := r.loadPlatform(ctx, p, srcs[i].Ref.String(), dt); err != nil {
 							return err
 						}
 					}
-					descs[i].Platform = p
+					srcs[i].Desc.Platform = p
 				case images.MediaTypeDockerSchema1Manifest:
 					return errors.Errorf("schema1 manifests are not allowed in manifest lists")
 				}
@@ -71,14 +74,14 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
 	}
 
 	// on single source, return original bytes
-	if len(descs) == 1 {
+	if len(srcs) == 1 {
-		if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
+		if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
-			return dts[0], descs[0], nil
+			return dts[0], srcs[0].Desc, nil
 		}
 	}
 
 	m := map[digest.Digest]int{}
-	newDescs := make([]ocispec.Descriptor, 0, len(descs))
+	newDescs := make([]ocispec.Descriptor, 0, len(srcs))
 
 	addDesc := func(d ocispec.Descriptor) {
 		idx, ok := m[d.Digest]
@@ -103,8 +106,8 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
 		}
 	}
 
-	for i, desc := range descs {
+	for i, src := range srcs {
-		switch desc.MediaType {
+		switch src.Desc.MediaType {
 		case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
 			var mfst ocispec.Index
 			if err := json.Unmarshal(dts[i], &mfst); err != nil {
@@ -114,7 +117,7 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
 				addDesc(d)
 			}
 		default:
-			addDesc(desc)
+			addDesc(src.Desc)
 		}
 	}
 
@@ -169,6 +172,37 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D
 	return err
 }
 
+func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) error {
+	dest = reference.TagNameOnly(dest)
+	p, err := r.resolver().Pusher(ctx, dest.String())
+	if err != nil {
+		return err
+	}
+
+	srcRef := reference.TagNameOnly(src.Ref)
+	f, err := r.resolver().Fetcher(ctx, srcRef.String())
+	if err != nil {
+		return err
+	}
+
+	refspec := reference.TrimNamed(src.Ref).String()
+	u, err := url.Parse("dummy://" + refspec)
+	if err != nil {
+		return err
+	}
+	source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/")
+	if src.Desc.Annotations == nil {
+		src.Desc.Annotations = make(map[string]string)
+	}
+	src.Desc.Annotations["containerd.io/distribution.source."+source] = repo
+
+	err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error {
 	var manifest ocispec.Manifest
 	if err := json.Unmarshal(dt, &manifest); err != nil {