imagetools: add create support

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

commands/imagetools/create.go

@@ -1,27 +1,214 @@
 package commands
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/util/appcontext"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/tonistiigi/buildx/util/imagetools"
+	"golang.org/x/sync/errgroup"
 )
 
 type createOptions struct {
 	files        []string
 	tags         []string
 	dryrun       bool
-	append bool
+	actionAppend bool
 }
 
 func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
-	return errors.Errorf("not-implemented")
+	if len(args) == 0 && len(in.files) == 0 {
+		return errors.Errorf("no sources specified")
+	}
+
+	if !in.dryrun && len(in.tags) == 0 {
+		return errors.Errorf("can't push with no tags specified, please set --tag or --dry-run")
+	}
+
+	fileArgs := make([]string, len(in.files))
+	for i, f := range in.files {
+		dt, err := ioutil.ReadFile(f)
+		if err != nil {
+			return err
+		}
+		fileArgs[i] = string(dt)
+	}
+
+	args = append(fileArgs, args...)
+
+	tags, err := parseRefs(in.tags)
+	if err != nil {
+		return err
+	}
+
+	if in.actionAppend && len(in.tags) > 0 {
+		args = append([]string{in.tags[0]}, args...)
+	}
+
+	srcs, err := parseSources(args)
+	if err != nil {
+		return err
+	}
+
+	repos := map[string]struct{}{}
+
+	for _, t := range tags {
+		repos[t.Name()] = struct{}{}
+	}
+
+	sourceRefs := false
+	for _, s := range srcs {
+		if s.Ref != nil {
+			repos[s.Ref.Name()] = struct{}{}
+			sourceRefs = true
+		}
+	}
+
+	if len(repos) == 0 {
+		return errors.Errorf("no repositories specified, please set a reference in tag or source")
+	}
+	if len(repos) > 1 {
+		return errors.Errorf("multiple repositories currently not supported, found %v", repos)
+	}
+
+	var repo string
+	for r := range repos {
+		repo = r
+	}
+
+	for i, s := range srcs {
+		if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
+			n, err := reference.ParseNormalizedNamed(repo)
+			if err != nil {
+				return err
+			}
+			r, err := reference.WithDigest(n, s.Desc.Digest)
+			if err != nil {
+				return err
+			}
+			srcs[i].Ref = r
+			sourceRefs = true
+		}
+	}
+
+	ctx := appcontext.Context()
+
+	r := imagetools.New(imagetools.Opt{
+		Auth: dockerCli.ConfigFile(),
+	})
+
+	if sourceRefs {
+		eg, ctx2 := errgroup.WithContext(ctx)
+		for i, s := range srcs {
+			if s.Ref == nil {
+				continue
+			}
+			func(i int) {
+				eg.Go(func() error {
+					_, desc, err := r.Resolve(ctx2, srcs[i].Ref.String())
+					if err != nil {
+						return err
+					}
+					srcs[i].Ref = nil
+					srcs[i].Desc = desc
+					return nil
+				})
+			}(i)
+		}
+		if err := eg.Wait(); err != nil {
+			return err
+		}
+	}
+
+	descs := make([]ocispec.Descriptor, len(srcs))
+	for i := range descs {
+		descs[i] = srcs[i].Desc
+	}
+
+	dt, desc, err := r.Combine(ctx, repo, descs)
+	if err != nil {
+		return err
+	}
+	_ = desc
+
+	if in.dryrun {
+		fmt.Printf("%s\n", dt)
+	}
+
+	return nil
+}
+
+type src struct {
+	Desc ocispec.Descriptor
+	Ref  reference.Named
+}
+
+func parseSources(in []string) ([]*src, error) {
+	out := make([]*src, len(in))
+	for i, in := range in {
+		s, err := parseSource(in)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, refereces and descriptors", in)
+		}
+		out[i] = s
+	}
+	return out, nil
+}
+
+func parseRefs(in []string) ([]reference.Named, error) {
+	refs := make([]reference.Named, len(in))
+	for i, in := range in {
+		n, err := reference.ParseNormalizedNamed(in)
+		if err != nil {
+			return nil, err
+		}
+		refs[i] = n
+	}
+	return refs, nil
+}
+
+func parseSource(in string) (*src, error) {
+	// source can be a digest, reference or a descriptor JSON
+	dgst, err := digest.Parse(in)
+	if err == nil {
+		return &src{
+			Desc: ocispec.Descriptor{
+				Digest: dgst,
+			},
+		}, nil
+	} else if strings.HasPrefix(in, "sha256") {
+		return nil, err
+	}
+
+	ref, err := reference.ParseNormalizedNamed(in)
+	if err == nil {
+		return &src{
+			Ref: ref,
+		}, nil
+	} else if !strings.HasPrefix(in, "{") {
+		return nil, err
+	}
+
+	var s src
+	if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &s, nil
 }
 
 func createCmd(dockerCli command.Cli) *cobra.Command {
 	var options createOptions
 
 	cmd := &cobra.Command{
-		Use:   "create [OPTIONS] [SOURCE...]",
+		Use:   "create [OPTIONS] [SOURCE] [SOURCE...]",
 		Short: "Create a new image based on source images",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runCreate(dockerCli, options, args)
@@ -33,7 +220,7 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
 	flags.StringArrayVarP(&options.files, "file", "f", []string{}, "Read source descriptor from file")
 	flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
 	flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
-	flags.BoolVar(&options.append, "append", false, "Append to existing manifest")
+	flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
 
 	_ = flags
 

util/imagetools/create.go

@@ -0,0 +1,186 @@
+package imagetools
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/containerd/containerd/images"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) {
+	ref, err := parseRef(in)
+	if err != nil {
+		return nil, ocispec.Descriptor{}, err
+	}
+
+	eg, ctx := errgroup.WithContext(ctx)
+
+	dts := make([][]byte, len(descs))
+	for i := range dts {
+		func(i int) {
+			eg.Go(func() error {
+				dt, err := r.GetDescriptor(ctx, ref.String(), descs[i])
+				if err != nil {
+					return err
+				}
+				dts[i] = dt
+
+				if descs[i].MediaType == "" {
+					mt, err := detectMediaType(dt)
+					if err != nil {
+						return err
+					}
+					descs[i].MediaType = mt
+				}
+
+				mt := descs[i].MediaType
+
+				switch mt {
+				case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+					if descs[i].Platform == nil {
+						cfg, err := r.loadConfig(ctx, in, dt)
+						if err != nil {
+							return err
+						}
+						descs[i].Platform = &ocispec.Platform{
+							OS:           cfg.OS,
+							Architecture: cfg.Architecture,
+						}
+					}
+				case images.MediaTypeDockerSchema1Manifest:
+					return errors.Errorf("schema1 manifests are not allowed in manifest lists")
+				}
+
+				return nil
+			})
+		}(i)
+	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, ocispec.Descriptor{}, err
+	}
+
+	// on single source, return original bytes
+	if len(descs) == 1 {
+		if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
+			return dts[0], descs[0], nil
+		}
+	}
+
+	m := map[digest.Digest]int{}
+	newDescs := make([]ocispec.Descriptor, 0, len(descs))
+
+	addDesc := func(d ocispec.Descriptor) {
+		idx, ok := m[d.Digest]
+		if ok {
+			old := newDescs[idx]
+			if old.MediaType == "" {
+				old.MediaType = d.MediaType
+			}
+			if d.Platform != nil {
+				old.Platform = d.Platform
+			}
+			if old.Annotations == nil {
+				old.Annotations = map[string]string{}
+			}
+			for k, v := range d.Annotations {
+				old.Annotations[k] = v
+			}
+			newDescs[idx] = old
+		} else {
+			m[d.Digest] = len(newDescs)
+			newDescs = append(newDescs, d)
+		}
+	}
+
+	for i, desc := range descs {
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+			var mfst ocispec.Index
+			if err := json.Unmarshal(dts[i], &mfst); err != nil {
+				return nil, ocispec.Descriptor{}, errors.WithStack(err)
+			}
+			for _, d := range mfst.Manifests {
+				addDesc(d)
+			}
+		default:
+			addDesc(desc)
+		}
+	}
+
+	mt := images.MediaTypeDockerSchema2ManifestList //ocispec.MediaTypeImageIndex
+	idx := struct {
+		// MediaType is reserved in the OCI spec but
+		// excluded from go types.
+		MediaType string `json:"mediaType,omitempty"`
+
+		ocispec.Index
+	}{
+		MediaType: mt,
+		Index: ocispec.Index{
+			Versioned: specs.Versioned{
+				SchemaVersion: 2,
+			},
+			Manifests: newDescs,
+		},
+	}
+
+	idxBytes, err := json.MarshalIndent(idx, "", "   ")
+	if err != nil {
+		return nil, ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal index")
+	}
+
+	return idxBytes, ocispec.Descriptor{
+		MediaType: mt,
+		Size:      int64(len(idxBytes)),
+		Digest:    digest.FromBytes(idxBytes),
+	}, nil
+}
+
+func (r *Resolver) loadConfig(ctx context.Context, in string, dt []byte) (*ocispec.Image, error) {
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(dt, &manifest); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	dt, err := r.GetDescriptor(ctx, in, manifest.Config)
+	if err != nil {
+		return nil, err
+	}
+
+	var img ocispec.Image
+	if err := json.Unmarshal(dt, &img); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return &img, nil
+}
+
+func detectMediaType(dt []byte) (string, error) {
+	var mfst struct {
+		MediaType string          `json:"mediaType"`
+		Config    json.RawMessage `json:"config"`
+		FSLayers  []string        `json:"fsLayers"`
+	}
+
+	if err := json.Unmarshal(dt, &mfst); err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	if mfst.MediaType != "" {
+		return mfst.MediaType, nil
+	}
+	if mfst.Config != nil {
+		return images.MediaTypeDockerSchema2Manifest, nil
+	}
+	if len(mfst.FSLayers) > 0 {
+		return images.MediaTypeDockerSchema1Manifest, nil
+	}
+
+	return images.MediaTypeDockerSchema2ManifestList, nil
+}

util/imagetools/inspect.go

@@ -35,35 +35,52 @@ func New(opt Opt) *Resolver {
 	}
 }
 
-func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) {
+func (r *Resolver) Resolve(ctx context.Context, in string) (string, ocispec.Descriptor, error) {
 	ref, err := parseRef(in)
 	if err != nil {
-		return nil, ocispec.Descriptor{}, err
+		return "", ocispec.Descriptor{}, err
 	}
 
 	in, desc, err := r.r.Resolve(ctx, ref.String())
+	if err != nil {
+		return "", ocispec.Descriptor{}, err
+	}
+
+	return in, desc, nil
+}
+
+func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) {
+	in, desc, err := r.Resolve(ctx, in)
 	if err != nil {
 		return nil, ocispec.Descriptor{}, err
 	}
 
-	fetcher, err := r.r.Fetcher(ctx, in)
+	dt, err := r.GetDescriptor(ctx, in, desc)
 	if err != nil {
 		return nil, ocispec.Descriptor{}, err
 	}
+	return dt, desc, nil
+}
+
+func (r *Resolver) GetDescriptor(ctx context.Context, in string, desc ocispec.Descriptor) ([]byte, error) {
+	fetcher, err := r.r.Fetcher(ctx, in)
+	if err != nil {
+		return nil, err
+	}
 
 	rc, err := fetcher.Fetch(ctx, desc)
 	if err != nil {
-		return nil, ocispec.Descriptor{}, err
+		return nil, err
 	}
 
 	buf := &bytes.Buffer{}
 	_, err = io.Copy(buf, rc)
 	rc.Close()
 	if err != nil {
-		return nil, ocispec.Descriptor{}, err
+		return nil, err
 	}
 
-	return buf.Bytes(), desc, nil
+	return buf.Bytes(), nil
 }
 
 func parseRef(s string) (reference.Named, error) {