imagetools inspect: add --format flag

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

vendor/github.com/moby/buildkit/util/buildinfo/types/types.go

@@ -0,0 +1,52 @@
+package binfotypes
+
+import (
+	srctypes "github.com/moby/buildkit/source/types"
+)
+
+// ImageConfigField defines the key of build dependencies.
+const ImageConfigField = "moby.buildkit.buildinfo.v1"
+
+// ImageConfig defines the structure of build dependencies
+// inside image config.
+type ImageConfig struct {
+	BuildInfo string `json:"moby.buildkit.buildinfo.v1,omitempty"`
+}
+
+// BuildInfo defines the main structure added to image config as
+// ImageConfigField key and returned in solver ExporterResponse as
+// exptypes.ExporterBuildInfo key.
+type BuildInfo struct {
+	// Frontend defines the frontend used to build.
+	Frontend string `json:"frontend,omitempty"`
+	// Attrs defines build request attributes.
+	Attrs map[string]*string `json:"attrs,omitempty"`
+	// Sources defines build dependencies.
+	Sources []Source `json:"sources,omitempty"`
+	// Deps defines context dependencies.
+	Deps map[string]BuildInfo `json:"deps,omitempty"`
+}
+
+// Source defines a build dependency.
+type Source struct {
+	// Type defines the SourceType source type (docker-image, git, http).
+	Type SourceType `json:"type,omitempty"`
+	// Ref is the reference of the source.
+	Ref string `json:"ref,omitempty"`
+	// Alias is a special field used to match with the actual source ref
+	// because frontend might have already transformed a string user typed
+	// before generating LLB.
+	Alias string `json:"alias,omitempty"`
+	// Pin is the source digest.
+	Pin string `json:"pin,omitempty"`
+}
+
+// SourceType contains source type.
+type SourceType string
+
+// List of source types.
+const (
+	SourceTypeDockerImage SourceType = srctypes.DockerImageScheme
+	SourceTypeGit         SourceType = srctypes.GitScheme
+	SourceTypeHTTP        SourceType = srctypes.HTTPScheme
+)

vendor/github.com/moby/buildkit/util/contentutil/buffer.go

@@ -0,0 +1,156 @@
+package contentutil
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// Buffer is a content provider and ingester that keeps data in memory
+type Buffer interface {
+	content.Provider
+	content.Ingester
+}
+
+// NewBuffer returns a new buffer
+func NewBuffer() Buffer {
+	return &buffer{
+		buffers: map[digest.Digest][]byte{},
+		refs:    map[string]struct{}{},
+	}
+}
+
+type buffer struct {
+	mu      sync.Mutex
+	buffers map[digest.Digest][]byte
+	refs    map[string]struct{}
+}
+
+func (b *buffer) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	var wOpts content.WriterOpts
+	for _, opt := range opts {
+		if err := opt(&wOpts); err != nil {
+			return nil, err
+		}
+	}
+	b.mu.Lock()
+	if _, ok := b.refs[wOpts.Ref]; ok {
+		return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked", wOpts.Ref)
+	}
+	b.mu.Unlock()
+	return &bufferedWriter{
+		main:     b,
+		digester: digest.Canonical.Digester(),
+		buffer:   bytes.NewBuffer(nil),
+		expected: wOpts.Desc.Digest,
+		releaseRef: func() {
+			b.mu.Lock()
+			delete(b.refs, wOpts.Ref)
+			b.mu.Unlock()
+		},
+	}, nil
+}
+
+func (b *buffer) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	r, err := b.getBytesReader(ctx, desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+	return &readerAt{Reader: r, Closer: ioutil.NopCloser(r), size: int64(r.Len())}, nil
+}
+
+func (b *buffer) getBytesReader(ctx context.Context, dgst digest.Digest) (*bytes.Reader, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if dt, ok := b.buffers[dgst]; ok {
+		return bytes.NewReader(dt), nil
+	}
+
+	return nil, errors.Wrapf(errdefs.ErrNotFound, "content %v", dgst)
+}
+
+func (b *buffer) addValue(k digest.Digest, dt []byte) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.buffers[k] = dt
+}
+
+type bufferedWriter struct {
+	main       *buffer
+	ref        string
+	offset     int64
+	total      int64
+	startedAt  time.Time
+	updatedAt  time.Time
+	buffer     *bytes.Buffer
+	expected   digest.Digest
+	digester   digest.Digester
+	releaseRef func()
+}
+
+func (w *bufferedWriter) Write(p []byte) (n int, err error) {
+	n, err = w.buffer.Write(p)
+	w.digester.Hash().Write(p[:n])
+	w.offset += int64(len(p))
+	w.updatedAt = time.Now()
+	return n, err
+}
+
+func (w *bufferedWriter) Close() error {
+	if w.buffer != nil {
+		w.releaseRef()
+		w.buffer = nil
+	}
+	return nil
+}
+
+func (w *bufferedWriter) Status() (content.Status, error) {
+	return content.Status{
+		Ref:       w.ref,
+		Offset:    w.offset,
+		Total:     w.total,
+		StartedAt: w.startedAt,
+		UpdatedAt: w.updatedAt,
+	}, nil
+}
+
+func (w *bufferedWriter) Digest() digest.Digest {
+	return w.digester.Digest()
+}
+
+func (w *bufferedWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opt ...content.Opt) error {
+	if w.buffer == nil {
+		return errors.Errorf("can't commit already committed or closed")
+	}
+	if s := int64(w.buffer.Len()); size > 0 && size != s {
+		return errors.Errorf("unexpected commit size %d, expected %d", s, size)
+	}
+	dgst := w.digester.Digest()
+	if expected != "" && expected != dgst {
+		return errors.Errorf("unexpected digest: %v != %v", dgst, expected)
+	}
+	if w.expected != "" && w.expected != dgst {
+		return errors.Errorf("unexpected digest: %v != %v", dgst, w.expected)
+	}
+	w.main.addValue(dgst, w.buffer.Bytes())
+	return w.Close()
+}
+
+func (w *bufferedWriter) Truncate(size int64) error {
+	if size != 0 {
+		return errors.New("Truncate: unsupported size")
+	}
+	w.offset = 0
+	w.digester.Hash().Reset()
+	w.buffer.Reset()
+	return nil
+}

vendor/github.com/moby/buildkit/util/contentutil/copy.go

@@ -0,0 +1,94 @@
+package contentutil
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/moby/buildkit/util/resolver/limited"
+	"github.com/moby/buildkit/util/resolver/retryhandler"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func Copy(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispecs.Descriptor, ref string, logger func([]byte)) error {
+	if _, err := retryhandler.New(limited.FetchHandler(ingester, &localFetcher{provider}, ref), logger)(ctx, desc); err != nil {
+		return err
+	}
+	return nil
+}
+
+type localFetcher struct {
+	content.Provider
+}
+
+func (f *localFetcher) Fetch(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) {
+	r, err := f.Provider.ReaderAt(ctx, desc)
+	if err != nil {
+		return nil, err
+	}
+	return &rc{ReaderAt: r}, nil
+}
+
+type rc struct {
+	content.ReaderAt
+	offset int64
+}
+
+func (r *rc) Read(b []byte) (int, error) {
+	n, err := r.ReadAt(b, r.offset)
+	r.offset += int64(n)
+	if n > 0 && err == io.EOF {
+		err = nil
+	}
+	return n, err
+}
+
+func (r *rc) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		r.offset = offset
+	case io.SeekCurrent:
+		r.offset += offset
+	case io.SeekEnd:
+		r.offset = r.Size() - offset
+	}
+	return r.offset, nil
+}
+
+func CopyChain(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispecs.Descriptor) error {
+	var m sync.Mutex
+	manifestStack := []ocispecs.Descriptor{}
+
+	filterHandler := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest,
+			images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
+			m.Lock()
+			manifestStack = append(manifestStack, desc)
+			m.Unlock()
+			return nil, images.ErrStopHandler
+		default:
+			return nil, nil
+		}
+	})
+	handlers := []images.Handler{
+		images.ChildrenHandler(provider),
+		filterHandler,
+		retryhandler.New(limited.FetchHandler(ingester, &localFetcher{provider}, ""), func(_ []byte) {}),
+	}
+
+	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
+		return errors.WithStack(err)
+	}
+
+	for i := len(manifestStack) - 1; i >= 0; i-- {
+		if err := Copy(ctx, ingester, provider, manifestStack[i], "", nil); err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return nil
+}

vendor/github.com/moby/buildkit/util/contentutil/fetcher.go

@@ -0,0 +1,73 @@
+package contentutil
+
+import (
+	"context"
+	"io"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/remotes"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func FromFetcher(f remotes.Fetcher) content.Provider {
+	return &fetchedProvider{
+		f: f,
+	}
+}
+
+type fetchedProvider struct {
+	f remotes.Fetcher
+}
+
+func (p *fetchedProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	rc, err := p.f.Fetch(ctx, desc)
+	if err != nil {
+		return nil, err
+	}
+
+	return &readerAt{Reader: rc, Closer: rc, size: desc.Size}, nil
+}
+
+type readerAt struct {
+	io.Reader
+	io.Closer
+	size   int64
+	offset int64
+}
+
+func (r *readerAt) ReadAt(b []byte, off int64) (int, error) {
+	if ra, ok := r.Reader.(io.ReaderAt); ok {
+		return ra.ReadAt(b, off)
+	}
+
+	if r.offset != off {
+		if seeker, ok := r.Reader.(io.Seeker); ok {
+			if _, err := seeker.Seek(off, io.SeekStart); err != nil {
+				return 0, err
+			}
+			r.offset = off
+		} else {
+			return 0, errors.Errorf("unsupported offset")
+		}
+	}
+
+	var totalN int
+	for len(b) > 0 {
+		n, err := r.Reader.Read(b)
+		if err == io.EOF && n == len(b) {
+			err = nil
+		}
+		r.offset += int64(n)
+		totalN += n
+		b = b[n:]
+		if err != nil {
+			return totalN, err
+		}
+	}
+	return totalN, nil
+}
+
+func (r *readerAt) Size() int64 {
+	return r.size
+}

vendor/github.com/moby/buildkit/util/contentutil/multiprovider.go

@@ -0,0 +1,92 @@
+package contentutil
+
+import (
+	"context"
+	"sync"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// NewMultiProvider creates a new mutable provider with a base provider
+func NewMultiProvider(base content.Provider) *MultiProvider {
+	return &MultiProvider{
+		base: base,
+		sub:  map[digest.Digest]content.Provider{},
+	}
+}
+
+// MultiProvider is a provider backed by a mutable map of providers
+type MultiProvider struct {
+	mu   sync.RWMutex
+	base content.Provider
+	sub  map[digest.Digest]content.Provider
+}
+
+func (mp *MultiProvider) SnapshotLabels(descs []ocispecs.Descriptor, index int) map[string]string {
+	if len(descs) < index {
+		return nil
+	}
+	desc := descs[index]
+	type snapshotLabels interface {
+		SnapshotLabels([]ocispecs.Descriptor, int) map[string]string
+	}
+
+	mp.mu.RLock()
+	if p, ok := mp.sub[desc.Digest]; ok {
+		mp.mu.RUnlock()
+		if cd, ok := p.(snapshotLabels); ok {
+			return cd.SnapshotLabels(descs, index)
+		}
+	} else {
+		mp.mu.RUnlock()
+	}
+	if cd, ok := mp.base.(snapshotLabels); ok {
+		return cd.SnapshotLabels(descs, index)
+	}
+	return nil
+}
+
+func (mp *MultiProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
+	type checkDescriptor interface {
+		CheckDescriptor(context.Context, ocispecs.Descriptor) error
+	}
+
+	mp.mu.RLock()
+	if p, ok := mp.sub[desc.Digest]; ok {
+		mp.mu.RUnlock()
+		if cd, ok := p.(checkDescriptor); ok {
+			return cd.CheckDescriptor(ctx, desc)
+		}
+	} else {
+		mp.mu.RUnlock()
+	}
+	if cd, ok := mp.base.(checkDescriptor); ok {
+		return cd.CheckDescriptor(ctx, desc)
+	}
+	return nil
+}
+
+// ReaderAt returns a content.ReaderAt
+func (mp *MultiProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	mp.mu.RLock()
+	if p, ok := mp.sub[desc.Digest]; ok {
+		mp.mu.RUnlock()
+		return p.ReaderAt(ctx, desc)
+	}
+	mp.mu.RUnlock()
+	if mp.base == nil {
+		return nil, errors.Wrapf(errdefs.ErrNotFound, "content %v", desc.Digest)
+	}
+	return mp.base.ReaderAt(ctx, desc)
+}
+
+// Add adds a new child provider for a specific digest
+func (mp *MultiProvider) Add(dgst digest.Digest, p content.Provider) {
+	mp.mu.Lock()
+	defer mp.mu.Unlock()
+	mp.sub[dgst] = p
+}

vendor/github.com/moby/buildkit/util/contentutil/pusher.go

@@ -0,0 +1,122 @@
+package contentutil
+
+import (
+	"context"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/remotes"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+func FromPusher(p remotes.Pusher) content.Ingester {
+	var mu sync.Mutex
+	c := sync.NewCond(&mu)
+	return &pushingIngester{
+		mu:     &mu,
+		c:      c,
+		p:      p,
+		active: map[digest.Digest]struct{}{},
+	}
+}
+
+type pushingIngester struct {
+	p remotes.Pusher
+
+	mu     *sync.Mutex
+	c      *sync.Cond
+	active map[digest.Digest]struct{}
+}
+
+// Writer implements content.Ingester. desc.MediaType must be set for manifest blobs.
+func (i *pushingIngester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	var wOpts content.WriterOpts
+	for _, opt := range opts {
+		if err := opt(&wOpts); err != nil {
+			return nil, err
+		}
+	}
+	if wOpts.Ref == "" {
+		return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty")
+	}
+
+	st := time.Now()
+
+	i.mu.Lock()
+	for {
+		if time.Since(st) > time.Hour {
+			i.mu.Unlock()
+			return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %v locked", wOpts.Desc.Digest)
+		}
+		if _, ok := i.active[wOpts.Desc.Digest]; ok {
+			i.c.Wait()
+		} else {
+			break
+		}
+	}
+
+	i.active[wOpts.Desc.Digest] = struct{}{}
+	i.mu.Unlock()
+
+	var once sync.Once
+	release := func() {
+		once.Do(func() {
+			i.mu.Lock()
+			delete(i.active, wOpts.Desc.Digest)
+			i.c.Broadcast()
+			i.mu.Unlock()
+		})
+	}
+
+	// pusher requires desc.MediaType to determine the PUT URL, especially for manifest blobs.
+	contentWriter, err := i.p.Push(ctx, wOpts.Desc)
+	if err != nil {
+		release()
+		return nil, err
+	}
+	runtime.SetFinalizer(contentWriter, func(_ content.Writer) {
+		release()
+	})
+	return &writer{
+		Writer:           contentWriter,
+		contentWriterRef: wOpts.Ref,
+		release:          release,
+	}, nil
+}
+
+type writer struct {
+	content.Writer          // returned from pusher.Push
+	contentWriterRef string // ref passed for Writer()
+	release          func()
+}
+
+func (w *writer) Status() (content.Status, error) {
+	st, err := w.Writer.Status()
+	if err != nil {
+		return st, err
+	}
+	if w.contentWriterRef != "" {
+		st.Ref = w.contentWriterRef
+	}
+	return st, nil
+}
+
+func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
+	err := w.Writer.Commit(ctx, size, expected, opts...)
+	if w.release != nil {
+		w.release()
+	}
+	return err
+}
+
+func (w *writer) Close() error {
+	err := w.Writer.Close()
+	if w.release != nil {
+		w.release()
+	}
+	return err
+}

vendor/github.com/moby/buildkit/util/contentutil/refs.go

@@ -0,0 +1,109 @@
+package contentutil
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/moby/buildkit/version"
+	"github.com/moby/locker"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func ProviderFromRef(ref string) (ocispecs.Descriptor, content.Provider, error) {
+	headers := http.Header{}
+	headers.Set("User-Agent", version.UserAgent())
+	remote := docker.NewResolver(docker.ResolverOptions{
+		Client:  http.DefaultClient,
+		Headers: headers,
+	})
+
+	name, desc, err := remote.Resolve(context.TODO(), ref)
+	if err != nil {
+		return ocispecs.Descriptor{}, nil, err
+	}
+
+	fetcher, err := remote.Fetcher(context.TODO(), name)
+	if err != nil {
+		return ocispecs.Descriptor{}, nil, err
+	}
+	return desc, FromFetcher(fetcher), nil
+}
+
+func IngesterFromRef(ref string) (content.Ingester, error) {
+	headers := http.Header{}
+	headers.Set("User-Agent", version.UserAgent())
+	remote := docker.NewResolver(docker.ResolverOptions{
+		Client:  http.DefaultClient,
+		Headers: headers,
+	})
+
+	p, err := remote.Pusher(context.TODO(), ref)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ingester{
+		locker: locker.New(),
+		pusher: &pusher{p},
+	}, nil
+}
+
+type pusher struct {
+	remotes.Pusher
+}
+
+type ingester struct {
+	locker *locker.Locker
+	pusher remotes.Pusher
+}
+
+func (w *ingester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	var wo content.WriterOpts
+	for _, o := range opts {
+		if err := o(&wo); err != nil {
+			return nil, err
+		}
+	}
+	if wo.Ref == "" {
+		return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty")
+	}
+	w.locker.Lock(wo.Ref)
+	var once sync.Once
+	unlock := func() {
+		once.Do(func() {
+			w.locker.Unlock(wo.Ref)
+		})
+	}
+	writer, err := w.pusher.Push(ctx, wo.Desc)
+	if err != nil {
+		unlock()
+		return nil, err
+	}
+	return &lockedWriter{unlock: unlock, Writer: writer}, nil
+}
+
+type lockedWriter struct {
+	unlock func()
+	content.Writer
+}
+
+func (w *lockedWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
+	err := w.Writer.Commit(ctx, size, expected, opts...)
+	if err == nil {
+		w.unlock()
+	}
+	return err
+}
+
+func (w *lockedWriter) Close() error {
+	err := w.Writer.Close()
+	w.unlock()
+	return err
+}

vendor/github.com/moby/buildkit/util/imageutil/config.go

@@ -0,0 +1,222 @@
+package imageutil
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/moby/buildkit/util/leaseutil"
+	"github.com/moby/buildkit/util/resolver/limited"
+	"github.com/moby/buildkit/util/resolver/retryhandler"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type ContentCache interface {
+	content.Ingester
+	content.Provider
+}
+
+var leasesMu sync.Mutex
+var leasesF []func(context.Context) error
+
+func CancelCacheLeases() {
+	leasesMu.Lock()
+	for _, f := range leasesF {
+		f(context.TODO())
+	}
+	leasesF = nil
+	leasesMu.Unlock()
+}
+
+func AddLease(f func(context.Context) error) {
+	leasesMu.Lock()
+	leasesF = append(leasesF, f)
+	leasesMu.Unlock()
+}
+
+func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, leaseManager leases.Manager, p *ocispecs.Platform) (digest.Digest, []byte, error) {
+	// TODO: fix buildkit to take interface instead of struct
+	var platform platforms.MatchComparer
+	if p != nil {
+		platform = platforms.Only(*p)
+	} else {
+		platform = platforms.Default()
+	}
+	ref, err := reference.Parse(str)
+	if err != nil {
+		return "", nil, errors.WithStack(err)
+	}
+
+	if leaseManager != nil {
+		ctx2, done, err := leaseutil.WithLease(ctx, leaseManager, leases.WithExpiration(5*time.Minute), leaseutil.MakeTemporary)
+		if err != nil {
+			return "", nil, errors.WithStack(err)
+		}
+		ctx = ctx2
+		defer func() {
+			// this lease is not deleted to allow other components to access manifest/config from cache. It will be deleted after 5 min deadline or on pruning inactive builder
+			AddLease(done)
+		}()
+	}
+
+	desc := ocispecs.Descriptor{
+		Digest: ref.Digest(),
+	}
+	if desc.Digest != "" {
+		ra, err := cache.ReaderAt(ctx, desc)
+		if err == nil {
+			desc.Size = ra.Size()
+			mt, err := DetectManifestMediaType(ra)
+			if err == nil {
+				desc.MediaType = mt
+			}
+		}
+	}
+	// use resolver if desc is incomplete
+	if desc.MediaType == "" {
+		_, desc, err = resolver.Resolve(ctx, ref.String())
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
+	fetcher, err := resolver.Fetcher(ctx, ref.String())
+	if err != nil {
+		return "", nil, err
+	}
+
+	if desc.MediaType == images.MediaTypeDockerSchema1Manifest {
+		return readSchema1Config(ctx, ref.String(), desc, fetcher, cache)
+	}
+
+	children := childrenConfigHandler(cache, platform)
+
+	handlers := []images.Handler{
+		retryhandler.New(limited.FetchHandler(cache, fetcher, str), func(_ []byte) {}),
+		children,
+	}
+	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
+		return "", nil, err
+	}
+	config, err := images.Config(ctx, cache, desc, platform)
+	if err != nil {
+		return "", nil, err
+	}
+
+	dt, err := content.ReadBlob(ctx, cache, config)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return desc.Digest, dt, nil
+}
+
+func childrenConfigHandler(provider content.Provider, platform platforms.MatchComparer) images.HandlerFunc {
+	return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
+		var descs []ocispecs.Descriptor
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest:
+			p, err := content.ReadBlob(ctx, provider, desc)
+			if err != nil {
+				return nil, err
+			}
+
+			// TODO(stevvooe): We just assume oci manifest, for now. There may be
+			// subtle differences from the docker version.
+			var manifest ocispecs.Manifest
+			if err := json.Unmarshal(p, &manifest); err != nil {
+				return nil, err
+			}
+
+			descs = append(descs, manifest.Config)
+		case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
+			p, err := content.ReadBlob(ctx, provider, desc)
+			if err != nil {
+				return nil, err
+			}
+
+			var index ocispecs.Index
+			if err := json.Unmarshal(p, &index); err != nil {
+				return nil, err
+			}
+
+			if platform != nil {
+				for _, d := range index.Manifests {
+					if d.Platform == nil || platform.Match(*d.Platform) {
+						descs = append(descs, d)
+					}
+				}
+			} else {
+				descs = append(descs, index.Manifests...)
+			}
+		case images.MediaTypeDockerSchema2Config, ocispecs.MediaTypeImageConfig, docker.LegacyConfigMediaType:
+			// childless data types.
+			return nil, nil
+		default:
+			return nil, errors.Errorf("encountered unknown type %v; children may not be fetched", desc.MediaType)
+		}
+
+		return descs, nil
+	}
+}
+
+// specs.MediaTypeImageManifest, // TODO: detect schema1/manifest-list
+func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
+	// TODO: schema1
+
+	dt := make([]byte, ra.Size())
+	if _, err := ra.ReadAt(dt, 0); err != nil {
+		return "", err
+	}
+
+	return DetectManifestBlobMediaType(dt)
+}
+
+func DetectManifestBlobMediaType(dt []byte) (string, error) {
+	var mfst struct {
+		MediaType *string         `json:"mediaType"`
+		Config    json.RawMessage `json:"config"`
+		Manifests json.RawMessage `json:"manifests"`
+		Layers    json.RawMessage `json:"layers"`
+	}
+
+	if err := json.Unmarshal(dt, &mfst); err != nil {
+		return "", err
+	}
+
+	mt := images.MediaTypeDockerSchema2ManifestList
+
+	if mfst.Config != nil || mfst.Layers != nil {
+		mt = images.MediaTypeDockerSchema2Manifest
+
+		if mfst.Manifests != nil {
+			return "", errors.Errorf("invalid ambiguous manifest and manifest list")
+		}
+	}
+
+	if mfst.MediaType != nil {
+		switch *mfst.MediaType {
+		case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
+			if mt != images.MediaTypeDockerSchema2ManifestList {
+				return "", errors.Errorf("mediaType in manifest does not match manifest contents")
+			}
+			mt = *mfst.MediaType
+		case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest:
+			if mt != images.MediaTypeDockerSchema2Manifest {
+				return "", errors.Errorf("mediaType in manifest does not match manifest contents")
+			}
+			mt = *mfst.MediaType
+		}
+	}
+	return mt, nil
+}

vendor/github.com/moby/buildkit/util/imageutil/schema1.go

@@ -0,0 +1,87 @@
+package imageutil
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/remotes"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func readSchema1Config(ctx context.Context, ref string, desc ocispecs.Descriptor, fetcher remotes.Fetcher, cache ContentCache) (digest.Digest, []byte, error) {
+	rc, err := fetcher.Fetch(ctx, desc)
+	if err != nil {
+		return "", nil, err
+	}
+	defer rc.Close()
+	dt, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to fetch schema1 manifest")
+	}
+	dt, err = convertSchema1ConfigMeta(dt)
+	if err != nil {
+		return "", nil, err
+	}
+	return desc.Digest, dt, nil
+}
+
+func convertSchema1ConfigMeta(in []byte) ([]byte, error) {
+	type history struct {
+		V1Compatibility string `json:"v1Compatibility"`
+	}
+	var m struct {
+		History []history `json:"history"`
+	}
+	if err := json.Unmarshal(in, &m); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal schema1 manifest")
+	}
+	if len(m.History) == 0 {
+		return nil, errors.Errorf("invalid schema1 manifest")
+	}
+
+	var img ocispecs.Image
+	if err := json.Unmarshal([]byte(m.History[0].V1Compatibility), &img); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal image from schema 1 history")
+	}
+
+	img.RootFS = ocispecs.RootFS{
+		Type: "layers", // filled in by exporter
+	}
+	img.History = make([]ocispecs.History, len(m.History))
+
+	for i := range m.History {
+		var h v1History
+		if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &h); err != nil {
+			return nil, errors.Wrap(err, "failed to unmarshal history")
+		}
+		img.History[len(m.History)-i-1] = ocispecs.History{
+			Author:     h.Author,
+			Comment:    h.Comment,
+			Created:    &h.Created,
+			CreatedBy:  strings.Join(h.ContainerConfig.Cmd, " "),
+			EmptyLayer: (h.ThrowAway != nil && *h.ThrowAway) || (h.Size != nil && *h.Size == 0),
+		}
+	}
+
+	dt, err := json.MarshalIndent(img, "", "   ")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal schema1 config")
+	}
+	return dt, nil
+}
+
+type v1History struct {
+	Author          string    `json:"author,omitempty"`
+	Created         time.Time `json:"created"`
+	Comment         string    `json:"comment,omitempty"`
+	ThrowAway       *bool     `json:"throwaway,omitempty"`
+	Size            *int      `json:"Size,omitempty"` // used before ThrowAway field
+	ContainerConfig struct {
+		Cmd []string `json:"Cmd,omitempty"`
+	} `json:"container_config,omitempty"`
+}

vendor/github.com/moby/buildkit/util/leaseutil/manager.go

@@ -0,0 +1,75 @@
+package leaseutil
+
+import (
+	"context"
+	"time"
+
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/namespaces"
+)
+
+func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error) {
+	_, ok := leases.FromContext(ctx)
+	if ok {
+		return ctx, func(context.Context) error {
+			return nil
+		}, nil
+	}
+
+	l, err := ls.Create(ctx, append([]leases.Opt{leases.WithRandomID(), leases.WithExpiration(time.Hour)}, opts...)...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	ctx = leases.WithLease(ctx, l.ID)
+	return ctx, func(ctx context.Context) error {
+		return ls.Delete(ctx, l)
+	}, nil
+}
+
+func MakeTemporary(l *leases.Lease) error {
+	if l.Labels == nil {
+		l.Labels = map[string]string{}
+	}
+	l.Labels["buildkit/lease.temporary"] = time.Now().UTC().Format(time.RFC3339Nano)
+	return nil
+}
+
+func WithNamespace(lm leases.Manager, ns string) leases.Manager {
+	return &nsLM{manager: lm, ns: ns}
+}
+
+type nsLM struct {
+	manager leases.Manager
+	ns      string
+}
+
+func (l *nsLM) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.Create(ctx, opts...)
+}
+
+func (l *nsLM) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.Delete(ctx, lease, opts...)
+}
+
+func (l *nsLM) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.List(ctx, filters...)
+}
+
+func (l *nsLM) AddResource(ctx context.Context, lease leases.Lease, resource leases.Resource) error {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.AddResource(ctx, lease, resource)
+}
+
+func (l *nsLM) DeleteResource(ctx context.Context, lease leases.Lease, resource leases.Resource) error {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.DeleteResource(ctx, lease, resource)
+}
+
+func (l *nsLM) ListResources(ctx context.Context, lease leases.Lease) ([]leases.Resource, error) {
+	ctx = namespaces.WithNamespace(ctx, l.ns)
+	return l.manager.ListResources(ctx, lease)
+}

vendor/github.com/moby/buildkit/util/resolver/limited/group.go

@@ -0,0 +1,175 @@
+package limited
+
+import (
+	"context"
+	"io"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/remotes"
+	"github.com/docker/distribution/reference"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/semaphore"
+)
+
+type contextKeyT string
+
+var contextKey = contextKeyT("buildkit/util/resolver/limited")
+
+var Default = New(4)
+
+type Group struct {
+	mu   sync.Mutex
+	size int
+	sem  map[string][2]*semaphore.Weighted
+}
+
+type req struct {
+	g   *Group
+	ref string
+}
+
+func (r *req) acquire(ctx context.Context, desc ocispecs.Descriptor) (context.Context, func(), error) {
+	if v := ctx.Value(contextKey); v != nil {
+		return ctx, func() {}, nil
+	}
+
+	ctx = context.WithValue(ctx, contextKey, struct{}{})
+
+	// json request get one additional connection
+	highPriority := strings.HasSuffix(desc.MediaType, "+json")
+
+	r.g.mu.Lock()
+	s, ok := r.g.sem[r.ref]
+	if !ok {
+		s = [2]*semaphore.Weighted{
+			semaphore.NewWeighted(int64(r.g.size)),
+			semaphore.NewWeighted(int64(r.g.size + 1)),
+		}
+		r.g.sem[r.ref] = s
+	}
+	r.g.mu.Unlock()
+	if !highPriority {
+		if err := s[0].Acquire(ctx, 1); err != nil {
+			return ctx, nil, err
+		}
+	}
+	if err := s[1].Acquire(ctx, 1); err != nil {
+		if !highPriority {
+			s[0].Release(1)
+		}
+		return ctx, nil, err
+	}
+	return ctx, func() {
+		s[1].Release(1)
+		if !highPriority {
+			s[0].Release(1)
+		}
+	}, nil
+}
+
+func New(size int) *Group {
+	return &Group{
+		size: size,
+		sem:  make(map[string][2]*semaphore.Weighted),
+	}
+}
+
+func (g *Group) req(ref string) *req {
+	return &req{g: g, ref: domain(ref)}
+}
+
+func (g *Group) WrapFetcher(f remotes.Fetcher, ref string) remotes.Fetcher {
+	return &fetcher{Fetcher: f, req: g.req(ref)}
+}
+
+func (g *Group) PushHandler(pusher remotes.Pusher, provider content.Provider, ref string) images.HandlerFunc {
+	ph := remotes.PushHandler(pusher, provider)
+	req := g.req(ref)
+	return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
+		ctx, release, err := req.acquire(ctx, desc)
+		if err != nil {
+			return nil, err
+		}
+		defer release()
+		return ph(ctx, desc)
+	}
+}
+
+type fetcher struct {
+	remotes.Fetcher
+	req *req
+}
+
+func (f *fetcher) Fetch(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) {
+	ctx, release, err := f.req.acquire(ctx, desc)
+	if err != nil {
+		return nil, err
+	}
+	rc, err := f.Fetcher.Fetch(ctx, desc)
+	if err != nil {
+		release()
+		return nil, err
+	}
+
+	rcw := &readCloser{ReadCloser: rc}
+	closer := func() {
+		if !rcw.closed {
+			logrus.Warnf("fetcher not closed cleanly: %s", desc.Digest)
+		}
+		release()
+	}
+	rcw.release = closer
+	runtime.SetFinalizer(rcw, func(rc *readCloser) {
+		rc.close()
+	})
+
+	if s, ok := rc.(io.Seeker); ok {
+		return &readCloserSeeker{rcw, s}, nil
+	}
+
+	return rcw, nil
+}
+
+type readCloserSeeker struct {
+	*readCloser
+	io.Seeker
+}
+
+type readCloser struct {
+	io.ReadCloser
+	once    sync.Once
+	closed  bool
+	release func()
+}
+
+func (r *readCloser) Close() error {
+	r.closed = true
+	r.close()
+	return r.ReadCloser.Close()
+}
+
+func (r *readCloser) close() {
+	r.once.Do(r.release)
+}
+
+func FetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, ref string) images.HandlerFunc {
+	return remotes.FetchHandler(ingester, Default.WrapFetcher(fetcher, ref))
+}
+
+func PushHandler(pusher remotes.Pusher, provider content.Provider, ref string) images.HandlerFunc {
+	return Default.PushHandler(pusher, provider, ref)
+}
+
+func domain(ref string) string {
+	if ref != "" {
+		if named, err := reference.ParseNormalizedNamed(ref); err == nil {
+			return reference.Domain(named)
+		}
+	}
+	return ref
+}

vendor/github.com/moby/buildkit/util/resolver/retryhandler/retry.go

@@ -0,0 +1,72 @@
+package retryhandler
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/images"
+	remoteserrors "github.com/containerd/containerd/remotes/errors"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
+	return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
+		backoff := time.Second
+		for {
+			descs, err := f(ctx, desc)
+			if err != nil {
+				select {
+				case <-ctx.Done():
+					return nil, err
+				default:
+					if !retryError(err) {
+						return nil, err
+					}
+				}
+				if logger != nil {
+					logger([]byte(fmt.Sprintf("error: %v\n", err.Error())))
+				}
+			} else {
+				return descs, nil
+			}
+			// backoff logic
+			if backoff >= 8*time.Second {
+				return nil, err
+			}
+			if logger != nil {
+				logger([]byte(fmt.Sprintf("retrying in %v\n", backoff)))
+			}
+			time.Sleep(backoff)
+			backoff *= 2
+		}
+	}
+}
+
+func retryError(err error) bool {
+	// Retry on 5xx errors
+	var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus
+	if errors.As(err, &errUnexpectedStatus) &&
+		errUnexpectedStatus.StatusCode >= 500 &&
+		errUnexpectedStatus.StatusCode <= 599 {
+		return true
+	}
+
+	if errors.Is(err, io.EOF) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) || errors.Is(err, net.ErrClosed) {
+		return true
+	}
+	// catches TLS timeout or other network-related temporary errors
+	if ne, ok := errors.Cause(err).(net.Error); ok && ne.Temporary() {
+		return true
+	}
+	// https://github.com/containerd/containerd/pull/4724
+	if errors.Cause(err).Error() == "no response" {
+		return true
+	}
+
+	return false
+}