test: add basic integration tests

Signed-off-by: Justin Chadwell <me@jedevc.com>

vendor/github.com/containerd/containerd/images/archive/exporter.go

@@ -0,0 +1,522 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package archive
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type exportOptions struct {
+	manifests          []ocispec.Descriptor
+	platform           platforms.MatchComparer
+	allPlatforms       bool
+	skipDockerManifest bool
+	blobRecordOptions  blobRecordOptions
+}
+
+// ExportOpt defines options for configuring exported descriptors
+type ExportOpt func(context.Context, *exportOptions) error
+
+// WithPlatform defines the platform to require manifest lists have
+// not exporting all platforms.
+// Additionally, platform is used to resolve image configs for
+// Docker v1.1, v1.2 format compatibility.
+func WithPlatform(p platforms.MatchComparer) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.platform = p
+		return nil
+	}
+}
+
+// WithAllPlatforms exports all manifests from a manifest list.
+// Missing content will fail the export.
+func WithAllPlatforms() ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.allPlatforms = true
+		return nil
+	}
+}
+
+// WithSkipDockerManifest skips creation of the Docker compatible
+// manifest.json file.
+func WithSkipDockerManifest() ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.skipDockerManifest = true
+		return nil
+	}
+}
+
+// WithImage adds the provided images to the exported archive.
+func WithImage(is images.Store, name string) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		img, err := is.Get(ctx, name)
+		if err != nil {
+			return err
+		}
+
+		img.Target.Annotations = addNameAnnotation(name, img.Target.Annotations)
+		o.manifests = append(o.manifests, img.Target)
+
+		return nil
+	}
+}
+
+// WithImages adds multiples images to the exported archive.
+func WithImages(imgs []images.Image) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		for _, img := range imgs {
+			img.Target.Annotations = addNameAnnotation(img.Name, img.Target.Annotations)
+			o.manifests = append(o.manifests, img.Target)
+		}
+
+		return nil
+	}
+}
+
+// WithManifest adds a manifest to the exported archive.
+// When names are given they will be set on the manifest in the
+// exported archive, creating an index record for each name.
+// When no names are provided, it is up to caller to put name annotation to
+// on the manifest descriptor if needed.
+func WithManifest(manifest ocispec.Descriptor, names ...string) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		if len(names) == 0 {
+			o.manifests = append(o.manifests, manifest)
+		}
+		for _, name := range names {
+			mc := manifest
+			mc.Annotations = addNameAnnotation(name, manifest.Annotations)
+			o.manifests = append(o.manifests, mc)
+		}
+
+		return nil
+	}
+}
+
+// BlobFilter returns false if the blob should not be included in the archive.
+type BlobFilter func(ocispec.Descriptor) bool
+
+// WithBlobFilter specifies BlobFilter.
+func WithBlobFilter(f BlobFilter) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.blobRecordOptions.blobFilter = f
+		return nil
+	}
+}
+
+// WithSkipNonDistributableBlobs excludes non-distributable blobs such as Windows base layers.
+func WithSkipNonDistributableBlobs() ExportOpt {
+	f := func(desc ocispec.Descriptor) bool {
+		return !images.IsNonDistributable(desc.MediaType)
+	}
+	return WithBlobFilter(f)
+}
+
+func addNameAnnotation(name string, base map[string]string) map[string]string {
+	annotations := map[string]string{}
+	for k, v := range base {
+		annotations[k] = v
+	}
+
+	annotations[images.AnnotationImageName] = name
+	annotations[ocispec.AnnotationRefName] = ociReferenceName(name)
+
+	return annotations
+}
+
+// Export implements Exporter.
+func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
+	var eo exportOptions
+	for _, opt := range opts {
+		if err := opt(ctx, &eo); err != nil {
+			return err
+		}
+	}
+
+	records := []tarRecord{
+		ociLayoutFile(""),
+		ociIndexRecord(eo.manifests),
+	}
+
+	algorithms := map[string]struct{}{}
+	dManifests := map[digest.Digest]*exportManifest{}
+	resolvedIndex := map[digest.Digest]digest.Digest{}
+	for _, desc := range eo.manifests {
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+			mt, ok := dManifests[desc.Digest]
+			if !ok {
+				// TODO(containerd): Skip if already added
+				r, err := getRecords(ctx, store, desc, algorithms, &eo.blobRecordOptions)
+				if err != nil {
+					return err
+				}
+				records = append(records, r...)
+
+				mt = &exportManifest{
+					manifest: desc,
+				}
+				dManifests[desc.Digest] = mt
+			}
+
+			name := desc.Annotations[images.AnnotationImageName]
+			if name != "" && !eo.skipDockerManifest {
+				mt.names = append(mt.names, name)
+			}
+		case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+			d, ok := resolvedIndex[desc.Digest]
+			if !ok {
+				if err := desc.Digest.Validate(); err != nil {
+					return err
+				}
+				records = append(records, blobRecord(store, desc, &eo.blobRecordOptions))
+
+				p, err := content.ReadBlob(ctx, store, desc)
+				if err != nil {
+					return err
+				}
+
+				var index ocispec.Index
+				if err := json.Unmarshal(p, &index); err != nil {
+					return err
+				}
+
+				var manifests []ocispec.Descriptor
+				for _, m := range index.Manifests {
+					if eo.platform != nil {
+						if m.Platform == nil || eo.platform.Match(*m.Platform) {
+							manifests = append(manifests, m)
+						} else if !eo.allPlatforms {
+							continue
+						}
+					}
+
+					r, err := getRecords(ctx, store, m, algorithms, &eo.blobRecordOptions)
+					if err != nil {
+						return err
+					}
+
+					records = append(records, r...)
+				}
+
+				if !eo.skipDockerManifest {
+					if len(manifests) >= 1 {
+						if len(manifests) > 1 {
+							sort.SliceStable(manifests, func(i, j int) bool {
+								if manifests[i].Platform == nil {
+									return false
+								}
+								if manifests[j].Platform == nil {
+									return true
+								}
+								return eo.platform.Less(*manifests[i].Platform, *manifests[j].Platform)
+							})
+						}
+						d = manifests[0].Digest
+						dManifests[d] = &exportManifest{
+							manifest: manifests[0],
+						}
+					} else if eo.platform != nil {
+						return fmt.Errorf("no manifest found for platform: %w", errdefs.ErrNotFound)
+					}
+				}
+				resolvedIndex[desc.Digest] = d
+			}
+			if d != "" {
+				if name := desc.Annotations[images.AnnotationImageName]; name != "" {
+					mt := dManifests[d]
+					mt.names = append(mt.names, name)
+				}
+
+			}
+		default:
+			return fmt.Errorf("only manifests may be exported: %w", errdefs.ErrInvalidArgument)
+		}
+	}
+
+	if len(dManifests) > 0 {
+		tr, err := manifestsRecord(ctx, store, dManifests)
+		if err != nil {
+			return fmt.Errorf("unable to create manifests file: %w", err)
+		}
+
+		records = append(records, tr)
+	}
+
+	if len(algorithms) > 0 {
+		records = append(records, directoryRecord("blobs/", 0755))
+		for alg := range algorithms {
+			records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
+		}
+	}
+
+	tw := tar.NewWriter(writer)
+	defer tw.Close()
+	return writeTar(ctx, tw, records)
+}
+
+func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}, brOpts *blobRecordOptions) ([]tarRecord, error) {
+	var records []tarRecord
+	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		if err := desc.Digest.Validate(); err != nil {
+			return nil, err
+		}
+		records = append(records, blobRecord(store, desc, brOpts))
+		algorithms[desc.Digest.Algorithm().String()] = struct{}{}
+		return nil, nil
+	}
+
+	childrenHandler := images.ChildrenHandler(store)
+
+	handlers := images.Handlers(
+		childrenHandler,
+		images.HandlerFunc(exportHandler),
+	)
+
+	// Walk sequentially since the number of fetches is likely one and doing in
+	// parallel requires locking the export handler
+	if err := images.Walk(ctx, handlers, desc); err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}
+
+type tarRecord struct {
+	Header *tar.Header
+	CopyTo func(context.Context, io.Writer) (int64, error)
+}
+
+type blobRecordOptions struct {
+	blobFilter BlobFilter
+}
+
+func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {
+	if opts != nil && opts.blobFilter != nil && !opts.blobFilter(desc) {
+		return tarRecord{}
+	}
+	path := path.Join("blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     path,
+			Mode:     0444,
+			Size:     desc.Size,
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			r, err := cs.ReaderAt(ctx, desc)
+			if err != nil {
+				return 0, fmt.Errorf("failed to get reader: %w", err)
+			}
+			defer r.Close()
+
+			// Verify digest
+			dgstr := desc.Digest.Algorithm().Digester()
+
+			n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
+			if err != nil {
+				return 0, fmt.Errorf("failed to copy to tar: %w", err)
+			}
+			if dgstr.Digest() != desc.Digest {
+				return 0, fmt.Errorf("unexpected digest %s copied", dgstr.Digest())
+			}
+			return n, nil
+		},
+	}
+}
+
+func directoryRecord(name string, mode int64) tarRecord {
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     name,
+			Mode:     mode,
+			Typeflag: tar.TypeDir,
+		},
+	}
+}
+
+func ociLayoutFile(version string) tarRecord {
+	if version == "" {
+		version = ocispec.ImageLayoutVersion
+	}
+	layout := ocispec.ImageLayout{
+		Version: version,
+	}
+
+	b, err := json.Marshal(layout)
+	if err != nil {
+		panic(err)
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     ocispec.ImageLayoutFile,
+			Mode:     0444,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}
+
+}
+
+func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord {
+	index := ocispec.Index{
+		Versioned: ocispecs.Versioned{
+			SchemaVersion: 2,
+		},
+		Manifests: manifests,
+	}
+
+	b, err := json.Marshal(index)
+	if err != nil {
+		panic(err)
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     "index.json",
+			Mode:     0644,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}
+}
+
+type exportManifest struct {
+	manifest ocispec.Descriptor
+	names    []string
+}
+
+func manifestsRecord(ctx context.Context, store content.Provider, manifests map[digest.Digest]*exportManifest) (tarRecord, error) {
+	mfsts := make([]struct {
+		Config   string
+		RepoTags []string
+		Layers   []string
+	}, len(manifests))
+
+	var i int
+	for _, m := range manifests {
+		p, err := content.ReadBlob(ctx, store, m.manifest)
+		if err != nil {
+			return tarRecord{}, err
+		}
+
+		var manifest ocispec.Manifest
+		if err := json.Unmarshal(p, &manifest); err != nil {
+			return tarRecord{}, err
+		}
+		if err := manifest.Config.Digest.Validate(); err != nil {
+			return tarRecord{}, fmt.Errorf("invalid manifest %q: %w", m.manifest.Digest, err)
+		}
+
+		dgst := manifest.Config.Digest
+		if err := dgst.Validate(); err != nil {
+			return tarRecord{}, err
+		}
+		mfsts[i].Config = path.Join("blobs", dgst.Algorithm().String(), dgst.Encoded())
+		for _, l := range manifest.Layers {
+			path := path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Encoded())
+			mfsts[i].Layers = append(mfsts[i].Layers, path)
+		}
+
+		for _, name := range m.names {
+			nname, err := familiarizeReference(name)
+			if err != nil {
+				return tarRecord{}, err
+			}
+
+			mfsts[i].RepoTags = append(mfsts[i].RepoTags, nname)
+		}
+
+		i++
+	}
+
+	b, err := json.Marshal(mfsts)
+	if err != nil {
+		return tarRecord{}, err
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     "manifest.json",
+			Mode:     0644,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}, nil
+}
+
+func writeTar(ctx context.Context, tw *tar.Writer, recordsWithEmpty []tarRecord) error {
+	var records []tarRecord
+	for _, r := range recordsWithEmpty {
+		if r.Header != nil {
+			records = append(records, r)
+		}
+	}
+	sort.Slice(records, func(i, j int) bool {
+		return records[i].Header.Name < records[j].Header.Name
+	})
+
+	var last string
+	for _, record := range records {
+		if record.Header.Name == last {
+			continue
+		}
+		last = record.Header.Name
+		if err := tw.WriteHeader(record.Header); err != nil {
+			return err
+		}
+		if record.CopyTo != nil {
+			n, err := record.CopyTo(ctx, tw)
+			if err != nil {
+				return err
+			}
+			if n != record.Header.Size {
+				return fmt.Errorf("unexpected copy size for %s", record.Header.Name)
+			}
+		} else if record.Header.Size > 0 {
+			return fmt.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
+		}
+	}
+	return nil
+}

vendor/github.com/containerd/containerd/images/archive/importer.go

@@ -0,0 +1,420 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package archive provides a Docker and OCI compatible importer
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+
+	"github.com/containerd/containerd/archive/compression"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/platforms"
+	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type importOpts struct {
+	compress bool
+}
+
+// ImportOpt is an option for importing an OCI index
+type ImportOpt func(*importOpts) error
+
+// WithImportCompression compresses uncompressed layers on import.
+// This is used for import formats which do not include the manifest.
+func WithImportCompression() ImportOpt {
+	return func(io *importOpts) error {
+		io.compress = true
+		return nil
+	}
+}
+
+// ImportIndex imports an index from a tar archive image bundle
+//   - implements Docker v1.1, v1.2 and OCI v1.
+//   - prefers OCI v1 when provided
+//   - creates OCI index for Docker formats
+//   - normalizes Docker references and adds as OCI ref name
+//     e.g. alpine:latest -> docker.io/library/alpine:latest
+//   - existing OCI reference names are untouched
+func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
+	var (
+		tr = tar.NewReader(reader)
+
+		ociLayout ocispec.ImageLayout
+		mfsts     []struct {
+			Config   string
+			RepoTags []string
+			Layers   []string
+		}
+		symlinks = make(map[string]string)
+		blobs    = make(map[string]ocispec.Descriptor)
+		iopts    importOpts
+	)
+
+	for _, o := range opts {
+		if err := o(&iopts); err != nil {
+			return ocispec.Descriptor{}, err
+		}
+	}
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return ocispec.Descriptor{}, err
+		}
+		if hdr.Typeflag == tar.TypeSymlink {
+			symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
+		}
+
+		//nolint:staticcheck // TypeRegA is deprecated but we may still receive an external tar with TypeRegA
+		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
+			if hdr.Typeflag != tar.TypeDir {
+				log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
+			}
+			continue
+		}
+
+		hdrName := path.Clean(hdr.Name)
+		if hdrName == ocispec.ImageLayoutFile {
+			if err = onUntarJSON(tr, &ociLayout); err != nil {
+				return ocispec.Descriptor{}, fmt.Errorf("untar oci layout %q: %w", hdr.Name, err)
+			}
+		} else if hdrName == "manifest.json" {
+			if err = onUntarJSON(tr, &mfsts); err != nil {
+				return ocispec.Descriptor{}, fmt.Errorf("untar manifest %q: %w", hdr.Name, err)
+			}
+		} else {
+			dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
+			if err != nil {
+				return ocispec.Descriptor{}, fmt.Errorf("failed to ingest %q: %w", hdr.Name, err)
+			}
+
+			blobs[hdrName] = ocispec.Descriptor{
+				Digest: dgst,
+				Size:   hdr.Size,
+			}
+		}
+	}
+
+	// If OCI layout was given, interpret the tar as an OCI layout.
+	// When not provided, the layout of the tar will be interpreted
+	// as Docker v1.1 or v1.2.
+	if ociLayout.Version != "" {
+		if ociLayout.Version != ocispec.ImageLayoutVersion {
+			return ocispec.Descriptor{}, fmt.Errorf("unsupported OCI version %s", ociLayout.Version)
+		}
+
+		idx, ok := blobs["index.json"]
+		if !ok {
+			return ocispec.Descriptor{}, fmt.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
+		}
+
+		idx.MediaType = ocispec.MediaTypeImageIndex
+		return idx, nil
+	}
+
+	if mfsts == nil {
+		return ocispec.Descriptor{}, errors.New("unrecognized image format")
+	}
+
+	for name, linkname := range symlinks {
+		desc, ok := blobs[linkname]
+		if !ok {
+			return ocispec.Descriptor{}, fmt.Errorf("no target for symlink layer from %q to %q", name, linkname)
+		}
+		blobs[name] = desc
+	}
+
+	idx := ocispec.Index{
+		Versioned: specs.Versioned{
+			SchemaVersion: 2,
+		},
+	}
+	for _, mfst := range mfsts {
+		config, ok := blobs[mfst.Config]
+		if !ok {
+			return ocispec.Descriptor{}, fmt.Errorf("image config %q not found", mfst.Config)
+		}
+		config.MediaType = images.MediaTypeDockerSchema2Config
+
+		layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
+		if err != nil {
+			return ocispec.Descriptor{}, fmt.Errorf("failed to resolve layers: %w", err)
+		}
+
+		manifest := struct {
+			SchemaVersion int                  `json:"schemaVersion"`
+			MediaType     string               `json:"mediaType"`
+			Config        ocispec.Descriptor   `json:"config"`
+			Layers        []ocispec.Descriptor `json:"layers"`
+		}{
+			SchemaVersion: 2,
+			MediaType:     images.MediaTypeDockerSchema2Manifest,
+			Config:        config,
+			Layers:        layers,
+		}
+
+		desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
+		if err != nil {
+			return ocispec.Descriptor{}, fmt.Errorf("write docker manifest: %w", err)
+		}
+
+		imgPlatforms, err := images.Platforms(ctx, store, desc)
+		if err != nil {
+			return ocispec.Descriptor{}, fmt.Errorf("unable to resolve platform: %w", err)
+		}
+		if len(imgPlatforms) > 0 {
+			// Only one platform can be resolved from non-index manifest,
+			// The platform can only come from the config included above,
+			// if the config has no platform it can be safely omitted.
+			desc.Platform = &imgPlatforms[0]
+
+			// If the image we've just imported is a Windows image without the OSVersion set,
+			// we could just assume it matches this host's OS Version. Without this, the
+			// children labels might not be set on the image content, leading to it being
+			// garbage collected, breaking the image.
+			// See: https://github.com/containerd/containerd/issues/5690
+			if desc.Platform.OS == "windows" && desc.Platform.OSVersion == "" {
+				platform := platforms.DefaultSpec()
+				desc.Platform.OSVersion = platform.OSVersion
+			}
+		}
+
+		if len(mfst.RepoTags) == 0 {
+			idx.Manifests = append(idx.Manifests, desc)
+		} else {
+			// Add descriptor per tag
+			for _, ref := range mfst.RepoTags {
+				mfstdesc := desc
+
+				normalized, err := normalizeReference(ref)
+				if err != nil {
+					return ocispec.Descriptor{}, err
+				}
+
+				mfstdesc.Annotations = map[string]string{
+					images.AnnotationImageName: normalized,
+					ocispec.AnnotationRefName:  ociReferenceName(normalized),
+				}
+
+				idx.Manifests = append(idx.Manifests, mfstdesc)
+			}
+		}
+	}
+
+	return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
+}
+
+const (
+	kib       = 1024
+	mib       = 1024 * kib
+	jsonLimit = 20 * mib
+)
+
+func onUntarJSON(r io.Reader, j interface{}) error {
+	return json.NewDecoder(io.LimitReader(r, jsonLimit)).Decode(j)
+}
+
+func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
+	dgstr := digest.Canonical.Digester()
+
+	if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
+		return "", err
+	}
+
+	return dgstr.Digest(), nil
+}
+
+func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
+	layers := make([]ocispec.Descriptor, len(layerFiles))
+	descs := map[digest.Digest]*ocispec.Descriptor{}
+	filters := []string{}
+	for i, f := range layerFiles {
+		desc, ok := blobs[f]
+		if !ok {
+			return nil, fmt.Errorf("layer %q not found", f)
+		}
+		layers[i] = desc
+		descs[desc.Digest] = &layers[i]
+		filters = append(filters, fmt.Sprintf("labels.\"%s\"==%s", labels.LabelUncompressed, desc.Digest.String()))
+	}
+
+	err := store.Walk(ctx, func(info content.Info) error {
+		dgst, ok := info.Labels[labels.LabelUncompressed]
+		if ok {
+			desc := descs[digest.Digest(dgst)]
+			if desc != nil {
+				desc.Digest = info.Digest
+				desc.Size = info.Size
+				mediaType, err := detectLayerMediaType(ctx, store, *desc)
+				if err != nil {
+					return fmt.Errorf("failed to detect media type of layer: %w", err)
+				}
+				desc.MediaType = mediaType
+			}
+		}
+		return nil
+	}, filters...)
+	if err != nil {
+		return nil, fmt.Errorf("failure checking for compressed blobs: %w", err)
+	}
+
+	for i, desc := range layers {
+		if desc.MediaType != "" {
+			continue
+		}
+		// Open blob, resolve media type
+		ra, err := store.ReaderAt(ctx, desc)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open %q (%s): %w", layerFiles[i], desc.Digest, err)
+		}
+		s, err := compression.DecompressStream(content.NewReader(ra))
+		if err != nil {
+			ra.Close()
+			return nil, fmt.Errorf("failed to detect compression for %q: %w", layerFiles[i], err)
+		}
+		if s.GetCompression() == compression.Uncompressed {
+			if compress {
+				if err := desc.Digest.Validate(); err != nil {
+					return nil, err
+				}
+				ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
+				labels := map[string]string{
+					labels.LabelUncompressed: desc.Digest.String(),
+				}
+				layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
+				if err != nil {
+					s.Close()
+					ra.Close()
+					return nil, err
+				}
+				layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
+			} else {
+				layers[i].MediaType = images.MediaTypeDockerSchema2Layer
+			}
+		} else {
+			layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
+		}
+		s.Close()
+		ra.Close()
+	}
+	return layers, nil
+}
+
+func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
+	w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
+	if err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("failed to open writer: %w", err)
+	}
+
+	defer func() {
+		w.Close()
+		if err != nil {
+			cs.Abort(ctx, ref)
+		}
+	}()
+	if err := w.Truncate(0); err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("failed to truncate writer: %w", err)
+	}
+
+	cw, err := compression.CompressStream(w, compression.Gzip)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+
+	if _, err := io.Copy(cw, r); err != nil {
+		return ocispec.Descriptor{}, err
+	}
+	if err := cw.Close(); err != nil {
+		return ocispec.Descriptor{}, err
+	}
+
+	cst, err := w.Status()
+	if err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("failed to get writer status: %w", err)
+	}
+
+	desc.Digest = w.Digest()
+	desc.Size = cst.Offset
+
+	if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
+		if !errdefs.IsAlreadyExists(err) {
+			return ocispec.Descriptor{}, fmt.Errorf("failed to commit: %w", err)
+		}
+	}
+
+	return desc, nil
+}
+
+func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+
+	desc := ocispec.Descriptor{
+		MediaType: mediaType,
+		Digest:    digest.FromBytes(manifestBytes),
+		Size:      int64(len(manifestBytes)),
+	}
+	if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
+		return ocispec.Descriptor{}, err
+	}
+
+	return desc, nil
+}
+
+func detectLayerMediaType(ctx context.Context, store content.Store, desc ocispec.Descriptor) (string, error) {
+	var mediaType string
+	// need to parse existing blob to use the proper media type
+	bytes := make([]byte, 10)
+	ra, err := store.ReaderAt(ctx, desc)
+	if err != nil {
+		return "", fmt.Errorf("failed to read content store to detect layer media type: %w", err)
+	}
+	defer ra.Close()
+	_, err = ra.ReadAt(bytes, 0)
+	if err != nil && err != io.EOF {
+		return "", fmt.Errorf("failed to read header bytes from layer to detect media type: %w", err)
+	}
+	if err == io.EOF {
+		// in the case of an empty layer then the media type should be uncompressed
+		return images.MediaTypeDockerSchema2Layer, nil
+	}
+	switch c := compression.DetectCompression(bytes); c {
+	case compression.Uncompressed:
+		mediaType = images.MediaTypeDockerSchema2Layer
+	default:
+		mediaType = images.MediaTypeDockerSchema2LayerGzip
+	}
+	return mediaType, nil
+}

vendor/github.com/containerd/containerd/images/archive/reference.go

@@ -0,0 +1,115 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package archive
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/containerd/containerd/reference"
+	distref "github.com/containerd/containerd/reference/docker"
+	"github.com/opencontainers/go-digest"
+)
+
+// FilterRefPrefix restricts references to having the given image
+// prefix. Tag-only references will have the prefix prepended.
+func FilterRefPrefix(image string) func(string) string {
+	return refTranslator(image, true)
+}
+
+// AddRefPrefix prepends the given image prefix to tag-only references,
+// while leaving returning full references unmodified.
+func AddRefPrefix(image string) func(string) string {
+	return refTranslator(image, false)
+}
+
+// refTranslator creates a reference which only has a tag or verifies
+// a full reference.
+func refTranslator(image string, checkPrefix bool) func(string) string {
+	return func(ref string) string {
+		if image == "" {
+			return ""
+		}
+		// Check if ref is full reference
+		if strings.ContainsAny(ref, "/:@") {
+			// If not prefixed, don't include image
+			if checkPrefix && !isImagePrefix(ref, image) {
+				return ""
+			}
+			return ref
+		}
+		return image + ":" + ref
+	}
+}
+
+func isImagePrefix(s, prefix string) bool {
+	if !strings.HasPrefix(s, prefix) {
+		return false
+	}
+	if len(s) > len(prefix) {
+		switch s[len(prefix)] {
+		case '/', ':', '@':
+			// Prevent matching partial namespaces
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeReference(ref string) (string, error) {
+	// TODO: Replace this function to not depend on reference package
+	normalized, err := distref.ParseDockerRef(ref)
+	if err != nil {
+		return "", fmt.Errorf("normalize image ref %q: %w", ref, err)
+	}
+
+	return normalized.String(), nil
+}
+
+func familiarizeReference(ref string) (string, error) {
+	named, err := distref.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse %q: %w", ref, err)
+	}
+	named = distref.TagNameOnly(named)
+
+	return distref.FamiliarString(named), nil
+}
+
+func ociReferenceName(name string) string {
+	// OCI defines the reference name as only a tag excluding the
+	// repository. The containerd annotation contains the full image name
+	// since the tag is insufficient for correctly naming and referring to an
+	// image
+	var ociRef string
+	if spec, err := reference.Parse(name); err == nil {
+		ociRef = spec.Object
+	} else {
+		ociRef = name
+	}
+
+	return ociRef
+}
+
+// DigestTranslator creates a digest reference by adding the
+// digest to an image name
+func DigestTranslator(prefix string) func(digest.Digest) string {
+	return func(dgst digest.Digest) string {
+		return prefix + "@" + dgst.String()
+	}
+}