history: add export command

Allow builds to be exported into .dockerbuild bundles
that can be shared and imported into Docker Desktop.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

commands/history/export.go

@@ -0,0 +1,137 @@
+package history
+
+import (
+	"context"
+	"io"
+	"os"
+	"slices"
+
+	"github.com/containerd/console"
+	"github.com/containerd/platforms"
+	"github.com/docker/buildx/builder"
+	"github.com/docker/buildx/localstate"
+	"github.com/docker/buildx/util/cobrautil/completion"
+	"github.com/docker/buildx/util/confutil"
+	"github.com/docker/buildx/util/desktop/bundle"
+	"github.com/docker/cli/cli/command"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+type exportOptions struct {
+	builder string
+	ref     string
+	output  string
+}
+
+func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error {
+	b, err := builder.New(dockerCli, builder.WithName(opts.builder))
+	if err != nil {
+		return err
+	}
+
+	nodes, err := b.LoadNodes(ctx, builder.WithData())
+	if err != nil {
+		return err
+	}
+	for _, node := range nodes {
+		if node.Err != nil {
+			return node.Err
+		}
+	}
+
+	recs, err := queryRecords(ctx, opts.ref, nodes, &queryOptions{
+		CompletedOnly: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(recs) == 0 {
+		if opts.ref == "" {
+			return errors.New("no records found")
+		}
+		return errors.Errorf("no record found for ref %q", opts.ref)
+	}
+
+	if opts.ref == "" {
+		slices.SortFunc(recs, func(a, b historyRecord) int {
+			return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
+		})
+	}
+
+	recs = recs[:1]
+
+	ls, err := localstate.New(confutil.NewConfig(dockerCli))
+	if err != nil {
+		return err
+	}
+
+	c, err := recs[0].node.Driver.Client(ctx)
+	if err != nil {
+		return err
+	}
+
+	toExport := make([]*bundle.Record, 0, len(recs))
+	for _, rec := range recs {
+		var defaultPlatform string
+		if p := rec.node.Platforms; len(p) > 0 {
+			defaultPlatform = platforms.FormatAll(platforms.Normalize(p[0]))
+		}
+
+		var stg *localstate.StateGroup
+		st, _ := ls.ReadRef(rec.node.Builder, rec.node.Name, rec.Ref)
+		if st != nil && st.GroupRef != "" {
+			stg, err = ls.ReadGroup(st.GroupRef)
+			if err != nil {
+				return err
+			}
+		}
+
+		toExport = append(toExport, &bundle.Record{
+			BuildHistoryRecord: rec.BuildHistoryRecord,
+			DefaultPlatform:    defaultPlatform,
+			LocalState:         st,
+			StateGroup:         stg,
+		})
+	}
+
+	var w io.Writer = os.Stdout
+	if opts.output != "" {
+		f, err := os.Create(opts.output)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create output file %q", opts.output)
+		}
+		defer f.Close()
+		w = f
+	} else {
+		if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
+			return errors.Errorf("refusing to write to console, use --output to specify a file")
+		}
+	}
+
+	return bundle.Export(ctx, c, w, toExport)
+}
+
+func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
+	var options exportOptions
+
+	cmd := &cobra.Command{
+		Use:   "export [OPTIONS] [REF]",
+		Short: "Export a build into Docker Desktop bundle",
+		Args:  cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) > 0 {
+				options.ref = args[0]
+			}
+			options.builder = *rootOpts.Builder
+			return runExport(cmd.Context(), dockerCli, options)
+		},
+		ValidArgsFunction: completion.Disable,
+	}
+
+	flags := cmd.Flags()
+	flags.StringVarP(&options.output, "output", "o", "", "Output file path")
+
+	return cmd
+}

commands/history/root.go

@@ -26,6 +26,7 @@ func RootCmd(rootcmd *cobra.Command, dockerCli command.Cli, opts RootOptions) *c
 		openCmd(dockerCli, opts),
 		traceCmd(dockerCli, opts),
 		importCmd(dockerCli, opts),
+		exportCmd(dockerCli, opts),
 	)
 
 	return cmd

docs/reference/buildx_history.md

@@ -7,6 +7,7 @@ Commands to work on build records
 
 | Name                                   | Description                                    |
 |:---------------------------------------|:-----------------------------------------------|
+| [`export`](buildx_history_export.md)   | Export a build into Docker Desktop bundle      |
 | [`import`](buildx_history_import.md)   | Import a build into Docker Desktop             |
 | [`inspect`](buildx_history_inspect.md) | Inspect a build                                |
 | [`logs`](buildx_history_logs.md)       | Print the logs of a build                      |

docs/reference/buildx_history_export.md

@@ -0,0 +1,16 @@
+# docker buildx history export
+
+<!---MARKER_GEN_START-->
+Export a build into Docker Desktop bundle
+
+### Options
+
+| Name             | Type     | Default | Description                              |
+|:-----------------|:---------|:--------|:-----------------------------------------|
+| `--builder`      | `string` |         | Override the configured builder instance |
+| `-D`, `--debug`  | `bool`   |         | Enable debug logging                     |
+| `-o`, `--output` | `string` |         | Output file path                         |
+
+
+<!---MARKER_GEN_END-->
+

util/desktop/bundle/export.go

@@ -0,0 +1,267 @@
+package bundle
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/content/proxy"
+	imgarchive "github.com/containerd/containerd/v2/core/images/archive"
+	"github.com/docker/buildx/localstate"
+	controlapi "github.com/moby/buildkit/api/services/control"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/util/contentutil"
+	"github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+const (
+	HistoryRecordMediaTypeV0 = "application/vnd.buildkit.historyrecord.v0"
+	RefDescriptorMediaType   = "vnd.export-build.descriptor.mediatype"
+)
+
+type Record struct {
+	*controlapi.BuildHistoryRecord
+
+	DefaultPlatform string
+	LocalState      *localstate.State      `json:"localState,omitempty"`
+	StateGroup      *localstate.StateGroup `json:"stateGroup,omitempty"`
+}
+
+func Export(ctx context.Context, c *client.Client, w io.Writer, records []*Record) error {
+	store := proxy.NewContentStore(c.ContentClient())
+	mp := contentutil.NewMultiProvider(store)
+
+	desc, err := export(ctx, mp, records)
+	if err != nil {
+		return errors.Wrap(err, "failed to export")
+	}
+
+	gz := gzip.NewWriter(w)
+	defer gz.Close()
+
+	if err := imgarchive.Export(ctx, mp, gz, imgarchive.WithManifest(desc), imgarchive.WithSkipDockerManifest()); err != nil {
+		return errors.Wrap(err, "failed to create dockerbuild archive")
+	}
+
+	return nil
+}
+
+func export(ctx context.Context, mp *contentutil.MultiProvider, records []*Record) (ocispecs.Descriptor, error) {
+	if len(records) == 1 {
+		desc, err := exportRecord(ctx, mp, records[0])
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to export record")
+		}
+		return desc, nil
+	}
+
+	var idx ocispecs.Index
+	idx.MediaType = ocispecs.MediaTypeImageIndex
+	idx.SchemaVersion = 2
+
+	for _, r := range records {
+		desc, err := exportRecord(ctx, mp, r)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to export record")
+		}
+		if desc.Annotations == nil {
+			desc.Annotations = make(map[string]string)
+		}
+		desc.Annotations["vnd.buildkit.history.reference"] = r.Ref
+		idx.Manifests = append(idx.Manifests, desc)
+	}
+
+	desc, err := writeJSON(ctx, mp, idx.MediaType, idx)
+	if err != nil {
+		return ocispecs.Descriptor{}, errors.Wrap(err, "failed to write index")
+	}
+
+	return desc, nil
+}
+
+func writeJSON(ctx context.Context, mp *contentutil.MultiProvider, mt string, data any) (ocispecs.Descriptor, error) {
+	dt, err := json.MarshalIndent(data, "", "  ")
+	if err != nil {
+		return ocispecs.Descriptor{}, errors.Wrap(err, "failed to marshal data")
+	}
+
+	desc := ocispecs.Descriptor{
+		MediaType: mt,
+		Size:      int64(len(dt)),
+		Digest:    digest.FromBytes(dt),
+	}
+
+	buf := contentutil.NewBuffer()
+	if err := content.WriteBlob(ctx, buf, "blob-"+desc.Digest.String(), bytes.NewReader(dt), desc); err != nil {
+		return ocispecs.Descriptor{}, errors.Wrap(err, "failed to write blob")
+	}
+
+	mp.Add(desc.Digest, buf)
+	return desc, nil
+}
+
+func sanitizeCacheImports(v string) (string, error) {
+	type cacheImport struct {
+		Type  string            `json:"Type"`
+		Attrs map[string]string `json:"Attrs"`
+	}
+	var arr []cacheImport
+	if err := json.Unmarshal([]byte(v), &arr); err != nil {
+		return "", errors.Wrap(err, "failed to unmarshal cache imports")
+	}
+	for i := range arr {
+		m := map[string]string{}
+		for k, v := range arr[i].Attrs {
+			if k == "scope" || k == "ref" {
+				m[k] = v
+			}
+		}
+		arr[i].Attrs = m
+	}
+	dt, err := json.Marshal(arr)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to marshal cache imports")
+	}
+	return string(dt), nil
+}
+
+func sanitizeRecord(rec *controlapi.BuildHistoryRecord) {
+	for k, v := range rec.FrontendAttrs {
+		if k == "cache-imports" {
+			v, err := sanitizeCacheImports(v)
+			if err != nil {
+				rec.FrontendAttrs[k] = ""
+			} else {
+				rec.FrontendAttrs[k] = v
+			}
+		}
+	}
+}
+
+func exportRecord(ctx context.Context, mp *contentutil.MultiProvider, record *Record) (ocispecs.Descriptor, error) {
+	var mfst ocispecs.Manifest
+	mfst.MediaType = ocispecs.MediaTypeImageManifest
+	mfst.SchemaVersion = 2
+
+	sanitizeRecord(record.BuildHistoryRecord)
+
+	visited := map[string]struct{}{}
+
+	if trace := record.Trace; trace != nil {
+		desc, err := loadDescriptor(ctx, mp, trace, visited)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to load trace descriptor")
+		}
+		desc, err = sanitizeTrace(ctx, mp, *desc)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to sanitize trace")
+		}
+		record.Trace.Digest = desc.Digest.String()
+		record.Trace.Size = desc.Size
+		mfst.Layers = append(mfst.Layers, *desc)
+	}
+
+	config, err := writeJSON(ctx, mp, HistoryRecordMediaTypeV0, record)
+	if err != nil {
+		return ocispecs.Descriptor{}, errors.Wrap(err, "failed to write history record")
+	}
+
+	mfst.Config = config
+
+	if logs := record.Logs; logs != nil {
+		desc, err := loadDescriptor(ctx, mp, logs, visited)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to load logs descriptor")
+		}
+		mfst.Layers = append(mfst.Layers, *desc)
+	}
+
+	if res := record.Result; res != nil {
+		results, err := loadResult(ctx, mp, res, visited)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to load result")
+		}
+		mfst.Layers = append(mfst.Layers, results...)
+	}
+
+	if exterr := record.ExternalError; exterr != nil {
+		desc, err := loadDescriptor(ctx, mp, exterr, visited)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to load external error descriptor")
+		}
+		mfst.Layers = append(mfst.Layers, *desc)
+	}
+
+	for _, res := range record.Results {
+		results, err := loadResult(ctx, mp, res, visited)
+		if err != nil {
+			return ocispecs.Descriptor{}, errors.Wrap(err, "failed to load result")
+		}
+		mfst.Layers = append(mfst.Layers, results...)
+	}
+
+	desc, err := writeJSON(ctx, mp, mfst.MediaType, mfst)
+	if err != nil {
+		return ocispecs.Descriptor{}, errors.Wrap(err, "failed to write manifest")
+	}
+
+	return desc, nil
+}
+
+func loadResult(ctx context.Context, ip content.InfoProvider, in *controlapi.BuildResultInfo, visited map[string]struct{}) ([]ocispecs.Descriptor, error) {
+	var out []ocispecs.Descriptor
+	for _, attest := range in.Attestations {
+		desc, err := loadDescriptor(ctx, ip, attest, visited)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load attestation descriptor")
+		}
+		if desc != nil {
+			out = append(out, *desc)
+		}
+	}
+	for _, r := range in.Results {
+		desc, err := loadDescriptor(ctx, ip, r, visited)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load result descriptor")
+		}
+		if desc != nil {
+			if desc.Annotations == nil {
+				desc.Annotations = make(map[string]string)
+			}
+			// Override media type to avoid containerd to walk children. Also
+			// keep original media type in annotations.
+			desc.Annotations[RefDescriptorMediaType] = desc.MediaType
+			desc.MediaType = "application/json"
+			out = append(out, *desc)
+		}
+	}
+	return out, nil
+}
+
+func loadDescriptor(ctx context.Context, ip content.InfoProvider, in *controlapi.Descriptor, visited map[string]struct{}) (*ocispecs.Descriptor, error) {
+	if _, ok := visited[in.Digest]; ok {
+		return nil, nil
+	}
+	visited[in.Digest] = struct{}{}
+
+	dgst, err := digest.Parse(in.Digest)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse digest")
+	}
+
+	if _, err := ip.Info(ctx, dgst); err != nil {
+		return nil, errors.Wrap(err, "failed to get info")
+	}
+
+	return &ocispecs.Descriptor{
+		MediaType:   in.MediaType,
+		Digest:      dgst,
+		Size:        in.Size,
+		Annotations: in.Annotations,
+	}, nil
+}

util/desktop/bundle/trace.go

@@ -0,0 +1,76 @@
+package bundle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/docker/buildx/util/otelutil"
+	"github.com/moby/buildkit/util/contentutil"
+	"github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+var (
+	sensitiveKeys = []string{"ghtoken", "token", "access_key_id", "secret_access_key", "session_token"}
+	reAttrs       = regexp.MustCompile(`(?i)(` + strings.Join(sensitiveKeys, "|") + `)=[^ ,]+`)
+	reGhs         = regexp.MustCompile(`ghs_[A-Za-z0-9]{36}`)
+)
+
+func sanitizeCommand(value string) string {
+	value = reAttrs.ReplaceAllString(value, "${1}=xxxxx")
+	// reGhs is just double proofing. Not really needed.
+	value = reGhs.ReplaceAllString(value, "xxxxx")
+	return value
+}
+
+func sanitizeTrace(ctx context.Context, mp *contentutil.MultiProvider, desc ocispecs.Descriptor) (*ocispecs.Descriptor, error) {
+	ra, err := mp.ReaderAt(ctx, desc)
+	if err != nil {
+		return nil, err
+	}
+	defer ra.Close()
+
+	buf := &bytes.Buffer{}
+	dec := json.NewDecoder(io.NewSectionReader(ra, 0, ra.Size()))
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", "  ")
+	for {
+		var obj otelutil.Span
+		if err := dec.Decode(&obj); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		for i, att := range obj.Attributes {
+			v := att.Value
+			if v.Type() == attribute.STRING {
+				obj.Attributes[i].Value = attribute.StringValue(sanitizeCommand(v.AsString()))
+			}
+		}
+
+		if err := enc.Encode(obj); err != nil {
+			return nil, err
+		}
+	}
+
+	buffer := contentutil.NewBuffer()
+	newDesc := ocispecs.Descriptor{
+		MediaType: desc.MediaType,
+		Size:      int64(buf.Len()),
+		Digest:    digest.FromBytes(buf.Bytes()),
+	}
+	if err := content.WriteBlob(ctx, buffer, "trace-sanitized", bytes.NewReader(buf.Bytes()), newDesc); err != nil {
+		return nil, err
+	}
+
+	mp.Add(newDesc.Digest, buffer)
+
+	return &newDesc, nil
+}