From 45dfb843614e656d3494a8e428378d8df96014da Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 20 Mar 2025 18:57:21 -0700 Subject: [PATCH] history: add support for exporting multiple and all records Signed-off-by: Tonis Tiigi --- commands/history/export.go | 77 +++++++++++++++--------- docs/reference/buildx_history_export.md | 1 + util/desktop/bundle/content.go | 78 +++++++++++++++++++++++++ util/desktop/bundle/export.go | 19 +++++- util/desktop/bundle/trace.go | 16 +++-- 5 files changed, 158 insertions(+), 33 deletions(-) create mode 100644 util/desktop/bundle/content.go diff --git a/commands/history/export.go b/commands/history/export.go index bad7ef37..c0c0f68d 100644 --- a/commands/history/export.go +++ b/commands/history/export.go @@ -14,14 +14,16 @@ import ( "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/desktop/bundle" "github.com/docker/cli/cli/command" + "github.com/moby/buildkit/client" "github.com/pkg/errors" "github.com/spf13/cobra" ) type exportOptions struct { builder string - ref string + refs []string output string + all bool } func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error { @@ -40,40 +42,60 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e } } - recs, err := queryRecords(ctx, opts.ref, nodes, &queryOptions{ - CompletedOnly: true, - }) - if err != nil { - return err + if len(opts.refs) == 0 { + opts.refs = []string{""} } - if len(recs) == 0 { - if opts.ref == "" { - return errors.New("no records found") - } - return errors.Errorf("no record found for ref %q", opts.ref) - } - - if opts.ref == "" { - slices.SortFunc(recs, func(a, b historyRecord) int { - return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime()) + var res []historyRecord + for _, ref := range opts.refs { + recs, err := queryRecords(ctx, ref, nodes, &queryOptions{ + CompletedOnly: true, }) - } + if err != nil { + return err + } - recs = recs[:1] + if len(recs) == 0 { + if ref == "" { + return errors.New("no records found") + } + return errors.Errorf("no record found for ref %q", ref) + } + + if ref == "" { + slices.SortFunc(recs, func(a, b historyRecord) int { + return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime()) + }) + } + + if opts.all { + res = append(res, recs...) + break + } else { + res = append(res, recs[0]) + } + } ls, err := localstate.New(confutil.NewConfig(dockerCli)) if err != nil { return err } - c, err := recs[0].node.Driver.Client(ctx) - if err != nil { - return err + visited := map[*builder.Node]struct{}{} + var clients []*client.Client + for _, rec := range res { + if _, ok := visited[rec.node]; ok { + continue + } + c, err := rec.node.Driver.Client(ctx) + if err != nil { + return err + } + clients = append(clients, c) } - toExport := make([]*bundle.Record, 0, len(recs)) - for _, rec := range recs { + toExport := make([]*bundle.Record, 0, len(res)) + for _, rec := range res { var defaultPlatform string if p := rec.node.Platforms; len(p) > 0 { defaultPlatform = platforms.FormatAll(platforms.Normalize(p[0])) @@ -110,7 +132,7 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e } } - return bundle.Export(ctx, c, w, toExport) + return bundle.Export(ctx, clients, w, toExport) } func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command { @@ -119,11 +141,11 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command { cmd := &cobra.Command{ Use: "export [OPTIONS] [REF]", Short: "Export a build into Docker Desktop bundle", - Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 { - options.ref = args[0] + if options.all && len(args) > 0 { + return errors.New("cannot specify refs when using --all") } + options.refs = args options.builder = *rootOpts.Builder return runExport(cmd.Context(), dockerCli, options) }, @@ -132,6 +154,7 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command { flags := cmd.Flags() flags.StringVarP(&options.output, "output", "o", "", "Output file path") + flags.BoolVar(&options.all, "all", false, "Export all records for the builder") return cmd } diff --git a/docs/reference/buildx_history_export.md b/docs/reference/buildx_history_export.md index 02e278d7..34b3bc2c 100644 --- a/docs/reference/buildx_history_export.md +++ b/docs/reference/buildx_history_export.md @@ -7,6 +7,7 @@ Export a build into Docker Desktop bundle | Name | Type | Default | Description | |:-----------------|:---------|:--------|:-----------------------------------------| +| `--all` | `bool` | | Export all records for the builder | | `--builder` | `string` | | Override the configured builder instance | | `-D`, `--debug` | `bool` | | Enable debug logging | | `-o`, `--output` | `string` | | Output file path | diff --git a/util/desktop/bundle/content.go b/util/desktop/bundle/content.go new file mode 100644 index 00000000..7a8a8fd4 --- /dev/null +++ b/util/desktop/bundle/content.go @@ -0,0 +1,78 @@ +package bundle + +import ( + "context" + + "github.com/containerd/containerd/v2/core/content" + cerrdefs "github.com/containerd/errdefs" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" +) + +type nsFallbackStore struct { + main content.Store + fb content.Store +} + +var _ content.Store = &nsFallbackStore{} + +func (c *nsFallbackStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { + info, err := c.main.Info(ctx, dgst) + if err != nil { + if cerrdefs.IsNotFound(err) { + return c.fb.Info(ctx, dgst) + } + } + return info, err +} + +func (c *nsFallbackStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) { + return c.main.Update(ctx, info, fieldpaths...) +} + +func (c *nsFallbackStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error { + seen := make(map[digest.Digest]struct{}) + err := c.main.Walk(ctx, func(i content.Info) error { + seen[i.Digest] = struct{}{} + return fn(i) + }, filters...) + if err != nil { + return err + } + return c.fb.Walk(ctx, func(i content.Info) error { + if _, ok := seen[i.Digest]; ok { + return nil + } + return fn(i) + }, filters...) +} + +func (c *nsFallbackStore) Delete(ctx context.Context, dgst digest.Digest) error { + return c.main.Delete(ctx, dgst) +} + +func (c *nsFallbackStore) Status(ctx context.Context, ref string) (content.Status, error) { + return c.main.Status(ctx, ref) +} + +func (c *nsFallbackStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) { + return c.main.ListStatuses(ctx, filters...) +} + +func (c *nsFallbackStore) Abort(ctx context.Context, ref string) error { + return c.main.Abort(ctx, ref) +} + +func (c *nsFallbackStore) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + ra, err := c.main.ReaderAt(ctx, desc) + if err != nil { + if cerrdefs.IsNotFound(err) { + return c.fb.ReaderAt(ctx, desc) + } + } + return ra, err +} + +func (c *nsFallbackStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + return c.main.Writer(ctx, opts...) +} diff --git a/util/desktop/bundle/export.go b/util/desktop/bundle/export.go index da522340..3ecb87a3 100644 --- a/util/desktop/bundle/export.go +++ b/util/desktop/bundle/export.go @@ -32,8 +32,23 @@ type Record struct { StateGroup *localstate.StateGroup `json:"stateGroup,omitempty"` } -func Export(ctx context.Context, c *client.Client, w io.Writer, records []*Record) error { - store := proxy.NewContentStore(c.ContentClient()) +func Export(ctx context.Context, c []*client.Client, w io.Writer, records []*Record) error { + var store content.Store + for _, c := range c { + s := proxy.NewContentStore(c.ContentClient()) + if store == nil { + store = s + break + } + store = &nsFallbackStore{ + main: store, + fb: s, + } + } + if store == nil { + return errors.New("no buildkit client found") + } + mp := contentutil.NewMultiProvider(store) desc, err := export(ctx, mp, records) diff --git a/util/desktop/bundle/trace.go b/util/desktop/bundle/trace.go index 23334a29..63f92037 100644 --- a/util/desktop/bundle/trace.go +++ b/util/desktop/bundle/trace.go @@ -7,6 +7,7 @@ import ( "io" "regexp" "strings" + "sync" "github.com/containerd/containerd/v2/core/content" "github.com/docker/buildx/util/otelutil" @@ -17,15 +18,22 @@ import ( ) var ( - sensitiveKeys = []string{"ghtoken", "token", "access_key_id", "secret_access_key", "session_token"} - reAttrs = regexp.MustCompile(`(?i)(` + strings.Join(sensitiveKeys, "|") + `)=[^ ,]+`) - reGhs = regexp.MustCompile(`ghs_[A-Za-z0-9]{36}`) + reOnce sync.Once + reAttrs, reGhs, reGhpat *regexp.Regexp ) func sanitizeCommand(value string) string { + reOnce.Do(func() { + sensitiveKeys := []string{"ghtoken", "token", "access_key_id", "secret_access_key", "session_token"} + reAttrs = regexp.MustCompile(`(?i)(` + strings.Join(sensitiveKeys, "|") + `)=[^ ,]+`) + reGhs = regexp.MustCompile(`(?:ghu|ghs)_[A-Za-z0-9]{36}`) + reGhpat = regexp.MustCompile(`github_pat_\w{82}`) + }) + value = reAttrs.ReplaceAllString(value, "${1}=xxxxx") - // reGhs is just double proofing. Not really needed. + // reGhs/reGhpat is just double proofing. Not really needed. value = reGhs.ReplaceAllString(value, "xxxxx") + value = reGhpat.ReplaceAllString(value, "xxxxx") return value }