Merge pull request #1434 from jedevc/resource-interpolation

Resource interpolation support

.dockerignore

@@ -1,3 +1 @@
 bin/
-cross-out/
-release-out/

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    open-pull-requests-limit: 10
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "dependencies"
+      - "bot"

.github/workflows/build.yml

@@ -1,5 +1,9 @@
 name: build
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
@@ -12,48 +16,119 @@ on:
     branches:
       - 'master'
       - 'v[0-9]*'
+    paths-ignore:
+      - 'README.md'
+      - 'docs/**'
 
 env:
   REPO_SLUG: "docker/buildx-bin"
-  RELEASE_OUT: "./release-out"
+  DESTDIR: "./bin"
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
       -
         name: Test
-        run: |
+        uses: docker/bake-action@v2
-          make test
-      -
-        name: Send to Codecov
-        uses: codecov/codecov-action@v2
         with:
-          file: ./coverage/coverage.txt
+          targets: test
+          set: |
+            *.cache-from=type=gha,scope=test
+            *.cache-to=type=gha,scope=test
       -
-        name: Build binaries
+        name: Upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ${{ env.DESTDIR }}/coverage
+
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.platforms.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Create matrix
+        id: platforms
         run: |
-          make release
+          echo ::set-output name=matrix::$(docker buildx bake binaries-cross --print | jq -cr '.target."binaries-cross".platforms')
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.platforms.outputs.matrix }}
+
+  binaries:
+    runs-on: ubuntu-latest
+    needs:
+      - prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    steps:
+      -
+        name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      -
+        name: Build
+        uses: docker/bake-action@v2
+        with:
+          targets: release
+          set: |
+            *.platform=${{ matrix.platform }}
+            *.cache-from=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }}
+            *.cache-to=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }},mode=max
       -
         name: Upload artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: buildx
-          path: ${{ env.RELEASE_OUT }}/*
+          path: ${{ env.DESTDIR }}/release/*
           if-no-files-found: error
+
+  bin-image:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
       -
         name: Docker meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: |
             ${{ env.REPO_SLUG }}
@@ -65,28 +140,57 @@ jobs:
       -
         name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push image
-        uses: docker/bake-action@v1
+        uses: docker/bake-action@v2
         with:
           files: |
             ./docker-bake.hcl
             ${{ steps.meta.outputs.bake-file }}
           targets: image-cross
           push: ${{ github.event_name != 'pull_request' }}
+          set: |
+            *.cache-from=type=gha,scope=bin-image
+            *.cache-to=type=gha,scope=bin-image,mode=max
+
+  release:
+    runs-on: ubuntu-latest
+    needs:
+      - binaries
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Download binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: buildx
+          path: ${{ env.DESTDIR }}
+      -
+        name: Create checksums
+        run: ./hack/hash-files
+      -
+        name: List artifacts
+        run: |
+          tree -nh ${{ env.DESTDIR }}
+      -
+        name: Check artifacts
+        run: |
+          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: GitHub Release
         if: startsWith(github.ref, 'refs/tags/v')
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           draft: true
-          files: ${{ env.RELEASE_OUT }}/*
+          files: ${{ env.DESTDIR }}/*
 
   buildkit-edge:
     runs-on: ubuntu-latest
@@ -94,19 +198,20 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         with:
+          version: latest
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: --debug
       -
         # Just run a bake target to check eveything runs fine
         name: Build
-        uses: docker/bake-action@v1
+        uses: docker/bake-action@v2
         with:
-          targets: binaries-cross
+          targets: binaries

.github/workflows/docs-release.yml

@@ -0,0 +1,56 @@
+name: docs-release
+
+on:
+  release:
+    types: [ released ]
+
+jobs:
+  open-pr:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout docs repo
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GHPAT_DOCS_DISPATCH }}
+          repository: docker/docs
+          ref: main
+      -
+        name: Prepare
+        run: |
+          rm -rf ./_data/buildx/*
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build docs
+        uses: docker/bake-action@v2
+        with:
+          source: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.event.release.name }}
+          targets: update-docs
+          set: |
+            *.output=/tmp/buildx-docs
+        env:
+          DOCS_FORMATS: yaml
+      -
+        name: Copy files
+        run: |
+          cp /tmp/buildx-docs/out/reference/*.yaml ./_data/buildx/
+      -
+        name: Commit changes
+        run: |
+          git add -A .
+      -
+        name: Create PR on docs repo
+        uses: peter-evans/create-pull-request@2b011faafdcbc9ceb11414d64d0573f37c774b04
+        with:
+          token: ${{ secrets.GHPAT_DOCS_DISPATCH }}
+          push-to-fork: docker-tools-robot/docker.github.io
+          commit-message: "build: update buildx reference to ${{ github.event.release.name }}"
+          signoff: true
+          branch: dispatch/buildx-ref-${{ github.event.release.name }}
+          delete-branch: true
+          title: Update buildx reference to ${{ github.event.release.name }}
+          body: |
+            Update the buildx reference documentation to keep in sync with the latest release `${{ github.event.release.name }}`
+          draft: false

.github/workflows/docs-upstream.yml

@@ -0,0 +1,61 @@
+# this workflow runs the remote validate bake target from docker/docker.github.io
+# to check if yaml reference docs and markdown files used in this repo are still valid
+# https://github.com/docker/docker.github.io/blob/98c7c9535063ae4cd2cd0a31478a21d16d2f07a3/docker-bake.hcl#L34-L36
+name: docs-upstream
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'v[0-9]*'
+    paths:
+      - '.github/workflows/docs-upstream.yml'
+      - 'docs/**'
+  pull_request:
+    paths:
+      - '.github/workflows/docs-upstream.yml'
+      - 'docs/**'
+
+jobs:
+  docs-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      -
+        name: Build reference YAML docs
+        uses: docker/bake-action@v2
+        with:
+          targets: update-docs
+          set: |
+            *.output=/tmp/buildx-docs
+            *.cache-from=type=gha,scope=docs-yaml
+            *.cache-to=type=gha,scope=docs-yaml,mode=max
+        env:
+          DOCS_FORMATS: yaml
+      -
+        name: Upload reference YAML docs
+        uses: actions/upload-artifact@v3
+        with:
+          name: docs-yaml
+          path: /tmp/buildx-docs/out/reference
+          retention-days: 1
+
+  validate:
+    uses: docker/docs/.github/workflows/validate-upstream.yml@main
+    needs:
+      - docs-yaml
+    with:
+      repo: https://github.com/${{ github.repository }}
+      data-files-id: docs-yaml
+      data-files-folder: buildx

.github/workflows/e2e.yml

@@ -1,5 +1,9 @@
 name: e2e
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
@@ -10,10 +14,50 @@ on:
     branches:
       - 'master'
       - 'v[0-9]*'
+    paths-ignore:
+      - 'README.md'
+      - 'docs/**'
+
+env:
+  DESTDIR: "./bin"
 
 jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      -
+        name: Build
+        uses: docker/bake-action@v2
+        with:
+          targets: binaries
+          set: |
+            *.cache-from=type=gha,scope=release
+            *.cache-from=type=gha,scope=binaries
+            *.cache-to=type=gha,scope=binaries
+      -
+        name: Rename binary
+        run: |
+          mv ${{ env.DESTDIR }}/build/buildx ${{ env.DESTDIR }}/build/docker-buildx
+      -
+        name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: binary
+          path: ${{ env.DESTDIR }}/build
+          if-no-files-found: error
+          retention-days: 7
+
   driver:
     runs-on: ubuntu-20.04
+    needs:
+      - build
     strategy:
       fail-fast: false
       matrix:
@@ -21,6 +65,7 @@ jobs:
           - docker
           - docker-container
           - kubernetes
+          - remote
         buildkit:
           - moby/buildkit:buildx-stable-1
           - moby/buildkit:master
@@ -36,6 +81,8 @@ jobs:
         include:
           - driver: kubernetes
             driver-opt: qemu.install=true
+          - driver: remote
+            endpoint: tcp://localhost:1234
         exclude:
           - driver: docker
             multi-node: mnode-true
@@ -43,18 +90,28 @@ jobs:
             buildkit-cfg: bkcfg-true
           - driver: docker-container
             multi-node: mnode-true
+          - driver: remote
+            multi-node: mnode-true
+          - driver: remote
+            buildkit-cfg: bkcfg-true
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
         if: matrix.driver == 'docker' || matrix.driver == 'docker-container'
       -
         name: Install buildx
+        uses: actions/download-artifact@v3
+        with:
+          name: binary
+          path: /home/runner/.docker/cli-plugins
+      -
+        name: Fix perms and check
         run: |
-          make install
+          chmod +x /home/runner/.docker/cli-plugins/docker-buildx
           docker buildx version
       -
         name: Init env vars
@@ -76,7 +133,7 @@ jobs:
       -
         name: Install k3s
         if: matrix.driver == 'kubernetes'
-        uses: debianmaster/actions-k3s@v1.0.3
+        uses: debianmaster/actions-k3s@b9cf3f599fd118699a3c8a0d18a2f2bda6cf4ce4
         id: k3s
         with:
           version: v1.21.2-k3s1
@@ -90,6 +147,17 @@ jobs:
         if: matrix.driver == 'kubernetes'
         run: |
           kubectl get nodes
+      -
+        name: Launch remote buildkitd
+        if: matrix.driver == 'remote'
+        run: |
+          docker run -d \
+            --privileged \
+            --name=remote-buildkit \
+            -p 1234:1234 \
+            ${{ matrix.buildkit }} \
+            --addr unix:///run/buildkit/buildkitd.sock \
+            --addr tcp://0.0.0.0:1234
       -
         name: Test
         run: |
@@ -98,4 +166,5 @@ jobs:
           BUILDKIT_IMAGE: ${{ matrix.buildkit }}
           DRIVER: ${{ matrix.driver }}
           DRIVER_OPT: ${{ matrix.driver-opt }}
+          ENDPOINT: ${{ matrix.endpoint }}
           PLATFORMS: ${{ matrix.platforms }}

.github/workflows/godev.yml

@@ -1,25 +0,0 @@
-# Workflow used to make a request to proxy.golang.org to refresh cache on https://pkg.go.dev/github.com/docker/buildx
-# when a released of buildx is produced
-name: godev
-
-on:
-  push:
-    tags:
-      - 'v*'
-
-jobs:
-  update:
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.13
-      -
-        name: Call pkg.go.dev
-        run: |
-          go get github.com/${GITHUB_REPOSITORY}@${GITHUB_REF#refs/tags/}
-        env:
-          GO111MODULE: on
-          GOPROXY: https://proxy.golang.org

.github/workflows/validate.yml

@@ -1,5 +1,9 @@
 name: validate
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
@@ -26,23 +30,13 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
       -
         name: Run
         run: |
           make ${{ matrix.target }}
-
-  validate-docs-yaml:
-    runs-on: ubuntu-latest
-    needs:
-      - validate
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Run
-        run: |
-          make docs
-        env:
-          FORMATS: yaml

.gitignore

@@ -1,4 +1 @@
-bin
+/bin
-coverage
-cross-out
-release-out

.golangci.yml

@@ -12,19 +12,29 @@ linters:
     - gofmt
     - govet
     - deadcode
+    - depguard
     - goimports
     - ineffassign
     - misspell
     - unused
     - varcheck
-    - golint
+    - revive
     - staticcheck
     - typecheck
-    - structcheck
+    - nolintlint
   disable-all: true
 
+linters-settings:
+  depguard:
+    list-type: blacklist
+    include-go-root: true
+    packages:
+      # The io/ioutil package has been deprecated.
+      # https://go.dev/doc/go1.16#ioutil
+      - io/ioutil
+
 issues:
   exclude-rules:
     - linters:
-        - golint
+        - revive
       text: "stutters"

.yamllint.yml

@@ -1,13 +0,0 @@
-ignore: |
-  /vendor
-
-extends: default
-
-yaml-files:
-  - '*.yaml'
-  - '*.yml'
-
-rules:
-  truthy: disable
-  line-length: disable
-  document-start: disable

Dockerfile

@@ -1,12 +1,13 @@
-# syntax=docker/dockerfile:1.3
+# syntax=docker/dockerfile:1.4
 
-ARG GO_VERSION=1.17
+ARG GO_VERSION=1.19
-ARG DOCKERD_VERSION=20.10.8
+ARG XX_VERSION=1.1.2
+ARG DOCKERD_VERSION=20.10.14
 
 FROM docker:$DOCKERD_VERSION AS dockerd-release
 
 # xx is a helper for cross-compilation
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.0.0 AS xx
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest
 
@@ -14,26 +15,30 @@ FROM golatest AS gobase
 COPY --from=xx / /
 RUN apk add --no-cache file git
 ENV GOFLAGS=-mod=vendor
+ENV CGO_ENABLED=0
 WORKDIR /src
 
 FROM gobase AS buildx-version
-RUN --mount=target=. \
+RUN --mount=type=bind,target=. <<EOT
-  PKG=github.com/docker/buildx VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \
+  set -e
-  echo "-X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" | tee /tmp/.ldflags; \
+  mkdir /buildx-version
-  echo -n "${VERSION}" | tee /tmp/.version;
+  echo -n "$(./hack/git-meta version)" | tee /buildx-version/version
+  echo -n "$(./hack/git-meta revision)" | tee /buildx-version/revision
+EOT
 
 FROM gobase AS buildx-build
-ENV CGO_ENABLED=0
-ARG LDFLAGS="-w -s"
 ARG TARGETPLATFORM
 RUN --mount=type=bind,target=. \
   --mount=type=cache,target=/root/.cache \
   --mount=type=cache,target=/go/pkg/mod \
-  --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildx-version \
+  --mount=type=bind,from=buildx-version,source=/buildx-version,target=/buildx-version <<EOT
-  set -x; xx-go build -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -o /usr/bin/buildx ./cmd/buildx && \
+  set -e
-  xx-verify --static /usr/bin/buildx
+  xx-go --wrap
+  DESTDIR=/usr/bin VERSION=$(cat /buildx-version/version) REVISION=$(cat /buildx-version/revision) GO_EXTRA_LDFLAGS="-s -w" ./hack/build
+  xx-verify --static /usr/bin/docker-buildx
+EOT
 
-FROM buildx-build AS test
+FROM gobase AS test
 RUN --mount=type=bind,target=. \
   --mount=type=cache,target=/root/.cache \
   --mount=type=cache,target=/go/pkg/mod \
@@ -44,13 +49,13 @@ FROM scratch AS test-coverage
 COPY --from=test /tmp/coverage.txt /coverage.txt
 
 FROM scratch AS binaries-unix
-COPY --from=buildx-build /usr/bin/buildx /
+COPY --link --from=buildx-build /usr/bin/docker-buildx /buildx
 
 FROM binaries-unix AS binaries-darwin
 FROM binaries-unix AS binaries-linux
 
 FROM scratch AS binaries-windows
-COPY --from=buildx-build /usr/bin/buildx /buildx.exe
+COPY --link --from=buildx-build /usr/bin/docker-buildx /buildx.exe
 
 FROM binaries-$TARGETOS AS binaries
 
@@ -59,8 +64,11 @@ FROM --platform=$BUILDPLATFORM alpine AS releaser
 WORKDIR /work
 ARG TARGETPLATFORM
 RUN --mount=from=binaries \
-  --mount=type=bind,source=/tmp/.version,target=/tmp/.version,from=buildx-version \
+  --mount=type=bind,from=buildx-version,source=/buildx-version,target=/buildx-version <<EOT
-  mkdir -p /out && cp buildx* "/out/buildx-$(cat /tmp/.version).$(echo $TARGETPLATFORM | sed 's/\//-/g')$(ls buildx* | sed -e 's/^buildx//')"
+  set -e
+  mkdir -p /out
+  cp buildx* "/out/buildx-$(cat /buildx-version/version).$(echo $TARGETPLATFORM | sed 's/\//-/g')$(ls buildx* | sed -e 's/^buildx//')"
+EOT
 
 FROM scratch AS release
 COPY --from=releaser /out/ /

MAINTAINERS

@@ -152,6 +152,7 @@ made through a pull request.
 		people = [
 			"akihirosuda",
 			"crazy-max",
+			"jedevc",
 			"tiborvass",
 			"tonistiigi",
 		]
@@ -188,6 +189,11 @@ made through a pull request.
 	Email = "contact@crazymax.dev"
 	GitHub = "crazy-max"
 
+	[people.jedevc]
+	Name = "Justin Chadwell"
+	Email = "me@jedevc.com"
+	GitHub = "jedevc"
+
 	[people.thajeztah]
 	Name = "Sebastiaan van Stijn"
 	Email = "github@gone.nl"

Makefile

@@ -4,59 +4,77 @@ else ifneq (, $(shell docker buildx version))
 	export BUILDX_CMD = docker buildx
 else ifneq (, $(shell which buildx))
 	export BUILDX_CMD = $(which buildx)
-else
-	$(error "Buildx is required: https://github.com/docker/buildx#installing")
 endif
 
-export BIN_OUT = ./bin
+export BUILDX_CMD ?= docker buildx
-export RELEASE_OUT = ./release-out
 
+.PHONY: all
+all: binaries
+
+.PHONY: build
+build:
+	./hack/build
+
+.PHONY: shell
 shell:
 	./hack/shell
 
+.PHONY: binaries
 binaries:
 	$(BUILDX_CMD) bake binaries
 
+.PHONY: binaries-cross
 binaries-cross:
 	$(BUILDX_CMD) bake binaries-cross
 
+.PHONY: install
 install: binaries
 	mkdir -p ~/.docker/cli-plugins
-	install bin/buildx ~/.docker/cli-plugins/docker-buildx
+	install bin/build/buildx ~/.docker/cli-plugins/docker-buildx
 
+.PHONY: release
 release:
 	./hack/release
 
+.PHONY: validate-all
 validate-all: lint test validate-vendor validate-docs
 
+.PHONY: lint
 lint:
 	$(BUILDX_CMD) bake lint
 
+.PHONY: test
 test:
 	$(BUILDX_CMD) bake test
 
+.PHONY: validate-vendor
 validate-vendor:
 	$(BUILDX_CMD) bake validate-vendor
 
+.PHONY: validate-docs
 validate-docs:
 	$(BUILDX_CMD) bake validate-docs
 
+.PHONY: validate-authors
 validate-authors:
 	$(BUILDX_CMD) bake validate-authors
 
+.PHONY: test-driver
 test-driver:
 	./hack/test-driver
 
+.PHONY: vendor
 vendor:
 	./hack/update-vendor
 
+.PHONY: docs
 docs:
 	./hack/update-docs
 
+.PHONY: authors
 authors:
 	$(BUILDX_CMD) bake update-authors
 
+.PHONY: mod-outdated
 mod-outdated:
 	$(BUILDX_CMD) bake mod-outdated
-
-.PHONY: shell binaries binaries-cross install release validate-all lint validate-vendor validate-docs validate-authors vendor docs authors

README.md

@@ -32,14 +32,19 @@ Key features:
   - [Building with buildx](#building-with-buildx)
   - [Working with builder instances](#working-with-builder-instances)
   - [Building multi-platform images](#building-multi-platform-images)
-  - [High-level build options](#high-level-build-options)
+- [Manuals](docs/manuals)
+  - [High-level build options with Bake](docs/manuals/bake/index.md)
+  - [Drivers](docs/manuals/drivers/index.md)
+  - [Exporters](docs/manuals/exporters/index.md)
+  - [Cache backends](docs/manuals/cache/backends/index.md)
 - [Guides](docs/guides)
   - [CI/CD](docs/guides/cicd.md)
   - [CNI networking](docs/guides/cni-networking.md)
-  - [Registry mirror](docs/guides/registry-mirror.md)
-  - [Resource limiting](docs/guides/resource-limiting.md)
   - [Using a custom network](docs/guides/custom-network.md)
   - [Using a custom registry configuration](docs/guides/custom-registry-config.md)
+  - [OpenTelemetry support](docs/guides/opentelemetry.md)
+  - [Registry mirror](docs/guides/registry-mirror.md)
+  - [Resource limiting](docs/guides/resource-limiting.md)
 - [Reference](docs/reference/buildx.md)
   - [`buildx bake`](docs/reference/buildx_bake.md)
   - [`buildx build`](docs/reference/buildx_build.md)
@@ -95,7 +100,7 @@ Rename the relevant binary and copy it to the destination matching your OS:
 | -------- | -------------------- | -----------------------------------------|
 | Linux    | `docker-buildx`      | `$HOME/.docker/cli-plugins`              |
 | macOS    | `docker-buildx`      | `$HOME/.docker/cli-plugins`              |
-| Windows  | `docker-buildx.exe`  | `%USERPROFILE%\.docker\cli-plugin`       |
+| Windows  | `docker-buildx.exe`  | `%USERPROFILE%\.docker\cli-plugins`      |
 
 Or copy it into one of these folders for installing it system-wide.
 
@@ -121,7 +126,8 @@ On Windows:
 Here is how to install and use Buildx inside a Dockerfile through the
 [`docker/buildx-bin`](https://hub.docker.com/r/docker/buildx-bin) image:
 
-```Dockerfile
+```dockerfile
+# syntax=docker/dockerfile:1
 FROM docker
 COPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 RUN docker buildx version
@@ -183,23 +189,17 @@ specifying target platform. In addition, Buildx also supports new features that
 are not yet available for regular `docker build` like building manifest lists,
 distributed caching, and exporting build results to OCI image tarballs.
 
-Buildx is supposed to be flexible and can be run in different configurations
+Buildx is flexible and can be run in different configurations that are exposed
-that are exposed through a driver concept. Currently, we support a
+through various "drivers". Each driver defines how and where a build should
-[`docker` driver](docs/reference/buildx_create.md#docker-driver) that uses
+run, and have different feature sets.
-the BuildKit library bundled into the Docker daemon binary, a
-[`docker-container` driver](docs/reference/buildx_create.md#docker-container-driver)
-that automatically launches BuildKit inside a Docker container and a
-[`kubernetes` driver](docs/reference/buildx_create.md#kubernetes-driver) to
-spin up pods with defined BuildKit container image to build your images. We
-plan to add more drivers in the future.
 
-The user experience of using buildx is very similar across drivers, but there
+We currently support the following drivers:
-are some features that are not currently supported by the `docker` driver,
+- The `docker` driver ([guide](docs/manuals/drivers/docker.md), [reference](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver))
-because the BuildKit library bundled into docker daemon currently uses a
+- The `docker-container` driver ([guide](docs/manuals/drivers/docker-container.md), [reference](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver))
-different storage component. In contrast, all images built with `docker` driver
+- The `kubernetes` driver ([guide](docs/manuals/drivers/kubernetes.md), [reference](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver))
-are automatically added to the `docker images` view by default, whereas when
+- The `remote` driver ([guide](docs/manuals/drivers/remote.md))
-using other drivers the method for outputting an image needs to be selected
+
-with `--output`.
+For more information on drivers, see the [drivers guide](docs/manuals/drivers/index.md).
 
 ## Working with builder instances
 
@@ -302,6 +302,7 @@ inside your Dockerfile and can be leveraged by the processes running as part
 of your build.
 
 ```dockerfile
+# syntax=docker/dockerfile:1
 FROM --platform=$BUILDPLATFORM golang:alpine AS build
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -315,26 +316,7 @@ cross-compilation helpers for more advanced use-cases.
 
 ## High-level build options
 
-Buildx also aims to provide support for high-level build concepts that go beyond
+See [`docs/manuals/bake/index.md`](docs/manuals/bake/index.md) for more details.
-invoking a single build command. We want to support building all the images in
-your application together and let the users define project specific reusable
-build flows that can then be easily invoked by anyone.
-
-BuildKit efficiently handles multiple concurrent build requests and
-de-duplicating work. The build commands can be combined with general-purpose
-command runners (for example, `make`). However, these tools generally invoke
-builds in sequence  and therefore cannot leverage the full potential of BuildKit
-parallelization, or combine BuildKit’s output for the user. For this use case,
-we have added a command called [`docker buildx bake`](docs/reference/buildx_bake.md).
-
-The `bake` command supports building images from compose files, similar to
-[`docker-compose build`](https://docs.docker.com/compose/reference/build/),
-but allowing all the services to be built concurrently as part of a single
-request.
-
-There is also support for custom build rules from HCL/JSON files allowing
-better code reuse and different target groups. The design of bake is in very
-early stages and we are looking for feedback from users.
 
 # Contributing
 

bake/bake.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/csv"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,7 +17,8 @@ import (
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
-	"github.com/docker/docker/pkg/urlutil"
+	"github.com/docker/cli/cli/config"
+	"github.com/docker/docker/builder/remotecontext/urlutil"
 	hcl "github.com/hashicorp/hcl/v2"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session/auth/authprovider"
@@ -65,12 +66,12 @@ func ReadLocalFiles(names []string) ([]File, error) {
 		var dt []byte
 		var err error
 		if n == "-" {
-			dt, err = ioutil.ReadAll(os.Stdin)
+			dt, err = io.ReadAll(os.Stdin)
 			if err != nil {
 				return nil, err
 			}
 		} else {
-			dt, err = ioutil.ReadFile(n)
+			dt, err = os.ReadFile(n)
 			if err != nil {
 				if isDefault && errors.Is(err, os.ErrNotExist) {
 					continue
@@ -83,53 +84,54 @@ func ReadLocalFiles(names []string) ([]File, error) {
 	return out, nil
 }
 
-func ReadTargets(ctx context.Context, files []File, targets, overrides []string, defaults map[string]string) (map[string]*Target, []*Group, error) {
+func ReadTargets(ctx context.Context, files []File, targets, overrides []string, defaults map[string]string) (map[string]*Target, map[string]*Group, error) {
 	c, err := ParseFiles(files, defaults)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	for i, t := range targets {
+		targets[i] = sanitizeTargetName(t)
+	}
+
 	o, err := c.newOverrides(overrides)
 	if err != nil {
 		return nil, nil, err
 	}
 	m := map[string]*Target{}
-	for _, n := range targets {
+	n := map[string]*Group{}
-		for _, n := range c.ResolveGroup(n) {
+	for _, target := range targets {
-			t, err := c.ResolveTarget(n, o)
+		ts, gs := c.ResolveGroup(target)
+		for _, tname := range ts {
+			t, err := c.ResolveTarget(tname, o)
 			if err != nil {
 				return nil, nil, err
 			}
 			if t != nil {
-				m[n] = t
+				m[tname] = t
+			}
+		}
+		for _, gname := range gs {
+			for _, group := range c.Groups {
+				if group.Name == gname {
+					n[gname] = group
+					break
+				}
 			}
 		}
 	}
 
-	var g []*Group
+	for _, target := range targets {
-	if len(targets) == 0 || (len(targets) == 1 && targets[0] == "default") {
+		if target == "default" {
-		for _, group := range c.Groups {
-			if group.Name != "default" {
 			continue
 		}
-			g = []*Group{{Targets: group.Targets}}
+		if _, ok := n["default"]; !ok {
+			n["default"] = &Group{Name: "default"}
 		}
-	} else {
+		n["default"].Targets = append(n["default"].Targets, target)
-		var gt []string
-		for _, target := range targets {
-			isGroup := false
-			for _, group := range c.Groups {
-				if target == group.Name {
-					gt = append(gt, group.Targets...)
-					isGroup = true
-					break
 	}
-			}
+	if g, ok := n["default"]; ok {
-			if !isGroup {
+		g.Targets = dedupSlice(g.Targets)
-				gt = append(gt, target)
-			}
-		}
-		g = []*Group{{Targets: dedupString(gt)}}
 	}
 
 	for name, t := range m {
@@ -138,10 +140,10 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string,
 		}
 	}
 
-	return m, g, nil
+	return m, n, nil
 }
 
-func dedupString(s []string) []string {
+func dedupSlice(s []string) []string {
 	if len(s) == 0 {
 		return s
 	}
@@ -156,21 +158,54 @@ func dedupString(s []string) []string {
 	return res
 }
 
+func dedupMap(ms ...map[string]string) map[string]string {
+	if len(ms) == 0 {
+		return nil
+	}
+	res := map[string]string{}
+	for _, m := range ms {
+		if len(m) == 0 {
+			continue
+		}
+		for k, v := range m {
+			if _, ok := res[k]; !ok {
+				res[k] = v
+			}
+		}
+	}
+	return res
+}
+
+func sliceToMap(env []string) (res map[string]string) {
+	res = make(map[string]string)
+	for _, s := range env {
+		kv := strings.SplitN(s, "=", 2)
+		key := kv[0]
+		switch {
+		case len(kv) == 1:
+			res[key] = ""
+		default:
+			res[key] = kv[1]
+		}
+	}
+	return
+}
+
 func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) {
 	defer func() {
 		err = formatHCLError(err, files)
 	}()
 
 	var c Config
-	var fs []*hcl.File
+	var composeFiles []File
+	var hclFiles []*hcl.File
 	for _, f := range files {
-		cfg, isCompose, composeErr := ParseComposeFile(f.Data, f.Name)
+		isCompose, composeErr := validateComposeFile(f.Data, f.Name)
 		if isCompose {
 			if composeErr != nil {
 				return nil, composeErr
 			}
-			c = mergeConfig(c, *cfg)
+			composeFiles = append(composeFiles, f)
-			c = dedupeConfig(c)
 		}
 		if !isCompose {
 			hf, isHCL, err := ParseHCLFile(f.Data, f.Name)
@@ -178,7 +213,7 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error)
 				if err != nil {
 					return nil, err
 				}
-				fs = append(fs, hf)
+				hclFiles = append(hclFiles, hf)
 			} else if composeErr != nil {
 				return nil, fmt.Errorf("failed to parse %s: parsing yaml: %v, parsing hcl: %w", f.Name, composeErr, err)
 			} else {
@@ -187,8 +222,17 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error)
 		}
 	}
 
-	if len(fs) > 0 {
+	if len(composeFiles) > 0 {
-		if err := hclparser.Parse(hcl.MergeFiles(fs), hclparser.Opt{
+		cfg, cmperr := ParseComposeFiles(composeFiles)
+		if cmperr != nil {
+			return nil, errors.Wrap(cmperr, "failed to parse compose file")
+		}
+		c = mergeConfig(c, *cfg)
+		c = dedupeConfig(c)
+	}
+
+	if len(hclFiles) > 0 {
+		if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{
 			LookupVar:     os.LookupEnv,
 			Vars:          defaults,
 			ValidateLabel: validateTargetName,
@@ -196,18 +240,25 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error)
 			return nil, err
 		}
 	}
+
 	return &c, nil
 }
 
 func dedupeConfig(c Config) Config {
 	c2 := c
+	c2.Groups = make([]*Group, 0, len(c2.Groups))
+	for _, g := range c.Groups {
+		g1 := *g
+		g1.Targets = dedupSlice(g1.Targets)
+		c2.Groups = append(c2.Groups, &g1)
+	}
 	c2.Targets = make([]*Target, 0, len(c2.Targets))
-	m := map[string]*Target{}
+	mt := map[string]*Target{}
 	for _, t := range c.Targets {
-		if t2, ok := m[t.Name]; ok {
+		if t2, ok := mt[t.Name]; ok {
 			t2.Merge(t)
 		} else {
-			m[t.Name] = t
+			mt[t.Name] = t
 			c2.Targets = append(c2.Targets, t)
 		}
 	}
@@ -218,22 +269,9 @@ func ParseFile(dt []byte, fn string) (*Config, error) {
 	return ParseFiles([]File{{Data: dt, Name: fn}}, nil)
 }
 
-func ParseComposeFile(dt []byte, fn string) (*Config, bool, error) {
-	fnl := strings.ToLower(fn)
-	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
-		cfg, err := ParseCompose(dt)
-		return cfg, true, err
-	}
-	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
-		return nil, false, nil
-	}
-	cfg, err := ParseCompose(dt)
-	return cfg, err == nil, err
-}
-
 type Config struct {
-	Groups  []*Group  `json:"group" hcl:"group,block"`
+	Groups  []*Group  `json:"group" hcl:"group,block" cty:"group"`
-	Targets []*Target `json:"target" hcl:"target,block"`
+	Targets []*Target `json:"target" hcl:"target,block" cty:"target"`
 }
 
 func mergeConfig(c1, c2 Config) Config {
@@ -332,6 +370,7 @@ func (c Config) loadLinks(name string, t *Target, m map[string]*Target, o map[st
 					return err
 				}
 				t2.Outputs = nil
+				t2.linked = true
 				m[target] = t2
 			}
 			if err := c.loadLinks(target, t2, m, o, visited); err != nil {
@@ -378,7 +417,7 @@ func (c Config) newOverrides(v []string) (map[string]map[string]Override, error)
 			o := t[kk[1]]
 
 			switch keys[1] {
-			case "output", "cache-to", "cache-from", "tags", "platform", "secrets", "ssh":
+			case "output", "cache-to", "cache-from", "tags", "platform", "secrets", "ssh", "attest":
 				if len(parts) == 2 {
 					o.ArrValue = append(o.ArrValue, parts[1])
 				}
@@ -411,13 +450,19 @@ func (c Config) newOverrides(v []string) (map[string]map[string]Override, error)
 	return m, nil
 }
 
-func (c Config) ResolveGroup(name string) []string {
+func (c Config) ResolveGroup(name string) ([]string, []string) {
-	return c.group(name, map[string]struct{}{})
+	targets, groups := c.group(name, map[string]visit{})
+	return dedupSlice(targets), dedupSlice(groups)
 }
 
-func (c Config) group(name string, visited map[string]struct{}) []string {
+type visit struct {
-	if _, ok := visited[name]; ok {
+	target []string
-		return nil
+	group  []string
+}
+
+func (c Config) group(name string, visited map[string]visit) ([]string, []string) {
+	if v, ok := visited[name]; ok {
+		return v.target, v.group
 	}
 	var g *Group
 	for _, group := range c.Groups {
@@ -427,19 +472,24 @@ func (c Config) group(name string, visited map[string]struct{}) []string {
 		}
 	}
 	if g == nil {
-		return []string{name}
+		return []string{name}, nil
 	}
-	visited[name] = struct{}{}
+	visited[name] = visit{}
 	targets := make([]string, 0, len(g.Targets))
+	groups := []string{name}
 	for _, t := range g.Targets {
-		tgroup := c.group(t, visited)
+		ttarget, tgroup := c.group(t, visited)
-		if len(tgroup) > 0 {
+		if len(ttarget) > 0 {
-			targets = append(targets, tgroup...)
+			targets = append(targets, ttarget...)
 		} else {
 			targets = append(targets, t)
 		}
+		if len(tgroup) > 0 {
+			groups = append(groups, tgroup...)
 		}
-	return targets
+	}
+	visited[name] = visit{target: targets, group: groups}
+	return targets, groups
 }
 
 func (c Config) ResolveTarget(name string, overrides map[string]map[string]Override) (*Target, error) {
@@ -497,39 +547,44 @@ func (c Config) target(name string, visited map[string]*Target, overrides map[st
 }
 
 type Group struct {
-	Name    string   `json:"-" hcl:"name,label"`
+	Name    string   `json:"-" hcl:"name,label" cty:"name"`
-	Targets []string `json:"targets" hcl:"targets"`
+	Targets []string `json:"targets" hcl:"targets" cty:"targets"`
 	// Target // TODO?
 }
 
 type Target struct {
-	Name string `json:"-" hcl:"name,label"`
+	Name string `json:"-" hcl:"name,label" cty:"name"`
 
 	// Inherits is the only field that cannot be overridden with --set
-	Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
+	Attest   []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
+	Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`
 
-	Context          *string           `json:"context,omitempty" hcl:"context,optional"`
+	Context          *string           `json:"context,omitempty" hcl:"context,optional" cty:"context"`
-	Contexts         map[string]string `json:"contexts,omitempty" hcl:"contexts,optional"`
+	Contexts         map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
-	Dockerfile       *string           `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
+	Dockerfile       *string           `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
-	DockerfileInline *string           `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
+	DockerfileInline *string           `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
-	Args             map[string]string `json:"args,omitempty" hcl:"args,optional"`
+	Args             map[string]string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
-	Labels           map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
+	Labels           map[string]string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
-	Tags             []string          `json:"tags,omitempty" hcl:"tags,optional"`
+	Tags             []string          `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
-	CacheFrom        []string          `json:"cache-from,omitempty"  hcl:"cache-from,optional"`
+	CacheFrom        []string          `json:"cache-from,omitempty"  hcl:"cache-from,optional" cty:"cache-from"`
-	CacheTo          []string          `json:"cache-to,omitempty"  hcl:"cache-to,optional"`
+	CacheTo          []string          `json:"cache-to,omitempty"  hcl:"cache-to,optional" cty:"cache-to"`
-	Target           *string           `json:"target,omitempty" hcl:"target,optional"`
+	Target           *string           `json:"target,omitempty" hcl:"target,optional" cty:"target"`
-	Secrets          []string          `json:"secret,omitempty" hcl:"secret,optional"`
+	Secrets          []string          `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
-	SSH              []string          `json:"ssh,omitempty" hcl:"ssh,optional"`
+	SSH              []string          `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
-	Platforms        []string          `json:"platforms,omitempty" hcl:"platforms,optional"`
+	Platforms        []string          `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
-	Outputs          []string          `json:"output,omitempty" hcl:"output,optional"`
+	Outputs          []string          `json:"output,omitempty" hcl:"output,optional" cty:"output"`
-	Pull             *bool             `json:"pull,omitempty" hcl:"pull,optional"`
+	Pull             *bool             `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
-	NoCache          *bool             `json:"no-cache,omitempty" hcl:"no-cache,optional"`
+	NoCache          *bool             `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
-	NetworkMode      *string           `json:"-" hcl:"-"`
+	NetworkMode      *string           `json:"-" hcl:"-" cty:"-"`
-	NoCacheFilter    []string          `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional"`
+	NoCacheFilter    []string          `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
-	// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
+	// IMPORTANT: if you add more fields here, do not forget to update newOverrides and docs/manuals/bake/file-definition.md.
+
+	// linked is a private field to mark a target used as a linked one
+	linked bool
 }
 
 func (t *Target) normalize() {
+	t.Attest = removeDupes(t.Attest)
 	t.Tags = removeDupes(t.Tags)
 	t.Secrets = removeDupes(t.Secrets)
 	t.SSH = removeDupes(t.SSH)
@@ -583,6 +638,9 @@ func (t *Target) Merge(t2 *Target) {
 	if t2.Target != nil {
 		t.Target = t2.Target
 	}
+	if t2.Attest != nil { // merge
+		t.Attest = append(t.Attest, t2.Attest...)
+	}
 	if t2.Secrets != nil { // merge
 		t.Secrets = append(t.Secrets, t2.Secrets...)
 	}
@@ -665,6 +723,8 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
 			t.Platforms = o.ArrValue
 		case "output":
 			t.Outputs = o.ArrValue
+		case "attest":
+			t.Attest = append(t.Attest, o.ArrValue...)
 		case "no-cache":
 			noCache, err := strconv.ParseBool(value)
 			if err != nil {
@@ -868,6 +928,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		NoCacheFilter: t.NoCacheFilter,
 		Pull:          pull,
 		NetworkMode:   networkMode,
+		Linked:        t.linked,
 	}
 
 	platforms, err := platformutil.Parse(t.Platforms)
@@ -876,7 +937,8 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 	}
 	bo.Platforms = platforms
 
-	bo.Session = append(bo.Session, authprovider.NewDockerAuthProvider(os.Stderr))
+	dockerConfig := config.LoadDefaultConfigFile(os.Stderr)
+	bo.Session = append(bo.Session, authprovider.NewDockerAuthProvider(dockerConfig))
 
 	secrets, err := buildflags.ParseSecretSpecs(t.Secrets)
 	if err != nil {
@@ -916,6 +978,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 	}
 	bo.Exports = outputs
 
+	attests, err := buildflags.ParseAttests(t.Attest)
+	if err != nil {
+		return nil, err
+	}
+	bo.Attests = attests
+
 	return bo, nil
 }
 
@@ -968,6 +1036,13 @@ func validateTargetName(name string) error {
 	return nil
 }
 
+func sanitizeTargetName(target string) string {
+	// as stipulated in compose spec, service name can contain a dot so as
+	// best-effort and to avoid any potential ambiguity, we replace the dot
+	// with an underscore.
+	return strings.ReplaceAll(target, ".", "_")
+}
+
 func sliceEqual(s1, s2 []string) bool {
 	if len(s1) != len(s2) {
 		return false

bake/bake_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -46,7 +47,7 @@ target "webapp" {
 		require.Nil(t, m["webapp"].Pull)
 
 		require.Equal(t, 1, len(g))
-		require.Equal(t, []string{"webapp"}, g[0].Targets)
+		require.Equal(t, []string{"webapp"}, g["default"].Targets)
 	})
 
 	t.Run("InvalidTargetOverrides", func(t *testing.T) {
@@ -87,7 +88,7 @@ target "webapp" {
 			require.Equal(t, m["webapp"].Args["VAR_INHERITED"], "override")
 
 			require.Equal(t, 1, len(g))
-			require.Equal(t, []string{"webapp"}, g[0].Targets)
+			require.Equal(t, []string{"webapp"}, g["default"].Targets)
 		})
 
 		// building leaf but overriding parent fields
@@ -101,7 +102,7 @@ target "webapp" {
 			require.Equal(t, m["webapp"].Args["VAR_INHERITED"], "override")
 			require.Equal(t, m["webapp"].Args["VAR_BOTH"], "webapp")
 			require.Equal(t, 1, len(g))
-			require.Equal(t, []string{"webapp"}, g[0].Targets)
+			require.Equal(t, []string{"webapp"}, g["default"].Targets)
 		})
 	})
 
@@ -113,7 +114,7 @@ target "webapp" {
 		require.NoError(t, err)
 		require.Equal(t, "foo", *m["webapp"].Context)
 		require.Equal(t, 1, len(g))
-		require.Equal(t, []string{"webapp"}, g[0].Targets)
+		require.Equal(t, []string{"webapp"}, g["default"].Targets)
 	})
 
 	t.Run("NoCacheOverride", func(t *testing.T) {
@@ -121,7 +122,7 @@ target "webapp" {
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].NoCache)
 		require.Equal(t, 1, len(g))
-		require.Equal(t, []string{"webapp"}, g[0].Targets)
+		require.Equal(t, []string{"webapp"}, g["default"].Targets)
 	})
 
 	t.Run("PullOverride", func(t *testing.T) {
@@ -129,12 +130,12 @@ target "webapp" {
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].Pull)
 		require.Equal(t, 1, len(g))
-		require.Equal(t, []string{"webapp"}, g[0].Targets)
+		require.Equal(t, []string{"webapp"}, g["default"].Targets)
 	})
 
 	t.Run("PatternOverride", func(t *testing.T) {
 		// same check for two cases
-		multiTargetCheck := func(t *testing.T, m map[string]*Target, g []*Group, err error) {
+		multiTargetCheck := func(t *testing.T, m map[string]*Target, g map[string]*Group, err error) {
 			require.NoError(t, err)
 			require.Equal(t, 2, len(m))
 			require.Equal(t, "foo", *m["webapp"].Dockerfile)
@@ -142,15 +143,15 @@ target "webapp" {
 			require.Equal(t, "foo", *m["webDEP"].Dockerfile)
 			require.Equal(t, "webDEP", m["webDEP"].Args["VAR_INHERITED"])
 			require.Equal(t, 1, len(g))
-			sort.Strings(g[0].Targets)
+			sort.Strings(g["default"].Targets)
-			require.Equal(t, []string{"webDEP", "webapp"}, g[0].Targets)
+			require.Equal(t, []string{"webDEP", "webapp"}, g["default"].Targets)
 		}
 
 		cases := []struct {
 			name      string
 			targets   []string
 			overrides []string
-			check     func(*testing.T, map[string]*Target, []*Group, error)
+			check     func(*testing.T, map[string]*Target, map[string]*Group, error)
 		}{
 			{
 				name:      "multi target single pattern",
@@ -168,20 +169,20 @@ target "webapp" {
 				name:      "single target",
 				targets:   []string{"webapp"},
 				overrides: []string{"web*.dockerfile=foo"},
-				check: func(t *testing.T, m map[string]*Target, g []*Group, err error) {
+				check: func(t *testing.T, m map[string]*Target, g map[string]*Group, err error) {
 					require.NoError(t, err)
 					require.Equal(t, 1, len(m))
 					require.Equal(t, "foo", *m["webapp"].Dockerfile)
 					require.Equal(t, "webDEP", m["webapp"].Args["VAR_INHERITED"])
 					require.Equal(t, 1, len(g))
-					require.Equal(t, []string{"webapp"}, g[0].Targets)
+					require.Equal(t, []string{"webapp"}, g["default"].Targets)
 				},
 			},
 			{
 				name:      "nomatch",
 				targets:   []string{"webapp"},
 				overrides: []string{"nomatch*.dockerfile=foo"},
-				check: func(t *testing.T, m map[string]*Target, g []*Group, err error) {
+				check: func(t *testing.T, m map[string]*Target, g map[string]*Group, err error) {
 					// NOTE: I am unsure whether failing to match should always error out
 					// instead of simply skipping that override.
 					// Let's enforce the error and we can relax it later if users complain.
@@ -278,9 +279,19 @@ services:
 `),
 	}
 
+	fp3 := File{
+		Name: "docker-compose3.yml",
+		Data: []byte(
+			`version: "3"
+services:
+  webapp:
+    entrypoint: echo 1
+`),
+	}
+
 	ctx := context.TODO()
 
-	m, g, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil, nil)
+	m, g, err := ReadTargets(ctx, []File{fp, fp2, fp3}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 3, len(m))
@@ -293,8 +304,69 @@ services:
 	require.Equal(t, "12", m["webapp"].Args["buildno2"])
 
 	require.Equal(t, 1, len(g))
-	sort.Strings(g[0].Targets)
+	sort.Strings(g["default"].Targets)
-	require.Equal(t, []string{"db", "newservice", "webapp"}, g[0].Targets)
+	require.Equal(t, []string{"db", "newservice", "webapp"}, g["default"].Targets)
+}
+
+func TestReadTargetsWithDotCompose(t *testing.T) {
+	t.Parallel()
+
+	fp := File{
+		Name: "docker-compose.yml",
+		Data: []byte(
+			`version: "3"
+services:
+  web.app:
+    build:
+      dockerfile: Dockerfile.webapp
+      args:
+        buildno: 1
+`),
+	}
+
+	fp2 := File{
+		Name: "docker-compose2.yml",
+		Data: []byte(
+			`version: "3"
+services:
+  web_app:
+    build:
+      args:
+        buildno2: 12
+`),
+	}
+
+	ctx := context.TODO()
+
+	m, _, err := ReadTargets(ctx, []File{fp}, []string{"web.app"}, nil, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(m))
+	_, ok := m["web_app"]
+	require.True(t, ok)
+	require.Equal(t, "Dockerfile.webapp", *m["web_app"].Dockerfile)
+	require.Equal(t, "1", m["web_app"].Args["buildno"])
+
+	m, _, err = ReadTargets(ctx, []File{fp2}, []string{"web_app"}, nil, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(m))
+	_, ok = m["web_app"]
+	require.True(t, ok)
+	require.Equal(t, "Dockerfile", *m["web_app"].Dockerfile)
+	require.Equal(t, "12", m["web_app"].Args["buildno2"])
+
+	m, g, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(m))
+	_, ok = m["web_app"]
+	require.True(t, ok)
+	require.Equal(t, "Dockerfile.webapp", *m["web_app"].Dockerfile)
+	require.Equal(t, ".", *m["web_app"].Context)
+	require.Equal(t, "1", m["web_app"].Args["buildno"])
+	require.Equal(t, "12", m["web_app"].Args["buildno2"])
+
+	require.Equal(t, 1, len(g))
+	sort.Strings(g["default"].Targets)
+	require.Equal(t, []string{"web_app"}, g["default"].Targets)
 }
 
 func TestHCLCwdPrefix(t *testing.T) {
@@ -321,7 +393,7 @@ func TestHCLCwdPrefix(t *testing.T) {
 	require.Equal(t, "foo", *m["app"].Context)
 
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"app"}, g[0].Targets)
+	require.Equal(t, []string{"app"}, g["default"].Targets)
 }
 
 func TestOverrideMerge(t *testing.T) {
@@ -446,6 +518,41 @@ func TestReadContextFromTargetUnknown(t *testing.T) {
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "failed to find target bar")
 }
+
+func TestReadEmptyTargets(t *testing.T) {
+	t.Parallel()
+
+	fp := File{
+		Name: "docker-bake.hcl",
+		Data: []byte(`target "app1" {}`),
+	}
+
+	fp2 := File{
+		Name: "docker-compose.yml",
+		Data: []byte(`
+services:
+  app2:
+    build: {}
+`),
+	}
+
+	ctx := context.TODO()
+
+	m, _, err := ReadTargets(ctx, []File{fp, fp2}, []string{"app1", "app2"}, nil, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 2, len(m))
+	_, ok := m["app1"]
+	require.True(t, ok)
+	_, ok = m["app2"]
+	require.True(t, ok)
+
+	require.Equal(t, "Dockerfile", *m["app1"].Dockerfile)
+	require.Equal(t, ".", *m["app1"].Context)
+	require.Equal(t, "Dockerfile", *m["app2"].Dockerfile)
+	require.Equal(t, ".", *m["app2"].Context)
+}
+
 func TestReadContextFromTargetChain(t *testing.T) {
 	ctx := context.TODO()
 	fp := File{
@@ -590,7 +697,7 @@ target "image" {
 	m, g, err := ReadTargets(ctx, []File{f}, []string{"image"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"image"}, g["default"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "test", *m["image"].Dockerfile)
 }
@@ -611,8 +718,9 @@ target "image" {
 
 	m, g, err := ReadTargets(ctx, []File{f}, []string{"foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"image"}, g["foo"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "test", *m["image"].Dockerfile)
 }
@@ -636,15 +744,17 @@ target "image" {
 
 	m, g, err := ReadTargets(ctx, []File{f}, []string{"foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"image"}, g["foo"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "test", *m["image"].Dockerfile)
 
 	m, g, err = ReadTargets(ctx, []File{f}, []string{"foo", "foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"image"}, g["foo"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "test", *m["image"].Dockerfile)
 }
@@ -723,7 +833,7 @@ services:
 	m, g, err := ReadTargets(ctx, []File{fhcl}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"image"}, g["default"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, 1, len(m["image"].Outputs))
 	require.Equal(t, "type=docker", m["image"].Outputs[0])
@@ -731,7 +841,7 @@ services:
 	m, g, err = ReadTargets(ctx, []File{fhcl}, []string{"image-release"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image-release"}, g[0].Targets)
+	require.Equal(t, []string{"image-release"}, g["default"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, 1, len(m["image-release"].Outputs))
 	require.Equal(t, "type=image,push=true", m["image-release"].Outputs[0])
@@ -739,7 +849,7 @@ services:
 	m, g, err = ReadTargets(ctx, []File{fhcl}, []string{"image", "image-release"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image", "image-release"}, g[0].Targets)
+	require.Equal(t, []string{"image", "image-release"}, g["default"].Targets)
 	require.Equal(t, 2, len(m))
 	require.Equal(t, ".", *m["image"].Context)
 	require.Equal(t, 1, len(m["image-release"].Outputs))
@@ -748,22 +858,22 @@ services:
 	m, g, err = ReadTargets(ctx, []File{fyml, fhcl}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"image"}, g["default"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, ".", *m["image"].Context)
 
 	m, g, err = ReadTargets(ctx, []File{fjson}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	require.Equal(t, []string{"image"}, g[0].Targets)
+	require.Equal(t, []string{"image"}, g["default"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, ".", *m["image"].Context)
 
 	m, g, err = ReadTargets(ctx, []File{fyml}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	sort.Strings(g[0].Targets)
+	sort.Strings(g["default"].Targets)
-	require.Equal(t, []string{"addon", "aws"}, g[0].Targets)
+	require.Equal(t, []string{"addon", "aws"}, g["default"].Targets)
 	require.Equal(t, 2, len(m))
 	require.Equal(t, "./Dockerfile", *m["addon"].Dockerfile)
 	require.Equal(t, "./aws.Dockerfile", *m["aws"].Dockerfile)
@@ -771,8 +881,8 @@ services:
 	m, g, err = ReadTargets(ctx, []File{fyml, fhcl}, []string{"addon", "aws"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	sort.Strings(g[0].Targets)
+	sort.Strings(g["default"].Targets)
-	require.Equal(t, []string{"addon", "aws"}, g[0].Targets)
+	require.Equal(t, []string{"addon", "aws"}, g["default"].Targets)
 	require.Equal(t, 2, len(m))
 	require.Equal(t, "./Dockerfile", *m["addon"].Dockerfile)
 	require.Equal(t, "./aws.Dockerfile", *m["aws"].Dockerfile)
@@ -780,8 +890,8 @@ services:
 	m, g, err = ReadTargets(ctx, []File{fyml, fhcl}, []string{"addon", "aws", "image"}, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(g))
-	sort.Strings(g[0].Targets)
+	sort.Strings(g["default"].Targets)
-	require.Equal(t, []string{"addon", "aws", "image"}, g[0].Targets)
+	require.Equal(t, []string{"addon", "aws", "image"}, g["default"].Targets)
 	require.Equal(t, 3, len(m))
 	require.Equal(t, ".", *m["image"].Context)
 	require.Equal(t, "./Dockerfile", *m["addon"].Dockerfile)
@@ -807,15 +917,17 @@ target "image" {
 
 	m, g, err := ReadTargets(ctx, []File{f}, []string{"foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"foo"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"foo"}, g["foo"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "bar", *m["foo"].Dockerfile)
 
 	m, g, err = ReadTargets(ctx, []File{f}, []string{"foo", "foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"foo"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"foo"}, g["foo"].Targets)
 	require.Equal(t, 1, len(m))
 	require.Equal(t, "bar", *m["foo"].Dockerfile)
 }
@@ -839,16 +951,18 @@ target "image" {
 
 	m, g, err := ReadTargets(ctx, []File{f}, []string{"foo"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"foo", "image"}, g[0].Targets)
+	require.Equal(t, []string{"foo"}, g["default"].Targets)
+	require.Equal(t, []string{"foo", "image"}, g["foo"].Targets)
 	require.Equal(t, 2, len(m))
 	require.Equal(t, "bar", *m["foo"].Dockerfile)
 	require.Equal(t, "type=docker", m["image"].Outputs[0])
 
 	m, g, err = ReadTargets(ctx, []File{f}, []string{"foo", "image"}, nil, nil)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(g))
+	require.Equal(t, 2, len(g))
-	require.Equal(t, []string{"foo", "image"}, g[0].Targets)
+	require.Equal(t, []string{"foo", "image"}, g["default"].Targets)
+	require.Equal(t, []string{"foo", "image"}, g["foo"].Targets)
 	require.Equal(t, 2, len(m))
 	require.Equal(t, "bar", *m["foo"].Dockerfile)
 	require.Equal(t, "type=docker", m["image"].Outputs[0])
@@ -909,7 +1023,7 @@ target "d" {
 			m, g, err := ReadTargets(ctx, []File{f}, []string{"d"}, tt.overrides, nil)
 			require.NoError(t, err)
 			require.Equal(t, 1, len(g))
-			require.Equal(t, []string{"d"}, g[0].Targets)
+			require.Equal(t, []string{"d"}, g["default"].Targets)
 			require.Equal(t, 1, len(m))
 			require.Equal(t, tt.want, m["d"].Args)
 		})
@@ -981,7 +1095,7 @@ group "default" {
 			m, g, err := ReadTargets(ctx, []File{f}, []string{"default"}, tt.overrides, nil)
 			require.NoError(t, err)
 			require.Equal(t, 1, len(g))
-			require.Equal(t, []string{"child1", "child2"}, g[0].Targets)
+			require.Equal(t, []string{"child1", "child2"}, g["default"].Targets)
 			require.Equal(t, 2, len(m))
 			require.Equal(t, tt.wantch1, m["child1"].Args)
 			require.Equal(t, []string{"type=docker"}, m["child1"].Outputs)
@@ -1045,3 +1159,134 @@ func TestTargetName(t *testing.T) {
 		})
 	}
 }
+
+func TestNestedGroupsWithSameTarget(t *testing.T) {
+	ctx := context.TODO()
+
+	f := File{
+		Name: "docker-bake.hcl",
+		Data: []byte(`
+group "a" {
+  targets = ["b", "c"]
+}
+
+group "b" {
+  targets = ["d"]
+}
+
+group "c" {
+  targets = ["b"]
+}
+
+target "d" {
+  context = "."
+  dockerfile = "./testdockerfile"
+}
+
+group "e" {
+  targets = ["a", "f"]
+}
+
+target "f" {
+  context = "./foo"
+}`)}
+
+	cases := []struct {
+		names   []string
+		targets []string
+		groups  []string
+		count   int
+	}{
+		{
+			names:   []string{"a"},
+			targets: []string{"a"},
+			groups:  []string{"default", "a", "b", "c"},
+			count:   1,
+		},
+		{
+			names:   []string{"b"},
+			targets: []string{"b"},
+			groups:  []string{"default", "b"},
+			count:   1,
+		},
+		{
+			names:   []string{"c"},
+			targets: []string{"c"},
+			groups:  []string{"default", "b", "c"},
+			count:   1,
+		},
+		{
+			names:   []string{"d"},
+			targets: []string{"d"},
+			groups:  []string{"default"},
+			count:   1,
+		},
+		{
+			names:   []string{"e"},
+			targets: []string{"e"},
+			groups:  []string{"default", "a", "b", "c", "e"},
+			count:   2,
+		},
+		{
+			names:   []string{"a", "e"},
+			targets: []string{"a", "e"},
+			groups:  []string{"default", "a", "b", "c", "e"},
+			count:   2,
+		},
+	}
+	for _, tt := range cases {
+		tt := tt
+		t.Run(strings.Join(tt.names, "+"), func(t *testing.T) {
+			m, g, err := ReadTargets(ctx, []File{f}, tt.names, nil, nil)
+			require.NoError(t, err)
+
+			var gnames []string
+			for _, g := range g {
+				gnames = append(gnames, g.Name)
+			}
+			sort.Strings(gnames)
+			sort.Strings(tt.groups)
+			require.Equal(t, tt.groups, gnames)
+
+			sort.Strings(g["default"].Targets)
+			sort.Strings(tt.targets)
+			require.Equal(t, tt.targets, g["default"].Targets)
+
+			require.Equal(t, tt.count, len(m))
+			require.Equal(t, ".", *m["d"].Context)
+			require.Equal(t, "./testdockerfile", *m["d"].Dockerfile)
+		})
+	}
+}
+
+func TestUnknownExt(t *testing.T) {
+	dt := []byte(`
+		target "app" {
+			context = "dir"
+			args = {
+				v1 = "foo"
+			}
+		}
+		`)
+	dt2 := []byte(`
+services:
+  app:
+    build:
+      dockerfile: Dockerfile-alternate
+      args:
+        v2: "bar"
+`)
+
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.foo"},
+		{Data: dt2, Name: "c2.bar"},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, "app", c.Targets[0].Name)
+	require.Equal(t, "foo", c.Targets[0].Args["v1"])
+	require.Equal(t, "bar", c.Targets[0].Args["v2"])
+	require.Equal(t, "dir", *c.Targets[0].Context)
+	require.Equal(t, "Dockerfile-alternate", *c.Targets[0].Dockerfile)
+}

bake/compose.go

@@ -1,49 +1,44 @@
 package bake
 
 import (
-	"fmt"
 	"os"
-	"reflect"
+	"path/filepath"
 	"strings"
 
+	"github.com/compose-spec/compose-go/dotenv"
 	"github.com/compose-spec/compose-go/loader"
 	compose "github.com/compose-spec/compose-go/types"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
 )
 
-func parseCompose(dt []byte) (*compose.Project, error) {
+func ParseComposeFiles(fs []File) (*Config, error) {
-	return loader.Load(compose.ConfigDetails{
+	envs, err := composeEnv()
-		ConfigFiles: []compose.ConfigFile{
+	if err != nil {
-			{
+		return nil, err
-				Content: dt,
+	}
-			},
+	var cfgs []compose.ConfigFile
-		},
+	for _, f := range fs {
-		Environment: envMap(os.Environ()),
+		cfgs = append(cfgs, compose.ConfigFile{
+			Filename: f.Name,
+			Content:  f.Data,
+		})
+	}
+	return ParseCompose(cfgs, envs)
+}
+
+func ParseCompose(cfgs []compose.ConfigFile, envs map[string]string) (*Config, error) {
+	cfg, err := loader.Load(compose.ConfigDetails{
+		ConfigFiles: cfgs,
+		Environment: envs,
 	}, func(options *loader.Options) {
 		options.SkipNormalization = true
 	})
-}
-
-func envMap(env []string) map[string]string {
-	result := make(map[string]string, len(env))
-	for _, s := range env {
-		kv := strings.SplitN(s, "=", 2)
-		if len(kv) != 2 {
-			continue
-		}
-		result[kv[0]] = kv[1]
-	}
-	return result
-}
-
-func ParseCompose(dt []byte) (*Config, error) {
-	cfg, err := parseCompose(dt)
 	if err != nil {
 		return nil, err
 	}
 
 	var c Config
-	var zeroBuildConfig compose.BuildConfig
 	if len(cfg.Services) > 0 {
 		c.Groups = []*Group{}
 		c.Targets = []*Target{}
@@ -51,17 +46,13 @@ func ParseCompose(dt []byte) (*Config, error) {
 		g := &Group{Name: "default"}
 
 		for _, s := range cfg.Services {
-
+			if s.Build == nil {
-			if s.Build == nil || reflect.DeepEqual(s.Build, zeroBuildConfig) {
-				// if not make sure they're setting an image or it's invalid d-c.yml
-				if s.Image == "" {
-					return nil, fmt.Errorf("compose file invalid: service %s has neither an image nor a build context specified. At least one must be provided", s.Name)
-				}
 				continue
 			}
 
-			if err = validateTargetName(s.Name); err != nil {
+			targetName := sanitizeTargetName(s.Name)
-				return nil, errors.Wrapf(err, "invalid service name %q", s.Name)
+			if err = validateTargetName(targetName); err != nil {
+				return nil, errors.Wrapf(err, "invalid service name %q", targetName)
 			}
 
 			var contextPathP *string
@@ -74,11 +65,22 @@ func ParseCompose(dt []byte) (*Config, error) {
 				dockerfilePath := s.Build.Dockerfile
 				dockerfilePathP = &dockerfilePath
 			}
-			g.Targets = append(g.Targets, s.Name)
+
+			var secrets []string
+			for _, bs := range s.Build.Secrets {
+				secret, err := composeToBuildkitSecret(bs, cfg.Secrets[bs.Source])
+				if err != nil {
+					return nil, err
+				}
+				secrets = append(secrets, secret)
+			}
+
+			g.Targets = append(g.Targets, targetName)
 			t := &Target{
-				Name:       s.Name,
+				Name:       targetName,
 				Context:    contextPathP,
 				Dockerfile: dockerfilePathP,
+				Tags:       s.Build.Tags,
 				Labels:     s.Build.Labels,
 				Args: flatten(s.Build.Args.Resolve(func(val string) (string, bool) {
 					if val, ok := s.Environment[val]; ok && val != nil {
@@ -88,7 +90,9 @@ func ParseCompose(dt []byte) (*Config, error) {
 					return val, ok
 				})),
 				CacheFrom:   s.Build.CacheFrom,
+				CacheTo:     s.Build.CacheTo,
 				NetworkMode: &s.Build.Network,
+				Secrets:     secrets,
 			}
 			if err = t.composeExtTarget(s.Build.Extensions); err != nil {
 				return nil, err
@@ -109,6 +113,86 @@ func ParseCompose(dt []byte) (*Config, error) {
 	return &c, nil
 }
 
+func validateComposeFile(dt []byte, fn string) (bool, error) {
+	envs, err := composeEnv()
+	if err != nil {
+		return true, err
+	}
+	fnl := strings.ToLower(fn)
+	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
+		return true, validateCompose(dt, envs)
+	}
+	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
+		return false, nil
+	}
+	err = validateCompose(dt, envs)
+	return err == nil, err
+}
+
+func validateCompose(dt []byte, envs map[string]string) error {
+	_, err := loader.Load(compose.ConfigDetails{
+		ConfigFiles: []compose.ConfigFile{
+			{
+				Content: dt,
+			},
+		},
+		Environment: envs,
+	}, func(options *loader.Options) {
+		options.SkipNormalization = true
+		// consistency is checked later in ParseCompose to ensure multiple
+		// compose files can be merged together
+		options.SkipConsistencyCheck = true
+	})
+	return err
+}
+
+func composeEnv() (map[string]string, error) {
+	envs := sliceToMap(os.Environ())
+	if wd, err := os.Getwd(); err == nil {
+		envs, err = loadDotEnv(envs, wd)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return envs, nil
+}
+
+func loadDotEnv(curenv map[string]string, workingDir string) (map[string]string, error) {
+	if curenv == nil {
+		curenv = make(map[string]string)
+	}
+
+	ef, err := filepath.Abs(filepath.Join(workingDir, ".env"))
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = os.Stat(ef); os.IsNotExist(err) {
+		return curenv, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	dt, err := os.ReadFile(ef)
+	if err != nil {
+		return nil, err
+	}
+
+	envs, err := dotenv.UnmarshalBytes(dt)
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range envs {
+		if _, set := curenv[k]; set {
+			continue
+		}
+		curenv[k] = v
+	}
+
+	return curenv, nil
+}
+
 func flatten(in compose.MappingWithEquals) compose.Mapping {
 	if len(in) == 0 {
 		return nil
@@ -123,89 +207,111 @@ func flatten(in compose.MappingWithEquals) compose.Mapping {
 	return out
 }
 
-// composeExtTarget converts Compose build extension x-bake to bake Target
+// xbake Compose build extension provides fields not (yet) available in
-// https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension
+// Compose build specification: https://github.com/compose-spec/compose-spec/blob/master/build.md
-func (t *Target) composeExtTarget(exts map[string]interface{}) error {
+type xbake struct {
-	if ext, ok := exts["x-bake"]; ok {
+	Tags          stringArray `yaml:"tags,omitempty"`
-		for key, val := range ext.(map[string]interface{}) {
+	CacheFrom     stringArray `yaml:"cache-from,omitempty"`
-			switch key {
+	CacheTo       stringArray `yaml:"cache-to,omitempty"`
-			case "tags":
+	Secrets       stringArray `yaml:"secret,omitempty"`
-				if res, k := val.(string); k {
+	SSH           stringArray `yaml:"ssh,omitempty"`
-					t.Tags = append(t.Tags, res)
+	Platforms     stringArray `yaml:"platforms,omitempty"`
+	Outputs       stringArray `yaml:"output,omitempty"`
+	Pull          *bool       `yaml:"pull,omitempty"`
+	NoCache       *bool       `yaml:"no-cache,omitempty"`
+	NoCacheFilter stringArray `yaml:"no-cache-filter,omitempty"`
+	Contexts      stringMap   `yaml:"contexts,omitempty"`
+	// don't forget to update documentation if you add a new field:
+	// docs/manuals/bake/compose-file.md#extension-field-with-x-bake
+}
+
+type stringMap map[string]string
+type stringArray []string
+
+func (sa *stringArray) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var multi []string
+	err := unmarshal(&multi)
+	if err != nil {
+		var single string
+		if err := unmarshal(&single); err != nil {
+			return err
+		}
+		*sa = strings.Fields(single)
 	} else {
-					for _, res := range val.([]interface{}) {
+		*sa = multi
-						t.Tags = append(t.Tags, res.(string))
-					}
-				}
-			case "cache-from":
-				t.CacheFrom = []string{} // Needed to override the main field
-				if res, k := val.(string); k {
-					t.CacheFrom = append(t.CacheFrom, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.CacheFrom = append(t.CacheFrom, res.(string))
-					}
-				}
-			case "cache-to":
-				if res, k := val.(string); k {
-					t.CacheTo = append(t.CacheTo, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.CacheTo = append(t.CacheTo, res.(string))
-					}
-				}
-			case "secret":
-				if res, k := val.(string); k {
-					t.Secrets = append(t.Secrets, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.Secrets = append(t.Secrets, res.(string))
-					}
-				}
-			case "ssh":
-				if res, k := val.(string); k {
-					t.SSH = append(t.SSH, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.SSH = append(t.SSH, res.(string))
-					}
-				}
-			case "platforms":
-				if res, k := val.(string); k {
-					t.Platforms = append(t.Platforms, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.Platforms = append(t.Platforms, res.(string))
-					}
-				}
-			case "output":
-				if res, k := val.(string); k {
-					t.Outputs = append(t.Outputs, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.Outputs = append(t.Outputs, res.(string))
-					}
-				}
-			case "pull":
-				if res, ok := val.(bool); ok {
-					t.Pull = &res
-				}
-			case "no-cache":
-				if res, ok := val.(bool); ok {
-					t.NoCache = &res
-				}
-			case "no-cache-filter":
-				if res, k := val.(string); k {
-					t.NoCacheFilter = append(t.NoCacheFilter, res)
-				} else {
-					for _, res := range val.([]interface{}) {
-						t.NoCacheFilter = append(t.NoCacheFilter, res.(string))
-					}
-				}
-			default:
-				return fmt.Errorf("compose file invalid: unkwown %s field for x-bake", key)
-			}
-		}
 	}
 	return nil
 }
+
+// composeExtTarget converts Compose build extension x-bake to bake Target
+// https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension
+func (t *Target) composeExtTarget(exts map[string]interface{}) error {
+	var xb xbake
+
+	ext, ok := exts["x-bake"]
+	if !ok || ext == nil {
+		return nil
+	}
+
+	yb, _ := yaml.Marshal(ext)
+	if err := yaml.Unmarshal(yb, &xb); err != nil {
+		return err
+	}
+
+	if len(xb.Tags) > 0 {
+		t.Tags = dedupSlice(append(t.Tags, xb.Tags...))
+	}
+	if len(xb.CacheFrom) > 0 {
+		t.CacheFrom = dedupSlice(append(t.CacheFrom, xb.CacheFrom...))
+	}
+	if len(xb.CacheTo) > 0 {
+		t.CacheTo = dedupSlice(append(t.CacheTo, xb.CacheTo...))
+	}
+	if len(xb.Secrets) > 0 {
+		t.Secrets = dedupSlice(append(t.Secrets, xb.Secrets...))
+	}
+	if len(xb.SSH) > 0 {
+		t.SSH = dedupSlice(append(t.SSH, xb.SSH...))
+	}
+	if len(xb.Platforms) > 0 {
+		t.Platforms = dedupSlice(append(t.Platforms, xb.Platforms...))
+	}
+	if len(xb.Outputs) > 0 {
+		t.Outputs = dedupSlice(append(t.Outputs, xb.Outputs...))
+	}
+	if xb.Pull != nil {
+		t.Pull = xb.Pull
+	}
+	if xb.NoCache != nil {
+		t.NoCache = xb.NoCache
+	}
+	if len(xb.NoCacheFilter) > 0 {
+		t.NoCacheFilter = dedupSlice(append(t.NoCacheFilter, xb.NoCacheFilter...))
+	}
+	if len(xb.Contexts) > 0 {
+		t.Contexts = dedupMap(t.Contexts, xb.Contexts)
+	}
+
+	return nil
+}
+
+// composeToBuildkitSecret converts secret from compose format to buildkit's
+// csv format.
+func composeToBuildkitSecret(inp compose.ServiceSecretConfig, psecret compose.SecretConfig) (string, error) {
+	if psecret.External.External {
+		return "", errors.Errorf("unsupported external secret %s", psecret.Name)
+	}
+
+	var bkattrs []string
+	if inp.Source != "" {
+		bkattrs = append(bkattrs, "id="+inp.Source)
+	}
+	if psecret.File != "" {
+		bkattrs = append(bkattrs, "src="+psecret.File)
+	}
+	if psecret.Environment != "" {
+		bkattrs = append(bkattrs, "env="+psecret.Environment)
+	}
+
+	return strings.Join(bkattrs, ","), nil
+}

bake/compose_test.go

@@ -2,9 +2,12 @@ package bake
 
 import (
 	"os"
+	"path/filepath"
 	"sort"
 	"testing"
 
+	compose "github.com/compose-spec/compose-go/types"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -23,13 +26,25 @@ services:
         none
       args:
         buildno: 123
+      cache_from:
+        - type=local,src=path/to/cache
+      cache_to:
+        - type=local,dest=path/to/cache
+      secrets:
+        - token
+        - aws
+secrets:
+  token:
+    environment: ENV_TOKEN
+  aws:
+    file: /root/.aws/credentials
 `)
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(c.Groups))
-	require.Equal(t, c.Groups[0].Name, "default")
+	require.Equal(t, "default", c.Groups[0].Name)
 	sort.Strings(c.Groups[0].Targets)
 	require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
 
@@ -39,13 +54,20 @@ services:
 	})
 	require.Equal(t, "db", c.Targets[0].Name)
 	require.Equal(t, "./db", *c.Targets[0].Context)
+	require.Equal(t, []string{"docker.io/tonistiigi/db"}, c.Targets[0].Tags)
 
 	require.Equal(t, "webapp", c.Targets[1].Name)
 	require.Equal(t, "./dir", *c.Targets[1].Context)
 	require.Equal(t, "Dockerfile-alternate", *c.Targets[1].Dockerfile)
 	require.Equal(t, 1, len(c.Targets[1].Args))
 	require.Equal(t, "123", c.Targets[1].Args["buildno"])
+	require.Equal(t, []string{"type=local,src=path/to/cache"}, c.Targets[1].CacheFrom)
+	require.Equal(t, []string{"type=local,dest=path/to/cache"}, c.Targets[1].CacheTo)
 	require.Equal(t, "none", *c.Targets[1].NetworkMode)
+	require.Equal(t, []string{
+		"id=token,env=ENV_TOKEN",
+		"id=aws,src=/root/.aws/credentials",
+	}, c.Targets[1].Secrets)
 }
 
 func TestNoBuildOutOfTreeService(t *testing.T) {
@@ -56,9 +78,10 @@ services:
     webapp:
         build: ./db
 `)
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, 1, len(c.Targets))
 }
 
 func TestParseComposeTarget(t *testing.T) {
@@ -74,7 +97,7 @@ services:
       target: webapp
 `)
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 2, len(c.Targets))
@@ -99,15 +122,15 @@ services:
       target: webapp
 `)
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 2, len(c.Targets))
 	sort.Slice(c.Targets, func(i, j int) bool {
 		return c.Targets[i].Name < c.Targets[j].Name
 	})
-	require.Equal(t, c.Targets[0].Name, "db")
+	require.Equal(t, "db", c.Targets[0].Name)
 	require.Equal(t, "db", *c.Targets[0].Target)
-	require.Equal(t, c.Targets[1].Name, "webapp")
+	require.Equal(t, "webapp", c.Targets[1].Name)
 	require.Equal(t, "webapp", *c.Targets[1].Target)
 }
 
@@ -133,28 +156,22 @@ services:
 	os.Setenv("ZZZ_BAR", "zzz_foo")
 	defer os.Unsetenv("ZZZ_BAR")
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, sliceToMap(os.Environ()))
 	require.NoError(t, err)
-	require.Equal(t, c.Targets[0].Args["FOO"], "bar")
+	require.Equal(t, "bar", c.Targets[0].Args["FOO"])
-	require.Equal(t, c.Targets[0].Args["BAR"], "zzz_foo")
+	require.Equal(t, "zzz_foo", c.Targets[0].Args["BAR"])
-	require.Equal(t, c.Targets[0].Args["BRB"], "FOO")
+	require.Equal(t, "FOO", c.Targets[0].Args["BRB"])
 }
 
-func TestBogusCompose(t *testing.T) {
+func TestInconsistentComposeFile(t *testing.T) {
 	var dt = []byte(`
 services:
-  db:
-    labels:
-      - "foo"
   webapp:
-    build:
+    entrypoint: echo 1
-      context: .
-      target: webapp
 `)
 
-	_, err := ParseCompose(dt)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "has neither an image nor a build context specified: invalid compose project")
 }
 
 func TestAdvancedNetwork(t *testing.T) {
@@ -178,10 +195,28 @@ networks:
           gateway: 10.5.0.254
 `)
 
-	_, err := ParseCompose(dt)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 }
 
+func TestTags(t *testing.T) {
+	var dt = []byte(`
+services:
+  example:
+    image: example
+    build:
+      context: .
+      dockerfile: Dockerfile
+      tags:
+        - foo
+        - bar
+`)
+
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
+	require.NoError(t, err)
+	require.Equal(t, []string{"foo", "bar"}, c.Targets[0].Tags)
+}
+
 func TestDependsOnList(t *testing.T) {
 	var dt = []byte(`
 version: "3.8"
@@ -214,7 +249,7 @@ networks:
     name: test-net
 `)
 
-	_, err := ParseCompose(dt)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 }
 
@@ -228,10 +263,16 @@ services:
       dockerfile: ./Dockerfile
       cache_from:
         - user/app:cache
+      cache_to:
+        - user/app:cache
+      tags:
+        - ct-addon:baz
       args:
         CT_ECR: foo
         CT_TAG: bar
       x-bake:
+        contexts:
+          alpine: docker-image://alpine:3.13
         tags:
           - ct-addon:foo
           - ct-addon:alp
@@ -240,7 +281,8 @@ services:
           - linux/arm64
         cache-from:
           - type=local,src=path/to/cache
-        cache-to: local,dest=path/to/cache
+        cache-to:
+          - type=local,dest=path/to/cache
         pull: true
 
   aws:
@@ -260,24 +302,56 @@ services:
         no-cache: true
 `)
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
 	require.Equal(t, 2, len(c.Targets))
 	sort.Slice(c.Targets, func(i, j int) bool {
 		return c.Targets[i].Name < c.Targets[j].Name
 	})
-	require.Equal(t, c.Targets[0].Args, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"})
+	require.Equal(t, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"}, c.Targets[0].Args)
-	require.Equal(t, c.Targets[0].Tags, []string{"ct-addon:foo", "ct-addon:alp"})
+	require.Equal(t, []string{"ct-addon:baz", "ct-addon:foo", "ct-addon:alp"}, c.Targets[0].Tags)
-	require.Equal(t, c.Targets[0].Platforms, []string{"linux/amd64", "linux/arm64"})
+	require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[0].Platforms)
-	require.Equal(t, c.Targets[0].CacheFrom, []string{"type=local,src=path/to/cache"})
+	require.Equal(t, []string{"user/app:cache", "type=local,src=path/to/cache"}, c.Targets[0].CacheFrom)
-	require.Equal(t, c.Targets[0].CacheTo, []string{"local,dest=path/to/cache"})
+	require.Equal(t, []string{"user/app:cache", "type=local,dest=path/to/cache"}, c.Targets[0].CacheTo)
-	require.Equal(t, c.Targets[0].Pull, newBool(true))
+	require.Equal(t, newBool(true), c.Targets[0].Pull)
-	require.Equal(t, c.Targets[1].Tags, []string{"ct-fake-aws:bar"})
+	require.Equal(t, map[string]string{"alpine": "docker-image://alpine:3.13"}, c.Targets[0].Contexts)
-	require.Equal(t, c.Targets[1].Secrets, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"})
+	require.Equal(t, []string{"ct-fake-aws:bar"}, c.Targets[1].Tags)
-	require.Equal(t, c.Targets[1].SSH, []string{"default"})
+	require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,src=/local/secret2"}, c.Targets[1].Secrets)
-	require.Equal(t, c.Targets[1].Platforms, []string{"linux/arm64"})
+	require.Equal(t, []string{"default"}, c.Targets[1].SSH)
-	require.Equal(t, c.Targets[1].Outputs, []string{"type=docker"})
+	require.Equal(t, []string{"linux/arm64"}, c.Targets[1].Platforms)
-	require.Equal(t, c.Targets[1].NoCache, newBool(true))
+	require.Equal(t, []string{"type=docker"}, c.Targets[1].Outputs)
+	require.Equal(t, newBool(true), c.Targets[1].NoCache)
+}
+
+func TestComposeExtDedup(t *testing.T) {
+	var dt = []byte(`
+services:
+  webapp:
+    image: app:bar
+    build:
+      cache_from:
+        - user/app:cache
+      cache_to:
+        - user/app:cache
+      tags:
+        - ct-addon:foo
+      x-bake:
+        tags:
+          - ct-addon:foo
+          - ct-addon:baz
+        cache-from:
+          - user/app:cache
+          - type=local,src=path/to/cache
+        cache-to:
+          - type=local,dest=path/to/cache
+`)
+
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, []string{"ct-addon:foo", "ct-addon:baz"}, c.Targets[0].Tags)
+	require.Equal(t, []string{"user/app:cache", "type=local,src=path/to/cache"}, c.Targets[0].CacheFrom)
+	require.Equal(t, []string{"user/app:cache", "type=local,dest=path/to/cache"}, c.Targets[0].CacheTo)
 }
 
 func TestEnv(t *testing.T) {
@@ -305,9 +379,54 @@ services:
       - ` + envf.Name() + `
 `)
 
-	c, err := ParseCompose(dt)
+	c, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
+	require.NoError(t, err)
+	require.Equal(t, map[string]string{"CT_ECR": "foo", "FOO": "bsdf -csdf", "NODE_ENV": "test"}, c.Targets[0].Args)
+}
+
+func TestDotEnv(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	err := os.WriteFile(filepath.Join(tmpdir, ".env"), []byte("FOO=bar"), 0644)
+	require.NoError(t, err)
+
+	var dt = []byte(`
+services:
+  scratch:
+    build:
+     context: .
+     args:
+        FOO:
+`)
+
+	chdir(t, tmpdir)
+	c, err := ParseComposeFiles([]File{{
+		Name: "docker-compose.yml",
+		Data: dt,
+	}})
+	require.NoError(t, err)
+	require.Equal(t, map[string]string{"FOO": "bar"}, c.Targets[0].Args)
+}
+
+func TestPorts(t *testing.T) {
+	var dt = []byte(`
+services:
+  foo:
+    build:
+     context: .
+    ports:
+      - 3306:3306
+  bar:
+    build:
+     context: .
+    ports:
+      - mode: ingress
+        target: 3306
+        published: "3306"
+        protocol: tcp
+`)
+	_, err := ParseCompose([]compose.ConfigFile{{Content: dt}}, nil)
 	require.NoError(t, err)
-	require.Equal(t, c.Targets[0].Args, map[string]string{"CT_ECR": "foo", "FOO": "bsdf -csdf", "NODE_ENV": "test"})
 }
 
 func newBool(val bool) *bool {
@@ -330,7 +449,7 @@ func TestServiceName(t *testing.T) {
 		},
 		{
 			svc:     "a.b",
-			wantErr: true,
+			wantErr: false,
 		},
 		{
 			svc:     "_a",
@@ -352,12 +471,12 @@ func TestServiceName(t *testing.T) {
 	for _, tt := range cases {
 		tt := tt
 		t.Run(tt.svc, func(t *testing.T) {
-			_, err := ParseCompose([]byte(`
+			_, err := ParseCompose([]compose.ConfigFile{{Content: []byte(`
 services:
   ` + tt.svc + `:
     build:
       context: .
-`))
+`)}}, nil)
 			if tt.wantErr {
 				require.Error(t, err)
 			} else {
@@ -366,3 +485,164 @@ services:
 		})
 	}
 }
+
+func TestValidateComposeSecret(t *testing.T) {
+	cases := []struct {
+		name    string
+		dt      []byte
+		wantErr bool
+	}{
+		{
+			name: "secret set by file",
+			dt: []byte(`
+secrets:
+  foo:
+    file: .secret
+`),
+			wantErr: false,
+		},
+		{
+			name: "secret set by environment",
+			dt: []byte(`
+secrets:
+  foo:
+    environment: TOKEN
+`),
+			wantErr: false,
+		},
+		{
+			name: "external secret",
+			dt: []byte(`
+secrets:
+  foo:
+    external: true
+`),
+			wantErr: false,
+		},
+		{
+			name: "unset secret",
+			dt: []byte(`
+secrets:
+  foo: {}
+`),
+			wantErr: true,
+		},
+		{
+			name: "undefined secret",
+			dt: []byte(`
+services:
+  foo:
+    build:
+      secrets:
+        - token
+`),
+			wantErr: true,
+		},
+	}
+	for _, tt := range cases {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := ParseCompose([]compose.ConfigFile{{Content: tt.dt}}, nil)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateComposeFile(t *testing.T) {
+	cases := []struct {
+		name      string
+		fn        string
+		dt        []byte
+		isCompose bool
+		wantErr   bool
+	}{
+		{
+			name: "empty service",
+			fn:   "docker-compose.yml",
+			dt: []byte(`
+services:
+  foo:
+`),
+			isCompose: true,
+			wantErr:   true,
+		},
+		{
+			name: "build",
+			fn:   "docker-compose.yml",
+			dt: []byte(`
+services:
+  foo:
+    build: .
+`),
+			isCompose: true,
+			wantErr:   false,
+		},
+		{
+			name: "image",
+			fn:   "docker-compose.yml",
+			dt: []byte(`
+services:
+  simple:
+    image: nginx
+`),
+			isCompose: true,
+			wantErr:   false,
+		},
+		{
+			name: "unknown ext",
+			fn:   "docker-compose.foo",
+			dt: []byte(`
+services:
+  simple:
+    image: nginx
+`),
+			isCompose: true,
+			wantErr:   false,
+		},
+		{
+			name: "hcl",
+			fn:   "docker-bake.hcl",
+			dt: []byte(`
+target "default" {
+  dockerfile = "test"
+}
+`),
+			isCompose: false,
+			wantErr:   false,
+		},
+	}
+	for _, tt := range cases {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			isCompose, err := validateComposeFile(tt.dt, tt.fn)
+			assert.Equal(t, tt.isCompose, isCompose)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+// chdir changes the current working directory to the named directory,
+// and then restore the original working directory at the end of the test.
+func chdir(t *testing.T, dir string) {
+	olddir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("chdir: %v", err)
+	}
+	if err := os.Chdir(dir); err != nil {
+		t.Fatalf("chdir %s: %v", dir, err)
+	}
+	t.Cleanup(func() {
+		if err := os.Chdir(olddir); err != nil {
+			t.Errorf("chdir to original working directory %s: %v", olddir, err)
+			os.Exit(1)
+		}
+	})
+}

bake/hcl_test.go

@@ -444,6 +444,94 @@ func TestHCLAttrs(t *testing.T) {
 	// attr-multifile
 }
 
+func TestHCLTargetAttrs(t *testing.T) {
+	dt := []byte(`
+		target "foo" {
+			dockerfile = "xxx"
+			context = target.bar.context
+			target = target.foo.dockerfile
+		}
+		
+		target "bar" {
+			dockerfile = target.foo.dockerfile
+			context = "yyy"
+			target = target.bar.context
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 2, len(c.Targets))
+	require.Equal(t, "foo", c.Targets[0].Name)
+	require.Equal(t, "bar", c.Targets[1].Name)
+
+	require.Equal(t, "xxx", *c.Targets[0].Dockerfile)
+	require.Equal(t, "yyy", *c.Targets[0].Context)
+	require.Equal(t, "xxx", *c.Targets[0].Target)
+
+	require.Equal(t, "xxx", *c.Targets[1].Dockerfile)
+	require.Equal(t, "yyy", *c.Targets[1].Context)
+	require.Equal(t, "yyy", *c.Targets[1].Target)
+}
+
+func TestHCLTargetGlobal(t *testing.T) {
+	dt := []byte(`
+		target "foo" {
+			dockerfile = "x"
+		}
+		x = target.foo.dockerfile
+		y = x
+		target "bar" {
+			dockerfile = y
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 2, len(c.Targets))
+	require.Equal(t, "foo", c.Targets[0].Name)
+	require.Equal(t, "bar", c.Targets[1].Name)
+
+	require.Equal(t, "x", *c.Targets[0].Dockerfile)
+	require.Equal(t, "x", *c.Targets[1].Dockerfile)
+}
+
+func TestHCLTargetAttrName(t *testing.T) {
+	dt := []byte(`
+		target "foo" {
+			dockerfile = target.foo.name
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, "foo", c.Targets[0].Name)
+	require.Equal(t, "foo", *c.Targets[0].Dockerfile)
+}
+
+func TestHCLTargetAttrEmptyChain(t *testing.T) {
+	dt := []byte(`
+		target "foo" {
+			# dockerfile = Dockerfile
+			context = target.foo.dockerfile
+			target = target.foo.context
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, "foo", c.Targets[0].Name)
+	require.Nil(t, c.Targets[0].Dockerfile)
+	require.Nil(t, c.Targets[0].Context)
+	require.Nil(t, c.Targets[0].Target)
+}
+
 func TestHCLAttrsCustomType(t *testing.T) {
 	dt := []byte(`
 		platforms=["linux/arm64", "linux/amd64"]
@@ -620,3 +708,177 @@ func TestHCLBuiltinVars(t *testing.T) {
 	require.Equal(t, "foo", *c.Targets[0].Context)
 	require.Equal(t, "test", *c.Targets[0].Dockerfile)
 }
+
+func TestCombineHCLAndJSONTargets(t *testing.T) {
+	c, err := ParseFiles([]File{
+		{
+			Name: "docker-bake.hcl",
+			Data: []byte(`
+group "default" {
+  targets = ["a"]
+}
+
+target "metadata-a" {}
+target "metadata-b" {}
+
+target "a" {
+  inherits = ["metadata-a"]
+  context = "."
+  target = "a"
+}
+
+target "b" {
+  inherits = ["metadata-b"]
+  context = "."
+  target = "b"
+}`),
+		},
+		{
+			Name: "metadata-a.json",
+			Data: []byte(`
+{
+  "target": [{
+    "metadata-a": [{
+      "tags": [
+        "app/a:1.0.0",
+        "app/a:latest"
+      ]
+    }]
+  }]
+}`),
+		},
+		{
+			Name: "metadata-b.json",
+			Data: []byte(`
+{
+  "target": [{
+    "metadata-b": [{
+      "tags": [
+        "app/b:1.0.0",
+        "app/b:latest"
+      ]
+    }]
+  }]
+}`),
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"a"}, c.Groups[0].Targets)
+
+	require.Equal(t, 4, len(c.Targets))
+
+	require.Equal(t, c.Targets[0].Name, "metadata-a")
+	require.Equal(t, []string{"app/a:1.0.0", "app/a:latest"}, c.Targets[0].Tags)
+
+	require.Equal(t, c.Targets[1].Name, "metadata-b")
+	require.Equal(t, []string{"app/b:1.0.0", "app/b:latest"}, c.Targets[1].Tags)
+
+	require.Equal(t, c.Targets[2].Name, "a")
+	require.Equal(t, ".", *c.Targets[2].Context)
+	require.Equal(t, "a", *c.Targets[2].Target)
+
+	require.Equal(t, c.Targets[3].Name, "b")
+	require.Equal(t, ".", *c.Targets[3].Context)
+	require.Equal(t, "b", *c.Targets[3].Target)
+}
+
+func TestCombineHCLAndJSONVars(t *testing.T) {
+	c, err := ParseFiles([]File{
+		{
+			Name: "docker-bake.hcl",
+			Data: []byte(`
+variable "ABC" {
+  default = "foo"
+}
+variable "DEF" {
+  default = ""
+}
+group "default" {
+  targets = ["one"]
+}
+target "one" {
+  args = {
+    a = "pre-${ABC}"
+  }
+}
+target "two" {
+  args = {
+    b = "pre-${DEF}"
+  }
+}`),
+		},
+		{
+			Name: "foo.json",
+			Data: []byte(`{"variable": {"DEF": {"default": "bar"}}, "target": { "one": { "args": {"a": "pre-${ABC}-${DEF}"}} } }`),
+		},
+		{
+			Name: "bar.json",
+			Data: []byte(`{"ABC": "ghi", "DEF": "jkl"}`),
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"one"}, c.Groups[0].Targets)
+
+	require.Equal(t, 2, len(c.Targets))
+
+	require.Equal(t, c.Targets[0].Name, "one")
+	require.Equal(t, map[string]string{"a": "pre-ghi-jkl"}, c.Targets[0].Args)
+
+	require.Equal(t, c.Targets[1].Name, "two")
+	require.Equal(t, map[string]string{"b": "pre-jkl"}, c.Targets[1].Args)
+}
+
+func TestEmptyVariableJSON(t *testing.T) {
+	dt := []byte(`{
+	  "variable": {
+	    "VAR": {}
+	  }
+	}`)
+	_, err := ParseFile(dt, "docker-bake.json")
+	require.NoError(t, err)
+}
+
+func TestFunctionNoParams(t *testing.T) {
+	dt := []byte(`
+		function "foo" {
+			result = "bar"
+		}
+		target "foo_target" {
+			args = {
+				test = foo()
+			}
+		}
+		`)
+
+	_, err := ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+}
+
+func TestFunctionNoResult(t *testing.T) {
+	dt := []byte(`
+		function "foo" {
+			params = ["a"]
+		}
+		`)
+
+	_, err := ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+}
+
+func TestVarUnsupportedType(t *testing.T) {
+	dt := []byte(`
+		variable "FOO" {
+			default = []
+		}
+		target "default" {}`)
+
+	t.Setenv("FOO", "bar")
+	_, err := ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+}

bake/hclparser/body.go

@@ -0,0 +1,103 @@
+package hclparser
+
+import (
+	"github.com/hashicorp/hcl/v2"
+)
+
+type filterBody struct {
+	body    hcl.Body
+	schema  *hcl.BodySchema
+	exclude bool
+}
+
+func FilterIncludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
+	return &filterBody{
+		body:   body,
+		schema: schema,
+	}
+}
+
+func FilterExcludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
+	return &filterBody{
+		body:    body,
+		schema:  schema,
+		exclude: true,
+	}
+}
+
+func (b *filterBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) {
+	if b.exclude {
+		schema = subtractSchemas(schema, b.schema)
+	} else {
+		schema = intersectSchemas(schema, b.schema)
+	}
+	content, _, diag := b.body.PartialContent(schema)
+	return content, diag
+}
+
+func (b *filterBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) {
+	if b.exclude {
+		schema = subtractSchemas(schema, b.schema)
+	} else {
+		schema = intersectSchemas(schema, b.schema)
+	}
+	return b.body.PartialContent(schema)
+}
+
+func (b *filterBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) {
+	return b.body.JustAttributes()
+}
+
+func (b *filterBody) MissingItemRange() hcl.Range {
+	return b.body.MissingItemRange()
+}
+
+func intersectSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
+	result := &hcl.BodySchema{}
+	for _, blockA := range a.Blocks {
+		for _, blockB := range b.Blocks {
+			if blockA.Type == blockB.Type {
+				result.Blocks = append(result.Blocks, blockA)
+				break
+			}
+		}
+	}
+	for _, attrA := range a.Attributes {
+		for _, attrB := range b.Attributes {
+			if attrA.Name == attrB.Name {
+				result.Attributes = append(result.Attributes, attrA)
+				break
+			}
+		}
+	}
+	return result
+}
+
+func subtractSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
+	result := &hcl.BodySchema{}
+	for _, blockA := range a.Blocks {
+		found := false
+		for _, blockB := range b.Blocks {
+			if blockA.Type == blockB.Type {
+				found = true
+				break
+			}
+		}
+		if !found {
+			result.Blocks = append(result.Blocks, blockA)
+		}
+	}
+	for _, attrA := range a.Attributes {
+		found := false
+		for _, attrB := range b.Attributes {
+			if attrA.Name == attrB.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			result.Attributes = append(result.Attributes, attrA)
+		}
+	}
+	return result
+}

bake/hclparser/expr.go

@@ -87,7 +87,7 @@ func appendJSONFuncCalls(exp hcl.Expression, m map[string]struct{}) error {
 		return nil
 	}
 	rng := src.FieldByName("SrcRange")
-	if val.IsZero() {
+	if rng.IsZero() {
 		return nil
 	}
 	var stringVal struct {

bake/hclparser/hclparser.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/pkg/errors"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
 )
 
 type Opt struct {
@@ -48,11 +49,17 @@ type parser struct {
 	attrs map[string]*hcl.Attribute
 	funcs map[string]*functionDef
 
+	blocks      map[string]map[string][]*hcl.Block
+	blockValues map[*hcl.Block]reflect.Value
+	blockTypes  map[string]reflect.Type
+
 	ectx *hcl.EvalContext
 
 	progress  map[string]struct{}
 	progressF map[string]struct{}
+	progressB map[*hcl.Block]map[string]struct{}
 	doneF     map[string]struct{}
+	doneB     map[*hcl.Block]map[string]struct{}
 }
 
 func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
@@ -79,6 +86,59 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
 		if _, ok := exclude[v.RootName()]; ok {
 			continue
 		}
+		if _, ok := p.blockTypes[v.RootName()]; ok {
+			blockType := v.RootName()
+
+			split := v.SimpleSplit().Rel
+			if len(split) == 0 {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   fmt.Sprintf("cannot access %s as a variable", blockType),
+						Subject:  exp.Range().Ptr(),
+						Context:  exp.Range().Ptr(),
+					},
+				}
+			}
+			blockName, ok := split[0].(hcl.TraverseAttr)
+			if !ok {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   fmt.Sprintf("cannot traverse %s without attribute", blockType),
+						Subject:  exp.Range().Ptr(),
+						Context:  exp.Range().Ptr(),
+					},
+				}
+			}
+			blocks := p.blocks[blockType][blockName.Name]
+			if len(blocks) == 0 {
+				continue
+			}
+
+			var target *hcl.BodySchema
+			if len(split) > 1 {
+				if attr, ok := split[1].(hcl.TraverseAttr); ok {
+					target = &hcl.BodySchema{
+						Attributes: []hcl.AttributeSchema{{Name: attr.Name}},
+						Blocks:     []hcl.BlockHeaderSchema{{Type: attr.Name}},
+					}
+				}
+			}
+			if err := p.resolveBlock(blocks[0], target); err != nil {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   err.Error(),
+						Subject:  v.SourceRange().Ptr(),
+						Context:  v.SourceRange().Ptr(),
+					},
+				}
+			}
+		} else {
 			if err := p.resolveValue(v.RootName()); err != nil {
 				return hcl.Diagnostics{
 					&hcl.Diagnostic{
@@ -91,10 +151,13 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
 				}
 			}
 		}
+	}
 
 	return nil
 }
 
+// resolveFunction forces evaluation of a function, storing the result into the
+// parser.
 func (p *parser) resolveFunction(name string) error {
 	if _, ok := p.doneF[name]; ok {
 		return nil
@@ -111,6 +174,13 @@ func (p *parser) resolveFunction(name string) error {
 	}
 	p.progressF[name] = struct{}{}
 
+	if f.Result == nil {
+		return errors.Errorf("empty result not allowed for %s", name)
+	}
+	if f.Params == nil {
+		return errors.Errorf("empty params not allowed for %s", name)
+	}
+
 	paramExprs, paramsDiags := hcl.ExprList(f.Params.Expr)
 	if paramsDiags.HasErrors() {
 		return paramsDiags
@@ -163,6 +233,8 @@ func (p *parser) resolveFunction(name string) error {
 	return nil
 }
 
+// resolveValue forces evaluation of a named value, storing the result into the
+// parser.
 func (p *parser) resolveValue(name string) (err error) {
 	if _, ok := p.ectx.Variables[name]; ok {
 		return nil
@@ -234,13 +306,164 @@ func (p *parser) resolveValue(name string) (err error) {
 			return nil
 		} else {
 			// TODO: support lists with csv values
-			return errors.Errorf("unsupported type %s for variable %s", v.Type(), name)
+			return errors.Errorf("unsupported type %s for variable %s", vv.Type().FriendlyName(), name)
 		}
 	}
 	v = &vv
 	return nil
 }
 
+// resolveBlock force evaluates a block, storing the result in the parser. If a
+// target schema is provided, only the attributes and blocks present in the
+// schema will be evaluated.
+func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) {
+	name := block.Labels[0]
+	if err := p.opt.ValidateLabel(name); err != nil {
+		return hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid name",
+				Detail:   err.Error(),
+				Subject:  &block.LabelRanges[0],
+			},
+		}
+	}
+
+	if _, ok := p.doneB[block]; !ok {
+		p.doneB[block] = map[string]struct{}{}
+	}
+	if _, ok := p.progressB[block]; !ok {
+		p.progressB[block] = map[string]struct{}{}
+	}
+
+	if target != nil {
+		// filter out attributes and blocks that are already evaluated
+		original := target
+		target = &hcl.BodySchema{}
+		for _, a := range original.Attributes {
+			if _, ok := p.doneB[block][a.Name]; !ok {
+				target.Attributes = append(target.Attributes, a)
+			}
+		}
+		for _, b := range original.Blocks {
+			if _, ok := p.doneB[block][b.Type]; !ok {
+				target.Blocks = append(target.Blocks, b)
+			}
+		}
+		if len(target.Attributes) == 0 && len(target.Blocks) == 0 {
+			return nil
+		}
+	}
+
+	if target != nil {
+		// detect reference cycles
+		for _, a := range target.Attributes {
+			if _, ok := p.progressB[block][a.Name]; ok {
+				return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name)
+			}
+		}
+		for _, b := range target.Blocks {
+			if _, ok := p.progressB[block][b.Type]; ok {
+				return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type)
+			}
+		}
+		for _, a := range target.Attributes {
+			p.progressB[block][a.Name] = struct{}{}
+		}
+		for _, b := range target.Blocks {
+			p.progressB[block][b.Type] = struct{}{}
+		}
+	}
+
+	// create a filtered body that contains only the target properties
+	body := func() hcl.Body {
+		if target != nil {
+			return FilterIncludeBody(block.Body, target)
+		}
+
+		filter := &hcl.BodySchema{}
+		for k := range p.doneB[block] {
+			filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k})
+			filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k})
+		}
+		return FilterExcludeBody(block.Body, filter)
+	}
+
+	// load dependencies from all targeted properties
+	t, ok := p.blockTypes[block.Type]
+	if !ok {
+		return nil
+	}
+	schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface())
+	content, _, diag := body().PartialContent(schema)
+	if diag.HasErrors() {
+		return diag
+	}
+	for _, a := range content.Attributes {
+		diag := p.loadDeps(a.Expr, nil)
+		if diag.HasErrors() {
+			return diag
+		}
+	}
+	for _, b := range content.Blocks {
+		err := p.resolveBlock(b, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	// decode!
+	var output reflect.Value
+	if prev, ok := p.blockValues[block]; ok {
+		output = prev
+	} else {
+		output = reflect.New(t)
+		setLabel(output, block.Labels[0]) // early attach labels, so we can reference them
+	}
+	diag = gohcl.DecodeBody(body(), p.ectx, output.Interface())
+	if diag.HasErrors() {
+		return diag
+	}
+	p.blockValues[block] = output
+
+	// mark all targeted properties as done
+	for _, a := range content.Attributes {
+		p.doneB[block][a.Name] = struct{}{}
+	}
+	for _, b := range content.Blocks {
+		p.doneB[block][b.Type] = struct{}{}
+	}
+	if target != nil {
+		for _, a := range target.Attributes {
+			p.doneB[block][a.Name] = struct{}{}
+		}
+		for _, b := range target.Blocks {
+			p.doneB[block][b.Type] = struct{}{}
+		}
+	}
+
+	// store the result into the evaluation context (so if can be referenced)
+	outputType, err := gocty.ImpliedType(output.Interface())
+	if err != nil {
+		return err
+	}
+	outputValue, err := gocty.ToCtyValue(output.Interface(), outputType)
+	if err != nil {
+		return err
+	}
+	var m map[string]cty.Value
+	if m2, ok := p.ectx.Variables[block.Type]; ok {
+		m = m2.AsValueMap()
+	}
+	if m == nil {
+		m = map[string]cty.Value{}
+	}
+	m[name] = outputValue
+	p.ectx.Variables[block.Type] = cty.MapVal(m)
+
+	return nil
+}
+
 func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	reserved := map[string]struct{}{}
 	schema, _ := gohcl.ImpliedBodySchema(val)
@@ -256,6 +479,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	if err := gohcl.DecodeBody(b, nil, &defs); err != nil {
 		return err
 	}
+	defsSchema, _ := gohcl.ImpliedBodySchema(defs)
 
 	if opt.LookupVar == nil {
 		opt.LookupVar = func(string) (string, bool) {
@@ -276,9 +500,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		attrs: map[string]*hcl.Attribute{},
 		funcs: map[string]*functionDef{},
 
+		blocks:      map[string]map[string][]*hcl.Block{},
+		blockValues: map[*hcl.Block]reflect.Value{},
+		blockTypes:  map[string]reflect.Type{},
+
 		progress:  map[string]struct{}{},
 		progressF: map[string]struct{}{},
+		progressB: map[*hcl.Block]map[string]struct{}{},
+
 		doneF: map[string]struct{}{},
+		doneB: map[*hcl.Block]map[string]struct{}{},
 		ectx: &hcl.EvalContext{
 			Variables: map[string]cty.Value{},
 			Functions: stdlibFunctions,
@@ -300,12 +531,20 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		p.funcs[v.Name] = v
 	}
 
-	attrs, diags := b.JustAttributes()
+	content, b, diags := b.PartialContent(schema)
 	if diags.HasErrors() {
-		for _, d := range diags {
-			if d.Detail != "Blocks are not allowed here." {
 		return diags
 	}
+
+	blocks, b, diags := b.PartialContent(defsSchema)
+	if diags.HasErrors() {
+		return diags
+	}
+
+	attrs, diags := b.JustAttributes()
+	if diags.HasErrors() {
+		if d := removeAttributesDiags(diags, reserved, p.vars); len(d) > 0 {
+			return d
 		}
 	}
 
@@ -321,22 +560,17 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		_ = p.resolveValue(k)
 	}
 
-	for k := range p.attrs {
+	for _, a := range content.Attributes {
-		if err := p.resolveValue(k); err != nil {
-			if diags, ok := err.(hcl.Diagnostics); ok {
-				return diags
-			}
 		return hcl.Diagnostics{
 			&hcl.Diagnostic{
 				Severity: hcl.DiagError,
 				Summary:  "Invalid attribute",
-					Detail:   err.Error(),
+				Detail:   "global attributes currently not supported",
-					Subject:  &p.attrs[k].Range,
+				Subject:  &a.Range,
-					Context:  &p.attrs[k].Range,
+				Context:  &a.Range,
 			},
 		}
 	}
-	}
 
 	for k := range p.vars {
 		if err := p.resolveValue(k); err != nil {
@@ -361,36 +595,32 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 			if diags, ok := err.(hcl.Diagnostics); ok {
 				return diags
 			}
+			var subject *hcl.Range
+			var context *hcl.Range
+			if p.funcs[k].Params != nil {
+				subject = &p.funcs[k].Params.Range
+				context = subject
+			} else {
+				for _, block := range blocks.Blocks {
+					if block.Type == "function" && len(block.Labels) == 1 && block.Labels[0] == k {
+						subject = &block.LabelRanges[0]
+						context = &block.DefRange
+						break
+					}
+				}
+			}
 			return hcl.Diagnostics{
 				&hcl.Diagnostic{
 					Severity: hcl.DiagError,
 					Summary:  "Invalid function",
 					Detail:   err.Error(),
-					Subject:  &p.funcs[k].Params.Range,
+					Subject:  subject,
-					Context:  &p.funcs[k].Params.Range,
+					Context:  context,
 				},
 			}
 		}
 	}
 
-	content, _, diags := b.PartialContent(schema)
-	if diags.HasErrors() {
-		return diags
-	}
-
-	for _, a := range content.Attributes {
-		return hcl.Diagnostics{
-			&hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Invalid attribute",
-				Detail:   "global attributes currently not supported",
-				Subject:  &a.Range,
-				Context:  &a.Range,
-			},
-		}
-	}
-
-	m := map[string]map[string][]*hcl.Block{}
 	for _, b := range content.Blocks {
 		if len(b.Labels) == 0 || len(b.Labels) > 1 {
 			return hcl.Diagnostics{
@@ -403,19 +633,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 				},
 			}
 		}
-		bm, ok := m[b.Type]
+		bm, ok := p.blocks[b.Type]
 		if !ok {
 			bm = map[string][]*hcl.Block{}
-			m[b.Type] = bm
+			p.blocks[b.Type] = bm
 		}
 
 		lbl := b.Labels[0]
 		bm[lbl] = append(bm[lbl], b)
 	}
 
-	vt := reflect.ValueOf(val).Elem().Type()
-	numFields := vt.NumField()
-
 	type value struct {
 		reflect.Value
 		idx int
@@ -427,9 +654,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	}
 	types := map[string]field{}
 
-	for i := 0; i < numFields; i++ {
+	vt := reflect.ValueOf(val).Elem().Type()
+	for i := 0; i < vt.NumField(); i++ {
 		tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
 
+		p.blockTypes[tags[0]] = vt.Field(i).Type.Elem().Elem()
 		types[tags[0]] = field{
 			idx:    i,
 			typ:    vt.Field(i).Type,
@@ -441,29 +670,29 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	for _, b := range content.Blocks {
 		v := reflect.ValueOf(val)
 
-		t, ok := types[b.Type]
+		err := p.resolveBlock(b, nil)
-		if !ok {
+		if err != nil {
-			continue
+			if diag, ok := err.(hcl.Diagnostics); ok {
-		}
-
-		vv := reflect.New(t.typ.Elem().Elem())
-		diag := gohcl.DecodeBody(b.Body, p.ectx, vv.Interface())
 				if diag.HasErrors() {
 					diags = append(diags, diag...)
 					continue
 				}
-
+			} else {
-		if err := opt.ValidateLabel(b.Labels[0]); err != nil {
 				return hcl.Diagnostics{
 					&hcl.Diagnostic{
 						Severity: hcl.DiagError,
-					Summary:  "Invalid name",
+						Summary:  "Invalid attribute",
 						Detail:   err.Error(),
 						Subject:  &b.LabelRanges[0],
+						Context:  &b.DefRange,
 					},
 				}
 			}
+		}
 
+		vv := p.blockValues[b]
+
+		t := types[b.Type]
 		lblIndex := setLabel(vv, b.Labels[0])
 
 		oldValue, exists := t.values[b.Labels[0]]
@@ -477,7 +706,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 					}
 				}
 			}
-
 		}
 		if exists {
 			if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
@@ -498,6 +726,23 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		return diags
 	}
 
+	for k := range p.attrs {
+		if err := p.resolveValue(k); err != nil {
+			if diags, ok := err.(hcl.Diagnostics); ok {
+				return diags
+			}
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid attribute",
+					Detail:   err.Error(),
+					Subject:  &p.attrs[k].Range,
+					Context:  &p.attrs[k].Range,
+				},
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -514,3 +759,33 @@ func setLabel(v reflect.Value, lbl string) int {
 	}
 	return -1
 }
+
+func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, vars map[string]*variable) hcl.Diagnostics {
+	var fdiags hcl.Diagnostics
+	for _, d := range diags {
+		if fout := func(d *hcl.Diagnostic) bool {
+			// https://github.com/docker/buildx/pull/541
+			if d.Detail == "Blocks are not allowed here." {
+				return true
+			}
+			for r := range reserved {
+				// JSON body objects don't handle repeated blocks like HCL but
+				// reserved name attributes should be allowed when multi bodies are merged.
+				// https://github.com/hashicorp/hcl/blob/main/json/spec.md#blocks
+				if strings.HasPrefix(d.Detail, fmt.Sprintf(`Argument "%s" was already set at `, r)) {
+					return true
+				}
+			}
+			for v := range vars {
+				// Do the same for global variables
+				if strings.HasPrefix(d.Detail, fmt.Sprintf(`Argument "%s" was already set at `, v)) {
+					return true
+				}
+			}
+			return false
+		}(d); !fout {
+			fdiags = append(fdiags, d)
+		}
+	}
+	return fdiags
+}

bake/hclparser/stdlib.go

@@ -1,12 +1,15 @@
 package hclparser
 
 import (
+	"time"
+
 	"github.com/hashicorp/go-cty-funcs/cidr"
 	"github.com/hashicorp/go-cty-funcs/crypto"
 	"github.com/hashicorp/go-cty-funcs/encoding"
 	"github.com/hashicorp/go-cty-funcs/uuid"
 	"github.com/hashicorp/hcl/v2/ext/tryfunc"
 	"github.com/hashicorp/hcl/v2/ext/typeexpr"
+	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
 	"github.com/zclconf/go-cty/cty/function/stdlib"
 )
@@ -96,6 +99,7 @@ var stdlibFunctions = map[string]function.Function{
 	"substr":                 stdlib.SubstrFunc,
 	"subtract":               stdlib.SubtractFunc,
 	"timeadd":                stdlib.TimeAddFunc,
+	"timestamp":              timestampFunc,
 	"title":                  stdlib.TitleFunc,
 	"trim":                   stdlib.TrimFunc,
 	"trimprefix":             stdlib.TrimPrefixFunc,
@@ -109,3 +113,14 @@ var stdlibFunctions = map[string]function.Function{
 	"values":                 stdlib.ValuesFunc,
 	"zipmap":                 stdlib.ZipmapFunc,
 }
+
+// timestampFunc constructs a function that returns a string representation of the current date and time.
+//
+// This function was imported from terraform's datetime utilities.
+var timestampFunc = function.New(&function.Spec{
+	Params: []function.Parameter{},
+	Type:   function.StaticReturnType(cty.String),
+	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+		return cty.StringVal(time.Now().UTC().Format(time.RFC3339)), nil
+	},
+})

bake/remote.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"strings"
 
-	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/util/progress"
 	"github.com/moby/buildkit/client"
@@ -20,7 +20,7 @@ type Input struct {
 	URL   string
 }
 
-func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
+func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
 	var filename string
 	st, ok := detectGitContext(url)
 	if !ok {
@@ -33,18 +33,18 @@ func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, na
 	inp := &Input{State: st, URL: url}
 	var files []File
 
-	var di *build.DriverInfo
+	var node *builder.Node
-	for _, d := range dis {
+	for _, n := range nodes {
-		if d.Err == nil {
+		if n.Err == nil {
-			di = &d
+			node = &n
 			continue
 		}
 	}
-	if di == nil {
+	if node == nil {
 		return nil, nil, nil
 	}
 
-	c, err := driver.Boot(ctx, ctx, di.Driver, pw)
+	c, err := driver.Boot(ctx, ctx, node.Driver, pw)
 	if err != nil {
 		return nil, nil, err
 	}

build/build.go

@@ -2,13 +2,14 @@ package build
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"crypto/rand"
+	_ "crypto/sha256" // ensure digests can be computed
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -17,9 +18,13 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/resolver"
@@ -27,15 +32,16 @@ import (
 	"github.com/docker/cli/opts"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
-	dockerclient "github.com/docker/docker/client"
+	"github.com/docker/docker/builder/remotecontext/urlutil"
 	"github.com/docker/docker/pkg/jsonmessage"
-	"github.com/docker/docker/pkg/urlutil"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/frontend/attestations"
 	gateway "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
+	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/entitlements"
@@ -54,10 +60,15 @@ var (
 	errDockerfileConflict = errors.New("ambiguous Dockerfile source: both stdin and flag correspond to Dockerfiles")
 )
 
+const (
+	printFallbackImage = "docker/dockerfile-upstream:1.4-outline@sha256:627443ff4e2d0f635d429cfc1da5388bcd5a70949c38adcd3cd7c4e5df67c73c"
+)
+
 type Options struct {
 	Inputs Inputs
 
 	Allow         []entitlements.Entitlement
+	Attests       map[string]*string
 	BuildArgs     map[string]string
 	CacheFrom     []client.CacheOptionsEntry
 	CacheTo       []client.CacheOptionsEntry
@@ -76,6 +87,15 @@ type Options struct {
 	Tags          []string
 	Target        string
 	Ulimits       *opts.UlimitOpt
+
+	// Linked marks this target as exclusively linked (not requested by the user).
+	Linked    bool
+	PrintFunc *PrintFunc
+}
+
+type PrintFunc struct {
+	Name   string
+	Format string
 }
 
 type Inputs struct {
@@ -92,28 +112,15 @@ type NamedContext struct {
 	State *llb.State
 }
 
-type DriverInfo struct {
+func filterAvailableNodes(nodes []builder.Node) ([]builder.Node, error) {
-	Driver      driver.Driver
+	out := make([]builder.Node, 0, len(nodes))
-	Name        string
-	Platform    []specs.Platform
-	Err         error
-	ImageOpt    imagetools.Opt
-	ProxyConfig map[string]string
-}
-
-type DockerAPI interface {
-	DockerAPI(name string) (dockerclient.APIClient, error)
-}
-
-func filterAvailableDrivers(drivers []DriverInfo) ([]DriverInfo, error) {
-	out := make([]DriverInfo, 0, len(drivers))
 	err := errors.Errorf("no drivers found")
-	for _, di := range drivers {
+	for _, n := range nodes {
-		if di.Err == nil && di.Driver != nil {
+		if n.Err == nil && n.Driver != nil {
-			out = append(out, di)
+			out = append(out, n)
 		}
-		if di.Err != nil {
+		if n.Err != nil {
-			err = di.Err
+			err = n.Err
 		}
 	}
 	if len(out) > 0 {
@@ -152,8 +159,8 @@ func allIndexes(l int) []int {
 	return out
 }
 
-func ensureBooted(ctx context.Context, drivers []DriverInfo, idxs []int, pw progress.Writer) ([]*client.Client, error) {
+func ensureBooted(ctx context.Context, nodes []builder.Node, idxs []int, pw progress.Writer) ([]*client.Client, error) {
-	clients := make([]*client.Client, len(drivers))
+	clients := make([]*client.Client, len(nodes))
 
 	baseCtx := ctx
 	eg, ctx := errgroup.WithContext(ctx)
@@ -161,7 +168,7 @@ func ensureBooted(ctx context.Context, drivers []DriverInfo, idxs []int, pw prog
 	for _, i := range idxs {
 		func(i int) {
 			eg.Go(func() error {
-				c, err := driver.Boot(ctx, baseCtx, drivers[i].Driver, pw)
+				c, err := driver.Boot(ctx, baseCtx, nodes[i].Driver, pw)
 				if err != nil {
 					return err
 				}
@@ -189,6 +196,10 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
 			pp = append(pp, p)
 			mm[idx] = pp
 		}
+		// if no platform is specified, use first driver
+		if len(mm) == 0 {
+			mm[0] = nil
+		}
 		dps := make([]driverPair, 0, 2)
 		for idx, pp := range mm {
 			dps = append(dps, driverPair{driverIndex: idx, platforms: pp})
@@ -198,8 +209,8 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
 	return m
 }
 
-func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
+func resolveDrivers(ctx context.Context, nodes []builder.Node, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
-	dps, clients, err := resolveDriversBase(ctx, drivers, opt, pw)
+	dps, clients, err := resolveDriversBase(ctx, nodes, opt, pw)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -239,10 +250,10 @@ func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Op
 	return dps, clients, nil
 }
 
-func resolveDriversBase(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
+func resolveDriversBase(ctx context.Context, nodes []builder.Node, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
 	availablePlatforms := map[string]int{}
-	for i, d := range drivers {
+	for i, node := range nodes {
-		for _, p := range d.Platform {
+		for _, p := range node.Platforms {
 			availablePlatforms[platforms.Format(p)] = i
 		}
 	}
@@ -260,12 +271,12 @@ func resolveDriversBase(ctx context.Context, drivers []DriverInfo, opt map[strin
 	}
 
 	// fast path
-	if len(drivers) == 1 || len(allPlatforms) == 0 {
+	if len(nodes) == 1 || len(allPlatforms) == 0 {
 		m := map[string][]driverPair{}
 		for k, opt := range opt {
 			m[k] = []driverPair{{driverIndex: 0, platforms: opt.Platforms}}
 		}
-		clients, err := ensureBooted(ctx, drivers, driverIndexes(m), pw)
+		clients, err := ensureBooted(ctx, nodes, driverIndexes(m), pw)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -275,7 +286,7 @@ func resolveDriversBase(ctx context.Context, drivers []DriverInfo, opt map[strin
 	// map based on existing platforms
 	if !undetectedPlatform {
 		m := splitToDriverPairs(availablePlatforms, opt)
-		clients, err := ensureBooted(ctx, drivers, driverIndexes(m), pw)
+		clients, err := ensureBooted(ctx, nodes, driverIndexes(m), pw)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -283,7 +294,7 @@ func resolveDriversBase(ctx context.Context, drivers []DriverInfo, opt map[strin
 	}
 
 	// boot all drivers in k
-	clients, err := ensureBooted(ctx, drivers, allIndexes(len(drivers)), pw)
+	clients, err := ensureBooted(ctx, nodes, allIndexes(len(nodes)), pw)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -345,8 +356,8 @@ func toRepoOnly(in string) (string, error) {
 	return strings.Join(out, ","), nil
 }
 
-func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
+func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
-	d := di.Driver
+	nodeDriver := node.Driver
 	defers := make([]func(), 0, 2)
 	releaseF := func() {
 		for _, f := range defers {
@@ -378,8 +389,8 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 	}
 
 	for _, e := range opt.CacheTo {
-		if e.Type != "inline" && !d.Features()[driver.CacheExport] {
+		if e.Type != "inline" && !nodeDriver.Features()[driver.CacheExport] {
-			return nil, nil, notSupported(d, driver.CacheExport)
+			return nil, nil, notSupported(nodeDriver, driver.CacheExport)
 		}
 	}
 
@@ -389,6 +400,10 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
 				continue
 			}
+		} else if e.Type == "s3" {
+			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.s3")) {
+				continue
+			}
 		}
 		cacheTo = append(cacheTo, e)
 	}
@@ -399,6 +414,10 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
 				continue
 			}
+		} else if e.Type == "s3" {
+			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.s3")) {
+				continue
+			}
 		}
 		cacheFrom = append(cacheFrom, e)
 	}
@@ -431,7 +450,7 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 	case 1:
 		// valid
 	case 0:
-		if d.IsMobyDriver() && !noDefaultLoad() {
+		if nodeDriver.IsMobyDriver() && !noDefaultLoad() {
 			// backwards compat for docker driver only:
 			// this ensures the build results in a docker image.
 			opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
@@ -480,15 +499,15 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 		if (e.Type == "local" || e.Type == "tar") && opt.ImageIDFile != "" {
 			return nil, nil, errors.Errorf("local and tar exporters are incompatible with image ID file")
 		}
-		if e.Type == "oci" && !d.Features()[driver.OCIExporter] {
+		if e.Type == "oci" && !nodeDriver.Features()[driver.OCIExporter] {
-			return nil, nil, notSupported(d, driver.OCIExporter)
+			return nil, nil, notSupported(nodeDriver, driver.OCIExporter)
 		}
 		if e.Type == "docker" {
 			if len(opt.Platforms) > 1 {
 				return nil, nil, errors.Errorf("docker exporter does not currently support exporting manifest lists")
 			}
 			if e.Output == nil {
-				if d.IsMobyDriver() {
+				if nodeDriver.IsMobyDriver() {
 					e.Type = "image"
 				} else {
 					w, cancel, err := dl(e.Attrs["context"])
@@ -498,11 +517,11 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 					defers = append(defers, cancel)
 					opt.Exports[i].Output = wrapWriteCloser(w)
 				}
-			} else if !d.Features()[driver.DockerExporter] {
+			} else if !nodeDriver.Features()[driver.DockerExporter] {
-				return nil, nil, notSupported(d, driver.DockerExporter)
+				return nil, nil, notSupported(nodeDriver, driver.DockerExporter)
 			}
 		}
-		if e.Type == "image" && d.IsMobyDriver() {
+		if e.Type == "image" && nodeDriver.IsMobyDriver() {
 			opt.Exports[i].Type = "moby"
 			if e.Attrs["push"] != "" {
 				if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
@@ -523,7 +542,7 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 	so.Exports = opt.Exports
 	so.Session = opt.Session
 
-	releaseLoad, err := LoadInputs(ctx, d, opt.Inputs, pw, &so)
+	releaseLoad, err := LoadInputs(ctx, nodeDriver, opt.Inputs, pw, &so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -555,20 +574,39 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 		so.FrontendAttrs["label:"+k] = v
 	}
 
-	for k, v := range di.ProxyConfig {
+	for k, v := range node.ProxyConfig {
 		if _, ok := opt.BuildArgs[k]; !ok {
 			so.FrontendAttrs["build-arg:"+k] = v
 		}
 	}
 
+	if len(opt.Attests) > 0 {
+		if !bopts.LLBCaps.Contains(apicaps.CapID("exporter.image.attestations")) {
+			return nil, nil, errors.Errorf("attestations are not supported by the current buildkitd")
+		}
+		for k, v := range opt.Attests {
+			if v == nil {
+				continue
+			}
+			so.FrontendAttrs[k] = *v
+		}
+	}
+	if _, ok := opt.Attests["attest:provenance"]; !ok {
+		so.FrontendAttrs["attest:provenance"] = "mode=min,inline-only=true"
+	}
+
+	for k, v := range getGitAttributes(ctx, opt.Inputs.ContextPath, opt.Inputs.DockerfilePath) {
+		so.FrontendAttrs[k] = v
+	}
+
 	// set platforms
 	if len(opt.Platforms) != 0 {
 		pp := make([]string, len(opt.Platforms))
 		for i, p := range opt.Platforms {
 			pp[i] = platforms.Format(p)
 		}
-		if len(pp) > 1 && !d.Features()[driver.MultiPlatform] {
+		if len(pp) > 1 && !nodeDriver.Features()[driver.MultiPlatform] {
-			return nil, nil, notSupported(d, driver.MultiPlatform)
+			return nil, nil, notSupported(nodeDriver, driver.MultiPlatform)
 		}
 		so.FrontendAttrs["platform"] = strings.Join(pp, ",")
 	}
@@ -582,15 +620,17 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 		so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
 	case "", "default":
 	default:
-		return nil, nil, errors.Errorf("network mode %q not supported by buildkit. You can define a custom network for your builder using the network driver-opt in buildx create.", opt.NetworkMode)
+		return nil, nil, errors.Errorf("network mode %q not supported by buildkit - you can define a custom network for your builder using the network driver-opt in buildx create", opt.NetworkMode)
 	}
 
 	// setup extrahosts
-	extraHosts, err := toBuildkitExtraHosts(opt.ExtraHosts)
+	extraHosts, err := toBuildkitExtraHosts(opt.ExtraHosts, nodeDriver.IsMobyDriver())
 	if err != nil {
 		return nil, nil, err
 	}
+	if len(extraHosts) > 0 {
 		so.FrontendAttrs["add-hosts"] = extraHosts
+	}
 
 	// setup shm size
 	if opt.ShmSize.Value() > 0 {
@@ -608,34 +648,193 @@ func toSolveOpt(ctx context.Context, di DriverInfo, multiDriver bool, opt Option
 	return &so, releaseF, nil
 }
 
-func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
+// ContainerConfig is configuration for a container to run.
-	if len(drivers) == 0 {
+type ContainerConfig struct {
+	ResultCtx *ResultContext
+
+	Stdin  io.ReadCloser
+	Stdout io.WriteCloser
+	Stderr io.WriteCloser
+	Tty    bool
+
+	Entrypoint []string
+	Cmd        []string
+	Env        []string
+	User       *string
+	Cwd        *string
+}
+
+// ResultContext is a build result with the client that built it.
+type ResultContext struct {
+	Client *client.Client
+	Res    *gateway.Result
+}
+
+// Invoke invokes a build result as a container.
+func Invoke(ctx context.Context, cfg ContainerConfig) error {
+	if cfg.ResultCtx == nil {
+		return errors.Errorf("result must be provided")
+	}
+	c, res := cfg.ResultCtx.Client, cfg.ResultCtx.Res
+
+	mainCtx := ctx
+
+	_, err := c.Build(context.TODO(), client.SolveOpt{}, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		ctx, cancel := context.WithCancel(ctx)
+		go func() {
+			<-mainCtx.Done()
+			cancel()
+		}()
+
+		if res.Ref == nil {
+			return nil, errors.Errorf("no reference is registered")
+		}
+		st, err := res.Ref.ToState()
+		if err != nil {
+			return nil, err
+		}
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+		imgRef, err := c.Solve(ctx, gateway.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return nil, err
+		}
+		ctr, err := c.NewContainer(ctx, gateway.NewContainerRequest{
+			Mounts: []gateway.Mount{
+				{
+					Dest:      "/",
+					MountType: pb.MountType_BIND,
+					Ref:       imgRef.Ref,
+				},
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		defer ctr.Release(context.TODO())
+
+		imgData := res.Metadata[exptypes.ExporterImageConfigKey]
+		var img *specs.Image
+		if len(imgData) > 0 {
+			img = &specs.Image{}
+			if err := json.Unmarshal(imgData, img); err != nil {
+				fmt.Println(err)
+				return nil, err
+			}
+		}
+
+		user := ""
+		if cfg.User != nil {
+			user = *cfg.User
+		} else if img != nil {
+			user = img.Config.User
+		}
+
+		cwd := ""
+		if cfg.Cwd != nil {
+			cwd = *cfg.Cwd
+		} else if img != nil {
+			cwd = img.Config.WorkingDir
+		}
+
+		env := []string{}
+		if img != nil {
+			env = append(env, img.Config.Env...)
+		}
+		env = append(env, cfg.Env...)
+
+		args := []string{}
+		if cfg.Entrypoint != nil {
+			args = append(args, cfg.Entrypoint...)
+		} else if img != nil {
+			args = append(args, img.Config.Entrypoint...)
+		}
+		if cfg.Cmd != nil {
+			args = append(args, cfg.Cmd...)
+		} else if img != nil {
+			args = append(args, img.Config.Cmd...)
+		}
+
+		proc, err := ctr.Start(ctx, gateway.StartRequest{
+			Args:   args,
+			Env:    env,
+			User:   user,
+			Cwd:    cwd,
+			Tty:    cfg.Tty,
+			Stdin:  cfg.Stdin,
+			Stdout: cfg.Stdout,
+			Stderr: cfg.Stderr,
+		})
+		if err != nil {
+			return nil, errors.Errorf("failed to start container: %v", err)
+		}
+		errCh := make(chan error)
+		doneCh := make(chan struct{})
+		go func() {
+			if err := proc.Wait(); err != nil {
+				errCh <- err
+				return
+			}
+			close(doneCh)
+		}()
+		select {
+		case <-doneCh:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case err := <-errCh:
+			return nil, err
+		}
+		return nil, nil
+	}, nil)
+	return err
+}
+
+func Build(ctx context.Context, nodes []builder.Node, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
+	return BuildWithResultHandler(ctx, nodes, opt, docker, configDir, w, nil, false)
+}
+
+func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) {
+	if len(nodes) == 0 {
 		return nil, errors.Errorf("driver required for build")
 	}
 
-	drivers, err = filterAvailableDrivers(drivers)
+	nodes, err = filterAvailableNodes(nodes)
 	if err != nil {
 		return nil, errors.Wrapf(err, "no valid drivers found")
 	}
 
 	var noMobyDriver driver.Driver
-	for _, d := range drivers {
+	for _, n := range nodes {
-		if !d.Driver.IsMobyDriver() {
+		if !n.Driver.IsMobyDriver() {
-			noMobyDriver = d.Driver
+			noMobyDriver = n.Driver
 			break
 		}
 	}
 
-	if noMobyDriver != nil && !noDefaultLoad() {
+	if noMobyDriver != nil && !noDefaultLoad() && noPrintFunc(opt) {
-		for _, opt := range opt {
+		var noOutputTargets []string
-			if len(opt.Exports) == 0 {
+		for name, opt := range opt {
-				logrus.Warnf("No output specified for %s driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load", noMobyDriver.Factory().Name())
+			if !opt.Linked && len(opt.Exports) == 0 {
-				break
+				noOutputTargets = append(noOutputTargets, name)
 			}
 		}
+		if len(noOutputTargets) > 0 && !allowNoOutput {
+			var warnNoOutputBuf bytes.Buffer
+			warnNoOutputBuf.WriteString("No output specified ")
+			if len(noOutputTargets) == 1 && noOutputTargets[0] == "default" {
+				warnNoOutputBuf.WriteString(fmt.Sprintf("with %s driver", noMobyDriver.Factory().Name()))
+			} else {
+				warnNoOutputBuf.WriteString(fmt.Sprintf("for %s target(s) with %s driver", strings.Join(noOutputTargets, ", "), noMobyDriver.Factory().Name()))
+			}
+			logrus.Warnf("%s. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load", warnNoOutputBuf.String())
+		}
 	}
 
-	m, clients, err := resolveDrivers(ctx, drivers, opt, w)
+	m, clients, err := resolveDrivers(ctx, nodes, opt, w)
 	if err != nil {
 		return nil, err
 	}
@@ -654,14 +853,14 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 	for k, opt := range opt {
 		multiDriver := len(m[k]) > 1
 		hasMobyDriver := false
-		for i, dp := range m[k] {
+		for i, np := range m[k] {
-			di := drivers[dp.driverIndex]
+			node := nodes[np.driverIndex]
-			if di.Driver.IsMobyDriver() {
+			if node.Driver.IsMobyDriver() {
 				hasMobyDriver = true
 			}
-			opt.Platforms = dp.platforms
+			opt.Platforms = np.platforms
-			so, release, err := toSolveOpt(ctx, di, multiDriver, opt, dp.bopts, configDir, w, func(name string) (io.WriteCloser, func(), error) {
+			so, release, err := toSolveOpt(ctx, node, multiDriver, opt, np.bopts, configDir, w, func(name string) (io.WriteCloser, func(), error) {
-				return newDockerLoader(ctx, docker, name, w)
+				return docker.LoadImage(ctx, name, w)
 			})
 			if err != nil {
 				return nil, err
@@ -762,9 +961,12 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 				resp[k] = res[0]
 				respMu.Unlock()
 				if len(res) == 1 {
-					digest := res[0].ExporterResponse["containerimage.digest"]
+					dgst := res[0].ExporterResponse[exptypes.ExporterImageDigestKey]
+					if v, ok := res[0].ExporterResponse[exptypes.ExporterImageConfigDigestKey]; ok {
+						dgst = v
+					}
 					if opt.ImageIDFile != "" {
-						return ioutil.WriteFile(opt.ImageIDFile, []byte(digest), 0644)
+						return os.WriteFile(opt.ImageIDFile, []byte(dgst), 0644)
 					}
 					return nil
 				}
@@ -774,7 +976,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 						descs := make([]specs.Descriptor, 0, len(res))
 
 						for _, r := range res {
-							s, ok := r.ExporterResponse["containerimage.digest"]
+							s, ok := r.ExporterResponse[exptypes.ExporterImageDigestKey]
 							if ok {
 								descs = append(descs, specs.Descriptor{
 									Digest:    digest.Digest(s),
@@ -786,7 +988,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 						if len(descs) > 0 {
 							var imageopt imagetools.Opt
 							for _, dp := range dps {
-								imageopt = drivers[dp.driverIndex].ImageOpt
+								imageopt = nodes[dp.driverIndex].ImageOpt
 								break
 							}
 							names := strings.Split(pushNames, ",")
@@ -808,12 +1010,26 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 
 							itpull := imagetools.New(imageopt)
 
-							dt, desc, err := itpull.Combine(ctx, names[0], descs)
+							ref, err := reference.ParseNormalizedNamed(names[0])
+							if err != nil {
+								return err
+							}
+							ref = reference.TagNameOnly(ref)
+
+							srcs := make([]*imagetools.Source, len(descs))
+							for i, desc := range descs {
+								srcs[i] = &imagetools.Source{
+									Desc: desc,
+									Ref:  ref,
+								}
+							}
+
+							dt, desc, err := itpull.Combine(ctx, srcs)
 							if err != nil {
 								return err
 							}
 							if opt.ImageIDFile != "" {
-								if err := ioutil.WriteFile(opt.ImageIDFile, []byte(desc.Digest), 0644); err != nil {
+								if err := os.WriteFile(opt.ImageIDFile, []byte(desc.Digest), 0644); err != nil {
 									return err
 								}
 							}
@@ -901,27 +1117,78 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 							FrontendInputs: frontendInputs,
 						}
 						so.Frontend = ""
-						so.FrontendAttrs = nil
+						so.FrontendAttrs = attestations.Filter(so.FrontendAttrs)
 						so.FrontendInputs = nil
 
 						ch, done := progress.NewChannel(pw)
 						defer func() { <-done }()
 
+						cc := c
+						var printRes map[string][]byte
 						rr, err := c.Build(ctx, so, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+							var isFallback bool
+							var origErr error
+							for {
+								if opt.PrintFunc != nil {
+									if _, ok := req.FrontendOpt["frontend.caps"]; !ok {
+										req.FrontendOpt["frontend.caps"] = "moby.buildkit.frontend.subrequests+forward"
+									} else {
+										req.FrontendOpt["frontend.caps"] += ",moby.buildkit.frontend.subrequests+forward"
+									}
+									req.FrontendOpt["requestid"] = "frontend." + opt.PrintFunc.Name
+									if isFallback {
+										req.FrontendOpt["build-arg:BUILDKIT_SYNTAX"] = printFallbackImage
+									}
+								}
 								res, err := c.Solve(ctx, req)
 								if err != nil {
+									if origErr != nil {
 										return nil, err
 									}
+									var reqErr *errdefs.UnsupportedSubrequestError
+									if !isFallback {
+										if errors.As(err, &reqErr) {
+											switch reqErr.Name {
+											case "frontend.outline", "frontend.targets":
+												isFallback = true
+												origErr = err
+												continue
+											}
+											return nil, err
+										}
+										// buildkit v0.8 vendored in Docker 20.10 does not support typed errors
+										if strings.Contains(err.Error(), "unsupported request frontend.outline") || strings.Contains(err.Error(), "unsupported request frontend.targets") {
+											isFallback = true
+											origErr = err
+											continue
+										}
+									}
+									return nil, err
+								}
+								if opt.PrintFunc != nil {
+									printRes = res.Metadata
+								}
 								results.Set(resultKey(dp.driverIndex, k), res)
+								if resultHandleFunc != nil {
+									resultHandleFunc(dp.driverIndex, &ResultContext{cc, res})
+								}
 								return res, nil
+							}
 						}, ch)
 						if err != nil {
 							return err
 						}
 						res[i] = rr
 
-						d := drivers[dp.driverIndex].Driver
+						if rr.ExporterResponse == nil {
-						if d.IsMobyDriver() {
+							rr.ExporterResponse = map[string]string{}
+						}
+						for k, v := range printRes {
+							rr.ExporterResponse[k] = string(v)
+						}
+
+						node := nodes[dp.driverIndex].Driver
+						if node.IsMobyDriver() {
 							for _, e := range so.Exports {
 								if e.Type == "moby" && e.Attrs["push"] != "" {
 									if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
@@ -930,13 +1197,27 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 											return errors.Errorf("tag is needed when pushing to registry")
 										}
 										pw := progress.ResetTime(pw)
-										for _, name := range strings.Split(pushNames, ",") {
+										pushList := strings.Split(pushNames, ",")
+										for _, name := range pushList {
 											if err := progress.Wrap(fmt.Sprintf("pushing %s with docker", name), pw.Write, func(l progress.SubLogger) error {
-												return pushWithMoby(ctx, d, name, l)
+												return pushWithMoby(ctx, node, name, l)
 											}); err != nil {
 												return err
 											}
 										}
+										remoteDigest, err := remoteDigestWithMoby(ctx, node, pushList[0])
+										if err == nil && remoteDigest != "" {
+											// old daemons might not have containerimage.config.digest set
+											// in response so use containerimage.digest value for it if available
+											if _, ok := rr.ExporterResponse[exptypes.ExporterImageConfigDigestKey]; !ok {
+												if v, ok := rr.ExporterResponse[exptypes.ExporterImageDigestKey]; ok {
+													rr.ExporterResponse[exptypes.ExporterImageConfigDigestKey] = v
+												}
+											}
+											rr.ExporterResponse[exptypes.ExporterImageDigestKey] = remoteDigest
+										} else if err != nil {
+											return err
+										}
 									}
 								}
 							}
@@ -1041,8 +1322,31 @@ func pushWithMoby(ctx context.Context, d driver.Driver, name string, l progress.
 	return nil
 }
 
+func remoteDigestWithMoby(ctx context.Context, d driver.Driver, name string) (string, error) {
+	api := d.Config().DockerAPI
+	if api == nil {
+		return "", errors.Errorf("invalid empty Docker API reference") // should never happen
+	}
+	creds, err := imagetools.RegistryAuthForRef(name, d.Config().Auth)
+	if err != nil {
+		return "", err
+	}
+	image, _, err := api.ImageInspectWithRaw(ctx, name)
+	if err != nil {
+		return "", err
+	}
+	if len(image.RepoDigests) == 0 {
+		return "", nil
+	}
+	remoteImage, err := api.DistributionInspect(ctx, name, creds)
+	if err != nil {
+		return "", err
+	}
+	return remoteImage.Descriptor.Digest.String(), nil
+}
+
 func createTempDockerfile(r io.Reader) (string, error) {
-	dir, err := ioutil.TempDir("", "dockerfile")
+	dir, err := os.MkdirTemp("", "dockerfile")
 	if err != nil {
 		return "", err
 	}
@@ -1101,7 +1405,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
 				}
 				// stdin is dockerfile
 				dockerfileReader = buf
-				inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
+				inp.ContextPath, _ = os.MkdirTemp("", "empty-dir")
 				toRemove = append(toRemove, inp.ContextPath)
 				target.LocalDirs["context"] = inp.ContextPath
 			}
@@ -1121,7 +1425,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
 
 	case urlutil.IsGitURL(inp.ContextPath), urlutil.IsURL(inp.ContextPath):
 		if inp.DockerfilePath == "-" {
-			return nil, errors.Errorf("Dockerfile from stdin is not supported with remote contexts")
+			dockerfileReader = inp.InStream
 		}
 		target.FrontendAttrs["context"] = inp.ContextPath
 	default:
@@ -1178,6 +1482,32 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
 			target.FrontendAttrs["context:"+k] = v.Path
 			continue
 		}
+
+		// handle OCI layout
+		if strings.HasPrefix(v.Path, "oci-layout://") {
+			pathAlone := strings.TrimPrefix(v.Path, "oci-layout://")
+			parts := strings.SplitN(pathAlone, "@", 2)
+			if len(parts) != 2 {
+				return nil, errors.Errorf("invalid oci-layout context %s, must be oci-layout:///path/to/layout@sha256:hash", v.Path)
+			}
+			localPath := parts[0]
+			dgst, err := digest.Parse(parts[1])
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid oci-layout context %s, does not have proper hash, must be oci-layout:///path/to/layout@sha256:hash", v.Path)
+			}
+			store, err := local.NewStore(localPath)
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid store at %s", localPath)
+			}
+			// now we can add it
+			if target.OCIStores == nil {
+				target.OCIStores = map[string]content.Store{}
+			}
+			target.OCIStores[k] = store
+
+			target.FrontendAttrs["context:"+k] = fmt.Sprintf("oci-layout:%s@%s", k, dgst.String())
+			continue
+		}
 		st, err := os.Stat(v.Path)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to get build context %v", k)
@@ -1263,6 +1593,7 @@ func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *c
 					so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
 				}
 			}
+			delete(so.FrontendAttrs, v)
 		}
 		if rr.Ref != nil {
 			st, err := rr.Ref.ToState()
@@ -1296,40 +1627,6 @@ func notSupported(d driver.Driver, f driver.Feature) error {
 
 type dockerLoadCallback func(name string) (io.WriteCloser, func(), error)
 
-func newDockerLoader(ctx context.Context, d DockerAPI, name string, status progress.Writer) (io.WriteCloser, func(), error) {
-	c, err := d.DockerAPI(name)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	pr, pw := io.Pipe()
-	done := make(chan struct{})
-
-	ctx, cancel := context.WithCancel(ctx)
-	var w *waitingWriter
-	w = &waitingWriter{
-		PipeWriter: pw,
-		f: func() {
-			resp, err := c.ImageLoad(ctx, pr, false)
-			defer close(done)
-			if err != nil {
-				pr.CloseWithError(err)
-				w.mu.Lock()
-				w.err = err
-				w.mu.Unlock()
-				return
-			}
-			prog := progress.WithPrefix(status, "", false)
-			progress.FromReader(prog, "importing to docker", resp.Body)
-		},
-		done:   done,
-		cancel: cancel,
-	}
-	return w, func() {
-		pr.Close()
-	}, nil
-}
-
 func noDefaultLoad() bool {
 	v, ok := os.LookupEnv("BUILDX_NO_DEFAULT_LOAD")
 	if !ok {
@@ -1342,34 +1639,6 @@ func noDefaultLoad() bool {
 	return b
 }
 
-type waitingWriter struct {
-	*io.PipeWriter
-	f      func()
-	once   sync.Once
-	mu     sync.Mutex
-	err    error
-	done   chan struct{}
-	cancel func()
-}
-
-func (w *waitingWriter) Write(dt []byte) (int, error) {
-	w.once.Do(func() {
-		go w.f()
-	})
-	return w.PipeWriter.Write(dt)
-}
-
-func (w *waitingWriter) Close() error {
-	err := w.PipeWriter.Close()
-	<-w.done
-	if err == nil {
-		w.mu.Lock()
-		defer w.mu.Unlock()
-		return w.err
-	}
-	return err
-}
-
 // handle https://github.com/moby/moby/pull/10858
 func handleLowercaseDockerfile(dir, p string) string {
 	if filepath.Base(p) != "Dockerfile" {
@@ -1419,15 +1688,24 @@ func tryNodeIdentifier(configDir string) (out string) {
 			if _, err := rand.Read(b); err != nil {
 				return out
 			}
-			if err := ioutil.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil {
+			if err := os.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil {
 				return out
 			}
 		}
 	}
 
-	dt, err := ioutil.ReadFile(sessionFile)
+	dt, err := os.ReadFile(sessionFile)
 	if err == nil {
 		return string(dt)
 	}
 	return
 }
+
+func noPrintFunc(opt map[string]Options) bool {
+	for _, v := range opt {
+		if v.PrintFunc != nil {
+			return false
+		}
+	}
+	return true
+}

build/git.go

@@ -0,0 +1,109 @@
+package build
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/docker/buildx/util/gitutil"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
+)
+
+const DockerfileLabel = "com.docker.image.source.entrypoint"
+
+func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath string) (res map[string]string) {
+	res = make(map[string]string)
+	if contextPath == "" {
+		return
+	}
+
+	setGitLabels := false
+	if v, ok := os.LookupEnv("BUILDX_GIT_LABELS"); ok {
+		if v == "full" { // backward compatibility with old "full" mode
+			setGitLabels = true
+		} else if v, _ := strconv.ParseBool(v); v {
+			setGitLabels = v
+		}
+	}
+	setGitInfo := true
+	if v, ok := os.LookupEnv("BUILDX_GIT_INFO"); ok {
+		if v, _ := strconv.ParseBool(v); v {
+			setGitInfo = v
+		}
+	}
+
+	if !setGitLabels && !setGitInfo {
+		return
+	}
+
+	// figure out in which directory the git command needs to run in
+	var wd string
+	if filepath.IsAbs(contextPath) {
+		wd = contextPath
+	} else {
+		cwd, _ := os.Getwd()
+		wd, _ = filepath.Abs(filepath.Join(cwd, contextPath))
+	}
+
+	gitc := gitutil.New(gitutil.WithContext(ctx), gitutil.WithWorkingDir(wd))
+	if !gitc.IsInsideWorkTree() {
+		logrus.Warnf("Unable to determine Git information")
+		return
+	}
+
+	var resRevision, resSource, resDockerfilePath string
+
+	if sha, err := gitc.FullCommit(); err == nil && sha != "" {
+		resRevision = sha
+		if gitc.IsDirty() {
+			resRevision += "-dirty"
+		}
+	}
+
+	if rurl, err := gitc.RemoteURL(); err == nil && rurl != "" {
+		resSource = rurl
+	}
+
+	if setGitLabels {
+		if root, err := gitc.RootDir(); err == nil && root != "" {
+			if dockerfilePath == "" {
+				dockerfilePath = filepath.Join(wd, "Dockerfile")
+			}
+			if !filepath.IsAbs(dockerfilePath) {
+				cwd, _ := os.Getwd()
+				dockerfilePath = filepath.Join(cwd, dockerfilePath)
+			}
+			dockerfilePath, _ = filepath.Rel(root, dockerfilePath)
+			if !strings.HasPrefix(dockerfilePath, "..") {
+				resDockerfilePath = dockerfilePath
+			}
+		}
+	}
+
+	if resSource != "" {
+		if setGitLabels {
+			res["label:"+specs.AnnotationSource] = resSource
+		}
+		if setGitInfo {
+			res["vcs:source"] = resSource
+		}
+	}
+	if resRevision != "" {
+		if setGitLabels {
+			res["label:"+specs.AnnotationRevision] = resRevision
+		}
+		if setGitInfo {
+			res["vcs:revision"] = resRevision
+		}
+	}
+	if resDockerfilePath != "" {
+		if setGitLabels {
+			res["label:"+DockerfileLabel] = resDockerfilePath
+		}
+	}
+
+	return
+}

build/git_test.go

@@ -0,0 +1,132 @@
+package build
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/docker/buildx/util/gitutil"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func setupTest(tb testing.TB) {
+	gitutil.Mktmp(tb)
+	gitutil.GitInit(tb)
+	df := []byte("FROM alpine:latest\n")
+	assert.NoError(tb, os.WriteFile("Dockerfile", df, 0644))
+	gitutil.GitAdd(tb, "Dockerfile")
+	gitutil.GitCommit(tb, "initial commit")
+}
+
+func TestGetGitAttributesNoContext(t *testing.T) {
+	setupTest(t)
+
+	gitattrs := getGitAttributes(context.Background(), "", "Dockerfile")
+	assert.Empty(t, gitattrs)
+}
+
+func TestGetGitAttributes(t *testing.T) {
+	cases := []struct {
+		name         string
+		envGitLabels string
+		envGitInfo   string
+		expected     []string
+	}{
+		{
+			name:         "default",
+			envGitLabels: "",
+			envGitInfo:   "",
+			expected: []string{
+				"vcs:revision",
+			},
+		},
+		{
+			name:         "gitinfo",
+			envGitLabels: "false",
+			envGitInfo:   "true",
+			expected: []string{
+				"vcs:revision",
+			},
+		},
+		{
+			name:         "gitlabels",
+			envGitLabels: "true",
+			envGitInfo:   "false",
+			expected: []string{
+				"label:" + DockerfileLabel,
+				"label:" + specs.AnnotationRevision,
+			},
+		},
+		{
+			name:         "both",
+			envGitLabels: "true",
+			envGitInfo:   "",
+			expected: []string{
+				"label:" + DockerfileLabel,
+				"label:" + specs.AnnotationRevision,
+				"vcs:revision",
+			},
+		},
+	}
+	for _, tt := range cases {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			setupTest(t)
+			if tt.envGitLabels != "" {
+				t.Setenv("BUILDX_GIT_LABELS", tt.envGitLabels)
+			}
+			if tt.envGitInfo != "" {
+				t.Setenv("BUILDX_GIT_INFO", tt.envGitInfo)
+			}
+			gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile")
+			for _, e := range tt.expected {
+				assert.Contains(t, gitattrs, e)
+				assert.NotEmpty(t, gitattrs[e])
+				if e == "label:"+DockerfileLabel {
+					assert.Equal(t, "Dockerfile", gitattrs[e])
+				}
+			}
+		})
+	}
+}
+
+func TestGetGitAttributesWithRemote(t *testing.T) {
+	setupTest(t)
+	gitutil.GitSetRemote(t, "git@github.com:docker/buildx.git")
+
+	t.Setenv("BUILDX_GIT_LABELS", "true")
+	gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile")
+	assert.Equal(t, 5, len(gitattrs))
+	assert.Contains(t, gitattrs, "label:"+DockerfileLabel)
+	assert.Equal(t, "Dockerfile", gitattrs["label:"+DockerfileLabel])
+	assert.Contains(t, gitattrs, "label:"+specs.AnnotationRevision)
+	assert.NotEmpty(t, gitattrs["label:"+specs.AnnotationRevision])
+	assert.Contains(t, gitattrs, "label:"+specs.AnnotationSource)
+	assert.Equal(t, "git@github.com:docker/buildx.git", gitattrs["label:"+specs.AnnotationSource])
+	assert.Contains(t, gitattrs, "vcs:revision")
+	assert.NotEmpty(t, gitattrs["vcs:revision"])
+	assert.Contains(t, gitattrs, "vcs:source")
+	assert.Equal(t, "git@github.com:docker/buildx.git", gitattrs["vcs:source"])
+}
+
+func TestGetGitAttributesDirty(t *testing.T) {
+	setupTest(t)
+
+	// make a change to test dirty flag
+	df := []byte("FROM alpine:edge\n")
+	assert.NoError(t, os.Mkdir("dir", 0755))
+	assert.NoError(t, os.WriteFile(filepath.Join("dir", "Dockerfile"), df, 0644))
+
+	t.Setenv("BUILDX_GIT_LABELS", "true")
+	gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile")
+	assert.Equal(t, 3, len(gitattrs))
+	assert.Contains(t, gitattrs, "label:"+DockerfileLabel)
+	assert.Equal(t, "Dockerfile", gitattrs["label:"+DockerfileLabel])
+	assert.Contains(t, gitattrs, "label:"+specs.AnnotationRevision)
+	assert.True(t, strings.HasSuffix(gitattrs["label:"+specs.AnnotationRevision], "-dirty"))
+	assert.Contains(t, gitattrs, "vcs:revision")
+	assert.True(t, strings.HasSuffix(gitattrs["vcs:revision"], "-dirty"))
+}

build/url.go

@@ -2,7 +2,7 @@ package build
 
 import (
 	"context"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/docker/buildx/driver"
@@ -53,11 +53,11 @@ func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url strin
 		if err != nil {
 			return nil, err
 		}
-		dir, err := ioutil.TempDir("", "buildx")
+		dir, err := os.MkdirTemp("", "buildx")
 		if err != nil {
 			return nil, err
 		}
-		if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil {
+		if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil {
 			return nil, err
 		}
 		out = dir

build/utils.go

@@ -11,8 +11,14 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
 	// archiveHeaderSize is the number of bytes in an archive header
-const archiveHeaderSize = 512
+	archiveHeaderSize = 512
+	// mobyHostGatewayName defines a special string which users can append to
+	// --add-host to add an extra entry in /etc/hosts that maps
+	// host.docker.internal to the host IP
+	mobyHostGatewayName = "host-gateway"
+)
 
 func isLocalDir(c string) bool {
 	st, err := os.Stat(c)
@@ -39,18 +45,23 @@ func isArchive(header []byte) bool {
 }
 
 // toBuildkitExtraHosts converts hosts from docker key:value format to buildkit's csv format
-func toBuildkitExtraHosts(inp []string) (string, error) {
+func toBuildkitExtraHosts(inp []string, mobyDriver bool) (string, error) {
 	if len(inp) == 0 {
 		return "", nil
 	}
 	hosts := make([]string, 0, len(inp))
 	for _, h := range inp {
-		parts := strings.Split(h, ":")
+		host, ip, ok := strings.Cut(h, ":")
-
+		if !ok || host == "" || ip == "" {
-		if len(parts) != 2 || parts[0] == "" || net.ParseIP(parts[1]) == nil {
 			return "", errors.Errorf("invalid host %s", h)
 		}
-		hosts = append(hosts, parts[0]+"="+parts[1])
+		// Skip IP address validation for "host-gateway" string with moby driver
+		if !mobyDriver || ip != mobyHostGatewayName {
+			if net.ParseIP(ip) == nil {
+				return "", errors.Errorf("invalid host %s", h)
+			}
+		}
+		hosts = append(hosts, host+"="+ip)
 	}
 	return strings.Join(hosts, ","), nil
 }

builder/builder.go

@@ -0,0 +1,292 @@
+package builder
+
+import (
+	"context"
+	"os"
+	"sort"
+	"sync"
+
+	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/store/storeutil"
+	"github.com/docker/buildx/util/dockerutil"
+	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/buildx/util/progress"
+	"github.com/docker/cli/cli/command"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+// Builder represents an active builder object
+type Builder struct {
+	*store.NodeGroup
+	driverFactory driverFactory
+	nodes         []Node
+	opts          builderOpts
+	err           error
+}
+
+type builderOpts struct {
+	dockerCli       command.Cli
+	name            string
+	txn             *store.Txn
+	contextPathHash string
+	validate        bool
+}
+
+// Option provides a variadic option for configuring the builder.
+type Option func(b *Builder)
+
+// WithName sets builder name.
+func WithName(name string) Option {
+	return func(b *Builder) {
+		b.opts.name = name
+	}
+}
+
+// WithStore sets a store instance used at init.
+func WithStore(txn *store.Txn) Option {
+	return func(b *Builder) {
+		b.opts.txn = txn
+	}
+}
+
+// WithContextPathHash is used for determining pods in k8s driver instance.
+func WithContextPathHash(contextPathHash string) Option {
+	return func(b *Builder) {
+		b.opts.contextPathHash = contextPathHash
+	}
+}
+
+// WithSkippedValidation skips builder context validation.
+func WithSkippedValidation() Option {
+	return func(b *Builder) {
+		b.opts.validate = false
+	}
+}
+
+// New initializes a new builder client
+func New(dockerCli command.Cli, opts ...Option) (_ *Builder, err error) {
+	b := &Builder{
+		opts: builderOpts{
+			dockerCli: dockerCli,
+			validate:  true,
+		},
+	}
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	if b.opts.txn == nil {
+		// if store instance is nil we create a short-lived one using the
+		// default store and ensure we release it on completion
+		var release func()
+		b.opts.txn, release, err = storeutil.GetStore(dockerCli)
+		if err != nil {
+			return nil, err
+		}
+		defer release()
+	}
+
+	if b.opts.name != "" {
+		if b.NodeGroup, err = storeutil.GetNodeGroup(b.opts.txn, dockerCli, b.opts.name); err != nil {
+			return nil, err
+		}
+	} else {
+		if b.NodeGroup, err = storeutil.GetCurrentInstance(b.opts.txn, dockerCli); err != nil {
+			return nil, err
+		}
+	}
+	if b.opts.validate {
+		if err = b.Validate(); err != nil {
+			return nil, err
+		}
+	}
+
+	return b, nil
+}
+
+// Validate validates builder context
+func (b *Builder) Validate() error {
+	if b.NodeGroup.DockerContext {
+		list, err := b.opts.dockerCli.ContextStore().List()
+		if err != nil {
+			return err
+		}
+		currentContext := b.opts.dockerCli.CurrentContext()
+		for _, l := range list {
+			if l.Name == b.Name && l.Name != currentContext {
+				return errors.Errorf("use `docker --context=%s buildx` to switch to context %q", l.Name, l.Name)
+			}
+		}
+	}
+	return nil
+}
+
+// ContextName returns builder context name if available.
+func (b *Builder) ContextName() string {
+	ctxbuilders, err := b.opts.dockerCli.ContextStore().List()
+	if err != nil {
+		return ""
+	}
+	for _, cb := range ctxbuilders {
+		if b.NodeGroup.Driver == "docker" && len(b.NodeGroup.Nodes) == 1 && b.NodeGroup.Nodes[0].Endpoint == cb.Name {
+			return cb.Name
+		}
+	}
+	return ""
+}
+
+// ImageOpt returns registry auth configuration
+func (b *Builder) ImageOpt() (imagetools.Opt, error) {
+	return storeutil.GetImageConfig(b.opts.dockerCli, b.NodeGroup)
+}
+
+// Boot bootstrap a builder
+func (b *Builder) Boot(ctx context.Context) (bool, error) {
+	toBoot := make([]int, 0, len(b.nodes))
+	for idx, d := range b.nodes {
+		if d.Err != nil || d.Driver == nil || d.DriverInfo == nil {
+			continue
+		}
+		if d.DriverInfo.Status != driver.Running {
+			toBoot = append(toBoot, idx)
+		}
+	}
+	if len(toBoot) == 0 {
+		return false, nil
+	}
+
+	printer, err := progress.NewPrinter(context.TODO(), os.Stderr, os.Stderr, progress.PrinterModeAuto)
+	if err != nil {
+		return false, err
+	}
+
+	baseCtx := ctx
+	eg, _ := errgroup.WithContext(ctx)
+	for _, idx := range toBoot {
+		func(idx int) {
+			eg.Go(func() error {
+				pw := progress.WithPrefix(printer, b.NodeGroup.Nodes[idx].Name, len(toBoot) > 1)
+				_, err := driver.Boot(ctx, baseCtx, b.nodes[idx].Driver, pw)
+				if err != nil {
+					b.nodes[idx].Err = err
+				}
+				return nil
+			})
+		}(idx)
+	}
+
+	err = eg.Wait()
+	err1 := printer.Wait()
+	if err == nil {
+		err = err1
+	}
+
+	return true, err
+}
+
+// Inactive checks if all nodes are inactive for this builder.
+func (b *Builder) Inactive() bool {
+	for _, d := range b.nodes {
+		if d.DriverInfo != nil && d.DriverInfo.Status == driver.Running {
+			return false
+		}
+	}
+	return true
+}
+
+// Err returns error if any.
+func (b *Builder) Err() error {
+	return b.err
+}
+
+type driverFactory struct {
+	driver.Factory
+	once sync.Once
+}
+
+// Factory returns the driver factory.
+func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
+	b.driverFactory.once.Do(func() {
+		if b.Driver != "" {
+			b.driverFactory.Factory, err = driver.GetFactory(b.Driver, true)
+			if err != nil {
+				return
+			}
+		} else {
+			// empty driver means nodegroup was implicitly created as a default
+			// driver for a docker context and allows falling back to a
+			// docker-container driver for older daemon that doesn't support
+			// buildkit (< 18.06).
+			ep := b.nodes[0].Endpoint
+			var dockerapi *dockerutil.ClientAPI
+			dockerapi, err = dockerutil.NewClientAPI(b.opts.dockerCli, b.nodes[0].Endpoint)
+			if err != nil {
+				return
+			}
+			// check if endpoint is healthy is needed to determine the driver type.
+			// if this fails then can't continue with driver selection.
+			if _, err = dockerapi.Ping(ctx); err != nil {
+				return
+			}
+			b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false)
+			if err != nil {
+				return
+			}
+			b.Driver = b.driverFactory.Factory.Name()
+		}
+	})
+	return b.driverFactory.Factory, err
+}
+
+// GetBuilders returns all builders
+func GetBuilders(dockerCli command.Cli, txn *store.Txn) ([]*Builder, error) {
+	storeng, err := txn.List()
+	if err != nil {
+		return nil, err
+	}
+
+	builders := make([]*Builder, len(storeng))
+	seen := make(map[string]struct{})
+	for i, ng := range storeng {
+		b, err := New(dockerCli,
+			WithName(ng.Name),
+			WithStore(txn),
+			WithSkippedValidation(),
+		)
+		if err != nil {
+			return nil, err
+		}
+		builders[i] = b
+		seen[b.NodeGroup.Name] = struct{}{}
+	}
+
+	contexts, err := dockerCli.ContextStore().List()
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(contexts, func(i, j int) bool {
+		return contexts[i].Name < contexts[j].Name
+	})
+
+	for _, c := range contexts {
+		// if a context has the same name as an instance from the store, do not
+		// add it to the builders list. An instance from the store takes
+		// precedence over context builders.
+		if _, ok := seen[c.Name]; ok {
+			continue
+		}
+		b, err := New(dockerCli,
+			WithName(c.Name),
+			WithStore(txn),
+			WithSkippedValidation(),
+		)
+		if err != nil {
+			return nil, err
+		}
+		builders = append(builders, b)
+	}
+
+	return builders, nil
+}

builder/node.go

@@ -0,0 +1,201 @@
+package builder
+
+import (
+	"context"
+
+	"github.com/docker/buildx/driver"
+	ctxkube "github.com/docker/buildx/driver/kubernetes/context"
+	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/store/storeutil"
+	"github.com/docker/buildx/util/dockerutil"
+	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/buildx/util/platformutil"
+	"github.com/moby/buildkit/util/grpcerrors"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc/codes"
+)
+
+type Node struct {
+	store.Node
+	Driver      driver.Driver
+	DriverInfo  *driver.Info
+	Platforms   []ocispecs.Platform
+	ImageOpt    imagetools.Opt
+	ProxyConfig map[string]string
+	Version     string
+	Err         error
+}
+
+// Nodes returns nodes for this builder.
+func (b *Builder) Nodes() []Node {
+	return b.nodes
+}
+
+// LoadNodes loads and returns nodes for this builder.
+// TODO: this should be a method on a Node object and lazy load data for each driver.
+func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err error) {
+	eg, _ := errgroup.WithContext(ctx)
+	b.nodes = make([]Node, len(b.NodeGroup.Nodes))
+
+	defer func() {
+		if b.err == nil && err != nil {
+			b.err = err
+		}
+	}()
+
+	factory, err := b.Factory(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	imageopt, err := b.ImageOpt()
+	if err != nil {
+		return nil, err
+	}
+
+	for i, n := range b.NodeGroup.Nodes {
+		func(i int, n store.Node) {
+			eg.Go(func() error {
+				node := Node{
+					Node:        n,
+					ProxyConfig: storeutil.GetProxyConfig(b.opts.dockerCli),
+				}
+				defer func() {
+					b.nodes[i] = node
+				}()
+
+				dockerapi, err := dockerutil.NewClientAPI(b.opts.dockerCli, n.Endpoint)
+				if err != nil {
+					node.Err = err
+					return nil
+				}
+
+				contextStore := b.opts.dockerCli.ContextStore()
+
+				var kcc driver.KubeClientConfig
+				kcc, err = ctxkube.ConfigFromContext(n.Endpoint, contextStore)
+				if err != nil {
+					// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
+					// try again with name="default".
+					// FIXME(@AkihiroSuda): n should retain real context name.
+					kcc, err = ctxkube.ConfigFromContext("default", contextStore)
+					if err != nil {
+						logrus.Error(err)
+					}
+				}
+
+				tryToUseKubeConfigInCluster := false
+				if kcc == nil {
+					tryToUseKubeConfigInCluster = true
+				} else {
+					if _, err := kcc.ClientConfig(); err != nil {
+						tryToUseKubeConfigInCluster = true
+					}
+				}
+				if tryToUseKubeConfigInCluster {
+					kccInCluster := driver.KubeClientConfigInCluster{}
+					if _, err := kccInCluster.ClientConfig(); err == nil {
+						logrus.Debug("using kube config in cluster")
+						kcc = kccInCluster
+					}
+				}
+
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash)
+				if err != nil {
+					node.Err = err
+					return nil
+				}
+				node.Driver = d
+				node.ImageOpt = imageopt
+
+				if withData {
+					if err := node.loadData(ctx); err != nil {
+						node.Err = err
+					}
+				}
+				return nil
+			})
+		}(i, n)
+	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	// TODO: This should be done in the routine loading driver data
+	if withData {
+		kubernetesDriverCount := 0
+		for _, d := range b.nodes {
+			if d.DriverInfo != nil && len(d.DriverInfo.DynamicNodes) > 0 {
+				kubernetesDriverCount++
+			}
+		}
+
+		isAllKubernetesDrivers := len(b.nodes) == kubernetesDriverCount
+		if isAllKubernetesDrivers {
+			var nodes []Node
+			var dynamicNodes []store.Node
+			for _, di := range b.nodes {
+				// dynamic nodes are used in Kubernetes driver.
+				// Kubernetes' pods are dynamically mapped to BuildKit Nodes.
+				if di.DriverInfo != nil && len(di.DriverInfo.DynamicNodes) > 0 {
+					for i := 0; i < len(di.DriverInfo.DynamicNodes); i++ {
+						diClone := di
+						if pl := di.DriverInfo.DynamicNodes[i].Platforms; len(pl) > 0 {
+							diClone.Platforms = pl
+						}
+						nodes = append(nodes, di)
+					}
+					dynamicNodes = append(dynamicNodes, di.DriverInfo.DynamicNodes...)
+				}
+			}
+
+			// not append (remove the static nodes in the store)
+			b.NodeGroup.Nodes = dynamicNodes
+			b.nodes = nodes
+			b.NodeGroup.Dynamic = true
+		}
+	}
+
+	return b.nodes, nil
+}
+
+func (n *Node) loadData(ctx context.Context) error {
+	if n.Driver == nil {
+		return nil
+	}
+	info, err := n.Driver.Info(ctx)
+	if err != nil {
+		return err
+	}
+	n.DriverInfo = info
+	if n.DriverInfo.Status == driver.Running {
+		driverClient, err := n.Driver.Client(ctx)
+		if err != nil {
+			return err
+		}
+		workers, err := driverClient.ListWorkers(ctx)
+		if err != nil {
+			return errors.Wrap(err, "listing workers")
+		}
+		for _, w := range workers {
+			n.Platforms = append(n.Platforms, w.Platforms...)
+		}
+		n.Platforms = platformutil.Dedupe(n.Platforms)
+		inf, err := driverClient.Info(ctx)
+		if err != nil {
+			if st, ok := grpcerrors.AsGRPCStatus(err); ok && st.Code() == codes.Unimplemented {
+				n.Version, err = n.Driver.Version(ctx)
+				if err != nil {
+					return errors.Wrap(err, "getting version")
+				}
+			}
+		} else {
+			n.Version = inf.BuildkitVersion.Version
+		}
+	}
+	return nil
+}

cmd/buildx/main.go

@@ -15,13 +15,8 @@ import (
 	cliflags "github.com/docker/cli/cli/flags"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/util/stack"
-	"github.com/moby/buildkit/util/tracing/detect"
-	"go.opentelemetry.io/otel"
 
-	_ "github.com/moby/buildkit/util/tracing/detect/delegated"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
-	_ "github.com/moby/buildkit/util/tracing/env"
-
-	// FIXME: "k8s.io/client-go/plugin/pkg/client/auth/azure" is excluded because of compilation error
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
@@ -29,55 +24,50 @@ import (
 	_ "github.com/docker/buildx/driver/docker"
 	_ "github.com/docker/buildx/driver/docker-container"
 	_ "github.com/docker/buildx/driver/kubernetes"
+	_ "github.com/docker/buildx/driver/remote"
 )
 
-var experimental string
-
 func init() {
 	seed.WithTimeAndRand()
 	stack.SetVersionInfo(version.Version, version.Revision)
-
-	detect.ServiceName = "buildx"
-	// do not log tracing errors to stdio
-	otel.SetErrorHandler(skipErrors{})
 }
 
-func main() {
+func runStandalone(cmd *command.DockerCli) error {
-	if os.Getenv("DOCKER_CLI_PLUGIN_ORIGINAL_CLI_COMMAND") == "" {
+	if err := cmd.Initialize(cliflags.NewClientOptions()); err != nil {
-		if len(os.Args) < 2 || os.Args[1] != manager.MetadataSubcommandName {
+		return err
-			dockerCli, err := command.NewDockerCli()
-			if err != nil {
-				fmt.Fprintln(os.Stderr, err)
-				os.Exit(1)
-			}
-			opts := cliflags.NewClientOptions()
-			dockerCli.Initialize(opts)
-			rootCmd := commands.NewRootCmd(os.Args[0], false, dockerCli)
-			if err := rootCmd.Execute(); err != nil {
-				os.Exit(1)
-			}
-			os.Exit(0)
 	}
+	rootCmd := commands.NewRootCmd(os.Args[0], false, cmd)
+	return rootCmd.Execute()
 }
 
-	dockerCli, err := command.NewDockerCli()
+func runPlugin(cmd *command.DockerCli) error {
-	if err != nil {
+	rootCmd := commands.NewRootCmd("buildx", true, cmd)
-		fmt.Fprintln(os.Stderr, err)
+	return plugin.RunPlugin(cmd, rootCmd, manager.Metadata{
-		os.Exit(1)
-	}
-
-	p := commands.NewRootCmd("buildx", true, dockerCli)
-	meta := manager.Metadata{
 		SchemaVersion: "0.1.0",
 		Vendor:        "Docker Inc.",
 		Version:       version.Version,
-		Experimental:  experimental != "",
+	})
+}
+
+func main() {
+	cmd, err := command.NewDockerCli()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	if plugin.RunningStandalone() {
+		err = runStandalone(cmd)
+	} else {
+		err = runPlugin(cmd)
+	}
+	if err == nil {
+		return
 	}
 
-	if err := plugin.RunPlugin(dockerCli, p, meta); err != nil {
 	if sterr, ok := err.(cli.StatusError); ok {
 		if sterr.Status != "" {
-				fmt.Fprintln(dockerCli.Err(), sterr.Status)
+			fmt.Fprintln(cmd.Err(), sterr.Status)
 		}
 		// StatusError should only be used for errors, and all errors should
 		// have a non-zero exit status, so never exit with 0
@@ -86,20 +76,15 @@ func main() {
 		}
 		os.Exit(sterr.StatusCode)
 	}
-		for _, s := range errdefs.Sources(err) {
-			s.Print(dockerCli.Err())
-		}
 
+	for _, s := range errdefs.Sources(err) {
+		s.Print(cmd.Err())
+	}
 	if debug.IsEnabled() {
-			fmt.Fprintf(dockerCli.Err(), "error: %+v", stack.Formatter(err))
+		fmt.Fprintf(cmd.Err(), "ERROR: %+v", stack.Formatter(err))
 	} else {
-			fmt.Fprintf(dockerCli.Err(), "error: %v\n", err)
+		fmt.Fprintf(cmd.Err(), "ERROR: %v\n", err)
 	}
 
 	os.Exit(1)
 }
-}
-
-type skipErrors struct{}
-
-func (skipErrors) Handle(err error) {}

cmd/buildx/tracing.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"github.com/moby/buildkit/util/tracing/detect"
+	"go.opentelemetry.io/otel"
+
+	_ "github.com/moby/buildkit/util/tracing/detect/delegated"
+	_ "github.com/moby/buildkit/util/tracing/env"
+)
+
+func init() {
+	detect.ServiceName = "buildx"
+	// do not log tracing errors to stdio
+	otel.SetErrorHandler(skipErrors{})
+}
+
+type skipErrors struct{}
+
+func (skipErrors) Handle(err error) {}

commands/bake.go

@@ -9,7 +9,10 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/buildx/bake"
 	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/builder"
+	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/confutil"
+	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
@@ -71,11 +74,20 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 	if in.pull != nil {
 		overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
 	}
+	if in.sbom != "" {
+		overrides = append(overrides, fmt.Sprintf("*.attest=%s", buildflags.CanonicalizeAttest("sbom", in.sbom)))
+	}
+	if in.provenance != "" {
+		overrides = append(overrides, fmt.Sprintf("*.attest=%s", buildflags.CanonicalizeAttest("provenance", in.provenance)))
+	}
 	contextPathHash, _ := os.Getwd()
 
 	ctx2, cancel := context.WithCancel(context.TODO())
 	defer cancel()
-	printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
+	printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
+	if err != nil {
+		return err
+	}
 
 	defer func() {
 		if printer != nil {
@@ -86,16 +98,30 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 		}
 	}()
 
-	dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
+	var nodes []builder.Node
-	if err != nil {
-		return err
-	}
-
 	var files []bake.File
 	var inp *bake.Input
 
+	// instance only needed for reading remote bake files or building
+	if url != "" || !in.printOnly {
+		b, err := builder.New(dockerCli,
+			builder.WithName(in.builder),
+			builder.WithContextPathHash(contextPathHash),
+		)
+		if err != nil {
+			return err
+		}
+		if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
+			return errors.Wrapf(err, "failed to update builder last activity time")
+		}
+		nodes, err = b.LoadNodes(ctx, false)
+		if err != nil {
+			return err
+		}
+	}
+
 	if url != "" {
-		files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
+		files, inp, err = bake.ReadRemoteFiles(ctx, nodes, url, in.files, printer)
 	} else {
 		files, err = bake.ReadLocalFiles(in.files)
 	}
@@ -104,8 +130,8 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 	}
 
 	tgts, grps, err := bake.ReadTargets(ctx, files, targets, overrides, map[string]string{
-		// Don't forget to update documentation if you add a new
+		// don't forget to update documentation if you add a new
-		// built-in variable: docs/reference/buildx_bake.md#built-in-variables
+		// built-in variable: docs/manuals/bake/file-definition.md#built-in-variables
 		"BAKE_CMD_CONTEXT":    cmdContext,
 		"BAKE_LOCAL_PLATFORM": platforms.DefaultString(),
 	})
@@ -120,17 +146,11 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 	}
 
 	if in.printOnly {
-		var defg map[string]*bake.Group
-		if len(grps) == 1 {
-			defg = map[string]*bake.Group{
-				"default": grps[0],
-			}
-		}
 		dt, err := json.MarshalIndent(struct {
 			Group  map[string]*bake.Group  `json:"group,omitempty"`
 			Target map[string]*bake.Target `json:"target"`
 		}{
-			defg,
+			grps,
 			tgts,
 		}, "", "  ")
 		if err != nil {
@@ -145,19 +165,12 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 		return nil
 	}
 
-	resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer)
+	resp, err := build.Build(ctx, nodes, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer)
 	if err != nil {
 		return wrapBuildError(err, true)
 	}
 
-	if len(in.metadataFile) > 0 && resp != nil {
+	if len(in.metadataFile) > 0 {
-		if len(resp) == 1 {
-			for _, r := range resp {
-				if err := writeMetadataFile(in.metadataFile, decodeExporterResponse(r.ExporterResponse)); err != nil {
-					return err
-				}
-			}
-		} else {
 		dt := make(map[string]interface{})
 		for t, r := range resp {
 			dt[t] = decodeExporterResponse(r.ExporterResponse)
@@ -166,7 +179,6 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 			return err
 		}
 	}
-	}
 
 	return err
 }
@@ -197,6 +209,8 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`)
 	flags.BoolVar(&options.printOnly, "print", false, "Print the options without building")
 	flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`)
+	flags.StringVar(&options.sbom, "sbom", "", `Shorthand for "--set=*.attest=type=sbom"`)
+	flags.StringVar(&options.provenance, "provenance", "", `Shorthand for "--set=*.attest=type=provenance"`)
 	flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`)
 
 	commonBuildFlags(&options.commonOptions, flags)

commands/build.go

@@ -4,22 +4,32 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/containerd/console"
 	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/builder"
+	"github.com/docker/buildx/monitor"
+	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/confutil"
+	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli-docs-tool/annotation"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/config"
 	dockeropts "github.com/docker/cli/opts"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/pkg/ioutils"
@@ -42,8 +52,10 @@ const defaultTargetName = "default"
 type buildOptions struct {
 	contextPath    string
 	dockerfileName string
+	printFunc      string
 
 	allow         []string
+	attests       []string
 	buildArgs     []string
 	cacheFrom     []string
 	cacheTo       []string
@@ -51,6 +63,7 @@ type buildOptions struct {
 	contexts      []string
 	extraHosts    []string
 	imageIDFile   string
+	invoke        string
 	labels        []string
 	networkMode   string
 	noCacheFilter []string
@@ -73,11 +86,11 @@ type commonOptions struct {
 	progress     string
 	pull         *bool
 
-	// golangci-lint#826
-	// nolint:structcheck
 	exportPush bool
-	// nolint:structcheck
 	exportLoad bool
+
+	sbom       string
+	provenance string
 }
 
 func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
@@ -104,7 +117,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		return errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together")
 	}
 
-	if in.quiet && in.progress != "auto" && in.progress != "quiet" {
+	if in.quiet && in.progress != progress.PrinterModeAuto && in.progress != progress.PrinterModeQuiet {
 		return errors.Errorf("progress=%s and quiet cannot be used together", in.progress)
 	} else if in.quiet {
 		in.progress = "quiet"
@@ -115,6 +128,11 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		return err
 	}
 
+	printFunc, err := parsePrintFunc(in.printFunc)
+	if err != nil {
+		return err
+	}
+
 	opts := build.Options{
 		Inputs: build.Inputs{
 			ContextPath:    in.contextPath,
@@ -134,6 +152,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		Tags:          in.tags,
 		Target:        in.target,
 		Ulimits:       in.ulimits,
+		PrintFunc:     printFunc,
 	}
 
 	platforms, err := platformutil.Parse(in.platforms)
@@ -142,7 +161,8 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 	}
 	opts.Platforms = platforms
 
-	opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(os.Stderr))
+	dockerConfig := config.LoadDefaultConfigFile(os.Stderr)
+	opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(dockerConfig))
 
 	secrets, err := buildflags.ParseSecretSpecs(in.secrets)
 	if err != nil {
@@ -198,9 +218,20 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 			}
 		}
 	}
-
 	opts.Exports = outputs
 
+	inAttests := append([]string{}, in.attests...)
+	if in.provenance != "" {
+		inAttests = append(inAttests, buildflags.CanonicalizeAttest("provenance", in.provenance))
+	}
+	if in.sbom != "" {
+		inAttests = append(inAttests, buildflags.CanonicalizeAttest("sbom", in.sbom))
+	}
+	opts.Attests, err = buildflags.ParseAttests(inAttests)
+	if err != nil {
+		return err
+	}
+
 	cacheImports, err := buildflags.ParseCacheEntry(in.cacheFrom)
 	if err != nil {
 		return err
@@ -225,47 +256,147 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		contextPathHash = in.contextPath
 	}
 
-	imageID, err := buildTargets(ctx, dockerCli, map[string]build.Options{defaultTargetName: opts}, in.progress, contextPathHash, in.builder, in.metadataFile)
+	b, err := builder.New(dockerCli,
+		builder.WithName(in.builder),
+		builder.WithContextPathHash(contextPathHash),
+	)
+	if err != nil {
+		return err
+	}
+	if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
+		return errors.Wrapf(err, "failed to update builder last activity time")
+	}
+	nodes, err := b.LoadNodes(ctx, false)
+	if err != nil {
+		return err
+	}
+
+	imageID, res, err := buildTargets(ctx, dockerCli, nodes, map[string]build.Options{defaultTargetName: opts}, in.progress, in.metadataFile, in.invoke != "")
 	err = wrapBuildError(err, false)
 	if err != nil {
 		return err
 	}
 
+	if in.invoke != "" {
+		cfg, err := parseInvokeConfig(in.invoke)
+		if err != nil {
+			return err
+		}
+		cfg.ResultCtx = res
+		con := console.Current()
+		if err := con.SetRaw(); err != nil {
+			return errors.Errorf("failed to configure terminal: %v", err)
+		}
+		err = monitor.RunMonitor(ctx, cfg, func(ctx context.Context) (*build.ResultContext, error) {
+			_, rr, err := buildTargets(ctx, dockerCli, nodes, map[string]build.Options{defaultTargetName: opts}, in.progress, in.metadataFile, true)
+			return rr, err
+		}, io.NopCloser(os.Stdin), nopCloser{os.Stdout}, nopCloser{os.Stderr})
+		if err != nil {
+			logrus.Warnf("failed to run monitor: %v", err)
+		}
+		con.Reset()
+	}
+
 	if in.quiet {
 		fmt.Println(imageID)
 	}
 	return nil
 }
 
-func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode, contextPathHash, instance string, metadataFile string) (imageID string, err error) {
+type nopCloser struct {
-	dis, err := getInstanceOrDefault(ctx, dockerCli, instance, contextPathHash)
+	io.WriteCloser
-	if err != nil {
-		return "", err
 }
 
+func (c nopCloser) Close() error { return nil }
+
+func buildTargets(ctx context.Context, dockerCli command.Cli, nodes []builder.Node, opts map[string]build.Options, progressMode string, metadataFile string, allowNoOutput bool) (imageID string, res *build.ResultContext, err error) {
 	ctx2, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
-	printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, progressMode)
+	printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, progressMode)
+	if err != nil {
+		return "", nil, err
+	}
 
-	resp, err := build.Build(ctx, dis, opts, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer)
+	var mu sync.Mutex
+	var idx int
+	resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer, func(driverIndex int, gotRes *build.ResultContext) {
+		mu.Lock()
+		defer mu.Unlock()
+		if res == nil || driverIndex < idx {
+			idx, res = driverIndex, gotRes
+		}
+	}, allowNoOutput)
 	err1 := printer.Wait()
 	if err == nil {
 		err = err1
 	}
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 
 	if len(metadataFile) > 0 && resp != nil {
 		if err := writeMetadataFile(metadataFile, decodeExporterResponse(resp[defaultTargetName].ExporterResponse)); err != nil {
-			return "", err
+			return "", nil, err
 		}
 	}
 
 	printWarnings(os.Stderr, printer.Warnings(), progressMode)
 
-	return resp[defaultTargetName].ExporterResponse["containerimage.digest"], err
+	for k := range resp {
+		if opts[k].PrintFunc != nil {
+			if err := printResult(opts[k].PrintFunc, resp[k].ExporterResponse); err != nil {
+				return "", nil, err
+			}
+		}
+	}
+
+	return resp[defaultTargetName].ExporterResponse["containerimage.digest"], res, err
+}
+
+func parseInvokeConfig(invoke string) (cfg build.ContainerConfig, err error) {
+	cfg.Tty = true
+	if invoke == "default" {
+		return cfg, nil
+	}
+
+	csvReader := csv.NewReader(strings.NewReader(invoke))
+	fields, err := csvReader.Read()
+	if err != nil {
+		return cfg, err
+	}
+	if len(fields) == 1 && !strings.Contains(fields[0], "=") {
+		cfg.Cmd = []string{fields[0]}
+		return cfg, nil
+	}
+	for _, field := range fields {
+		parts := strings.SplitN(field, "=", 2)
+		if len(parts) != 2 {
+			return cfg, errors.Errorf("invalid value %s", field)
+		}
+		key := strings.ToLower(parts[0])
+		value := parts[1]
+		switch key {
+		case "args":
+			cfg.Cmd = append(cfg.Cmd, value) // TODO: support JSON
+		case "entrypoint":
+			cfg.Entrypoint = append(cfg.Entrypoint, value) // TODO: support JSON
+		case "env":
+			cfg.Env = append(cfg.Env, value)
+		case "user":
+			cfg.User = &value
+		case "cwd":
+			cfg.Cwd = &value
+		case "tty":
+			cfg.Tty, err = strconv.ParseBool(value)
+			if err != nil {
+				return cfg, errors.Errorf("failed to parse tty: %v", err)
+			}
+		default:
+			return cfg, errors.Errorf("unknown key %q", key)
+		}
+	}
+	return cfg, nil
 }
 
 func printWarnings(w io.Writer, warnings []client.VertexWarning, mode string) {
@@ -371,6 +502,10 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build")
 
+	if isExperimental() {
+		flags.StringVar(&options.printFunc, "print", "", "Print result of information request (e.g., outline, targets) [experimental]")
+	}
+
 	flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--output=type=registry"`)
 
 	flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success")
@@ -389,6 +524,14 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	flags.Var(options.ulimits, "ulimit", "Ulimit options")
 
+	flags.StringArrayVar(&options.attests, "attest", []string{}, `Attestation parameters (format: "type=sbom,generator=image")`)
+	flags.StringVar(&options.sbom, "sbom", "", `Shorthand for "--attest=type=sbom"`)
+	flags.StringVar(&options.provenance, "provenance", "", `Shortand for "--attest=type=provenance"`)
+
+	if isExperimental() {
+		flags.StringVar(&options.invoke, "invoke", "", "Invoke a command after the build [experimental]")
+	}
+
 	// hidden flags
 	var ignore string
 	var ignoreSlice []string
@@ -444,7 +587,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) {
 	options.noCache = flags.Bool("no-cache", false, "Do not use cache when building the image")
 	flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)
-	options.pull = flags.Bool("pull", false, "Always attempt to pull a newer version of the image")
+	options.pull = flags.Bool("pull", false, "Always attempt to pull all referenced images")
 	flags.StringVar(&options.metadataFile, "metadata-file", "", "Write build result metadata to the file")
 }
 
@@ -500,6 +643,34 @@ func parseContextNames(values []string) (map[string]build.NamedContext, error) {
 	return result, nil
 }
 
+func parsePrintFunc(str string) (*build.PrintFunc, error) {
+	if str == "" {
+		return nil, nil
+	}
+	csvReader := csv.NewReader(strings.NewReader(str))
+	fields, err := csvReader.Read()
+	if err != nil {
+		return nil, err
+	}
+	f := &build.PrintFunc{}
+	for _, field := range fields {
+		parts := strings.SplitN(field, "=", 2)
+		if len(parts) == 2 {
+			if parts[0] == "format" {
+				f.Format = parts[1]
+			} else {
+				return nil, errors.Errorf("invalid print field: %s", field)
+			}
+		} else {
+			if f.Name != "" {
+				return nil, errors.Errorf("invalid print value: %s", str)
+			}
+			f.Name = field
+		}
+	}
+	return f, nil
+}
+
 func writeMetadataFile(filename string, dt interface{}) error {
 	b, err := json.MarshalIndent(dt, "", "  ")
 	if err != nil {
@@ -556,3 +727,20 @@ func (w *wrapped) Error() string {
 func (w *wrapped) Unwrap() error {
 	return w.err
 }
+
+func isExperimental() bool {
+	if v, ok := os.LookupEnv("BUILDX_EXPERIMENTAL"); ok {
+		vv, _ := strconv.ParseBool(v)
+		return vv
+	}
+	return false
+}
+
+func updateLastActivity(dockerCli command.Cli, ng *store.NodeGroup) error {
+	txn, release, err := storeutil.GetStore(dockerCli)
+	if err != nil {
+		return err
+	}
+	defer release()
+	return txn.UpdateLastActivity(ng)
+}

commands/create.go

@@ -10,12 +10,17 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/driver"
+	remoteutil "github.com/docker/buildx/driver/remote/util"
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/cobrautil"
+	"github.com/docker/buildx/util/confutil"
+	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
+	dopts "github.com/docker/cli/opts"
 	"github.com/google/shlex"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
@@ -60,22 +65,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		}
 	}
 
-	driverName := in.driver
-	if driverName == "" {
-		f, err := driver.GetDefaultFactory(ctx, dockerCli.Client(), true)
-		if err != nil {
-			return err
-		}
-		if f == nil {
-			return errors.Errorf("no valid drivers found")
-		}
-		driverName = f.Name()
-	}
-
-	if driver.GetFactory(driverName, true) == nil {
-		return errors.Errorf("failed to find driver %q", in.driver)
-	}
-
 	txn, release, err := storeutil.GetStore(dockerCli)
 	if err != nil {
 		return err
@@ -90,6 +79,19 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		}
 	}
 
+	if !in.actionLeave && !in.actionAppend {
+		contexts, err := dockerCli.ContextStore().List()
+		if err != nil {
+			return err
+		}
+		for _, c := range contexts {
+			if c.Name == name {
+				logrus.Warnf("instance name %q already exists as context builder", name)
+				break
+			}
+		}
+	}
+
 	ng, err := txn.NodeGroupByName(name)
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
@@ -97,29 +99,62 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 				logrus.Warnf("failed to find %q for append, creating a new instance instead", in.name)
 			}
 			if in.actionLeave {
-				return errors.Errorf("failed to find instance %q for leave", name)
+				return errors.Errorf("failed to find instance %q for leave", in.name)
 			}
 		} else {
 			return err
 		}
 	}
 
+	buildkitHost := os.Getenv("BUILDKIT_HOST")
+
+	driverName := in.driver
+	if driverName == "" {
+		if ng != nil {
+			driverName = ng.Driver
+		} else if len(args) == 0 && buildkitHost != "" {
+			driverName = "remote"
+		} else {
+			var arg string
+			if len(args) > 0 {
+				arg = args[0]
+			}
+			f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true)
+			if err != nil {
+				return err
+			}
+			if f == nil {
+				return errors.Errorf("no valid drivers found")
+			}
+			driverName = f.Name()
+		}
+	}
+
 	if ng != nil {
 		if in.nodeName == "" && !in.actionAppend {
-			return errors.Errorf("existing instance for %s but no append mode, specify --node to make changes for existing instances", name)
+			return errors.Errorf("existing instance for %q but no append mode, specify --node to make changes for existing instances", name)
 		}
+		if driverName != ng.Driver {
+			return errors.Errorf("existing instance for %q but has mismatched driver %q", name, ng.Driver)
+		}
+	}
+
+	if _, err := driver.GetFactory(driverName, true); err != nil {
+		return err
+	}
+
+	ngOriginal := ng
+	if ngOriginal != nil {
+		ngOriginal = ngOriginal.Copy()
 	}
 
 	if ng == nil {
 		ng = &store.NodeGroup{
 			Name:   name,
+			Driver: driverName,
 		}
 	}
 
-	if ng.Driver == "" || in.driver != "" {
-		ng.Driver = driverName
-	}
-
 	var flags []string
 	if in.flags != "" {
 		flags, err = shlex.Split(in.flags)
@@ -129,44 +164,72 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	}
 
 	var ep string
+	var setEp bool
 	if in.actionLeave {
 		if err := ng.Leave(in.nodeName); err != nil {
 			return err
 		}
 	} else {
+		switch {
+		case driverName == "kubernetes":
 			if len(args) > 0 {
-			ep, err = validateEndpoint(dockerCli, args[0])
+				logrus.Warnf("kubernetes driver does not support endpoint args %q", args[0])
-			if err != nil {
-				return err
 			}
-		} else {
-			if dockerCli.CurrentContext() == "default" && dockerCli.DockerEndpoint().TLSData != nil {
-				return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`")
-			}
-
-			ep, err = storeutil.GetCurrentEndpoint(dockerCli)
-			if err != nil {
-				return err
-			}
-		}
-
-		if in.driver == "kubernetes" {
 			// naming endpoint to make --append works
 			ep = (&url.URL{
-				Scheme: in.driver,
+				Scheme: driverName,
 				Path:   "/" + in.name,
 				RawQuery: (&url.Values{
 					"deployment": {in.nodeName},
 					"kubeconfig": {os.Getenv("KUBECONFIG")},
 				}).Encode(),
 			}).String()
+			setEp = false
+		case driverName == "remote":
+			if len(args) > 0 {
+				ep = args[0]
+			} else if buildkitHost != "" {
+				ep = buildkitHost
+			} else {
+				return errors.Errorf("no remote endpoint provided")
+			}
+			ep, err = validateBuildkitEndpoint(ep)
+			if err != nil {
+				return err
+			}
+			setEp = true
+		case len(args) > 0:
+			ep, err = validateEndpoint(dockerCli, args[0])
+			if err != nil {
+				return err
+			}
+			setEp = true
+		default:
+			if dockerCli.CurrentContext() == "default" && dockerCli.DockerEndpoint().TLSData != nil {
+				return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`")
+			}
+			ep, err = dockerutil.GetCurrentEndpoint(dockerCli)
+			if err != nil {
+				return err
+			}
+			setEp = false
 		}
 
 		m, err := csvToMap(in.driverOpts)
 		if err != nil {
 			return err
 		}
-		if err := ng.Update(in.nodeName, ep, in.platform, len(args) > 0, in.actionAppend, flags, in.configFile, m); err != nil {
+
+		if in.configFile == "" {
+			// if buildkit config is not provided, check if the default one is
+			// available and use it
+			if f, ok := confutil.DefaultConfigFile(dockerCli); ok {
+				logrus.Warnf("Using default BuildKit config in %s", f)
+				in.configFile = f
+			}
+		}
+
+		if err := ng.Update(in.nodeName, ep, in.platform, setEp, in.actionAppend, flags, in.configFile, m); err != nil {
 			return err
 		}
 	}
@@ -175,8 +238,41 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		return err
 	}
 
+	b, err := builder.New(dockerCli,
+		builder.WithName(ng.Name),
+		builder.WithStore(txn),
+		builder.WithSkippedValidation(),
+	)
+	if err != nil {
+		return err
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+
+	nodes, err := b.LoadNodes(timeoutCtx, true)
+	if err != nil {
+		return err
+	}
+
+	for _, node := range nodes {
+		if err := node.Err; err != nil {
+			err := errors.Errorf("failed to initialize builder %s (%s): %s", ng.Name, node.Name, err)
+			var err2 error
+			if ngOriginal == nil {
+				err2 = txn.Remove(ng.Name)
+			} else {
+				err2 = txn.Save(ngOriginal)
+			}
+			if err2 != nil {
+				logrus.Warnf("Could not rollback to previous state: %s", err2)
+			}
+			return err
+		}
+	}
+
 	if in.use && ep != "" {
-		current, err := storeutil.GetCurrentEndpoint(dockerCli)
+		current, err := dockerutil.GetCurrentEndpoint(dockerCli)
 		if err != nil {
 			return err
 		}
@@ -185,17 +281,8 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		}
 	}
 
-	ngi := &nginfo{ng: ng}
-
-	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
-	defer cancel()
-
-	if err = loadNodeGroupData(timeoutCtx, dockerCli, ngi); err != nil {
-		return err
-	}
-
 	if in.bootstrap {
-		if _, err = boot(ctx, ngi); err != nil {
+		if _, err = b.Boot(ctx); err != nil {
 			return err
 		}
 	}
@@ -208,7 +295,7 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
 	var options createOptions
 
 	var drivers bytes.Buffer
-	for _, d := range driver.GetFactories() {
+	for _, d := range driver.GetFactories(true) {
 		if len(drivers.String()) > 0 {
 			drivers.WriteString(", ")
 		}
@@ -246,6 +333,9 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
 }
 
 func csvToMap(in []string) (map[string]string, error) {
+	if len(in) == 0 {
+		return nil, nil
+	}
 	m := make(map[string]string, len(in))
 	for _, s := range in {
 		csvReader := csv.NewReader(strings.NewReader(s))
@@ -263,3 +353,27 @@ func csvToMap(in []string) (map[string]string, error) {
 	}
 	return m, nil
 }
+
+// validateEndpoint validates that endpoint is either a context or a docker host
+func validateEndpoint(dockerCli command.Cli, ep string) (string, error) {
+	dem, err := dockerutil.GetDockerEndpoint(dockerCli, ep)
+	if err == nil && dem != nil {
+		if ep == "default" {
+			return dem.Host, nil
+		}
+		return ep, nil
+	}
+	h, err := dopts.ParseHost(true, ep)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse endpoint %s", ep)
+	}
+	return h, nil
+}
+
+// validateBuildkitEndpoint validates that endpoint is a valid buildkit host
+func validateBuildkitEndpoint(ep string) (string, error) {
+	if err := remoteutil.IsValidEndpoint(ep); err != nil {
+		return "", err
+	}
+	return ep, nil
+}

commands/create_test.go

@@ -0,0 +1,26 @@
+package commands
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCsvToMap(t *testing.T) {
+	d := []string{
+		"\"tolerations=key=foo,value=bar;key=foo2,value=bar2\",replicas=1",
+		"namespace=default",
+	}
+	r, err := csvToMap(d)
+
+	require.NoError(t, err)
+
+	require.Contains(t, r, "tolerations")
+	require.Equal(t, r["tolerations"], "key=foo,value=bar;key=foo2,value=bar2")
+
+	require.Contains(t, r, "replicas")
+	require.Equal(t, r["replicas"], "1")
+
+	require.Contains(t, r, "namespace")
+	require.Equal(t, r["namespace"], "default")
+}

commands/diskusage.go

@@ -8,7 +8,7 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/builder"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/opts"
@@ -33,25 +33,29 @@ func runDiskUsage(dockerCli command.Cli, opts duOptions) error {
 		return err
 	}
 
-	dis, err := getInstanceOrDefault(ctx, dockerCli, opts.builder, "")
+	b, err := builder.New(dockerCli, builder.WithName(opts.builder))
 	if err != nil {
 		return err
 	}
 
-	for _, di := range dis {
+	nodes, err := b.LoadNodes(ctx, false)
-		if di.Err != nil {
+	if err != nil {
 		return err
 	}
+	for _, node := range nodes {
+		if node.Err != nil {
+			return node.Err
+		}
 	}
 
-	out := make([][]*client.UsageInfo, len(dis))
+	out := make([][]*client.UsageInfo, len(nodes))
 
 	eg, ctx := errgroup.WithContext(ctx)
-	for i, di := range dis {
+	for i, node := range nodes {
-		func(i int, di build.DriverInfo) {
+		func(i int, node builder.Node) {
 			eg.Go(func() error {
-				if di.Driver != nil {
+				if node.Driver != nil {
-					c, err := di.Driver.Client(ctx)
+					c, err := node.Driver.Client(ctx)
 					if err != nil {
 						return err
 					}
@@ -64,7 +68,7 @@ func runDiskUsage(dockerCli command.Cli, opts duOptions) error {
 				}
 				return nil
 			})
-		}(i, di)
+		}(i, node)
 	}
 
 	if err := eg.Wait(); err != nil {

commands/imagetools/create.go

@@ -1,14 +1,15 @@
 package commands
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 
-	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/builder"
-	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/distribution/reference"
 	"github.com/moby/buildkit/util/appcontext"
@@ -25,6 +26,7 @@ type createOptions struct {
 	tags         []string
 	dryrun       bool
 	actionAppend bool
+	progress     string
 }
 
 func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
@@ -38,7 +40,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 
 	fileArgs := make([]string, len(in.files))
 	for i, f := range in.files {
-		dt, err := ioutil.ReadFile(f)
+		dt, err := os.ReadFile(f)
 		if err != nil {
 			return err
 		}
@@ -78,53 +80,43 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	if len(repos) == 0 {
 		return errors.Errorf("no repositories specified, please set a reference in tag or source")
 	}
-	if len(repos) > 1 {
-		return errors.Errorf("multiple repositories currently not supported, found %v", repos)
-	}
 
-	var repo string
+	var defaultRepo *string
-	for r := range repos {
+	if len(repos) == 1 {
-		repo = r
+		for repo := range repos {
+			defaultRepo = &repo
+		}
 	}
 
 	for i, s := range srcs {
-		if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
+		if s.Ref == nil {
-			n, err := reference.ParseNormalizedNamed(repo)
+			if defaultRepo == nil {
+				return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i])
+			}
+			n, err := reference.ParseNormalizedNamed(*defaultRepo)
 			if err != nil {
 				return err
 			}
+			if s.Desc.MediaType == "" && s.Desc.Digest != "" {
 				r, err := reference.WithDigest(n, s.Desc.Digest)
 				if err != nil {
 					return err
 				}
 				srcs[i].Ref = r
 				sourceRefs = true
+			} else {
+				srcs[i].Ref = reference.TagNameOnly(n)
+			}
 		}
 	}
 
 	ctx := appcontext.Context()
 
-	txn, release, err := storeutil.GetStore(dockerCli)
+	b, err := builder.New(dockerCli, builder.WithName(in.builder))
 	if err != nil {
 		return err
 	}
-	defer release()
+	imageopt, err := b.ImageOpt()
-
-	var ng *store.NodeGroup
-
-	if in.builder != "" {
-		ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
-		if err != nil {
-			return err
-		}
-	} else {
-		ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
-		if err != nil {
-			return err
-		}
-	}
-
-	imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
 	if err != nil {
 		return err
 	}
@@ -143,7 +135,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 					if err != nil {
 						return err
 					}
-					srcs[i].Ref = nil
 					if srcs[i].Desc.Digest == "" {
 						srcs[i].Desc = desc
 					} else {
@@ -162,12 +153,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		}
 	}
 
-	descs := make([]ocispec.Descriptor, len(srcs))
+	dt, desc, err := r.Combine(ctx, srcs)
-	for i := range descs {
-		descs[i] = srcs[i].Desc
-	}
-
-	dt, desc, err := r.Combine(ctx, repo, descs)
 	if err != nil {
 		return err
 	}
@@ -180,23 +166,52 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	// new resolver cause need new auth
 	r = imagetools.New(imageopt)
 
-	for _, t := range tags {
+	ctx2, cancel := context.WithCancel(context.TODO())
-		if err := r.Push(ctx, t, desc, dt); err != nil {
+	defer cancel()
+	printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
+	if err != nil {
 		return err
 	}
-		fmt.Println(t.String())
+
+	eg, _ := errgroup.WithContext(ctx)
+	pw := progress.WithPrefix(printer, "internal", true)
+
+	for _, t := range tags {
+		t := t
+		eg.Go(func() error {
+			return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
+				eg2, _ := errgroup.WithContext(ctx)
+				for _, s := range srcs {
+					if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
+						continue
+					}
+					s := s
+					eg2.Go(func() error {
+						sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String())))
+						return r.Copy(ctx, s, t)
+					})
 				}
 
-	return nil
+				if err := eg2.Wait(); err != nil {
+					return err
+				}
+				sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String())))
+				return r.Push(ctx, t, desc, dt)
+			})
+		})
 	}
 
-type src struct {
+	err = eg.Wait()
-	Desc ocispec.Descriptor
+	err1 := printer.Wait()
-	Ref  reference.Named
+	if err == nil {
+		err = err1
 	}
 
-func parseSources(in []string) ([]*src, error) {
+	return err
-	out := make([]*src, len(in))
+}
+
+func parseSources(in []string) ([]*imagetools.Source, error) {
+	out := make([]*imagetools.Source, len(in))
 	for i, in := range in {
 		s, err := parseSource(in)
 		if err != nil {
@@ -219,11 +234,11 @@ func parseRefs(in []string) ([]reference.Named, error) {
 	return refs, nil
 }
 
-func parseSource(in string) (*src, error) {
+func parseSource(in string) (*imagetools.Source, error) {
 	// source can be a digest, reference or a descriptor JSON
 	dgst, err := digest.Parse(in)
 	if err == nil {
-		return &src{
+		return &imagetools.Source{
 			Desc: ocispec.Descriptor{
 				Digest: dgst,
 			},
@@ -234,14 +249,14 @@ func parseSource(in string) (*src, error) {
 
 	ref, err := reference.ParseNormalizedNamed(in)
 	if err == nil {
-		return &src{
+		return &imagetools.Source{
 			Ref: ref,
 		}, nil
 	} else if !strings.HasPrefix(in, "{") {
 		return nil, err
 	}
 
-	var s src
+	var s imagetools.Source
 	if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -255,7 +270,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
 		Use:   "create [OPTIONS] [SOURCE] [SOURCE...]",
 		Short: "Create a new image based on source images",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			options.builder = opts.Builder
+			options.builder = *opts.Builder
 			return runCreate(dockerCli, options, args)
 		},
 	}
@@ -265,6 +280,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
 	flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
 	flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
 	flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
+	flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)
 
 	return cmd
 }

commands/imagetools/inspect.go

@@ -1,8 +1,7 @@
 package commands
 
 import (
-	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/builder"
-	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/cli-docs-tool/annotation"
 	"github.com/docker/cli/cli"
@@ -25,27 +24,11 @@ func runInspect(dockerCli command.Cli, in inspectOptions, name string) error {
 		return errors.Errorf("format and raw cannot be used together")
 	}
 
-	txn, release, err := storeutil.GetStore(dockerCli)
+	b, err := builder.New(dockerCli, builder.WithName(in.builder))
 	if err != nil {
 		return err
 	}
-	defer release()
+	imageopt, err := b.ImageOpt()
-
-	var ng *store.NodeGroup
-
-	if in.builder != "" {
-		ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
-		if err != nil {
-			return err
-		}
-	} else {
-		ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
-		if err != nil {
-			return err
-		}
-	}
-
-	imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
 	if err != nil {
 		return err
 	}
@@ -66,7 +49,7 @@ func inspectCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
 		Short: "Show details of an image in the registry",
 		Args:  cli.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			options.builder = rootOpts.Builder
+			options.builder = *rootOpts.Builder
 			return runInspect(dockerCli, options, args[0])
 		},
 	}

commands/imagetools/root.go

@@ -6,7 +6,7 @@ import (
 )
 
 type RootOptions struct {
-	Builder string
+	Builder *string
 }
 
 func RootCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {

commands/inspect.go

@@ -8,8 +8,7 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/builder"
-	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
@@ -25,87 +24,71 @@ type inspectOptions struct {
 func runInspect(dockerCli command.Cli, in inspectOptions) error {
 	ctx := appcontext.Context()
 
-	txn, release, err := storeutil.GetStore(dockerCli)
+	b, err := builder.New(dockerCli,
+		builder.WithName(in.builder),
+		builder.WithSkippedValidation(),
+	)
 	if err != nil {
 		return err
 	}
-	defer release()
-
-	var ng *store.NodeGroup
-
-	if in.builder != "" {
-		ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
-		if err != nil {
-			return err
-		}
-	} else {
-		ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
-		if err != nil {
-			return err
-		}
-	}
-
-	if ng == nil {
-		ng = &store.NodeGroup{
-			Name: "default",
-			Nodes: []store.Node{{
-				Name:     "default",
-				Endpoint: "default",
-			}},
-		}
-	}
-
-	ngi := &nginfo{ng: ng}
 
 	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	err = loadNodeGroupData(timeoutCtx, dockerCli, ngi)
+	nodes, err := b.LoadNodes(timeoutCtx, true)
-
-	var bootNgi *nginfo
 	if in.bootstrap {
 		var ok bool
-		ok, err = boot(ctx, ngi)
+		ok, err = b.Boot(ctx)
 		if err != nil {
 			return err
 		}
-		bootNgi = ngi
 		if ok {
-			ngi = &nginfo{ng: ng}
+			nodes, err = b.LoadNodes(timeoutCtx, true)
-			err = loadNodeGroupData(ctx, dockerCli, ngi)
 		}
 	}
 
 	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
-	fmt.Fprintf(w, "Name:\t%s\n", ngi.ng.Name)
+	fmt.Fprintf(w, "Name:\t%s\n", b.Name)
-	fmt.Fprintf(w, "Driver:\t%s\n", ngi.ng.Driver)
+	fmt.Fprintf(w, "Driver:\t%s\n", b.Driver)
+	if !b.NodeGroup.LastActivity.IsZero() {
+		fmt.Fprintf(w, "Last Activity:\t%v\n", b.NodeGroup.LastActivity)
+	}
+
 	if err != nil {
 		fmt.Fprintf(w, "Error:\t%s\n", err.Error())
-	} else if ngi.err != nil {
+	} else if b.Err() != nil {
-		fmt.Fprintf(w, "Error:\t%s\n", ngi.err.Error())
+		fmt.Fprintf(w, "Error:\t%s\n", b.Err().Error())
 	}
 	if err == nil {
 		fmt.Fprintln(w, "")
 		fmt.Fprintln(w, "Nodes:")
 
-		for i, n := range ngi.ng.Nodes {
+		for i, n := range nodes {
 			if i != 0 {
 				fmt.Fprintln(w, "")
 			}
 			fmt.Fprintf(w, "Name:\t%s\n", n.Name)
 			fmt.Fprintf(w, "Endpoint:\t%s\n", n.Endpoint)
-			if err := ngi.drivers[i].di.Err; err != nil {
+
+			var driverOpts []string
+			for k, v := range n.DriverOpts {
+				driverOpts = append(driverOpts, fmt.Sprintf("%s=%q", k, v))
+			}
+			if len(driverOpts) > 0 {
+				fmt.Fprintf(w, "Driver Options:\t%s\n", strings.Join(driverOpts, " "))
+			}
+
+			if err := n.Err; err != nil {
 				fmt.Fprintf(w, "Error:\t%s\n", err.Error())
-			} else if err := ngi.drivers[i].err; err != nil {
-				fmt.Fprintf(w, "Error:\t%s\n", err.Error())
-			} else if bootNgi != nil && len(bootNgi.drivers) > i && bootNgi.drivers[i].err != nil {
-				fmt.Fprintf(w, "Error:\t%s\n", bootNgi.drivers[i].err.Error())
 			} else {
-				fmt.Fprintf(w, "Status:\t%s\n", ngi.drivers[i].info.Status)
+				fmt.Fprintf(w, "Status:\t%s\n", nodes[i].DriverInfo.Status)
 				if len(n.Flags) > 0 {
 					fmt.Fprintf(w, "Flags:\t%s\n", strings.Join(n.Flags, " "))
 				}
-				fmt.Fprintf(w, "Platforms:\t%s\n", strings.Join(platformutil.FormatInGroups(n.Platforms, ngi.drivers[i].platforms), ", "))
+				if nodes[i].Version != "" {
+					fmt.Fprintf(w, "Buildkit:\t%s\n", nodes[i].Version)
+				}
+				fmt.Fprintf(w, "Platforms:\t%s\n", strings.Join(platformutil.FormatInGroups(n.Node.Platforms, n.Platforms), ", "))
 			}
 		}
 	}

commands/ls.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 	"text/tabwriter"
 	"time"
 
-	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/cobrautil"
 	"github.com/docker/buildx/util/platformutil"
@@ -32,45 +31,24 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
 	}
 	defer release()
 
-	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	current, err := storeutil.GetCurrentInstance(txn, dockerCli)
+	if err != nil {
+		return err
+	}
+
+	builders, err := builder.GetBuilders(dockerCli, txn)
+	if err != nil {
+		return err
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	ll, err := txn.List()
+	eg, _ := errgroup.WithContext(timeoutCtx)
-	if err != nil {
-		return err
-	}
-
-	builders := make([]*nginfo, len(ll))
-	for i, ng := range ll {
-		builders[i] = &nginfo{ng: ng}
-	}
-
-	list, err := dockerCli.ContextStore().List()
-	if err != nil {
-		return err
-	}
-	ctxbuilders := make([]*nginfo, len(list))
-	for i, l := range list {
-		ctxbuilders[i] = &nginfo{ng: &store.NodeGroup{
-			Name: l.Name,
-			Nodes: []store.Node{{
-				Name:     l.Name,
-				Endpoint: l.Name,
-			}},
-		}}
-	}
-
-	builders = append(builders, ctxbuilders...)
-
-	eg, _ := errgroup.WithContext(ctx)
-
 	for _, b := range builders {
-		func(b *nginfo) {
+		func(b *builder.Builder) {
 			eg.Go(func() error {
-				err = loadNodeGroupData(ctx, dockerCli, b)
+				_, _ = b.LoadNodes(timeoutCtx, true)
-				if b.err == nil && err != nil {
-					b.err = err
-				}
 				return nil
 			})
 		}(b)
@@ -80,61 +58,62 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
 		return err
 	}
 
-	currentName := "default"
+	w := tabwriter.NewWriter(dockerCli.Out(), 0, 0, 1, ' ', 0)
-	current, err := storeutil.GetCurrentInstance(txn, dockerCli)
+	fmt.Fprintf(w, "NAME/NODE\tDRIVER/ENDPOINT\tSTATUS\tBUILDKIT\tPLATFORMS\n")
-	if err != nil {
-		return err
-	}
-	if current != nil {
-		currentName = current.Name
-		if current.Name == "default" {
-			currentName = current.Nodes[0].Endpoint
-		}
-	}
 
-	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	printErr := false
-	fmt.Fprintf(w, "NAME/NODE\tDRIVER/ENDPOINT\tSTATUS\tPLATFORMS\n")
-
-	currentSet := false
 	for _, b := range builders {
-		if !currentSet && b.ng.Name == currentName {
+		if current.Name == b.Name {
-			b.ng.Name += " *"
+			b.Name += " *"
-			currentSet = true
+		}
+		if ok := printBuilder(w, b); !ok {
+			printErr = true
 		}
-		printngi(w, b)
 	}
 
 	w.Flush()
 
+	if printErr {
+		_, _ = fmt.Fprintf(dockerCli.Err(), "\n")
+		for _, b := range builders {
+			if b.Err() != nil {
+				_, _ = fmt.Fprintf(dockerCli.Err(), "Cannot load builder %s: %s\n", b.Name, strings.TrimSpace(b.Err().Error()))
+			} else {
+				for _, d := range b.Nodes() {
+					if d.Err != nil {
+						_, _ = fmt.Fprintf(dockerCli.Err(), "Failed to get status for %s (%s): %s\n", b.Name, d.Name, strings.TrimSpace(d.Err.Error()))
+					}
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
-func printngi(w io.Writer, ngi *nginfo) {
+func printBuilder(w io.Writer, b *builder.Builder) (ok bool) {
+	ok = true
 	var err string
-	if ngi.err != nil {
+	if b.Err() != nil {
-		err = ngi.err.Error()
+		ok = false
-	}
+		err = "error"
-	fmt.Fprintf(w, "%s\t%s\t%s\t\n", ngi.ng.Name, ngi.ng.Driver, err)
-	if ngi.err == nil {
-		for idx, n := range ngi.ng.Nodes {
-			d := ngi.drivers[idx]
-			var err string
-			if d.err != nil {
-				err = d.err.Error()
-			} else if d.di.Err != nil {
-				err = d.di.Err.Error()
 	}
+	fmt.Fprintf(w, "%s\t%s\t%s\t\t\n", b.Name, b.Driver, err)
+	if b.Err() == nil {
+		for _, n := range b.Nodes() {
 			var status string
-			if d.info != nil {
+			if n.DriverInfo != nil {
-				status = d.info.Status.String()
+				status = n.DriverInfo.Status.String()
 			}
-			if err != "" {
+			if n.Err != nil {
-				fmt.Fprintf(w, "  %s\t%s\t%s\n", n.Name, n.Endpoint, err)
+				ok = false
+				fmt.Fprintf(w, "  %s\t%s\t%s\t\t\n", n.Name, n.Endpoint, "error")
 			} else {
-				fmt.Fprintf(w, "  %s\t%s\t%s\t%s\n", n.Name, n.Endpoint, status, strings.Join(platformutil.FormatInGroups(n.Platforms, d.platforms), ", "))
+				fmt.Fprintf(w, "  %s\t%s\t%s\t%s\t%s\n", n.Name, n.Endpoint, status, n.Version, strings.Join(platformutil.FormatInGroups(n.Node.Platforms, n.Platforms), ", "))
 			}
 		}
 	}
+	return
 }
 
 func lsCmd(dockerCli command.Cli) *cobra.Command {

commands/print.go

@@ -0,0 +1,48 @@
+package commands
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/docker/buildx/build"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/frontend/subrequests/outline"
+	"github.com/moby/buildkit/frontend/subrequests/targets"
+)
+
+func printResult(f *build.PrintFunc, res map[string]string) error {
+	switch f.Name {
+	case "outline":
+		return printValue(outline.PrintOutline, outline.SubrequestsOutlineDefinition.Version, f.Format, res)
+	case "targets":
+		return printValue(targets.PrintTargets, targets.SubrequestsTargetsDefinition.Version, f.Format, res)
+	case "subrequests.describe":
+		return printValue(subrequests.PrintDescribe, subrequests.SubrequestsDescribeDefinition.Version, f.Format, res)
+	default:
+		if dt, ok := res["result.txt"]; ok {
+			fmt.Print(dt)
+		} else {
+			log.Printf("%s %+v", f, res)
+		}
+	}
+	return nil
+}
+
+type printFunc func([]byte, io.Writer) error
+
+func printValue(printer printFunc, version string, format string, res map[string]string) error {
+	if format == "json" {
+		fmt.Fprintln(os.Stdout, res["result.json"])
+		return nil
+	}
+
+	if res["version"] != "" && versions.LessThan(version, res["version"]) && res["result.txt"] != "" {
+		// structure is too new and we don't know how to print it
+		fmt.Fprint(os.Stdout, res["result.txt"])
+		return nil
+	}
+	return printer([]byte(res["result.json"]), os.Stdout)
+}

commands/prune.go

@@ -7,7 +7,7 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/builder"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/opts"
@@ -54,15 +54,19 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) error {
 		return nil
 	}
 
-	dis, err := getInstanceOrDefault(ctx, dockerCli, opts.builder, "")
+	b, err := builder.New(dockerCli, builder.WithName(opts.builder))
 	if err != nil {
 		return err
 	}
 
-	for _, di := range dis {
+	nodes, err := b.LoadNodes(ctx, false)
-		if di.Err != nil {
+	if err != nil {
 		return err
 	}
+	for _, node := range nodes {
+		if node.Err != nil {
+			return node.Err
+		}
 	}
 
 	ch := make(chan client.UsageInfo)
@@ -90,11 +94,11 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) error {
 	}()
 
 	eg, ctx := errgroup.WithContext(ctx)
-	for _, di := range dis {
+	for _, node := range nodes {
-		func(di build.DriverInfo) {
+		func(node builder.Node) {
 			eg.Go(func() error {
-				if di.Driver != nil {
+				if node.Driver != nil {
-					c, err := di.Driver.Client(ctx)
+					c, err := node.Driver.Client(ctx)
 					if err != nil {
 						return err
 					}
@@ -109,7 +113,7 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) error {
 				}
 				return nil
 			})
-		}(di)
+		}(node)
 	}
 
 	if err := eg.Wait(); err != nil {
@@ -138,7 +142,7 @@ func pruneCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	}
 
 	flags := cmd.Flags()
-	flags.BoolVarP(&options.all, "all", "a", false, "Remove all unused images, not just dangling ones")
+	flags.BoolVarP(&options.all, "all", "a", false, "Include internal/frontend images")
 	flags.Var(&options.filter, "filter", `Provide filter values (e.g., "until=24h")`)
 	flags.Var(&options.keepStorage, "keep-storage", "Amount of disk space to keep for cache")
 	flags.BoolVar(&options.verbose, "verbose", false, "Provide a more verbose output")
@@ -155,9 +159,9 @@ func toBuildkitPruneInfo(f filters.Args) (*client.PruneInfo, error) {
 	if len(untilValues) > 0 && len(unusedForValues) > 0 {
 		return nil, errors.Errorf("conflicting filters %q and %q", "until", "unused-for")
 	}
-	filterKey := "until"
+	untilKey := "until"
 	if len(unusedForValues) > 0 {
-		filterKey = "unused-for"
+		untilKey = "unused-for"
 	}
 	untilValues = append(untilValues, unusedForValues...)
 
@@ -168,23 +172,27 @@ func toBuildkitPruneInfo(f filters.Args) (*client.PruneInfo, error) {
 		var err error
 		until, err = time.ParseDuration(untilValues[0])
 		if err != nil {
-			return nil, errors.Wrapf(err, "%q filter expects a duration (e.g., '24h')", filterKey)
+			return nil, errors.Wrapf(err, "%q filter expects a duration (e.g., '24h')", untilKey)
 		}
 	default:
 		return nil, errors.Errorf("filters expect only one value")
 	}
 
-	bkFilter := make([]string, 0, f.Len())
+	filters := make([]string, 0, f.Len())
-	for _, field := range f.Keys() {
+	for _, filterKey := range f.Keys() {
-		values := f.Get(field)
+		if filterKey == untilKey {
+			continue
+		}
+
+		values := f.Get(filterKey)
 		switch len(values) {
 		case 0:
-			bkFilter = append(bkFilter, field)
+			filters = append(filters, filterKey)
 		case 1:
-			if field == "id" {
+			if filterKey == "id" {
-				bkFilter = append(bkFilter, field+"~="+values[0])
+				filters = append(filters, filterKey+"~="+values[0])
 			} else {
-				bkFilter = append(bkFilter, field+"=="+values[0])
+				filters = append(filters, filterKey+"=="+values[0])
 			}
 		default:
 			return nil, errors.Errorf("filters expect only one value")
@@ -192,6 +200,6 @@ func toBuildkitPruneInfo(f filters.Args) (*client.PruneInfo, error) {
 	}
 	return &client.PruneInfo{
 		KeepDuration: until,
-		Filter:       []string{strings.Join(bkFilter, ",")},
+		Filter:       []string{strings.Join(filters, ",")},
 	}, nil
 }

commands/rm.go

@@ -2,8 +2,10 @@ package commands
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/cli/cli"
@@ -43,30 +45,33 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return rmAllInactive(ctx, txn, dockerCli, in)
 	}
 
-	if in.builder != "" {
+	b, err := builder.New(dockerCli,
-		ng, err := storeutil.GetNodeGroup(txn, dockerCli, in.builder)
+		builder.WithName(in.builder),
+		builder.WithStore(txn),
+		builder.WithSkippedValidation(),
+	)
 	if err != nil {
 		return err
 	}
-		err1 := rm(ctx, dockerCli, in, ng)
-		if err := txn.Remove(ng.Name); err != nil {
-			return err
-		}
-		return err1
-	}
 
-	ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
+	nodes, err := b.LoadNodes(ctx, false)
 	if err != nil {
 		return err
 	}
-	if ng != nil {
+
-		err1 := rm(ctx, dockerCli, in, ng)
+	if cb := b.ContextName(); cb != "" {
-		if err := txn.Remove(ng.Name); err != nil {
+		return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", cb)
+	}
+
+	err1 := rm(ctx, nodes, in)
+	if err := txn.Remove(b.Name); err != nil {
 		return err
 	}
+	if err1 != nil {
 		return err1
 	}
 
+	_, _ = fmt.Fprintf(dockerCli.Err(), "%s removed\n", b.Name)
 	return nil
 }
 
@@ -98,60 +103,56 @@ func rmCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	return cmd
 }
 
-func rm(ctx context.Context, dockerCli command.Cli, in rmOptions, ng *store.NodeGroup) error {
+func rm(ctx context.Context, nodes []builder.Node, in rmOptions) (err error) {
-	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
+	for _, node := range nodes {
-	if err != nil {
+		if node.Driver == nil {
-		return err
-	}
-	for _, di := range dis {
-		if di.Driver == nil {
 			continue
 		}
 		// Do not stop the buildkitd daemon when --keep-daemon is provided
 		if !in.keepDaemon {
-			if err := di.Driver.Stop(ctx, true); err != nil {
+			if err := node.Driver.Stop(ctx, true); err != nil {
 				return err
 			}
 		}
-		if err := di.Driver.Rm(ctx, true, !in.keepState, !in.keepDaemon); err != nil {
+		if err := node.Driver.Rm(ctx, true, !in.keepState, !in.keepDaemon); err != nil {
 			return err
 		}
-		if di.Err != nil {
+		if node.Err != nil {
-			err = di.Err
+			err = node.Err
 		}
 	}
 	return err
 }
 
 func rmAllInactive(ctx context.Context, txn *store.Txn, dockerCli command.Cli, in rmOptions) error {
-	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	builders, err := builder.GetBuilders(dockerCli, txn)
-	defer cancel()
-
-	ll, err := txn.List()
 	if err != nil {
 		return err
 	}
 
-	builders := make([]*nginfo, len(ll))
+	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
-	for i, ng := range ll {
+	defer cancel()
-		builders[i] = &nginfo{ng: ng}
-	}
 
-	eg, _ := errgroup.WithContext(ctx)
+	eg, _ := errgroup.WithContext(timeoutCtx)
 	for _, b := range builders {
-		func(b *nginfo) {
+		func(b *builder.Builder) {
 			eg.Go(func() error {
-				if err := loadNodeGroupData(ctx, dockerCli, b); err != nil {
+				nodes, err := b.LoadNodes(timeoutCtx, true)
-					return errors.Wrapf(err, "cannot load %s", b.ng.Name)
+				if err != nil {
+					return errors.Wrapf(err, "cannot load %s", b.Name)
 				}
-				if b.ng.Dynamic {
+				if cb := b.ContextName(); cb != "" {
+					return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", cb)
+				}
+				if b.Dynamic {
 					return nil
 				}
-				if b.inactive() {
+				if b.Inactive() {
-					rmerr := rm(ctx, dockerCli, in, b.ng)
+					rmerr := rm(ctx, nodes, in)
-					if err := txn.Remove(b.ng.Name); err != nil {
+					if err := txn.Remove(b.Name); err != nil {
 						return err
 					}
+					_, _ = fmt.Fprintf(dockerCli.Err(), "%s removed\n", b.Name)
 					return rmerr
 				}
 				return nil

commands/root.go

@@ -6,6 +6,7 @@ import (
 	imagetoolscmd "github.com/docker/buildx/commands/imagetools"
 	"github.com/docker/buildx/util/logutil"
 	"github.com/docker/cli-docs-tool/annotation"
+	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli-plugins/plugin"
 	"github.com/docker/cli/cli/command"
 	"github.com/sirupsen/logrus"
@@ -26,6 +27,14 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 		cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
 			return plugin.PersistentPreRunE(cmd, args)
 		}
+	} else {
+		// match plugin behavior for standalone mode
+		// https://github.com/docker/cli/blob/6c9eb708fa6d17765d71965f90e1c59cea686ee9/cli-plugins/plugin/plugin.go#L117-L127
+		cmd.SilenceUsage = true
+		cmd.SilenceErrors = true
+		cmd.TraverseChildren = true
+		cmd.DisableFlagsInUseLine = true
+		cli.DisableFlagsInUseLine(cmd)
 	}
 
 	logrus.SetFormatter(&logutil.Formatter{})
@@ -46,6 +55,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 		logrus.WarnLevel,
 	},
 		"commandConn.CloseWrite:",
+		"commandConn.CloseRead:",
 	))
 
 	addCommands(cmd, dockerCli)
@@ -74,7 +84,7 @@ func addCommands(cmd *cobra.Command, dockerCli command.Cli) {
 		versionCmd(dockerCli),
 		pruneCmd(dockerCli, opts),
 		duCmd(dockerCli, opts),
-		imagetoolscmd.RootCmd(dockerCli, imagetoolscmd.RootOptions{Builder: opts.builder}),
+		imagetoolscmd.RootCmd(dockerCli, imagetoolscmd.RootOptions{Builder: &opts.builder}),
 	)
 }
 

commands/stop.go

@@ -3,8 +3,7 @@ package commands
 import (
 	"context"
 
-	"github.com/docker/buildx/store"
+	"github.com/docker/buildx/builder"
-	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/moby/buildkit/util/appcontext"
@@ -18,32 +17,19 @@ type stopOptions struct {
 func runStop(dockerCli command.Cli, in stopOptions) error {
 	ctx := appcontext.Context()
 
-	txn, release, err := storeutil.GetStore(dockerCli)
+	b, err := builder.New(dockerCli,
+		builder.WithName(in.builder),
+		builder.WithSkippedValidation(),
+	)
 	if err != nil {
 		return err
 	}
-	defer release()
+	nodes, err := b.LoadNodes(ctx, false)
-
-	if in.builder != "" {
-		ng, err := storeutil.GetNodeGroup(txn, dockerCli, in.builder)
 	if err != nil {
 		return err
 	}
-		if err := stop(ctx, dockerCli, ng); err != nil {
-			return err
-		}
-		return nil
-	}
 
-	ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
+	return stop(ctx, nodes)
-	if err != nil {
-		return err
-	}
-	if ng != nil {
-		return stop(ctx, dockerCli, ng)
-	}
-
-	return stopCurrent(ctx, dockerCli)
 }
 
 func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
@@ -65,37 +51,15 @@ func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	return cmd
 }
 
-func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup) error {
+func stop(ctx context.Context, nodes []builder.Node) (err error) {
-	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
+	for _, node := range nodes {
-	if err != nil {
+		if node.Driver != nil {
-		return err
+			if err := node.Driver.Stop(ctx, true); err != nil {
-	}
-	for _, di := range dis {
-		if di.Driver != nil {
-			if err := di.Driver.Stop(ctx, true); err != nil {
 				return err
 			}
 		}
-		if di.Err != nil {
+		if node.Err != nil {
-			err = di.Err
+			err = node.Err
-		}
-	}
-	return err
-}
-
-func stopCurrent(ctx context.Context, dockerCli command.Cli) error {
-	dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
-	if err != nil {
-		return err
-	}
-	for _, di := range dis {
-		if di.Driver != nil {
-			if err := di.Driver.Stop(ctx, true); err != nil {
-				return err
-			}
-		}
-		if di.Err != nil {
-			err = di.Err
 		}
 	}
 	return err

commands/use.go

@@ -4,6 +4,7 @@ import (
 	"os"
 
 	"github.com/docker/buildx/store/storeutil"
+	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/pkg/errors"
@@ -29,7 +30,7 @@ func runUse(dockerCli command.Cli, in useOptions) error {
 				return errors.Errorf("run `docker context use default` to switch to default context")
 			}
 			if in.builder == "default" || in.builder == dockerCli.CurrentContext() {
-				ep, err := storeutil.GetCurrentEndpoint(dockerCli)
+				ep, err := dockerutil.GetCurrentEndpoint(dockerCli)
 				if err != nil {
 					return err
 				}
@@ -52,7 +53,7 @@ func runUse(dockerCli command.Cli, in useOptions) error {
 		return errors.Wrapf(err, "failed to find instance %q", in.builder)
 	}
 
-	ep, err := storeutil.GetCurrentEndpoint(dockerCli)
+	ep, err := dockerutil.GetCurrentEndpoint(dockerCli)
 	if err != nil {
 		return err
 	}

commands/util.go

@@ -1,443 +0,0 @@
-package commands
-
-import (
-	"context"
-	"net/url"
-	"os"
-	"strings"
-
-	"github.com/docker/buildx/build"
-	"github.com/docker/buildx/driver"
-	"github.com/docker/buildx/store"
-	"github.com/docker/buildx/store/storeutil"
-	"github.com/docker/buildx/util/platformutil"
-	"github.com/docker/buildx/util/progress"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/cli/cli/context/docker"
-	"github.com/docker/cli/cli/context/kubernetes"
-	ctxstore "github.com/docker/cli/cli/context/store"
-	dopts "github.com/docker/cli/opts"
-	dockerclient "github.com/docker/docker/client"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
-	"k8s.io/client-go/tools/clientcmd"
-)
-
-// validateEndpoint validates that endpoint is either a context or a docker host
-func validateEndpoint(dockerCli command.Cli, ep string) (string, error) {
-	de, err := storeutil.GetDockerEndpoint(dockerCli, ep)
-	if err == nil && de != "" {
-		if ep == "default" {
-			return de, nil
-		}
-		return ep, nil
-	}
-	h, err := dopts.ParseHost(true, ep)
-	if err != nil {
-		return "", errors.Wrapf(err, "failed to parse endpoint %s", ep)
-	}
-	return h, nil
-}
-
-// driversForNodeGroup returns drivers for a nodegroup instance
-func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, contextPathHash string) ([]build.DriverInfo, error) {
-	eg, _ := errgroup.WithContext(ctx)
-
-	dis := make([]build.DriverInfo, len(ng.Nodes))
-
-	var f driver.Factory
-	if ng.Driver != "" {
-		f = driver.GetFactory(ng.Driver, true)
-		if f == nil {
-			return nil, errors.Errorf("failed to find driver %q", f)
-		}
-	} else {
-		dockerapi, err := clientForEndpoint(dockerCli, ng.Nodes[0].Endpoint)
-		if err != nil {
-			return nil, err
-		}
-		f, err = driver.GetDefaultFactory(ctx, dockerapi, false)
-		if err != nil {
-			return nil, err
-		}
-		ng.Driver = f.Name()
-	}
-	imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
-	if err != nil {
-		return nil, err
-	}
-
-	for i, n := range ng.Nodes {
-		func(i int, n store.Node) {
-			eg.Go(func() error {
-				di := build.DriverInfo{
-					Name:        n.Name,
-					Platform:    n.Platforms,
-					ProxyConfig: storeutil.GetProxyConfig(dockerCli),
-				}
-				defer func() {
-					dis[i] = di
-				}()
-				dockerapi, err := clientForEndpoint(dockerCli, n.Endpoint)
-				if err != nil {
-					di.Err = err
-					return nil
-				}
-				// TODO: replace the following line with dockerclient.WithAPIVersionNegotiation option in clientForEndpoint
-				dockerapi.NegotiateAPIVersion(ctx)
-
-				contextStore := dockerCli.ContextStore()
-
-				var kcc driver.KubeClientConfig
-				kcc, err = configFromContext(n.Endpoint, contextStore)
-				if err != nil {
-					// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
-					// try again with name="default".
-					// FIXME: n should retain real context name.
-					kcc, err = configFromContext("default", contextStore)
-					if err != nil {
-						logrus.Error(err)
-					}
-				}
-
-				tryToUseKubeConfigInCluster := false
-				if kcc == nil {
-					tryToUseKubeConfigInCluster = true
-				} else {
-					if _, err := kcc.ClientConfig(); err != nil {
-						tryToUseKubeConfigInCluster = true
-					}
-				}
-				if tryToUseKubeConfigInCluster {
-					kccInCluster := driver.KubeClientConfigInCluster{}
-					if _, err := kccInCluster.ClientConfig(); err == nil {
-						logrus.Debug("using kube config in cluster")
-						kcc = kccInCluster
-					}
-				}
-
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
-				if err != nil {
-					di.Err = err
-					return nil
-				}
-				di.Driver = d
-				di.ImageOpt = imageopt
-				return nil
-			})
-		}(i, n)
-	}
-
-	if err := eg.Wait(); err != nil {
-		return nil, err
-	}
-
-	return dis, nil
-}
-
-func configFromContext(endpointName string, s ctxstore.Reader) (clientcmd.ClientConfig, error) {
-	if strings.HasPrefix(endpointName, "kubernetes://") {
-		u, _ := url.Parse(endpointName)
-		if kubeconfig := u.Query().Get("kubeconfig"); kubeconfig != "" {
-			_ = os.Setenv(clientcmd.RecommendedConfigPathEnvVar, kubeconfig)
-		}
-		rules := clientcmd.NewDefaultClientConfigLoadingRules()
-		apiConfig, err := rules.Load()
-		if err != nil {
-			return nil, err
-		}
-		return clientcmd.NewDefaultClientConfig(*apiConfig, &clientcmd.ConfigOverrides{}), nil
-	}
-	return kubernetes.ConfigFromContext(endpointName, s)
-}
-
-// clientForEndpoint returns a docker client for an endpoint
-func clientForEndpoint(dockerCli command.Cli, name string) (dockerclient.APIClient, error) {
-	list, err := dockerCli.ContextStore().List()
-	if err != nil {
-		return nil, err
-	}
-	for _, l := range list {
-		if l.Name == name {
-			dep, ok := l.Endpoints["docker"]
-			if !ok {
-				return nil, errors.Errorf("context %q does not have a Docker endpoint", name)
-			}
-			epm, ok := dep.(docker.EndpointMeta)
-			if !ok {
-				return nil, errors.Errorf("endpoint %q is not of type EndpointMeta, %T", dep, dep)
-			}
-			ep, err := docker.WithTLSData(dockerCli.ContextStore(), name, epm)
-			if err != nil {
-				return nil, err
-			}
-			clientOpts, err := ep.ClientOpts()
-			if err != nil {
-				return nil, err
-			}
-			return dockerclient.NewClientWithOpts(clientOpts...)
-		}
-	}
-
-	ep := docker.Endpoint{
-		EndpointMeta: docker.EndpointMeta{
-			Host: name,
-		},
-	}
-
-	clientOpts, err := ep.ClientOpts()
-	if err != nil {
-		return nil, err
-	}
-
-	return dockerclient.NewClientWithOpts(clientOpts...)
-}
-
-func getInstanceOrDefault(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) {
-	var defaultOnly bool
-
-	if instance == "default" && instance != dockerCli.CurrentContext() {
-		return nil, errors.Errorf("use `docker --context=default buildx` to switch to default context")
-	}
-	if instance == "default" || instance == dockerCli.CurrentContext() {
-		instance = ""
-		defaultOnly = true
-	}
-	list, err := dockerCli.ContextStore().List()
-	if err != nil {
-		return nil, err
-	}
-	for _, l := range list {
-		if l.Name == instance {
-			return nil, errors.Errorf("use `docker --context=%s buildx` to switch to context %s", instance, instance)
-		}
-	}
-
-	if instance != "" {
-		return getInstanceByName(ctx, dockerCli, instance, contextPathHash)
-	}
-	return getDefaultDrivers(ctx, dockerCli, defaultOnly, contextPathHash)
-}
-
-func getInstanceByName(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) {
-	txn, release, err := storeutil.GetStore(dockerCli)
-	if err != nil {
-		return nil, err
-	}
-	defer release()
-
-	ng, err := txn.NodeGroupByName(instance)
-	if err != nil {
-		return nil, err
-	}
-	return driversForNodeGroup(ctx, dockerCli, ng, contextPathHash)
-}
-
-// getDefaultDrivers returns drivers based on current cli config
-func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly bool, contextPathHash string) ([]build.DriverInfo, error) {
-	txn, release, err := storeutil.GetStore(dockerCli)
-	if err != nil {
-		return nil, err
-	}
-	defer release()
-
-	if !defaultOnly {
-		ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
-		if err != nil {
-			return nil, err
-		}
-
-		if ng != nil {
-			return driversForNodeGroup(ctx, dockerCli, ng, contextPathHash)
-		}
-	}
-
-	imageopt, err := storeutil.GetImageConfig(dockerCli, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), imageopt.Auth, nil, nil, nil, nil, nil, contextPathHash)
-	if err != nil {
-		return nil, err
-	}
-	return []build.DriverInfo{
-		{
-			Name:        "default",
-			Driver:      d,
-			ImageOpt:    imageopt,
-			ProxyConfig: storeutil.GetProxyConfig(dockerCli),
-		},
-	}, nil
-}
-
-func loadInfoData(ctx context.Context, d *dinfo) error {
-	if d.di.Driver == nil {
-		return nil
-	}
-	info, err := d.di.Driver.Info(ctx)
-	if err != nil {
-		return err
-	}
-	d.info = info
-	if info.Status == driver.Running {
-		c, err := d.di.Driver.Client(ctx)
-		if err != nil {
-			return err
-		}
-		workers, err := c.ListWorkers(ctx)
-		if err != nil {
-			return errors.Wrap(err, "listing workers")
-		}
-		for _, w := range workers {
-			d.platforms = append(d.platforms, w.Platforms...)
-		}
-		d.platforms = platformutil.Dedupe(d.platforms)
-	}
-	return nil
-}
-
-func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo) error {
-	eg, _ := errgroup.WithContext(ctx)
-
-	dis, err := driversForNodeGroup(ctx, dockerCli, ngi.ng, "")
-	if err != nil {
-		return err
-	}
-	ngi.drivers = make([]dinfo, len(dis))
-	for i, di := range dis {
-		d := di
-		ngi.drivers[i].di = &d
-		func(d *dinfo) {
-			eg.Go(func() error {
-				if err := loadInfoData(ctx, d); err != nil {
-					d.err = err
-				}
-				return nil
-			})
-		}(&ngi.drivers[i])
-	}
-
-	if eg.Wait(); err != nil {
-		return err
-	}
-
-	kubernetesDriverCount := 0
-
-	for _, di := range ngi.drivers {
-		if di.info != nil && len(di.info.DynamicNodes) > 0 {
-			kubernetesDriverCount++
-		}
-	}
-
-	isAllKubernetesDrivers := len(ngi.drivers) == kubernetesDriverCount
-
-	if isAllKubernetesDrivers {
-		var drivers []dinfo
-		var dynamicNodes []store.Node
-
-		for _, di := range ngi.drivers {
-			// dynamic nodes are used in Kubernetes driver.
-			// Kubernetes pods are dynamically mapped to BuildKit Nodes.
-			if di.info != nil && len(di.info.DynamicNodes) > 0 {
-				for i := 0; i < len(di.info.DynamicNodes); i++ {
-					// all []dinfo share *build.DriverInfo and *driver.Info
-					diClone := di
-					if pl := di.info.DynamicNodes[i].Platforms; len(pl) > 0 {
-						diClone.platforms = pl
-					}
-					drivers = append(drivers, di)
-				}
-				dynamicNodes = append(dynamicNodes, di.info.DynamicNodes...)
-			}
-		}
-
-		// not append (remove the static nodes in the store)
-		ngi.ng.Nodes = dynamicNodes
-		ngi.drivers = drivers
-		ngi.ng.Dynamic = true
-	}
-
-	return nil
-}
-
-func dockerAPI(dockerCli command.Cli) *api {
-	return &api{dockerCli: dockerCli}
-}
-
-type api struct {
-	dockerCli command.Cli
-}
-
-func (a *api) DockerAPI(name string) (dockerclient.APIClient, error) {
-	if name == "" {
-		name = a.dockerCli.CurrentContext()
-	}
-	return clientForEndpoint(a.dockerCli, name)
-}
-
-type dinfo struct {
-	di        *build.DriverInfo
-	info      *driver.Info
-	platforms []specs.Platform
-	err       error
-}
-
-type nginfo struct {
-	ng      *store.NodeGroup
-	drivers []dinfo
-	err     error
-}
-
-// inactive checks if all nodes are inactive for this builder
-func (n *nginfo) inactive() bool {
-	for idx := range n.ng.Nodes {
-		d := n.drivers[idx]
-		if d.info != nil && d.info.Status == driver.Running {
-			return false
-		}
-	}
-	return true
-}
-
-func boot(ctx context.Context, ngi *nginfo) (bool, error) {
-	toBoot := make([]int, 0, len(ngi.drivers))
-	for i, d := range ngi.drivers {
-		if d.err != nil || d.di.Err != nil || d.di.Driver == nil || d.info == nil {
-			continue
-		}
-		if d.info.Status != driver.Running {
-			toBoot = append(toBoot, i)
-		}
-	}
-	if len(toBoot) == 0 {
-		return false, nil
-	}
-
-	printer := progress.NewPrinter(context.TODO(), os.Stderr, os.Stderr, "auto")
-
-	baseCtx := ctx
-	eg, _ := errgroup.WithContext(ctx)
-	for _, idx := range toBoot {
-		func(idx int) {
-			eg.Go(func() error {
-				pw := progress.WithPrefix(printer, ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
-				_, err := driver.Boot(ctx, baseCtx, ngi.drivers[idx].di.Driver, pw)
-				if err != nil {
-					ngi.drivers[idx].err = err
-				}
-				return nil
-			})
-		}(idx)
-	}
-
-	err := eg.Wait()
-	err1 := printer.Wait()
-	if err == nil {
-		err = err1
-	}
-
-	return true, err
-}

docker-bake.hcl

@@ -1,17 +1,14 @@
 variable "GO_VERSION" {
-  default = "1.17"
+  default = "1.19"
-}
-variable "BIN_OUT" {
-  default = "./bin"
-}
-variable "RELEASE_OUT" {
-  default = "./release-out"
 }
 variable "DOCS_FORMATS" {
   default = "md"
 }
+variable "DESTDIR" {
+  default = "./bin"
+}
 
-// Special target: https://github.com/docker/metadata-action#bake-definition
+# Special target: https://github.com/docker/metadata-action#bake-definition
 target "meta-helper" {
   tags = ["docker/buildx-bin:local"]
 }
@@ -20,6 +17,7 @@ target "_common" {
   args = {
     GO_VERSION = GO_VERSION
     BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1
+    BUILDX_EXPERIMENTAL = 1
   }
 }
 
@@ -89,24 +87,20 @@ target "mod-outdated" {
   inherits = ["_common"]
   dockerfile = "./hack/dockerfiles/vendor.Dockerfile"
   target = "outdated"
-  args = {
+  no-cache-filter = ["outdated"]
-    // used to invalidate cache for outdated run stage
-    // can be dropped when https://github.com/moby/buildkit/issues/1213 fixed
-    _RANDOM = uuidv4()
-  }
   output = ["type=cacheonly"]
 }
 
 target "test" {
   inherits = ["_common"]
   target = "test-coverage"
-  output = ["./coverage"]
+  output = ["${DESTDIR}/coverage"]
 }
 
 target "binaries" {
   inherits = ["_common"]
   target = "binaries"
-  output = [BIN_OUT]
+  output = ["${DESTDIR}/build"]
   platforms = ["local"]
 }
 
@@ -130,7 +124,7 @@ target "binaries-cross" {
 target "release" {
   inherits = ["binaries-cross"]
   target = "release"
-  output = [RELEASE_OUT]
+  output = ["${DESTDIR}/release"]
 }
 
 target "image" {

docs/generate.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/docker/buildx/driver/docker"
 	_ "github.com/docker/buildx/driver/docker-container"
 	_ "github.com/docker/buildx/driver/kubernetes"
+	_ "github.com/docker/buildx/driver/remote"
 )
 
 const defaultSourcePath = "docs/reference/"

docs/guides/cicd.md

@@ -19,13 +19,13 @@ jobs:
     steps:
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v2 
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -40,7 +40,7 @@ jobs:
 In this example we are also using 3 other actions:
 
 * [`setup-buildx`](https://github.com/docker/setup-buildx-action) action will create and boot a builder using by
-default the `docker-container` [builder driver](../reference/buildx_create.md#driver).
+default the `docker-container` [builder driver](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver).
 This is **not required but recommended** using it to be able to build multi-platform images, export cache, etc.
 * [`setup-qemu`](https://github.com/docker/setup-qemu-action) action can be useful if you want
 to add emulation support with QEMU to be able to build against more platforms.

docs/guides/cni-networking.md

@@ -11,7 +11,7 @@ Now build this image:
 $ docker buildx build --tag buildkit-cni:local --load .
 ```
 
-Then [create a `docker-container` builder](../reference/buildx_create.md) that
+Then [create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/) that
 will use this image:
 
 ```console

docs/guides/color-output.md

@@ -0,0 +1,20 @@
+# Color output controls
+
+Buildx has support for modifying the colors that are used to output information
+to the terminal. You can set the environment variable `BUILDKIT_COLORS` to
+something like `run=123,20,245:error=yellow:cancel=blue:warning=white` to set
+the colors that you would like to use:
+
+![Progress output custom colors](https://user-images.githubusercontent.com/1951866/180584033-24522385-cafd-4a54-a4a2-18f5ce74eb27.png)
+
+Setting `NO_COLOR` to anything will disable any colorized output as recommended
+by [no-color.org](https://no-color.org/):
+
+![Progress output no color](https://user-images.githubusercontent.com/1951866/180584037-e28f9997-dd4c-49cf-8b26-04864815de19.png)
+
+> **Note**
+>
+> Parsing errors will be reported but ignored. This will result in default
+> color values being used where needed.
+
+See also [the list of pre-defined colors](https://github.com/moby/buildkit/blob/master/util/progress/progressui/colors.go).

docs/guides/custom-network.md

@@ -7,8 +7,8 @@ named `foonet`:
 $ docker network create foonet
 ```
 
-[Create a `docker-container` builder](../reference/buildx_create.md) named
+[Create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/)
-`mybuilder` that will use this network:
+named `mybuilder` that will use this network:
 
 ```console
 $ docker buildx create --use \
@@ -17,7 +17,7 @@ $ docker buildx create --use \
   --driver-opt "network=foonet"
 ```
 
-Boot and [inspect `mybuilder`](../reference/buildx_inspect.md):
+Boot and [inspect `mybuilder`](https://docs.docker.com/engine/reference/commandline/buildx_inspect/):
 
 ```console
 $ docker buildx inspect --bootstrap
@@ -26,23 +26,9 @@ $ docker buildx inspect --bootstrap
 [Inspect the builder container](https://docs.docker.com/engine/reference/commandline/inspect/)
 and see what network is being used:
 
+{% raw %}
 ```console
 $ docker inspect buildx_buildkit_mybuilder0 --format={{.NetworkSettings.Networks}}
 map[foonet:0xc00018c0c0]
 ```
-
+{% endraw %}
-## What's `buildx_buildkit_mybuilder0`?
-
-`buildx_buildkit_mybuilder0` is the container name. It can be broken down like this:
-
-* `buildx_buildkit_` is a hardcoded prefix
-* `mybuilder0` is the name of the node (defaults to builder name + position in the list of nodes)
-
-```console
-$ docker buildx ls
-NAME/NODE     DRIVER/ENDPOINT                        STATUS                 PLATFORMS
-mybuilder *   docker-container
-  mybuilder0  unix:///var/run/docker.sock            running                linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/mips64le, linux/mips64, linux/arm/v7, linux/arm/v6     
-default       docker
-  default     default                                running                linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6
-```

docs/guides/custom-registry-config.md

@@ -1,6 +1,6 @@
 # Using a custom registry configuration
 
-If you [create a `docker-container` or `kubernetes` builder](../reference/buildx_create.md) and
+If you [create a `docker-container` or `kubernetes` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/) and
 have specified certificates for registries in the [BuildKit daemon configuration](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md),
 the files will be copied into the container under `/etc/buildkit/certs` and
 configuration will be updated to reflect that.
@@ -8,7 +8,8 @@ configuration will be updated to reflect that.
 Take the following `buildkitd.toml` configuration that will be used for
 pushing an image to this registry using self-signed certificates:
 
-```toml"
+```toml
+# /etc/buildkitd.toml
 debug = true
 [registry."myregistry.com"]
   ca=["/etc/certs/myregistry.pem"]
@@ -16,11 +17,10 @@ debug = true
     key="/etc/certs/myregistry_key.pem"
     cert="/etc/certs/myregistry_cert.pem"
 ```
-> `/etc/buildkitd.toml`
 
 Here we have configured a self-signed certificate for `myregistry.com` registry.
 
-Now [create a `docker-container` builder](../reference/buildx_create.md)
+Now [create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/)
 that will use this BuildKit configuration:
 
 ```console

docs/guides/opentelemetry.md

@@ -9,7 +9,7 @@ First create a Jaeger container:
 $ docker run -d --name jaeger -p "6831:6831/udp" -p "16686:16686" jaegertracing/all-in-one
 ```
 
-Then [create a `docker-container` builder](../reference/buildx_create.md)
+Then [create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/)
 that will use the Jaeger instance via the `JAEGER_TRACE` env var:
 
 ```console
@@ -20,7 +20,7 @@ $ docker buildx create --use \
   --driver-opt "env.JAEGER_TRACE=localhost:6831"
 ```
 
-Boot and [inspect `mybuilder`](../reference/buildx_inspect.md):
+Boot and [inspect `mybuilder`](https://docs.docker.com/engine/reference/commandline/buildx_inspect/):
 
 ```console
 $ docker buildx inspect --bootstrap
@@ -28,4 +28,4 @@ $ docker buildx inspect --bootstrap
 
 Buildx commands should be traced at `http://127.0.0.1:16686/`:
 
-![](https://user-images.githubusercontent.com/1951866/124468052-ef085400-dd98-11eb-84ab-7ac8e261dd52.png)
+![OpenTelemetry Buildx Bake](https://user-images.githubusercontent.com/1951866/124468052-ef085400-dd98-11eb-84ab-7ac8e261dd52.png)

docs/guides/registry-mirror.md

@@ -1,19 +1,21 @@
 # Registry mirror
 
 You can define a registry mirror to use for your builds by providing a [BuildKit daemon configuration](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md)
-while creating a builder with the [`--config` flags](../reference/buildx_create.md#config).
+while creating a builder with the [`--config` flags](https://docs.docker.com/engine/reference/commandline/buildx_create/#config).
 
 ```toml
+# /etc/buildkitd.toml
 debug = true
 [registry."docker.io"]
   mirrors = ["mirror.gcr.io"]
 ```
-> `/etc/buildkitd.toml`
 
-> :information_source: `debug = true` has been added to be able to debug requests
+> **Note**
-in the BuildKit daemon and see if the mirror is effectively used.
+>
+> `debug = true` has been added to be able to debug requests
+> in the BuildKit daemon and see if the mirror is effectively used.
 
-Then [create a `docker-container` builder](../reference/buildx_create.md)
+Then [create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/)
 that will use this BuildKit configuration:
 
 ```console
@@ -23,7 +25,7 @@ $ docker buildx create --use \
   --config /etc/buildkitd.toml
 ```
 
-Boot and [inspect `mybuilder`](../reference/buildx_inspect.md):
+Boot and [inspect `mybuilder`](https://docs.docker.com/engine/reference/commandline/buildx_inspect/):
 
 ```console
 $ docker buildx inspect --bootstrap

docs/guides/resource-limiting.md

@@ -4,15 +4,15 @@
 
 You can limit the parallelism of the BuildKit solver, which is particularly useful
 for low-powered machines, using a [BuildKit daemon configuration](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md)
-while creating a builder with the [`--config` flags](../reference/buildx_create.md#config).
+while creating a builder with the [`--config` flags](https://docs.docker.com/engine/reference/commandline/buildx_create/#config).
 
 ```toml
+# /etc/buildkitd.toml
 [worker.oci]
   max-parallelism = 4
 ```
-> `/etc/buildkitd.toml`
 
-Now you can [create a `docker-container` builder](../reference/buildx_create.md)
+Now you can [create a `docker-container` builder](https://docs.docker.com/engine/reference/commandline/buildx_create/)
 that will use this BuildKit configuration to limit parallelism.
 
 ```console

docs/manuals/README.md

@@ -0,0 +1,14 @@
+# Buildx manuals 📚
+
+This directory contains a bunch of useful docs for how to use Buildx features.
+
+> **Note**
+>
+> The markdown files in this directory (excluding this README) are reused
+> downstream by the
+> [Docker documentation repository](https://github.com/docker/docs).
+>
+> If you wish to contribute to these docs, be sure to first review the
+> [documentation contribution guidelines](https://docs.docker.com/contribute/overview/).
+>
+> Thank you!

docs/manuals/bake/build-contexts.md

@@ -0,0 +1,3 @@
+# Defining additional build contexts and linking targets
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake/build-contexts)

docs/manuals/bake/compose-file.md

@@ -0,0 +1,3 @@
+# Building from Compose file
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake/compose-file)

docs/manuals/bake/configuring-build.md

@@ -0,0 +1,3 @@
+# Configuring builds
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake/configuring-build)

docs/manuals/bake/file-definition.md

@@ -0,0 +1,3 @@
+# Bake file definition
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake/file-definition)

docs/manuals/bake/hcl-funcs.md

@@ -0,0 +1,3 @@
+# User defined HCL functions
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake/hcl-funcs)

docs/manuals/bake/index.md

@@ -0,0 +1,3 @@
+# High-level build options with Bake
+
+Moved to [docs.docker.com](https://docs.docker.com/build/customize/bake)

docs/manuals/cache/backends/azblob.md

@@ -0,0 +1,3 @@
+# Azure Blob Storage cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/azblob)

docs/manuals/cache/backends/gha.md

@@ -0,0 +1,3 @@
+# GitHub Actions cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/gha)

docs/manuals/cache/backends/index.md

@@ -0,0 +1,3 @@
+# Cache storage backends
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends)

docs/manuals/cache/backends/inline.md

@@ -0,0 +1,3 @@
+# Inline cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/inline)

docs/manuals/cache/backends/local.md

@@ -0,0 +1,3 @@
+# Local cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/local)

docs/manuals/cache/backends/registry.md

@@ -0,0 +1,3 @@
+# Registry cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/registry)

docs/manuals/cache/backends/s3.md

@@ -0,0 +1,3 @@
+# Amazon S3 cache storage
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/cache/backends/s3)

docs/manuals/drivers/docker-container.md

@@ -0,0 +1,3 @@
+# Docker container driver
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/drivers/docker-container)

docs/manuals/drivers/docker.md

@@ -0,0 +1,3 @@
+# Docker driver
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/drivers/docker)

docs/manuals/drivers/index.md

@@ -0,0 +1,3 @@
+# Buildx drivers overview
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/drivers)

docs/manuals/drivers/kubernetes.md

@@ -0,0 +1,3 @@
+# Kubernetes driver
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/drivers/kubernetes)

docs/manuals/drivers/remote.md

@@ -0,0 +1,3 @@
+# Remote driver
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/drivers/remote)

docs/manuals/exporters/image-registry.md

@@ -0,0 +1,3 @@
+# Image and registry exporters
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/exporters/image-registry)

docs/manuals/exporters/index.md

@@ -0,0 +1,3 @@
+# Exporters overview
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/exporters)

docs/manuals/exporters/local-tar.md

@@ -0,0 +1,3 @@
+# Local and tar exporters
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/exporters/local-tar)

docs/manuals/exporters/oci-docker.md

@@ -0,0 +1,3 @@
+# OCI and Docker exporters
+
+Moved to [docs.docker.com](https://docs.docker.com/build/building/exporters/oci-docker)

docs/reference/buildx_bake.md

@@ -9,7 +9,7 @@ Build from a file
 
 ### Aliases
 
-`bake`, `f`
+`docker buildx bake`, `docker buildx f`
 
 ### Options
 
@@ -22,8 +22,10 @@ Build from a file
 | [`--no-cache`](#no-cache) |  |  | Do not use cache when building the image |
 | [`--print`](#print) |  |  | Print the options without building |
 | [`--progress`](#progress) | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
-| [`--pull`](#pull) |  |  | Always attempt to pull a newer version of the image |
+| `--provenance` | `string` |  | Shorthand for `--set=*.attest=type=provenance` |
+| [`--pull`](#pull) |  |  | Always attempt to pull all referenced images |
 | `--push` |  |  | Shorthand for `--set=*.output=type=registry` |
+| `--sbom` | `string` |  | Shorthand for `--set=*.attest=type=sbom` |
 | [`--set`](#set) | `stringArray` |  | Override target value (e.g., `targetpattern.key=value`) |
 
 
@@ -34,12 +36,14 @@ Build from a file
 Bake is a high-level build command. Each specified target will run in parallel
 as part of the build.
 
-Read [High-level build options](https://github.com/docker/buildx#high-level-build-options)
+Read [High-level build options with Bake](https://docs.docker.com/build/customize/bake/)
-for introduction.
+guide for introduction to writing bake files.
 
-Please note that `buildx bake` command may receive backwards incompatible
+> **Note**
-features in the future if needed. We are looking for feedback on improving the
+>
-command and extending the functionality further.
+> `buildx bake` command may receive backwards incompatible features in the future
+> if needed. We are looking for feedback on improving the command and extending
+> the functionality further.
 
 ## Examples
 
@@ -49,166 +53,42 @@ Same as [`buildx --builder`](buildx.md#builder).
 
 ### <a name="file"></a> Specify a build definition file (-f, --file)
 
-By default, `buildx bake` looks for build definition files in the current
+Use the `-f` / `--file` option to specify the build definition file to use.
-directory, the following are parsed:
+The file can be an HCL, JSON or Compose file. If multiple files are specified
-
-- `docker-compose.yml`
-- `docker-compose.yaml`
-- `docker-bake.json`
-- `docker-bake.override.json`
-- `docker-bake.hcl`
-- `docker-bake.override.hcl`
-
-Use the `-f` / `--file` option to specify the build definition file to use. The
-file can be a Docker Compose, JSON or HCL file. If multiple files are specified
 they are all read and configurations are combined.
 
-The following example uses a Docker Compose file named `docker-compose.dev.yaml`
+You can pass the names of the targets to build, to build only specific target(s).
-as build definition file, and builds all targets in the file:
+The following example builds the `db` and `webapp-release` targets that are
+defined in the `docker-bake.dev.hcl` file:
 
-```console
+```hcl
-$ docker buildx bake -f docker-compose.dev.yaml
+# docker-bake.dev.hcl
-
+group "default" {
-[+] Building 66.3s (30/30) FINISHED
+  targets = ["db", "webapp-dev"]
- => [frontend internal] load build definition from Dockerfile  0.1s
- => => transferring dockerfile: 36B                            0.0s
- => [backend internal] load build definition from Dockerfile   0.2s
- => => transferring dockerfile: 3.73kB                         0.0s
- => [database internal] load build definition from Dockerfile  0.1s
- => => transferring dockerfile: 5.77kB                         0.0s
- ...
-```
-
-Pass the names of the targets to build, to build only specific target(s). The
-following example builds the `backend` and `database` targets that are defined
-in the `docker-compose.dev.yaml` file, skipping the build for the `frontend`
-target:
-
-```console
-$ docker buildx bake -f docker-compose.dev.yaml backend database
-
-[+] Building 2.4s (13/13) FINISHED
- => [backend internal] load build definition from Dockerfile  0.1s
- => => transferring dockerfile: 81B                           0.0s
- => [database internal] load build definition from Dockerfile 0.2s
- => => transferring dockerfile: 36B                           0.0s
- => [backend internal] load .dockerignore                     0.3s
- ...
-```
-
-You can also use a remote `git` bake definition:
-
-```console
-$ docker buildx bake "https://github.com/docker/cli.git#v20.10.11" --print
-#1 [internal] load git source https://github.com/docker/cli.git#v20.10.11
-#1 0.745 e8f1871b077b64bcb4a13334b7146492773769f7       refs/tags/v20.10.11
-#1 2.022 From https://github.com/docker/cli
-#1 2.022  * [new tag]         v20.10.11  -> v20.10.11
-#1 DONE 2.9s
-{
-  "group": {
-    "default": {
-      "targets": [
-        "binary"
-      ]
 }
-  },
-  "target": {
-    "binary": {
-      "context": "https://github.com/docker/cli.git#v20.10.11",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "BASE_VARIANT": "alpine",
-        "GO_STRIP": "",
-        "VERSION": ""
-      },
-      "target": "binary",
-      "platforms": [
-        "local"
-      ],
-      "output": [
-        "build"
-      ]
-    }
-  }
-}
-```
 
-As you can see the context is fixed to `https://github.com/docker/cli.git` even if
+target "webapp-dev" {
-[no context is actually defined](https://github.com/docker/cli/blob/2776a6d694f988c0c1df61cad4bfac0f54e481c8/docker-bake.hcl#L17-L26)
+  dockerfile = "Dockerfile.webapp"
-in the definition.
+  tags = ["docker.io/username/webapp"]
+}
 
-If you want to access the main context for bake command from a bake file
+target "webapp-release" {
-that has been imported remotely, you can use the `BAKE_CMD_CONTEXT` builtin var:
+  inherits = ["webapp-dev"]
+  platforms = ["linux/amd64", "linux/arm64"]
+}
 
-```console
+target "db" {
-$ cat https://raw.githubusercontent.com/tonistiigi/buildx/remote-test/docker-bake.hcl
+  dockerfile = "Dockerfile.db"
-target "default" {
+  tags = ["docker.io/username/db"]
-  context = BAKE_CMD_CONTEXT
-  dockerfile-inline = <<EOT
-FROM alpine
-WORKDIR /src
-COPY . .
-RUN ls -l && stop
-EOT
 }
 ```
 
 ```console
-$ docker buildx bake "https://github.com/tonistiigi/buildx.git#remote-test" --print
+$ docker buildx bake -f docker-bake.dev.hcl db webapp-release
-{
-  "target": {
-    "default": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "dockerfile-inline": "FROM alpine\nWORKDIR /src\nCOPY . .\nRUN ls -l \u0026\u0026 stop\n"
-    }
-  }
-}
 ```
 
-```console
+See our [file definition](https://docs.docker.com/build/customize/bake/file-definition/)
-$ touch foo bar
+guide for more details.
-$ docker buildx bake "https://github.com/tonistiigi/buildx.git#remote-test"
-...
- > [4/4] RUN ls -l && stop:
-#8 0.101 total 0
-#8 0.102 -rw-r--r--    1 root     root             0 Jul 27 18:47 bar
-#8 0.102 -rw-r--r--    1 root     root             0 Jul 27 18:47 foo
-#8 0.102 /bin/sh: stop: not found
-```
-
-```console
-$ docker buildx bake "https://github.com/tonistiigi/buildx.git#remote-test" "https://github.com/docker/cli.git#v20.10.11" --print
-#1 [internal] load git source https://github.com/tonistiigi/buildx.git#remote-test
-#1 0.429 577303add004dd7efeb13434d69ea030d35f7888       refs/heads/remote-test
-#1 CACHED
-{
-  "target": {
-    "default": {
-      "context": "https://github.com/docker/cli.git#v20.10.11",
-      "dockerfile": "Dockerfile",
-      "dockerfile-inline": "FROM alpine\nWORKDIR /src\nCOPY . .\nRUN ls -l \u0026\u0026 stop\n"
-    }
-  }
-}
-```
-
-```console
-$ docker buildx bake "https://github.com/tonistiigi/buildx.git#remote-test" "https://github.com/docker/cli.git#v20.10.11"
-...
- > [4/4] RUN ls -l && stop:
-#8 0.136 drwxrwxrwx    5 root     root          4096 Jul 27 18:31 kubernetes
-#8 0.136 drwxrwxrwx    3 root     root          4096 Jul 27 18:31 man
-#8 0.136 drwxrwxrwx    2 root     root          4096 Jul 27 18:31 opts
-#8 0.136 -rw-rw-rw-    1 root     root          1893 Jul 27 18:31 poule.yml
-#8 0.136 drwxrwxrwx    7 root     root          4096 Jul 27 18:31 scripts
-#8 0.136 drwxrwxrwx    3 root     root          4096 Jul 27 18:31 service
-#8 0.136 drwxrwxrwx    2 root     root          4096 Jul 27 18:31 templates
-#8 0.136 drwxrwxrwx   10 root     root          4096 Jul 27 18:31 vendor
-#8 0.136 -rwxrwxrwx    1 root     root          9620 Jul 27 18:31 vendor.conf
-#8 0.136 /bin/sh: stop: not found
-```
 
 ### <a name="no-cache"></a> Do not use cache when building the image (--no-cache)
 
@@ -243,26 +123,7 @@ $ docker buildx bake -f docker-bake.hcl --print db
 
 ### <a name="progress"></a> Set type of progress output (--progress)
 
-Same as [`build --progress`](buildx_build.md#progress). Set type of progress
+Same as [`build --progress`](buildx_build.md#progress).
-output (auto, plain, tty). Use plain to show container output (default "auto").
-
-> You can also use the `BUILDKIT_PROGRESS` environment variable to set its value.
-
-The following example uses `plain` output during the build:
-
-```console
-$ docker buildx bake --progress=plain
-
-#2 [backend internal] load build definition from Dockerfile.test
-#2 sha256:de70cb0bb6ed8044f7b9b1b53b67f624e2ccfb93d96bb48b70c1fba562489618
-#2 ...
-
-#1 [database internal] load build definition from Dockerfile.test
-#1 sha256:453cb50abd941762900a1212657a35fc4aad107f5d180b0ee9d93d6b74481bce
-#1 transferring dockerfile: 36B done
-#1 DONE 0.1s
-...
-```
 
 ### <a name="pull"></a> Always attempt to pull a newer version of the image (--pull)
 
@@ -286,701 +147,20 @@ $ docker buildx bake --set foo*.no-cache              # bypass caching only for
 ```
 
 Complete list of overridable fields:
-`args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `labels`, `no-cache`,
-`output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
 
-### File definition
+* `args`
-
+* `cache-from`
-In addition to compose files, bake supports a JSON and an equivalent HCL file
+* `cache-to`
-format for defining build groups and targets.
+* `context`
-
+* `dockerfile`
-A target reflects a single docker build invocation with the same options that
+* `labels`
-you would specify for `docker build`. A group is a grouping of targets.
+* `no-cache`
-
+* `no-cache-filter`
-Multiple files can include the same target and final build options will be
+* `output`
-determined by merging them together.
+* `platform`
-
+* `pull`
-In the case of compose files, each service corresponds to a target.
+* `push`
-
+* `secrets`
-A group can specify its list of targets with the `targets` option. A target can
+* `ssh`
-inherit build options by setting the `inherits` option to the list of targets or
+* `tags`
-groups to inherit from.
+* `target`
-
-Note: Design of bake command is work in progress, the user experience may change
-based on feedback.
-
-HCL definition example:
-
-```hcl
-group "default" {
-    targets = ["db", "webapp-dev"]
-}
-
-target "webapp-dev" {
-    dockerfile = "Dockerfile.webapp"
-    tags = ["docker.io/username/webapp"]
-}
-
-target "webapp-release" {
-    inherits = ["webapp-dev"]
-    platforms = ["linux/amd64", "linux/arm64"]
-}
-
-target "db" {
-    dockerfile = "Dockerfile.db"
-    tags = ["docker.io/username/db"]
-}
-```
-
-Complete list of valid target fields:
-
-`args`, `cache-from`, `cache-to`, `context`, `contexts`, `dockerfile`, `inherits`, `labels`,
-`no-cache`, `no-cache-filter`, `output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
-
-### Global scope attributes
-
-You can define global scope attributes in HCL/JSON and use them for code reuse
-and setting values for variables. This means you can do a "data-only" HCL file
-with the values you want to set/override and use it in the list of regular
-output files.
-
-```hcl
-# docker-bake.hcl
-variable "FOO" {
-    default = "abc"
-}
-
-target "app" {
-    args = {
-        v1 = "pre-${FOO}"
-    }
-}
-```
-
-You can use this file directly:
-
-```console
-$ docker buildx bake --print app
-{
-  "group": {
-    "default": {
-      "targets": [
-        "app"
-      ]
-    }
-  },
-  "target": {
-    "app": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "v1": "pre-abc"
-      }
-    }
-  }
-}
-```
-
-Or create an override configuration file:
-
-```hcl
-# env.hcl
-WHOAMI="myuser"
-FOO="def-${WHOAMI}"
-```
-
-And invoke bake together with both of the files:
-
-```console
-$ docker buildx bake -f docker-bake.hcl -f env.hcl --print app
-{
-  "group": {
-    "default": {
-      "targets": [
-        "app"
-      ]
-    }
-  },
-  "target": {
-    "app": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "v1": "pre-def-myuser"
-      }
-    }
-  }
-}
-```
-
-### HCL variables and functions
-
-Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable),
-the HCL file format also supports variable block definitions. These can be used
-to define variables with values provided by the current environment, or a
-default value when unset.
-
-A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hclparser/stdlib.go)
-provided by [go-cty](https://github.com/zclconf/go-cty/tree/main/cty/function/stdlib)
-are available for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/main/ext/userfunc)
-are also supported.
-
-#### Using interpolation to tag an image with the git sha
-
-Bake supports variable blocks which are assigned to matching environment
-variables or default values.
-
-```hcl
-# docker-bake.hcl
-variable "TAG" {
-    default = "latest"
-}
-
-group "default" {
-    targets = ["webapp"]
-}
-
-target "webapp" {
-    tags = ["docker.io/username/webapp:${TAG}"]
-}
-```
-
-alternatively, in json format:
-
-```json
-{
-  "variable": {
-    "TAG": {
-      "default": "latest"
-    }
-  }
-  "group": {
-    "default": {
-      "targets": ["webapp"]
-    }
-  },
-  "target": {
-    "webapp": {
-      "tags": ["docker.io/username/webapp:${TAG}"]
-    }
-  }
-}
-```
-
-```console
-$ docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "tags": [
-        "docker.io/username/webapp:latest"
-      ]
-    }
-  }
-}
-```
-
-```console
-$ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "tags": [
-        "docker.io/username/webapp:985e9e9"
-      ]
-    }
-  }
-}
-```
-
-#### Using the `add` function
-
-You can use [`go-cty` stdlib functions](https://github.com/zclconf/go-cty/tree/main/cty/function/stdlib).
-Here we are using the `add` function.
-
-```hcl
-# docker-bake.hcl
-variable "TAG" {
-    default = "latest"
-}
-
-group "default" {
-    targets = ["webapp"]
-}
-
-target "webapp" {
-    args = {
-        buildno = "${add(123, 1)}"
-    }
-}
-```
-
-```console
-$ docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "buildno": "124"
-      }
-    }
-  }
-}
-```
-
-#### Defining an `increment` function
-
-It also supports [user defined functions](https://github.com/hashicorp/hcl/tree/main/ext/userfunc).
-The following example defines a simple an `increment` function.
-
-```hcl
-# docker-bake.hcl
-function "increment" {
-    params = [number]
-    result = number + 1
-}
-
-group "default" {
-    targets = ["webapp"]
-}
-
-target "webapp" {
-    args = {
-        buildno = "${increment(123)}"
-    }
-}
-```
-
-```console
-$ docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "buildno": "124"
-      }
-    }
-  }
-}
-```
-
-#### Only adding tags if a variable is not empty using an `notequal`
-
-Here we are using the conditional `notequal` function which is just for
-symmetry with the `equal` one.
-
-```hcl
-# docker-bake.hcl
-variable "TAG" {default="" }
-
-group "default" {
-    targets = [
-        "webapp",
-    ]
-}
-
-target "webapp" {
-    context="."
-    dockerfile="Dockerfile"
-    tags = [
-        "my-image:latest",
-        notequal("",TAG) ? "my-image:${TAG}": "",
-    ]
-}
-```
-
-```console
-$ docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "tags": [
-        "my-image:latest"
-      ]
-    }
-  }
-}
-```
-
-#### Using variables in functions
-
-You can refer variables to other variables like the target blocks can. Stdlib
-functions can also be called but user functions can't at the moment.
-
-```hcl
-# docker-bake.hcl
-variable "REPO" {
-    default = "user/repo"
-}
-
-function "tag" {
-    params = [tag]
-    result = ["${REPO}:${tag}"]
-}
-
-target "webapp" {
-    tags = tag("v1")
-}
-```
-
-```console
-$ docker buildx bake --print webapp
-{
-  "group": {
-    "default": {
-      "targets": [
-        "webapp"
-      ]
-    }
-  },
-  "target": {
-    "webapp": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "tags": [
-        "user/repo:v1"
-      ]
-    }
-  }
-}
-```
-
-#### Using variables in variables across files
-
-When multiple files are specified, one file can use variables defined in
-another file.
-
-```hcl
-# docker-bake1.hcl
-variable "FOO" {
-    default = upper("${BASE}def")
-}
-
-variable "BAR" {
-    default = "-${FOO}-"
-}
-
-target "app" {
-    args = {
-        v1 = "pre-${BAR}"
-    }
-}
-```
-
-```hcl
-# docker-bake2.hcl
-variable "BASE" {
-    default = "abc"
-}
-
-target "app" {
-    args = {
-        v2 = "${FOO}-post"
-    }
-}
-```
-
-```console
-$ docker buildx bake -f docker-bake1.hcl -f docker-bake2.hcl --print app
-{
-  "group": {
-    "default": {
-      "targets": [
-        "app"
-      ]
-    }
-  },
-  "target": {
-    "app": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "v1": "pre--ABCDEF-",
-        "v2": "ABCDEF-post"
-      }
-    }
-  }
-}
-```
-
-#### Using typed variables
-
-Non-string variables are also accepted. The value passed with env is parsed
-into suitable type first.
-
-```hcl
-# docker-bake.hcl
-variable "FOO" {
-    default = 3
-}
-
-variable "IS_FOO" {
-    default = true
-}
-
-target "app" {
-    args = {
-        v1 = FOO > 5 ? "higher" : "lower" 
-        v2 = IS_FOO ? "yes" : "no"
-    }
-}
-```
-
-```console
-$ docker buildx bake --print app
-{
-  "group": {
-    "default": {
-      "targets": [
-        "app"
-      ]
-    }
-  },
-  "target": {
-    "app": {
-      "context": ".",
-      "dockerfile": "Dockerfile",
-      "args": {
-        "v1": "lower",
-        "v2": "yes"
-      }
-    }
-  }
-}
-```
-
-### Defining additional build contexts and linking targets
-
-In addition to the main `context` key that defines the build context each target can also define additional named contexts with a map defined with key `contexts`. These values map to the `--build-context` flag in the [build command](buildx_build.md#build-context).
-
-Inside the Dockerfile these contexts can be used with the `FROM` instruction or `--from` flag.
-
-The value can be a local source directory, container image (with docker-image:// prefix), Git URL, HTTP URL or a name of another target in the Bake file (with target: prefix).
-
-#### Pinning alpine image
-
-```Dockerfile
-# Dockerfile
-FROM alpine
-RUN echo "Hello world"
-```
-
-```hcl
-# docker-bake.hcl
-target "app" {
-    contexts = {
-        alpine = "docker-image://alpine:3.13"
-    }
-}
-```
-
-#### Using a secondary source directory
-
-```Dockerfile
-# Dockerfile
-
-FROM scratch AS src
-
-FROM golang
-COPY --from=src . .
-```
-
-```hcl
-# docker-bake.hcl
-target "app" {
-    contexts = {
-        src = "../path/to/source"
-    }
-}
-```
-
-#### Using a result of one target as a base image in another target
-
-To use a result of one target as a build context of another, specity the target name with `target:` prefix.
-
-```Dockerfile
-# Dockerfile
-FROM baseapp
-RUN echo "Hello world"
-```
-
-```hcl
-# docker-bake.hcl
-
-target "base" {
-    dockerfile = "baseapp.Dockerfile"
-}
-
-target "app" {
-    contexts = {
-        baseapp = "target:base"
-    }
-}
-```
-
-Please note that in most cases you should just use a single multi-stage Dockerfile with multiple targets for similar behavior. This case is recommended when you have multiple Dockerfiles that can't be easily merged into one.
-
-### Extension field with Compose
-
-[Special extension](https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension)
-field `x-bake` can be used in your compose file to evaluate fields that are not
-(yet) available in the [build definition](https://github.com/compose-spec/compose-spec/blob/master/build.md#build-definition).
-
-```yaml
-# docker-compose.yml
-services:
-  addon:
-    image: ct-addon:bar
-    build:
-      context: .
-      dockerfile: ./Dockerfile
-      args:
-        CT_ECR: foo
-        CT_TAG: bar
-      x-bake:
-        tags:
-          - ct-addon:foo
-          - ct-addon:alp
-        platforms:
-          - linux/amd64
-          - linux/arm64
-        cache-from:
-          - user/app:cache
-          - type=local,src=path/to/cache
-        cache-to: type=local,dest=path/to/cache
-        pull: true
-
-  aws:
-    image: ct-fake-aws:bar
-    build:
-      dockerfile: ./aws.Dockerfile
-      args:
-        CT_ECR: foo
-        CT_TAG: bar
-      x-bake:
-        secret:
-          - id=mysecret,src=./secret
-          - id=mysecret2,src=./secret2
-        platforms: linux/arm64
-        output: type=docker
-        no-cache: true
-```
-
-```console
-$ docker buildx bake --print
-{
-  "group": {
-    "default": {
-      "targets": [
-        "aws",
-        "addon"
-      ]
-    }
-  },
-  "target": {
-    "addon": {
-      "context": ".",
-      "dockerfile": "./Dockerfile",
-      "args": {
-        "CT_ECR": "foo",
-        "CT_TAG": "bar"
-      },
-      "tags": [
-        "ct-addon:foo",
-        "ct-addon:alp"
-      ],
-      "cache-from": [
-        "user/app:cache",
-        "type=local,src=path/to/cache"
-      ],
-      "cache-to": [
-        "type=local,dest=path/to/cache"
-      ],
-      "platforms": [
-        "linux/amd64",
-        "linux/arm64"
-      ],
-      "pull": true
-    },
-    "aws": {
-      "context": ".",
-      "dockerfile": "./aws.Dockerfile",
-      "args": {
-        "CT_ECR": "foo",
-        "CT_TAG": "bar"
-      },
-      "tags": [
-        "ct-fake-aws:bar"
-      ],
-      "secret": [
-        "id=mysecret,src=./secret",
-        "id=mysecret2,src=./secret2"
-      ],
-      "platforms": [
-        "linux/arm64"
-      ],
-      "output": [
-        "type=docker"
-      ],
-      "no-cache": true
-    }
-  }
-}
-```
-
-Complete list of valid fields for `x-bake`:
-
-`tags`, `cache-from`, `cache-to`, `secret`, `ssh`, `platforms`, `output`,
-`pull`, `no-cache`, `no-cache-filter`
-
-### Built-in variables
-
-* `BAKE_CMD_CONTEXT` can be used to access the main `context` for bake command
-from a bake file that has been [imported remotely](#file).
-* `BAKE_LOCAL_PLATFORM` returns the current platform's default platform
-specification (e.g. `linux/amd64`).

docs/reference/buildx_build.md

@@ -9,7 +9,7 @@ Start a build
 
 ### Aliases
 
-`build`, `b`
+`docker buildx build`, `docker buildx b`
 
 ### Options
 
@@ -17,6 +17,7 @@ Start a build
 | --- | --- | --- | --- |
 | [`--add-host`](https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host) | `stringSlice` |  | Add a custom host-to-IP mapping (format: `host:ip`) |
 | [`--allow`](#allow) | `stringSlice` |  | Allow extra privileged entitlement (e.g., `network.host`, `security.insecure`) |
+| `--attest` | `stringArray` |  | Attestation parameters (format: `type=sbom,generator=image`) |
 | [`--build-arg`](#build-arg) | `stringArray` |  | Set build-time variables |
 | [`--build-context`](#build-context) | `stringArray` |  | Additional build contexts (e.g., name=path) |
 | [`--builder`](#builder) | `string` |  | Override the configured builder instance |
@@ -25,6 +26,7 @@ Start a build
 | [`--cgroup-parent`](https://docs.docker.com/engine/reference/commandline/build/#use-a-custom-parent-cgroup---cgroup-parent) | `string` |  | Optional parent cgroup for the container |
 | [`-f`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f), [`--file`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f) | `string` |  | Name of the Dockerfile (default: `PATH/Dockerfile`) |
 | `--iidfile` | `string` |  | Write the image ID to the file |
+| `--invoke` | `string` |  | Invoke a command after the build [experimental] |
 | `--label` | `stringArray` |  | Set metadata for an image |
 | [`--load`](#load) |  |  | Shorthand for `--output=type=docker` |
 | [`--metadata-file`](#metadata-file) | `string` |  | Write build result metadata to the file |
@@ -33,10 +35,13 @@ Start a build
 | `--no-cache-filter` | `stringArray` |  | Do not cache specified stages |
 | [`-o`](#output), [`--output`](#output) | `stringArray` |  | Output destination (format: `type=local,dest=path`) |
 | [`--platform`](#platform) | `stringArray` |  | Set target platform for build |
+| `--print` | `string` |  | Print result of information request (e.g., outline, targets) [experimental] |
 | [`--progress`](#progress) | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
-| `--pull` |  |  | Always attempt to pull a newer version of the image |
+| `--provenance` | `string` |  | Shortand for `--attest=type=provenance` |
+| `--pull` |  |  | Always attempt to pull all referenced images |
 | [`--push`](#push) |  |  | Shorthand for `--output=type=registry` |
 | `-q`, `--quiet` |  |  | Suppress the build output and print image ID on success |
+| `--sbom` | `string` |  | Shorthand for `--attest=type=sbom` |
 | [`--secret`](#secret) | `stringArray` |  | Secret to expose to the build (format: `id=mysecret[,src=/local/secret]`) |
 | [`--shm-size`](#shm-size) | `bytes` | `0` | Size of `/dev/shm` |
 | [`--ssh`](#ssh) | `stringArray` |  | SSH agent socket or keys to expose to the build (format: `default\|<id>[=<socket>\|<key>[,<key>]]`) |
@@ -47,6 +52,9 @@ Start a build
 
 <!---MARKER_GEN_END-->
 
+Flags marked with `[experimental]` need to be explicitly enabled by setting the
+`BUILDX_EXPERIMENTAL=1` environment variable.
+
 ## Description
 
 The `buildx build` command starts a build using BuildKit. This command is similar
@@ -54,19 +62,7 @@ to the UI of `docker build` command and takes the same flags and arguments.
 
 For documentation on most of these flags, refer to the [`docker build`
 documentation](https://docs.docker.com/engine/reference/commandline/build/). In
-here we’ll document a subset of the new flags.
+here we'll document a subset of the new flags.
-
-### Built-in build args
-
-* `BUILDKIT_INLINE_BUILDINFO_ATTRS=<bool>` inline build info attributes in image config or not
-* `BUILDKIT_INLINE_CACHE=<bool>` inline cache metadata to image config or not
-* `BUILDKIT_MULTI_PLATFORM=<bool>` opt into determnistic output regardless of multi-platform output or not
-
-```shell
-docker buildx build --build-arg BUILDKIT_MULTI_PLATFORM=1 .
-```
-
-Other useful built-in args can be found in [dockerfile frontend docs](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#built-in-build-args).
 
 ## Examples
 
@@ -80,7 +76,7 @@ Allow extra privileged entitlement. List of entitlements:
 
 - `network.host` - Allows executions with host networking.
 - `security.insecure` - Allows executions without sandbox. See
-  [related Dockerfile extensions](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---securityinsecuresandbox).
+  [related Dockerfile extensions](https://docs.docker.com/engine/reference/builder/#run---securitysandbox).
 
 For entitlements to be enabled, the `buildkitd` daemon also needs to allow them
 with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_create.md#buildkitd-flags))
@@ -99,14 +95,17 @@ Same as [`docker build` command](https://docs.docker.com/engine/reference/comman
 There are also useful built-in build args like:
 
 * `BUILDKIT_CONTEXT_KEEP_GIT_DIR=<bool>` trigger git context to keep the `.git` directory
+* `BUILDKIT_INLINE_BUILDINFO_ATTRS=<bool>` inline build info attributes in image config or not
 * `BUILDKIT_INLINE_CACHE=<bool>` inline cache metadata to image config or not
-* `BUILDKIT_MULTI_PLATFORM=<bool>` opt into determnistic output regardless of multi-platform output or not
+* `BUILDKIT_MULTI_PLATFORM=<bool>` opt into deterministic output regardless of multi-platform output or not
 
 ```console
 $ docker buildx build --build-arg BUILDKIT_MULTI_PLATFORM=1 .
 ```
 
-More built-in build args can be found in [dockerfile frontend docs](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#built-in-build-args).
+> **Note**
+>
+> More built-in build args can be found in [Dockerfile reference docs](https://docs.docker.com/engine/reference/builder/#buildkit-built-in-build-args).
 
 ### <a name="build-context"></a> Additional build contexts (--build-context)
 
@@ -117,7 +116,7 @@ More built-in build args can be found in [dockerfile frontend docs](https://gith
 Define additional build context with specified contents. In Dockerfile the context can be accessed when `FROM name` or `--from=name` is used.
 When Dockerfile defines a stage with the same name it is overwritten.
 
-The value can be a local source directory, container image (with docker-image:// prefix), Git or HTTP URL.
+The value can be a local source directory, [local OCI layout compliant directory](https://github.com/opencontainers/image-spec/blob/main/image-layout.md), container image (with docker-image:// prefix), Git or HTTP URL.
 
 Replace `alpine:latest` with a pinned one:
 
@@ -132,11 +131,38 @@ $ docker buildx build --build-context project=path/to/project/source .
 # docker buildx build --build-context project=https://github.com/myuser/project.git .
 ```
 
-```Dockerfile
+```dockerfile
+# syntax=docker/dockerfile:1
 FROM alpine
 COPY --from=project myfile /
 ```
 
+#### <a name="source-oci-layout"></a> Source image from OCI layout directory
+
+Source an image from a local [OCI layout compliant directory](https://github.com/opencontainers/image-spec/blob/main/image-layout.md):
+
+```console
+$ docker buildx build --build-context foo=oci-layout:///path/to/local/layout@sha256:abcd12345 .
+```
+
+```dockerfile
+# syntax=docker/dockerfile:1
+FROM alpine
+RUN apk add git
+COPY --from=foo myfile /
+
+FROM foo
+```
+
+The OCI layout directory must be compliant with the [OCI layout specification](https://github.com/opencontainers/image-spec/blob/main/image-layout.md). It looks _solely_ for hashes. It does not
+do any form of `image:tag` resolution to find the hash of the manifest; that is up to you.
+
+The format of the `--build-context` must be: `<context>=oci-layout://<path-to-local-layout>@sha256:<hash-of-manifest>`, where:
+
+* `context` is the name of the build context as used in the `Dockerfile`.
+* `path-to-local-layout` is the path on the local machine, where you are running `docker build`, to the spec-compliant OCI layout.
+* `hash-of-manifest` is the hash of the manifest for the image. It can be a single-architecture manifest or a multi-architecture index.
+
 ### <a name="builder"></a> Override the configured builder instance (--builder)
 
 Same as [`buildx --builder`](buildx.md#builder).
@@ -148,7 +174,7 @@ Same as [`buildx --builder`](buildx.md#builder).
 ```
 
 Use an external cache source for a build. Supported types are `registry`,
-`local` and `gha`.
+`local`, `gha` and `s3`.
 
 - [`registry` source](https://github.com/moby/buildkit#registry-push-image-and-cache-separately)
   can import cache from a cache manifest or (special) image configuration on the
@@ -158,6 +184,9 @@ Use an external cache source for a build. Supported types are `registry`,
 - [`gha` source](https://github.com/moby/buildkit#github-actions-cache-experimental)
   can import cache from a previously exported cache with `--cache-to` in your
   GitHub repository
+- [`s3` source](https://github.com/moby/buildkit#s3-cache-experimental)
+  can import cache from a previously exported cache with `--cache-to` in your
+  S3 bucket
 
 If no type is specified, `registry` exporter is used with a specified reference.
 
@@ -169,6 +198,7 @@ $ docker buildx build --cache-from=user/app .
 $ docker buildx build --cache-from=type=registry,ref=user/app .
 $ docker buildx build --cache-from=type=local,src=path/to/cache .
 $ docker buildx build --cache-from=type=gha .
+$ docker buildx build --cache-from=type=s3,region=eu-west-1,bucket=mybucket .
 ```
 
 More info about cache exporters and available attributes: https://github.com/moby/buildkit#export-cache
@@ -180,15 +210,17 @@ More info about cache exporters and available attributes: https://github.com/mob
 ```
 
 Export build cache to an external cache destination. Supported types are
-`registry`, `local`, `inline` and `gha`.
+`registry`, `local`, `inline`, `gha` and `s3`.
 
 - [`registry` type](https://github.com/moby/buildkit#registry-push-image-and-cache-separately) exports build cache to a cache manifest in the registry.
-- [`local` type](https://github.com/moby/buildkit#local-directory-1) type
+- [`local` type](https://github.com/moby/buildkit#local-directory-1) exports
-  exports cache to a local directory on the client.
+  cache to a local directory on the client.
 - [`inline` type](https://github.com/moby/buildkit#inline-push-image-and-cache-together)
-  type writes the cache metadata into the image configuration.
+  writes the cache metadata into the image configuration.
 - [`gha` type](https://github.com/moby/buildkit#github-actions-cache-experimental)
-  type exports cache through the [Github Actions Cache service API](https://github.com/tonistiigi/go-actions-cache/blob/master/api.md#authentication).
+  exports cache through the [GitHub Actions Cache service API](https://github.com/tonistiigi/go-actions-cache/blob/master/api.md#authentication).
+- [`s3` type](https://github.com/moby/buildkit#s3-cache-experimental) exports
+  cache to a S3 bucket.
 
 `docker` driver currently only supports exporting inline cache metadata to image
 configuration. Alternatively, `--build-arg BUILDKIT_INLINE_CACHE=1` can be used
@@ -206,6 +238,7 @@ $ docker buildx build --cache-to=type=inline .
 $ docker buildx build --cache-to=type=registry,ref=user/app .
 $ docker buildx build --cache-to=type=local,dest=path/to/cache .
 $ docker buildx build --cache-to=type=gha .
+$ docker buildx build --cache-to=type=s3,region=eu-west-1,bucket=mybucket .
 ```
 
 More info about cache exporters and available attributes: https://github.com/moby/buildkit#export-cache
@@ -398,8 +431,9 @@ $ docker buildx build --platform=darwin .
 Set type of progress output (auto, plain, tty). Use plain to show container
 output (default "auto").
 
-> You can also use the `BUILDKIT_PROGRESS` environment variable to set
+> **Note**
-> its value.
+>
+> You can also use the `BUILDKIT_PROGRESS` environment variable to set its value.
 
 The following example uses `plain` output during the build:
 
@@ -416,6 +450,11 @@ $ docker buildx build --load --progress=plain .
 ...
 ```
 
+> **Note**
+>
+> Check also our [Color output controls guide](https://github.com/docker/buildx/blob/master/docs/guides/color-output.md)
+> for modifying the colors that are used to output information to the terminal.
+
 ### <a name="push"></a> Push the build result to a registry (--push)
 
 Shorthand for [`--output=type=registry`](#registry). Will automatically push the
@@ -428,7 +467,7 @@ build result to registry.
 ```
 
 Exposes secret to the build. The secret can be used by the build using
-[`RUN --mount=type=secret` mount](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#run---mounttypesecret).
+[`RUN --mount=type=secret` mount](https://docs.docker.com/engine/reference/builder/#run---mounttypesecret).
 
 If `type` is unset it will be detected. Supported types are:
 
@@ -440,7 +479,7 @@ Attribute keys:
 - `src`, `source` - Secret filename. `id` used if unset.
 
 ```dockerfile
-# syntax=docker/dockerfile:1.3
+# syntax=docker/dockerfile:1
 FROM python:3
 RUN pip install awscli
 RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
@@ -459,7 +498,7 @@ Attribute keys:
 - `env` - Secret environment variable. `id` used if unset, otherwise will look for `src`, `source` if `id` unset.
 
 ```dockerfile
-# syntax=docker/dockerfile:1.3
+# syntax=docker/dockerfile:1
 FROM node:alpine
 RUN --mount=type=bind,target=. \
   --mount=type=secret,id=SECRET_TOKEN \
@@ -486,12 +525,12 @@ This can be useful when some commands in your Dockerfile need specific SSH
 authentication (e.g., cloning a private repository).
 
 `--ssh` exposes SSH agent socket or keys to the build and can be used with the
-[`RUN --mount=type=ssh` mount](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#run---mounttypessh).
+[`RUN --mount=type=ssh` mount](https://docs.docker.com/engine/reference/builder/#run---mounttypessh).
 
 Example to access Gitlab using an SSH agent socket:
 
 ```dockerfile
-# syntax=docker/dockerfile:1.3
+# syntax=docker/dockerfile:1
 FROM alpine
 RUN apk add --no-cache openssh-client
 RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan gitlab.com >> ~/.ssh/known_hosts

docs/reference/buildx_create.md

@@ -15,7 +15,7 @@ Create a new builder instance
 | `--bootstrap` |  |  | Boot builder after creation |
 | [`--buildkitd-flags`](#buildkitd-flags) | `string` |  | Flags for buildkitd daemon |
 | [`--config`](#config) | `string` |  | BuildKit config file |
-| [`--driver`](#driver) | `string` |  | Driver to use (available: `docker`, `docker-container`, `kubernetes`) |
+| [`--driver`](#driver) | `string` |  | Driver to use (available: `docker-container`, `kubernetes`, `remote`) |
 | [`--driver-opt`](#driver-opt) | `stringArray` |  | Options for the driver |
 | [`--leave`](#leave) |  |  | Remove a node from builder instead of changing it |
 | [`--name`](#name) | `string` |  | Builder instance name |
@@ -79,6 +79,11 @@ Specifies the configuration file for the buildkitd daemon to use. The configurat
 can be overridden by [`--buildkitd-flags`](#buildkitd-flags).
 See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
 
+If the configuration file is not specified, will look for one by default in:
+* `$BUILDX_CONFIG/buildkitd.default.toml`
+* `$DOCKER_CONFIG/buildx/buildkitd.default.toml`
+* `~/.docker/buildx/buildkitd.default.toml`
+
 Note that if you create a `docker-container` builder and have specified
 certificates for registries in the `buildkitd.toml` configuration, the files
 will be copied into the container under `/etc/buildkit/certs` and configuration
@@ -118,20 +123,39 @@ Unlike `docker` driver, built images will not automatically appear in
 `docker images` and [`build --load`](buildx_build.md#load) needs to be used
 to achieve that.
 
+#### `remote` driver
+
+Uses a remote instance of buildkitd over an arbitrary connection. With this
+driver, you manually create and manage instances of buildkit yourself, and
+configure buildx to point at it.
+
+Unlike `docker` driver, built images will not automatically appear in
+`docker images` and [`build --load`](buildx_build.md#load) needs to be used
+to achieve that.
+
 ### <a name="driver-opt"></a> Set additional driver-specific options (--driver-opt)
 
 ```
 --driver-opt OPTIONS
 ```
 
-Passes additional driver-specific options. Details for each driver:
+Passes additional driver-specific options.
+
+Note: When using quoted values for example for the `nodeselector` or
+`tolerations` options, ensure that quotes are escaped correctly for your shell.
+
+#### `docker` driver
+
+No driver options.
+
+#### `docker-container` driver
 
-- `docker` - No driver options
-- `docker-container`
 - `image=IMAGE` - Sets the container image to be used for running buildkit.
 - `network=NETMODE` - Sets the network mode for running the buildkit container.
 - `cgroup-parent=CGROUP` - Sets the cgroup parent of the buildkit container if docker is using the "cgroupfs" driver. Defaults to `/docker/buildx`.
-- `kubernetes`
+
+#### `kubernetes` driver
+
 - `image=IMAGE` - Sets the container image to be used for running buildkit.
 - `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
 - `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
@@ -139,12 +163,20 @@ Passes additional driver-specific options. Details for each driver:
 - `requests.memory` - Sets the request memory value specified in bytes or with a valid suffix. Example `requests.memory=500Mi`, `requests.memory=4G`
 - `limits.cpu` - Sets the limit CPU value specified in units of Kubernetes CPU. Example `limits.cpu=100m`, `limits.cpu=2`
 - `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G`
-  - `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
+- `"nodeselector=label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
-  - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
+- `"tolerations=key=foo,value=bar;key=foo2,operator=exists;key=foo3,effect=NoSchedule"` - Sets the `Pod` tolerations. Accepts the same values as the kube manifest tolera>tions. Key-value pairs are separated by `,`, tolerations are separated by `;`. No Defaults. Example `tolerations=operator=exists`
+- `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. Needs Kubernetes 1.19 or later. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
 - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
 - `qemu.install=(true|false)` - Install QEMU emulation for multi platforms support.
 - `qemu.image=IMAGE` - Sets the QEMU emulation image. Defaults to `tonistiigi/binfmt:latest`
 
+#### `remote` driver
+
+- `key=KEY` - Sets the TLS client key.
+- `cert=CERT` - Sets the TLS client certificate to present to buildkitd.
+- `cacert=CACERT` - Sets the TLS certificate authority used for validation.
+- `servername=SERVER` - Sets the TLS server name to be used in requests (defaults to the endpoint hostname).
+
 ### <a name="leave"></a> Remove a node from a builder (--leave)
 
 The `--leave` flag changes the action of the command to remove a node from a

docs/reference/buildx_imagetools_create.md

@@ -15,6 +15,7 @@ Create a new image based on source images
 | [`--builder`](#builder) | `string` |  | Override the configured builder instance |
 | [`--dry-run`](#dry-run) |  |  | Show final image instead of pushing |
 | [`-f`](#file), [`--file`](#file) | `stringArray` |  | Read source descriptor from file |
+| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
 | [`-t`](#tag), [`--tag`](#tag) | `stringArray` |  | Set reference for new image |
 
 

docs/reference/buildx_imagetools_inspect.md

@@ -178,6 +178,8 @@ $ docker buildx imagetools inspect moby/buildkit:master --format "{{json .Manife
 {
   "schemaVersion": 2,
   "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+  "digest": "sha256:79d97f205e2799d99a3a8ae2a1ef17acb331e11784262c3faada847dc6972c52",
+  "size": 2010,
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",

docs/reference/buildx_inspect.md

@@ -43,6 +43,10 @@ name of the builder to inspect to get information about that builder.
 The following example shows information about a builder instance named
 `elated_tesla`:
 
+> **Note**
+>
+> Asterisk `*` next to node build platform(s) indicate they had been set manually during `buildx create`. Otherwise, it had been autodetected.
+
 ```console
 $ docker buildx inspect elated_tesla
 
@@ -53,10 +57,12 @@ Nodes:
 Name:      elated_tesla0
 Endpoint:  unix:///var/run/docker.sock
 Status:    running
+Buildkit:  v0.10.3
 Platforms: linux/amd64
 
 Name:      elated_tesla1
 Endpoint:  ssh://ubuntu@1.2.3.4
 Status:    running
-Platforms: linux/arm64, linux/arm/v7, linux/arm/v6
+Buildkit:  v0.10.3
+Platforms: linux/arm64*, linux/arm/v7, linux/arm/v6
 ```

docs/reference/buildx_ls.md

@@ -16,13 +16,12 @@ Lists all builder instances and the nodes for each instance
 
 ```console
 $ docker buildx ls
-
+NAME/NODE       DRIVER/ENDPOINT             STATUS  BUILDKIT PLATFORMS
-NAME/NODE       DRIVER/ENDPOINT             STATUS  PLATFORMS
 elated_tesla *  docker-container
-  elated_tesla0 unix:///var/run/docker.sock running linux/amd64
+  elated_tesla0 unix:///var/run/docker.sock running v0.10.3  linux/amd64
-  elated_tesla1 ssh://ubuntu@1.2.3.4        running linux/arm64*, linux/arm/v7, linux/arm/v6
+  elated_tesla1 ssh://ubuntu@1.2.3.4        running v0.10.3  linux/arm64*, linux/arm/v7, linux/arm/v6
 default         docker
-  default       default                     running linux/amd64
+  default       default                     running 20.10.14 linux/amd64
 ```
 
 Each builder has one or more nodes associated with it. The current builder's

docs/reference/buildx_prune.md

@@ -11,7 +11,7 @@ Remove build cache
 
 | Name | Type | Default | Description |
 | --- | --- | --- | --- |
-| `-a`, `--all` |  |  | Remove all unused images, not just dangling ones |
+| `-a`, `--all` |  |  | Include internal/frontend images |
 | [`--builder`](#builder) | `string` |  | Override the configured builder instance |
 | `--filter` | `filter` |  | Provide filter values (e.g., `until=24h`) |
 | `-f`, `--force` |  |  | Do not prompt for confirmation |
@@ -21,6 +21,26 @@ Remove build cache
 
 <!---MARKER_GEN_END-->
 
+## Description
+
+Clears the build cache of the selected builder.
+
+You can finely control what cache data is kept using:
+
+- The `--filter=until=<duration>` flag to keep images that have been used in
+  the last `<duration>` time.
+
+  `<duration>` is a duration string, e.g. `24h` or `2h30m`, with allowable
+  units of `(h)ours`, `(m)inutes` and `(s)econds`.
+
+- The `--keep-storage=<size>` flag to keep `<size>` bytes of data in the cache.
+
+  `<size>` is a human-readable memory string, e.g. `128mb`, `2gb`, etc. Units
+  are case-insensitive.
+
+- The `--all` flag to allow clearing internal helper images and frontend images
+  set using the `#syntax=` directive or the `BUILDKIT_SYNTAX` build argument.
+
 ## Examples
 
 ### <a name="builder"></a> Override the configured builder instance (--builder)

driver/docker-container/driver.go

@@ -4,11 +4,11 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/docker/buildx/driver"
@@ -16,7 +16,6 @@ import (
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
-	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
@@ -37,7 +36,6 @@ const (
 type Driver struct {
 	driver.InitConfig
 	factory      driver.Factory
-	userNSRemap  bool // true if dockerd is running with userns-remap mode
 	netMode      string
 	image        string
 	cgroupParent string
@@ -84,13 +82,13 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if err != nil {
 			return err
 		}
-		rc, err := d.DockerAPI.ImageCreate(ctx, imageName, types.ImageCreateOptions{
+		rc, err := d.DockerAPI.ImageCreate(ctx, imageName, dockertypes.ImageCreateOptions{
 			RegistryAuth: ra,
 		})
 		if err != nil {
 			return err
 		}
-		_, err = io.Copy(ioutil.Discard, rc)
+		_, err = io.Copy(io.Discard, rc)
 		return err
 	}); err != nil {
 		// image pulling failed, check if it exists in local image store.
@@ -121,13 +119,11 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 				},
 			},
 		}
-		if d.userNSRemap {
-			hc.UsernsMode = "host"
-		}
 		if d.netMode != "" {
 			hc.NetworkMode = container.NetworkMode(d.netMode)
 		}
-		if info, err := d.DockerAPI.Info(ctx); err == nil && info.CgroupDriver == "cgroupfs" {
+		if info, err := d.DockerAPI.Info(ctx); err == nil {
+			if info.CgroupDriver == "cgroupfs" {
 				// Place all buildkit containers inside this cgroup by default so limits can be attached
 				// to all build activity on the host.
 				hc.CgroupParent = "/docker/buildx"
@@ -135,6 +131,19 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 					hc.CgroupParent = d.cgroupParent
 				}
 			}
+
+			secOpts, err := dockertypes.DecodeSecurityOptions(info.SecurityOptions)
+			if err != nil {
+				return err
+			}
+			for _, f := range secOpts {
+				if f.Name == "userns" {
+					hc.UsernsMode = "host"
+					break
+				}
+			}
+
+		}
 		_, err := d.DockerAPI.ContainerCreate(ctx, cfg, hc, &network.NetworkingConfig{}, nil, d.Name)
 		if err != nil {
 			return err
@@ -186,7 +195,7 @@ func (d *Driver) wait(ctx context.Context, l progress.SubLogger) error {
 }
 
 func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
-	rc, err := d.DockerAPI.ContainerLogs(ctx, d.Name, types.ContainerLogsOptions{
+	rc, err := d.DockerAPI.ContainerLogs(ctx, d.Name, dockertypes.ContainerLogsOptions{
 		ShowStdout: true, ShowStderr: true,
 	})
 	if err != nil {
@@ -219,7 +228,7 @@ func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) e
 }
 
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
-	execConfig := types.ExecConfig{
+	execConfig := dockertypes.ExecConfig{
 		Cmd:          cmd,
 		AttachStdin:  true,
 		AttachStdout: true,
@@ -235,7 +244,7 @@ func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, erro
 		return "", nil, errors.New("exec ID empty")
 	}
 
-	resp, err := d.DockerAPI.ContainerExecAttach(ctx, execID, types.ExecStartCheck{})
+	resp, err := d.DockerAPI.ContainerExecAttach(ctx, execID, dockertypes.ExecStartCheck{})
 	if err != nil {
 		return "", nil, err
 	}
@@ -262,7 +271,7 @@ func (d *Driver) run(ctx context.Context, cmd []string, stdout, stderr io.Writer
 }
 
 func (d *Driver) start(ctx context.Context, l progress.SubLogger) error {
-	return d.DockerAPI.ContainerStart(ctx, d.Name, types.ContainerStartOptions{})
+	return d.DockerAPI.ContainerStart(ctx, d.Name, dockertypes.ContainerStartOptions{})
 }
 
 func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
@@ -287,13 +296,29 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
 	}, nil
 }
 
+func (d *Driver) Version(ctx context.Context) (string, error) {
+	bufStdout := &bytes.Buffer{}
+	bufStderr := &bytes.Buffer{}
+	if err := d.run(ctx, []string{"buildkitd", "--version"}, bufStdout, bufStderr); err != nil {
+		if bufStderr.Len() > 0 {
+			return "", errors.Wrap(err, bufStderr.String())
+		}
+		return "", err
+	}
+	version := strings.Fields(bufStdout.String())
+	if len(version) != 4 {
+		return "", errors.Errorf("unexpected version format: %s", bufStdout.String())
+	}
+	return version[2], nil
+}
+
 func (d *Driver) Stop(ctx context.Context, force bool) error {
 	info, err := d.Info(ctx)
 	if err != nil {
 		return err
 	}
 	if info.Status == driver.Running {
-		return d.DockerAPI.ContainerStop(ctx, d.Name, nil)
+		return d.DockerAPI.ContainerStop(ctx, d.Name, container.StopOptions{})
 	}
 	return nil
 }
@@ -341,11 +366,14 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 		return nil, err
 	}
 
-	td, _ := exp.(client.TracerDelegate)
+	var opts []client.ClientOpt
-
+	opts = append(opts, client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
-	return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
 		return conn, nil
-	}), client.WithTracerDelegate(td))
+	}))
+	if td, ok := exp.(client.TracerDelegate); ok {
+		opts = append(opts, client.WithTracerDelegate(td))
+	}
+	return client.New(ctx, "", opts...)
 }
 
 func (d *Driver) Factory() driver.Factory {

driver/docker-container/factory.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/docker/buildx/driver"
-	dockertypes "github.com/docker/docker/api/types"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 )
@@ -29,7 +28,7 @@ func (*factory) Usage() string {
 	return "docker-container"
 }
 
-func (*factory) Priority(ctx context.Context, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
 	if api == nil {
 		return priorityUnsupported
 	}
@@ -41,20 +40,6 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		return nil, errors.Errorf("%s driver requires docker API access", f.Name())
 	}
 	d := &Driver{factory: f, InitConfig: cfg}
-	dockerInfo, err := cfg.DockerAPI.Info(ctx)
-	if err != nil {
-		return nil, err
-	}
-	secOpts, err := dockertypes.DecodeSecurityOptions(dockerInfo.SecurityOptions)
-	if err != nil {
-		return nil, err
-	}
-	for _, f := range secOpts {
-		if f.Name == "userns" {
-			d.userNSRemap = true
-			break
-		}
-	}
 	for k, v := range cfg.DriverOpts {
 		switch {
 		case k == "network":

driver/docker/driver.go

@@ -29,6 +29,14 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
 	}, nil
 }
 
+func (d *Driver) Version(ctx context.Context) (string, error) {
+	v, err := d.DockerAPI.ServerVersion(ctx)
+	if err != nil {
+		return "", errors.Wrapf(driver.ErrNotConnecting, err.Error())
+	}
+	return v.Version, nil
+}
+
 func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
@@ -46,11 +54,22 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 }
 
 func (d *Driver) Features() map[driver.Feature]bool {
+	var useContainerdSnapshotter bool
+	ctx := context.Background()
+	c, err := d.Client(ctx)
+	if err == nil {
+		workers, _ := c.ListWorkers(ctx)
+		for _, w := range workers {
+			if _, ok := w.Labels["org.mobyproject.buildkit.worker.snapshotter"]; ok {
+				useContainerdSnapshotter = true
+			}
+		}
+	}
 	return map[driver.Feature]bool{
-		driver.OCIExporter:    false,
+		driver.OCIExporter:    useContainerdSnapshotter,
-		driver.DockerExporter: false,
+		driver.DockerExporter: useContainerdSnapshotter,
-		driver.CacheExport:    false,
+		driver.CacheExport:    useContainerdSnapshotter,
-		driver.MultiPlatform:  false,
+		driver.MultiPlatform:  useContainerdSnapshotter,
 	}
 }
 

driver/docker/factory.go

@@ -26,7 +26,7 @@ func (*factory) Usage() string {
 	return "docker"
 }
 
-func (*factory) Priority(ctx context.Context, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
 	if api == nil {
 		return priorityUnsupported
 	}

driver/driver.go

@@ -53,6 +53,7 @@ type Driver interface {
 	Factory() Factory
 	Bootstrap(context.Context, progress.Logger) error
 	Info(context.Context) (*Info, error)
+	Version(context.Context) (string, error)
 	Stop(ctx context.Context, force bool) error
 	Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error
 	Client(ctx context.Context) (*client.Client, error)

driver/kubernetes/context/constants.go

@@ -1,4 +1,4 @@
-package kubernetes
+package context
 
 const (
 	// KubernetesEndpoint is the kubernetes endpoint name in a stored context

driver/kubernetes/context/endpoint_test.go

@@ -0,0 +1,224 @@
+package context
+
+import (
+	"os"
+	"testing"
+
+	"github.com/docker/cli/cli/context"
+	"github.com/docker/cli/cli/context/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+)
+
+func testEndpoint(server, defaultNamespace string, ca, cert, key []byte, skipTLSVerify bool) Endpoint {
+	var tlsData *context.TLSData
+	if ca != nil || cert != nil || key != nil {
+		tlsData = &context.TLSData{
+			CA:   ca,
+			Cert: cert,
+			Key:  key,
+		}
+	}
+	return Endpoint{
+		EndpointMeta: EndpointMeta{
+			EndpointMetaBase: context.EndpointMetaBase{
+				Host:          server,
+				SkipTLSVerify: skipTLSVerify,
+			},
+			DefaultNamespace: defaultNamespace,
+		},
+		TLSData: tlsData,
+	}
+}
+
+var testStoreCfg = store.NewConfig(
+	func() interface{} {
+		return &map[string]interface{}{}
+	},
+	store.EndpointTypeGetter(KubernetesEndpoint, func() interface{} { return &EndpointMeta{} }),
+)
+
+func TestSaveLoadContexts(t *testing.T) {
+	storeDir, err := os.MkdirTemp("", "test-load-save-k8-context")
+	require.NoError(t, err)
+	defer os.RemoveAll(storeDir)
+	store := store.New(storeDir, testStoreCfg)
+	require.NoError(t, save(store, testEndpoint("https://test", "test", nil, nil, nil, false), "raw-notls"))
+	require.NoError(t, save(store, testEndpoint("https://test", "test", nil, nil, nil, true), "raw-notls-skip"))
+	require.NoError(t, save(store, testEndpoint("https://test", "test", []byte("ca"), []byte("cert"), []byte("key"), true), "raw-tls"))
+
+	kcFile, err := os.CreateTemp(os.TempDir(), "test-load-save-k8-context")
+	require.NoError(t, err)
+	defer os.Remove(kcFile.Name())
+	defer kcFile.Close()
+	cfg := clientcmdapi.NewConfig()
+	cfg.AuthInfos["user"] = clientcmdapi.NewAuthInfo()
+	cfg.Contexts["context1"] = clientcmdapi.NewContext()
+	cfg.Clusters["cluster1"] = clientcmdapi.NewCluster()
+	cfg.Contexts["context2"] = clientcmdapi.NewContext()
+	cfg.Clusters["cluster2"] = clientcmdapi.NewCluster()
+	cfg.AuthInfos["user"].ClientCertificateData = []byte("cert")
+	cfg.AuthInfos["user"].ClientKeyData = []byte("key")
+	cfg.Clusters["cluster1"].Server = "https://server1"
+	cfg.Clusters["cluster1"].InsecureSkipTLSVerify = true
+	cfg.Clusters["cluster2"].Server = "https://server2"
+	cfg.Clusters["cluster2"].CertificateAuthorityData = []byte("ca")
+	cfg.Contexts["context1"].AuthInfo = "user"
+	cfg.Contexts["context1"].Cluster = "cluster1"
+	cfg.Contexts["context1"].Namespace = "namespace1"
+	cfg.Contexts["context2"].AuthInfo = "user"
+	cfg.Contexts["context2"].Cluster = "cluster2"
+	cfg.Contexts["context2"].Namespace = "namespace2"
+	cfg.CurrentContext = "context1"
+	cfgData, err := clientcmd.Write(*cfg)
+	require.NoError(t, err)
+	_, err = kcFile.Write(cfgData)
+	require.NoError(t, err)
+	kcFile.Close()
+
+	epDefault, err := FromKubeConfig(kcFile.Name(), "", "")
+	require.NoError(t, err)
+	epContext2, err := FromKubeConfig(kcFile.Name(), "context2", "namespace-override")
+	require.NoError(t, err)
+	require.NoError(t, save(store, epDefault, "embed-default-context"))
+	require.NoError(t, save(store, epContext2, "embed-context2"))
+
+	rawNoTLSMeta, err := store.GetMetadata("raw-notls")
+	require.NoError(t, err)
+	rawNoTLSSkipMeta, err := store.GetMetadata("raw-notls-skip")
+	require.NoError(t, err)
+	rawTLSMeta, err := store.GetMetadata("raw-tls")
+	require.NoError(t, err)
+	embededDefaultMeta, err := store.GetMetadata("embed-default-context")
+	require.NoError(t, err)
+	embededContext2Meta, err := store.GetMetadata("embed-context2")
+	require.NoError(t, err)
+
+	rawNoTLS := EndpointFromContext(rawNoTLSMeta)
+	rawNoTLSSkip := EndpointFromContext(rawNoTLSSkipMeta)
+	rawTLS := EndpointFromContext(rawTLSMeta)
+	embededDefault := EndpointFromContext(embededDefaultMeta)
+	embededContext2 := EndpointFromContext(embededContext2Meta)
+
+	rawNoTLSEP, err := rawNoTLS.WithTLSData(store, "raw-notls")
+	require.NoError(t, err)
+	checkClientConfig(t, rawNoTLSEP, "https://test", "test", nil, nil, nil, false)
+	rawNoTLSSkipEP, err := rawNoTLSSkip.WithTLSData(store, "raw-notls-skip")
+	require.NoError(t, err)
+	checkClientConfig(t, rawNoTLSSkipEP, "https://test", "test", nil, nil, nil, true)
+	rawTLSEP, err := rawTLS.WithTLSData(store, "raw-tls")
+	require.NoError(t, err)
+	checkClientConfig(t, rawTLSEP, "https://test", "test", []byte("ca"), []byte("cert"), []byte("key"), true)
+	embededDefaultEP, err := embededDefault.WithTLSData(store, "embed-default-context")
+	require.NoError(t, err)
+	checkClientConfig(t, embededDefaultEP, "https://server1", "namespace1", nil, []byte("cert"), []byte("key"), true)
+	embededContext2EP, err := embededContext2.WithTLSData(store, "embed-context2")
+	require.NoError(t, err)
+	checkClientConfig(t, embededContext2EP, "https://server2", "namespace-override", []byte("ca"), []byte("cert"), []byte("key"), false)
+}
+
+func checkClientConfig(t *testing.T, ep Endpoint, server, namespace string, ca, cert, key []byte, skipTLSVerify bool) {
+	config := ep.KubernetesConfig()
+	cfg, err := config.ClientConfig()
+	require.NoError(t, err)
+	ns, _, _ := config.Namespace()
+	assert.Equal(t, server, cfg.Host)
+	assert.Equal(t, namespace, ns)
+	assert.Equal(t, ca, cfg.CAData)
+	assert.Equal(t, cert, cfg.CertData)
+	assert.Equal(t, key, cfg.KeyData)
+	assert.Equal(t, skipTLSVerify, cfg.Insecure)
+}
+
+func save(s store.Writer, ep Endpoint, name string) error {
+	meta := store.Metadata{
+		Endpoints: map[string]interface{}{
+			KubernetesEndpoint: ep.EndpointMeta,
+		},
+		Name: name,
+	}
+	if err := s.CreateOrUpdate(meta); err != nil {
+		return err
+	}
+	return s.ResetEndpointTLSMaterial(name, KubernetesEndpoint, ep.TLSData.ToStoreTLSData())
+}
+
+func TestSaveLoadGKEConfig(t *testing.T) {
+	storeDir, err := os.MkdirTemp("", t.Name())
+	require.NoError(t, err)
+	defer os.RemoveAll(storeDir)
+	store := store.New(storeDir, testStoreCfg)
+	cfg, err := clientcmd.LoadFromFile("fixtures/gke-kubeconfig")
+	require.NoError(t, err)
+	clientCfg := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{})
+	expectedCfg, err := clientCfg.ClientConfig()
+	require.NoError(t, err)
+	ep, err := FromKubeConfig("fixtures/gke-kubeconfig", "", "")
+	require.NoError(t, err)
+	require.NoError(t, save(store, ep, "gke-context"))
+	persistedMetadata, err := store.GetMetadata("gke-context")
+	require.NoError(t, err)
+	persistedEPMeta := EndpointFromContext(persistedMetadata)
+	assert.True(t, persistedEPMeta != nil)
+	persistedEP, err := persistedEPMeta.WithTLSData(store, "gke-context")
+	require.NoError(t, err)
+	persistedCfg := persistedEP.KubernetesConfig()
+	actualCfg, err := persistedCfg.ClientConfig()
+	require.NoError(t, err)
+	assert.Equal(t, expectedCfg.AuthProvider, actualCfg.AuthProvider)
+}
+
+func TestSaveLoadEKSConfig(t *testing.T) {
+	storeDir, err := os.MkdirTemp("", t.Name())
+	require.NoError(t, err)
+	defer os.RemoveAll(storeDir)
+	store := store.New(storeDir, testStoreCfg)
+	cfg, err := clientcmd.LoadFromFile("fixtures/eks-kubeconfig")
+	require.NoError(t, err)
+	clientCfg := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{})
+	expectedCfg, err := clientCfg.ClientConfig()
+	require.NoError(t, err)
+	ep, err := FromKubeConfig("fixtures/eks-kubeconfig", "", "")
+	require.NoError(t, err)
+	require.NoError(t, save(store, ep, "eks-context"))
+	persistedMetadata, err := store.GetMetadata("eks-context")
+	require.NoError(t, err)
+	persistedEPMeta := EndpointFromContext(persistedMetadata)
+	assert.True(t, persistedEPMeta != nil)
+	persistedEP, err := persistedEPMeta.WithTLSData(store, "eks-context")
+	require.NoError(t, err)
+	persistedCfg := persistedEP.KubernetesConfig()
+	actualCfg, err := persistedCfg.ClientConfig()
+	require.NoError(t, err)
+	assert.Equal(t, expectedCfg.ExecProvider, actualCfg.ExecProvider)
+}
+
+func TestSaveLoadK3SConfig(t *testing.T) {
+	storeDir, err := os.MkdirTemp("", t.Name())
+	require.NoError(t, err)
+	defer os.RemoveAll(storeDir)
+	store := store.New(storeDir, testStoreCfg)
+	cfg, err := clientcmd.LoadFromFile("fixtures/k3s-kubeconfig")
+	require.NoError(t, err)
+	clientCfg := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{})
+	expectedCfg, err := clientCfg.ClientConfig()
+	require.NoError(t, err)
+	ep, err := FromKubeConfig("fixtures/k3s-kubeconfig", "", "")
+	require.NoError(t, err)
+	require.NoError(t, save(store, ep, "k3s-context"))
+	persistedMetadata, err := store.GetMetadata("k3s-context")
+	require.NoError(t, err)
+	persistedEPMeta := EndpointFromContext(persistedMetadata)
+	assert.True(t, persistedEPMeta != nil)
+	persistedEP, err := persistedEPMeta.WithTLSData(store, "k3s-context")
+	require.NoError(t, err)
+	persistedCfg := persistedEP.KubernetesConfig()
+	actualCfg, err := persistedCfg.ClientConfig()
+	require.NoError(t, err)
+	assert.True(t, len(actualCfg.Username) > 0)
+	assert.True(t, len(actualCfg.Password) > 0)
+	assert.Equal(t, expectedCfg.Username, actualCfg.Username)
+	assert.Equal(t, expectedCfg.Password, actualCfg.Password)
+}

driver/kubernetes/context/fixtures/eks-kubeconfig

@@ -0,0 +1,23 @@
+  apiVersion: v1
+  clusters:
+  - cluster:
+      server: https://some-server      
+    name: kubernetes
+  contexts:
+  - context:
+      cluster: kubernetes
+      user: aws
+    name: aws
+  current-context: aws
+  kind: Config
+  preferences: {}
+  users:
+  - name: aws
+    user:
+      exec:
+        apiVersion: client.authentication.k8s.io/v1alpha1
+        command: heptio-authenticator-aws
+        args:
+          - "token"
+          - "-i"
+          - "eks-cf"