Merge pull request #676 from tonistiigi/vol-delete

don't error on deleting old build containers without state volume

.dockerignore

@@ -1,2 +1,3 @@
 bin/
 cross-out/
+release-out/

.github/workflows/build.yml

@@ -12,9 +12,10 @@ on:
       - 'master'
 
 env:
+  REPO_SLUG: "docker/buildx-bin"
   REPO_SLUG_ORIGIN: "moby/buildkit:master"
   CACHEKEY_BINARIES: "binaries"
-  PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
+  PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64"
 
 jobs:
   base:
@@ -118,12 +119,14 @@ jobs:
         run: |
           make cross
         env:
-          TARGETPLATFORM: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
+          TARGETPLATFORM: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64
           CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
 
   binaries:
     runs-on: ubuntu-latest
     needs: [test, cross]
+    env:
+      RELEASE_OUT: ./release-out
     steps:
       -
         name: Checkout
@@ -158,21 +161,46 @@ jobs:
       -
         name: Build ${{ steps.prep.outputs.tag }}
         run: |
-          ./hack/release "${{ steps.prep.outputs.tag }}" release-out
+          ./hack/release ${{ env.RELEASE_OUT }}
         env:
-          PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
+          PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64
           CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
-      -
-        name: Move artifacts
-        run: |
-          mv ./release-out/**/* ./release-out/
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
           name: buildx
-          path: ./release-out/*
+          path: ${{ env.RELEASE_OUT }}/*
           if-no-files-found: error
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ env.REPO_SLUG }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+      -
+        name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          target: binaries
+          push: ${{ github.event_name != 'pull_request' }}
+          cache-from: type=local,src=/tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
+          platforms: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
       -
         name: GitHub Release
         if: startsWith(github.ref, 'refs/tags/v')
@@ -181,5 +209,5 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           draft: true
-          files: ./release-out/*
+          files: ${{ env.RELEASE_OUT }}/*
           name: ${{ steps.prep.outputs.tag }}

.github/workflows/validate.yml

@@ -23,6 +23,7 @@ jobs:
         target:
           - lint
           - validate-vendor
+          - validate-docs
     steps:
       -
         name: Checkout

.gitignore

@@ -1,3 +1,4 @@
 bin
 coverage
 cross-out
+release-out

.golangci.yml

@@ -0,0 +1,30 @@
+run:
+  timeout: 10m
+  skip-files:
+    - ".*\\.pb\\.go$"
+
+  modules-download-mode: vendor
+
+  build-tags:
+
+linters:
+  enable:
+    - gofmt
+    - govet
+    - deadcode
+    - goimports
+    - ineffassign
+    - misspell
+    - unused
+    - varcheck
+    - golint
+    - staticcheck
+    - typecheck
+    - structcheck
+  disable-all: true
+
+issues:
+  exclude-rules:
+    - linters:
+        - golint
+      text: "stutters"

Dockerfile

@@ -1,15 +1,25 @@
-# syntax=docker/dockerfile:1.1-experimental
+# syntax=docker/dockerfile:1.2
 
-ARG DOCKERD_VERSION=19.03-rc
+ARG DOCKERD_VERSION=19.03
 ARG CLI_VERSION=19.03
 
 FROM docker:$DOCKERD_VERSION AS dockerd-release
 
-# xgo is a helper for golang cross-compilation
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:golang@sha256:6f7d999551dd471b58f70716754290495690efa8421e0a1fcf18eb11d0c0a537 AS xgo
+# xx is a helper for cross-compilation
+FROM --platform=$BUILDPLATFORM tonistiigi/xx@sha256:21a61be4744f6531cb5f33b0e6f40ede41fa3a1b8c82d5946178f80cc84bfc04 AS xx
 
-FROM --platform=$BUILDPLATFORM golang:1.13-alpine AS gobase
-COPY --from=xgo / /
+FROM --platform=$BUILDPLATFORM golang:1.16-alpine AS golatest
+
+FROM golatest AS go-linux
+FROM golatest AS go-darwin
+FROM golatest AS go-windows-amd64
+FROM golatest AS go-windows-386
+FROM golatest AS go-windows-arm
+FROM --platform=$BUILDPLATFORM golang:1.17beta1-alpine AS go-windows-arm64
+FROM go-windows-${TARGETARCH} AS go-windows
+
+FROM go-${TARGETOS} AS gobase
+COPY --from=xx / /
 RUN apk add --no-cache file git
 ENV GOFLAGS=-mod=vendor
 WORKDIR /src
@@ -26,8 +36,8 @@ ARG TARGETPLATFORM
 RUN --mount=target=. --mount=target=/root/.cache,type=cache \
   --mount=target=/go/pkg/mod,type=cache \
   --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildx-version \
-  set -x; go build -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildx ./cmd/buildx && \
-  file /usr/bin/buildx && file /usr/bin/buildx | egrep "statically linked|Mach-O|Windows"
+  set -x; xx-go build -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildx ./cmd/buildx && \
+  xx-verify --static /usr/bin/buildx
 
 FROM buildx-build AS integration-tests
 COPY . .

MAINTAINERS

@@ -150,6 +150,8 @@ made through a pull request.
 	[Org.Maintainers]
 
 		people = [
+			"akihirosuda",
+			"crazy-max",
 			"tiborvass",
 			"tonistiigi",
 		]
@@ -176,6 +178,16 @@ made through a pull request.
 # All other sections should refer to people by their canonical key
 # in the people section.
 
+	[people.akihirosuda]
+	Name = "Akihiro Suda"
+	Email = "akihiro.suda.cz@hco.ntt.co.jp"
+	GitHub = "AkihiroSuda"
+
+	[people.crazy-max]
+	Name = "Kevin Alvarez"
+	Email = "contact@crazymax.dev"
+	GitHub = "crazy-max"
+
 	[people.thajeztah]
 	Name = "Sebastiaan van Stijn"
 	Email = "github@gone.nl"

Makefile

@@ -12,7 +12,7 @@ cross:
 
 install: binaries
 	mkdir -p ~/.docker/cli-plugins
-	cp bin/buildx ~/.docker/cli-plugins/docker-buildx
+	install bin/buildx ~/.docker/cli-plugins/docker-buildx
 
 lint:
 	./hack/lint
@@ -23,12 +23,18 @@ test:
 validate-vendor:
 	./hack/validate-vendor
 	
-validate-all: lint test validate-vendor
+validate-docs:
+	./hack/validate-docs
+	
+validate-all: lint test validate-vendor validate-docs
 
 vendor:
 	./hack/update-vendor
 
+docs:
+	./hack/update-docs
+
 generate-authors:
 	./hack/generate-authors
 
-.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors
+.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs

README.md

@@ -14,43 +14,45 @@ Key features:
 - Multiple builder instance support
 - Multi-node builds for cross-platform images
 - Compose build support
-- WIP: High-level build constructs (`bake`)
+- High-level build constructs (`bake`)
 - In-container driver support (both Docker and Kubernetes)
 
 # Table of Contents
 
 - [Installing](#installing)
+  - [Docker](#docker)
+  - [Binary release](#binary-release)
+  - [From `Dockerfile`](#from-dockerfile)
 - [Building](#building)
-    + [with Docker 18.09+](#with-docker-1809)
-    + [with buildx or Docker 19.03](#with-buildx-or-docker-1903)
+  - [with Docker 18.09+](#with-docker-1809)
+  - [with buildx or Docker 19.03](#with-buildx-or-docker-1903)
 - [Getting started](#getting-started)
-  * [Building with buildx](#building-with-buildx)
-  * [Working with builder instances](#working-with-builder-instances)
-  * [Building multi-platform images](#building-multi-platform-images)
-  * [High-level build options](#high-level-build-options)
-- [Documentation](#documentation)
-    + [`buildx build [OPTIONS] PATH | URL | -`](#buildx-build-options-path--url---)
-    + [`buildx create [OPTIONS] [CONTEXT|ENDPOINT]`](#buildx-create-options-contextendpoint)
-    + [`buildx use NAME`](#buildx-use-name)
-    + [`buildx inspect [NAME]`](#buildx-inspect-name)
-    + [`buildx ls`](#buildx-ls)
-    + [`buildx stop [NAME]`](#buildx-stop-name)
-    + [`buildx rm [NAME]`](#buildx-rm-name)
-    + [`buildx bake [OPTIONS] [TARGET...]`](#buildx-bake-options-target)
-    + [`buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`](#buildx-imagetools-create-options-source-source)
-    + [`buildx imagetools inspect NAME`](#buildx-imagetools-inspect-name)
+  - [Building with buildx](#building-with-buildx)
+  - [Working with builder instances](#working-with-builder-instances)
+  - [Building multi-platform images](#building-multi-platform-images)
+  - [High-level build options](#high-level-build-options)
+- [Documentation](docs/reference)
+  - [`buildx build [OPTIONS] PATH | URL | -`](docs/reference/buildx_build.md)
+  - [`buildx create [OPTIONS] [CONTEXT|ENDPOINT]`](docs/reference/buildx_create.md)
+  - [`buildx use NAME`](docs/reference/buildx_use.md)
+  - [`buildx inspect [NAME]`](docs/reference/buildx_inspect.md)
+  - [`buildx ls`](docs/reference/buildx_ls.md)
+  - [`buildx stop [NAME]`](docs/reference/buildx_stop.md)
+  - [`buildx rm [NAME]`](docs/reference/buildx_rm.md)
+  - [`buildx bake [OPTIONS] [TARGET...]`](docs/reference/buildx_bake.md)
+  - [`buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`](docs/reference/buildx_imagetools_create.md)
+  - [`buildx imagetools inspect NAME`](docs/reference/buildx_imagetools_inspect.md)
 - [Setting buildx as default builder in Docker 19.03+](#setting-buildx-as-default-builder-in-docker-1903)
 - [Contributing](#contributing)
 
 
 # Installing
 
-Using `buildx` as a docker CLI plugin requires using Docker 19.03. A limited set of functionality works with older versions of Docker when invoking the binary directly.
+Using `buildx` as a docker CLI plugin requires using Docker 19.03 or newer. A limited set of functionality works with older versions of Docker when invoking the binary directly.
 
-### Docker CE
+### Docker
 
-`buildx` comes bundled with Docker CE starting with 19.03, but requires experimental mode to be enabled on the Docker CLI.
-To enable it, `"experimental": "enabled"` can be added to the CLI configuration file `~/.docker/config.json`. An alternative is to set the `DOCKER_CLI_EXPERIMENTAL=enabled` environment variable.
+`buildx` comes bundled with Docker Desktop and in latest Docker CE packages, but may not be included in all Linux distros (in which case follow the binary release instructions).
 
 ### Binary release
 
@@ -61,15 +63,21 @@ Change the permission to execute:
 chmod a+x ~/.docker/cli-plugins/docker-buildx
 ```
 
+### From `Dockerfile`
+
+Here is how to use buildx inside a Dockerfile through the [`docker/buildx-bin`](https://hub.docker.com/r/docker/buildx-bin) image:
+
+```Dockerfile
+FROM docker
+COPY --from=docker/buildx-bin:latest /buildx /usr/libexec/docker/cli-plugins/docker-buildx
+RUN docker buildx version
+```
+
+
 # Building
 
-### with Docker 18.09+
-```
-$ git clone git://github.com/docker/buildx && cd buildx
-$ make install
-```
 
-### with buildx or Docker 19.03
+### with buildx or Docker 19.03+
 ```
 $ export DOCKER_BUILDKIT=1
 $ docker build --platform=local -o . git://github.com/docker/buildx
@@ -77,6 +85,12 @@ $ mkdir -p ~/.docker/cli-plugins
 $ mv buildx ~/.docker/cli-plugins/docker-buildx
 ```
 
+### with Docker 18.09+
+```
+$ git clone git://github.com/docker/buildx && cd buildx
+$ make install
+```
+
 # Getting started
 
 ## Building with buildx
@@ -96,7 +110,7 @@ Buildx will always build using the BuildKit engine and does not require `DOCKER_
 
 Buildx build command supports the features available for `docker build` including the new features in Docker 19.03 such as outputs configuration, inline build caching or specifying target platform. In addition, buildx supports new features not yet available for regular `docker build` like building manifest lists, distributed caching, exporting build results to OCI image tarballs etc.
 
-Buildx is supposed to be flexible and can be run in different configurations that are exposed through a driver concept. Currently, we support a "docker" driver that uses the BuildKit library bundled into the docker daemon binary, and a "docker-container" driver that automatically launches BuildKit inside a Docker container. We plan to add more drivers in the future, for example, one that would allow running buildx inside an (unprivileged) container.
+Buildx is supposed to be flexible and can be run in different configurations that are exposed through a driver concept. Currently, we support a "docker" driver that uses the BuildKit library bundled into the Docker daemon binary, and a "docker-container" driver that automatically launches BuildKit inside a Docker container. We plan to add more drivers in the future, for example, one that would allow running buildx inside an (unprivileged) container.
 
 The user experience of using buildx is very similar across drivers, but there are some features that are not currently supported by the "docker" driver, because the BuildKit library bundled into docker daemon currently uses a different storage component. In contrast, all images built with "docker" driver are automatically added to the "docker images" view by default, whereas when using other drivers the method for outputting an image needs to be selected with `--output`.
 
@@ -109,7 +123,7 @@ Buildx allows you to create new instances of isolated builders. This can be used
 
 New instances can be created with `docker buildx create` command. This will create a new builder instance with a single node based on your current configuration. To use a remote node you can specify the `DOCKER_HOST` or remote context name while creating the new builder. After creating a new instance you can manage its lifecycle with the `inspect`, `stop` and `rm` commands and list all available builders with `ls`. After creating a new builder you can also append new nodes to it.
 
-To switch between different builders use `docker buildx use <name>`. After running this command the build commands would automatically keep using this builder.
+To switch between different builders, use `docker buildx use <name>`. After running this command the build commands would automatically keep using this builder.
 
 Docker 19.03 also features a new `docker context` command that can be used for giving names for remote Docker API endpoints. Buildx integrates with `docker context` so that all of your contexts automatically get a default builder instance. While creating a new builder instance or when adding a node to it you can also set the context name as the target.
 
@@ -121,7 +135,7 @@ When invoking a build, the `--platform` flag can be used to specify the target p
 
 Multi-platform images can be built by mainly three different strategies that are all supported by buildx and Dockerfiles. You can use the QEMU emulation support in the kernel, build on multiple native nodes using the same builder instance or use a stage in Dockerfile to cross-compile to different architectures.
 
-QEMU is the easiest way to get started if your node already supports it (e.g. if you are using Docker Desktop). It requires no changes to your Dockerfile and BuildKit will automatically detect the secondary architectures that are available. When BuildKit needs to run a binary for a different architecture it will automatically load it through a binary registered in the binfmt_misc handler. For QEMU binaries registered with binfmt_misc on the host OS to work transparently inside containers they must be registed with the fix_binary flag. This requires a kernel >= 4.8 and binfmt-support >= 2.1.7. You can check for proper registration by checking if `F` is among the flags in `/proc/sys/fs/binfmt_misc/qemu-*`. While Docker Desktop comes preconfigured with binfmt_misc support for additional platforms, for other installations it likely needs to be installed using [`tonistiigi/binfmt`](https://github.com/tonistiigi/binfmt) image.
+QEMU is the easiest way to get started if your node already supports it (e.g. if you are using Docker Desktop). It requires no changes to your Dockerfile and BuildKit will automatically detect the secondary architectures that are available. When BuildKit needs to run a binary for a different architecture it will automatically load it through a binary registered in the binfmt_misc handler. For QEMU binaries registered with binfmt_misc on the host OS to work transparently inside containers they must be registered with the fix_binary flag. This requires a kernel >= 4.8 and binfmt-support >= 2.1.7. You can check for proper registration by checking if `F` is among the flags in `/proc/sys/fs/binfmt_misc/qemu-*`. While Docker Desktop comes preconfigured with binfmt_misc support for additional platforms, for other installations it likely needs to be installed using [`tonistiigi/binfmt`](https://github.com/tonistiigi/binfmt) image.
 
 ```
 $ docker run --privileged --rm tonistiigi/binfmt --install all
@@ -159,696 +173,7 @@ Currently, the bake command supports building images from compose files, similar
 
 There is also support for custom build rules from HCL/JSON files allowing better code reuse and different target groups. The design of bake is in very early stages and we are looking for feedback from users.
 
-
-
-# Documentation
-
-### `buildx build [OPTIONS] PATH | URL | -`
-
-The `buildx build` command starts a build using BuildKit. This command is similar to the UI of `docker build` command and takes the same flags and arguments.
-
-Options:
-
-| Flag | Description |
-| --- | --- |
-| --add-host []         | Add a custom host-to-IP mapping (host:ip)
-| --allow []        | Allow extra privileged entitlement, e.g. network.host, security.insecure
-| --build-arg []    | Set build-time variables
-| --cache-from []   | External cache sources (eg. user/app:cache, type=local,src=path/to/dir)
-| --cache-to []     | Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)
-| --file string              | Name of the Dockerfile (Default is 'PATH/Dockerfile')
-| --iidfile string           | Write the image ID to the file
-| --label []        | Set metadata for an image
-| --load                     | Shorthand for --output=type=docker
-| --network string           | Set the networking mode for the RUN instructions during build (default "default")
-| --no-cache                 | Do not use cache when building the image
-| --output []       | Output destination (format: type=local,dest=path)
-| --platform []     | Set target platform for build
-| --progress string          | Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto")
-| --pull                     | Always attempt to pull a newer version of the image
-| --push                     | Shorthand for --output=type=registry
-| --secret []       | Secret file to expose to the build: id=mysecret,src=/local/secret
-| --ssh []          | SSH agent socket or keys to expose to the build (format: default\|&#60;id&#62;[=&#60;socket&#62;\|&#60;key&#62;[,&#60;key&#62;]])
-| --tag []          | Name and optionally a tag in the 'name:tag' format
-| --target string            | Set the target build stage to build.
-
-For documentation on most of these flags refer to `docker build` documentation in https://docs.docker.com/engine/reference/commandline/build/ . In here we’ll document a subset of the new flags.
-
-####  ` --platform=value[,value]`
-
-Set the target platform for the build. All `FROM` commands inside the Dockerfile without their own `--platform` flag will pull base images for this platform and this value will also be the platform of the resulting image. The default value will be the current platform of the buildkit daemon. 
-
-When using `docker-container` driver with `buildx`, this flag can accept multiple values as an input separated by a comma. With multiple values the result will be built for all of the specified platforms and joined together into a single manifest list.
-
-If the`Dockerfile` needs to invoke the `RUN` command, the builder needs runtime support for the specified platform. In a clean setup, you can only execute `RUN` commands for your system architecture. If your kernel supports binfmt_misc https://en.wikipedia.org/wiki/Binfmt_misc   launchers for secondary architectures buildx will pick them up automatically. Docker desktop releases come with binfmt_misc automatically configured for `arm64` and `arm` architectures. You can see what runtime platforms your current builder instance supports by running `docker buildx inspect --bootstrap`.
-
-Inside a `Dockerfile`, you can access the current platform value through `TARGETPLATFORM` build argument. Please refer to `docker build` documentation for the full description of automatic platform argument variants https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope .
-
-The formatting for the platform specifier is defined in https://github.com/containerd/containerd/blob/v1.2.6/platforms/platforms.go#L63  .
-
-Examples:
-```
-docker buildx build --platform=linux/arm64 .
-docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 .
-docker buildx build --platform=darwin .
-```
-
-#### `-o, --output=[PATH,-,type=TYPE[,KEY=VALUE]`
-
-Sets the export action for the build result. In `docker build` all builds finish by creating a container image and exporting it to `docker images`. `buildx` makes this step configurable allowing results to be exported directly to the client, oci image tarballs, registry etc.
-
-Buildx with `docker` driver currently only supports local, tarball exporter and image exporter. `docker-container` driver supports all the exporters.
-
-If just the path is specified as a value, `buildx` will use the local exporter with this path as the destination. If the value is “-”, `buildx` will use `tar` exporter and write to `stdout`.
-
-Examples:
-
-```
-docker buildx build -o . .
-docker buildx build -o outdir .
-docker buildx build -o - - > out.tar
-docker buildx build -o type=docker .
-docker buildx build -o type=docker,dest=- . > myimage.tar
-docker buildx build -t tonistiigi/foo -o type=registry 
-````
-
-Supported exported types are:
-
-##### `local`
-
-The `local` export type writes all result files to a directory on the client. The new files will be owned by the current user. On multi-platform builds, all results will be put in subdirectories by their platform.
-
-Attribute key:
-
-- `dest` - destination directory where files will be written
-
-##### `tar`
-
-The `tar` export type writes all result files as a single tarball on the client. On multi-platform builds all results will be put in subdirectories by their platform.
-
-Attribute key:
-
-- `dest` - destination path where tarball will be written. “-” writes to stdout.
-
-##### `oci`
-
-The `oci` export type writes the result image or manifest list as an OCI image layout tarball https://github.com/opencontainers/image-spec/blob/master/image-layout.md on the client.
-
-Attribute key:
-
-- `dest` - destination path where tarball will be written. “-” writes to stdout.
-
-##### `docker`
-
-The `docker` export type writes the single-platform result image as a Docker image specification tarball https://github.com/moby/moby/blob/master/image/spec/v1.2.md on the client. Tarballs created by this exporter are also OCI compatible.
-
-Currently, multi-platform images cannot be exported with the `docker` export type. The most common usecase for multi-platform images is to directly push to a registry (see [`registry`](#registry)).
-
-Attribute keys:
-
-- `dest` - destination path where tarball will be written. If not specified the tar will be loaded automatically to the current docker instance.
-- `context` - name for the docker context where to import the result
-
-##### `image`
-
-The `image` exporter writes the build result as an image or a manifest list. When using `docker` driver the image will appear in `docker images`. Optionally image can be automatically pushed to a registry by specifying attributes.
-
-Attribute keys:
-
-- `name` - name (references) for the new image.
-- `push` - boolean to automatically push the image.
-
-##### `registry` 
-
-The `registry` exporter is a shortcut for `type=image,push=true`.
-
-
-#### `--push`
-
-Shorthand for [`--output=type=registry`](#registry). Will automatically push the build result to registry.
-
-#### `--load`
-
-Shorthand for [`--output=type=docker`](#docker). Will automatically load the single-platform build result to `docker images`.
-
-#### `--cache-from=[NAME|type=TYPE[,KEY=VALUE]]`
-
-Use an external cache source for a build. Supported types are `registry` and `local`. The `registry` source can import cache from a cache manifest or (special) image configuration on the registry. The `local` source can import cache from local files previously exported with `--cache-to`.
-
-If no type is specified, `registry` exporter is used with a specified reference.
-
-`docker` driver currently only supports importing build cache from the registry.
-
-Examples:
-```
-docker buildx build --cache-from=user/app:cache .
-docker buildx build --cache-from=user/app .
-docker buildx build --cache-from=type=registry,ref=user/app .
-docker buildx build --cache-from=type=local,src=path/to/cache .
-```
-
-#### `--cache-to=[NAME|type=TYPE[,KEY=VALUE]]`
-
-Export build cache to an external cache destination. Supported types are `registry`, `local` and `inline`. Registry exports build cache to a cache manifest in the registry, local exports cache to a local directory on the client and inline writes the cache metadata into the image configuration.
-
-`docker` driver currently only supports exporting inline cache metadata to image configuration. Alternatively, `--build-arg BUILDKIT_INLINE_CACHE=1` can be used to trigger inline cache exporter.
-
-Attribute key:
-
-- `mode` - Specifies how many layers are exported with the cache. “min” on only exports layers already in the final build build stage, “max” exports layers for all stages. Metadata is always exported for the whole build.
-
-Examples:
-```
-docker buildx build --cache-to=user/app:cache .
-docker buildx build --cache-to=type=inline .
-docker buildx build --cache-to=type=registry,ref=user/app .
-docker buildx build --cache-to=type=local,dest=path/to/cache .
-```
-
-#### `--allow=ENTITLEMENT`
-
-Allow extra privileged entitlement. List of entitlements:
-
-- `network.host` - Allows executions with host networking.
-- `security.insecure` - Allows executions without sandbox. See [related Dockerfile extensions](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---securityinsecuresandbox).
-
-For entitlements to be enabled, the `buildkitd` daemon also needs to allow them with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](#--buildkitd-flags-flags))
-
-Example:
-```
-$ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
-$ docker buildx build --allow security.insecure .
-```
-
-### `buildx create [OPTIONS] [CONTEXT|ENDPOINT]`
-
-Create makes a new builder instance pointing to a docker context or endpoint, where context is the name of a context from `docker context ls` and endpoint is the address for docker socket (eg. `DOCKER_HOST` value).
-
-By default, the current docker configuration is used for determining the context/endpoint value.
-
-Builder instances are isolated environments where builds can be invoked. All docker contexts also get the default builder instance.
-
-Options:
-
-| Flag | Description |
-| --- | --- |
-| --append                 | Append a node to builder instead of changing it
-| --buildkitd-flags string | Flags for buildkitd daemon
-| --config string          | BuildKit config file
-| --driver string          | Driver to use (eg. docker-container)
-| --driver-opt stringArray | Options for the driver
-| --leave                  | Remove a node from builder instead of changing it
-| --name string            | Builder instance name
-| --node string            | Create/modify node with given name
-| --platform stringArray   | Fixed platforms for current node
-| --use                    | Set the current builder instance
-
-#### `--append`
-
-Changes the action of the command to appends a new node to an existing builder specified by `--name`. Buildx will choose an appropriate node for a build based on the platforms it supports.
-
-Example:
-```
-$ docker buildx create mycontext1
-eager_beaver
-$ docker buildx create --name eager_beaver --append mycontext2
-eager_beaver
-```
-
-#### `--buildkitd-flags FLAGS`
-
-Adds flags when starting the buildkitd daemon. They take precedence over the configuration file specified by [`--config`](#--config-file). See `buildkitd --help` for the available flags.
-
-Example:
-```
---buildkitd-flags '--debug --debugaddr 0.0.0.0:6666'
-```
-
-#### `--config FILE`
-
-Specifies the configuration file for the buildkitd daemon to use. The configuration can be overridden by [`--buildkitd-flags`](#--buildkitd-flags-flags). See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
-
-#### `--driver DRIVER`
-
-Sets the builder driver to be used. There are two available drivers, each have their own specificities.
-
-- `docker` - Uses the builder that is built into the docker daemon. With this driver, the [`--load`](#--load) flag is implied by default on `buildx build`. However, building multi-platform images or exporting cache is not currently supported.
-
-- `docker-container` - Uses a buildkit container that will be spawned via docker. With this driver, both building multi-platform images and exporting cache are supported. However, images built will not automatically appear in `docker images` (see [`build --load`](#--load)).
-
-- `kubernetes` - Uses a kubernetes pods. With this driver , you can spin up pods with defined buildkit container image to build your images. 
-
-
-#### `--driver-opt OPTIONS`
-
-Passes additional driver-specific options. Details for each driver:
-
-- `docker` - No driver options
-- `docker-container`
-  - `image=IMAGE` - Sets the container image to be used for running buildkit.
-  - `network=NETMODE` - Sets the network mode for running the buildkit container.
-  - Example:
-    ```
-    --driver docker-container --driver-opt image=moby/buildkit:master,network=host
-    ```
-- `kubernetes`
-  - `image=IMAGE` - Sets the container image to be used for running buildkit.
-  - `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
-  - `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
-  - `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
-  - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
-  - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
-
-#### `--leave`
-
-Changes the action of the command to removes a node from a builder. The builder needs to be specified with `--name` and node that is removed is set with `--node`.
-
-Example:
-```
-docker buildx create --name mybuilder --node mybuilder0 --leave
-```
-
-#### `--name NAME`
-
-Specifies the name of the builder to be created or modified. If none is specified, one will be automatically generated.
-
-#### `--node NODE`
-
-Specifies the name of the node to be created or modified. If none is specified, it is the name of the builder it belongs to, with an index number suffix.
-
-#### `--platform PLATFORMS`
-
-Sets the platforms supported by the node. It expects a comma-separated list of platforms of the form OS/architecture/variant. The node will also automatically detect the platforms it supports, but manual values take priority over the detected ones and can be used when multiple nodes support building for the same platform.
-
-Example:
-```
-docker buildx create --platform linux/amd64
-docker buildx create --platform linux/arm64,linux/arm/v8
-```
-
-#### `--use`
-
-Automatically switches the current builder to the newly created one. Equivalent to running `docker buildx use $(docker buildx create ...)`.
-
-### `buildx use NAME`
-
-Switches the current builder instance. Build commands invoked after this command will run on a specified builder. Alternatively, a context name can be used to switch to the default builder of that context.
-
-### `buildx inspect [NAME]`
-
-Shows information about the current or specified builder.
-
-Example:
-```
-Name:   elated_tesla
-Driver: docker-container
-
-Nodes:
-Name:      elated_tesla0
-Endpoint:  unix:///var/run/docker.sock
-Status:    running
-Platforms: linux/amd64
-
-Name:      elated_tesla1
-Endpoint:  ssh://ubuntu@1.2.3.4
-Status:    running
-Platforms: linux/arm64, linux/arm/v7, linux/arm/v6
-```
-
-#### `--bootstrap`
-
-Ensures that the builder is running before inspecting it. If the driver is `docker-container`, then `--bootstrap` starts the buildkit container and waits until it is operational. Bootstrapping is automatically done during build, it is thus not necessary. The same BuildKit container is used during the lifetime of the associated builder node (as displayed in `buildx ls`).
-
-### `buildx ls`
-
-Lists all builder instances and the nodes for each instance
-
-Example:
-
-```
-docker buildx ls
-NAME/NODE       DRIVER/ENDPOINT             STATUS  PLATFORMS
-elated_tesla *  docker-container
-  elated_tesla0 unix:///var/run/docker.sock running linux/amd64
-  elated_tesla1 ssh://ubuntu@1.2.3.4        running linux/arm64, linux/arm/v7, linux/arm/v6
-default         docker
-  default       default                     running linux/amd64
-```
-
-Each builder has one or more nodes associated with it. The current builder’s name is marked with a `*`.
-
-### `buildx stop [NAME]`
-
-Stops the specified or current builder. This will not prevent buildx build to restart the builder. The implementation of stop depends on the driver.
-
-### `buildx rm [NAME]`
-
-Removes the specified or current builder. It is a no-op attempting to remove the default builder.
-
-### `buildx bake [OPTIONS] [TARGET...]`
-
-Bake is a high-level build command.
-
-Each specified target will run in parallel as part of the build.
-
-Options:
-
-| Flag | Description |
-| --- | --- |
-|  -f, --file stringArray  | Build definition file
-|      --load              | Shorthand for --set=*.output=type=docker
-|      --no-cache          | Do not use cache when building the image
-|      --print             | Print the options without building
-|      --progress string   | Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto")
-|      --pull              | Always attempt to pull a newer version of the image
-|      --push              | Shorthand for --set=*.output=type=registry
-|      --set stringArray   | Override target value (eg: targetpattern.key=value)
-
-#### `-f, --file FILE`
-
-Specifies the bake definition file. The file can be a Docker Compose, JSON or HCL file. If multiple files are specified they are all read and configurations are combined. By default, if no files are specified, the following are parsed:
-docker-compose.yml
-docker-compose.yaml
-docker-bake.json
-docker-bake.override.json
-docker-bake.hcl
-docker-bake.override.hcl
-
-#### `--no-cache`
-
-Same as `build --no-cache`. Do not use cache when building the image.
-
-#### `--print`
-
-Prints the resulting options of the targets desired to be built, in a JSON format, without starting a build.
-
-```
-$ docker buildx bake -f docker-bake.hcl --print db
-{
-   "target": {
-      "db": {
-         "context": "./",
-         "dockerfile": "Dockerfile",
-         "tags": [
-            "docker.io/tiborvass/db"
-         ]
-      }
-   }
-}
-```
-
-#### `--progress`
-
-Same as `build --progress`. Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto").
-
-#### `--pull`
-
-Same as `build --pull`.
-
-#### `--set targetpattern.key[.subkey]=value`
-
-Override target configurations from command line. The pattern matching syntax is defined in https://golang.org/pkg/path/#Match.
-
-Example:
-```
-docker buildx bake --set target.args.mybuildarg=value
-docker buildx bake --set target.platform=linux/arm64
-docker buildx bake --set foo*.args.mybuildarg=value	# overrides build arg for all targets starting with 'foo'
-docker buildx bake --set *.platform=linux/arm64		# overrides platform for all targets
-docker buildx bake --set foo*.no-cache                  # bypass caching only for targets starting with 'foo'
-```
-
-Complete list of overridable fields:
-	args, cache-from, cache-to, context, dockerfile, labels, no-cache, output, platform, pull, secrets, ssh, tags, target
-
-#### File definition
-
-In addition to compose files, bake supports a JSON and an equivalent HCL file format for defining build groups and targets.
-
-A target reflects a single docker build invocation with the same options that you would specify for `docker build`. A group is a grouping of targets.
-
-Multiple files can include the same target and final build options will be determined by merging them together. 
-
-In the case of compose files, each service corresponds to a target.
-
-A group can specify its list of targets with the `targets` option. A target can inherit build options by setting the `inherits` option to the list of targets or groups to inherit from.
-
-Note: Design of bake command is work in progress, the user experience may change based on feedback.
-
-
-
-Example HCL defintion:
-
-```
-group "default" {
-	targets = ["db", "webapp-dev"]
-}
-
-target "webapp-dev" {
-	dockerfile = "Dockerfile.webapp"
-	tags = ["docker.io/username/webapp"]
-}
-
-target "webapp-release" {
-	inherits = ["webapp-dev"]
-	platforms = ["linux/amd64", "linux/arm64"]
-}
-
-target "db" {
-	dockerfile = "Dockerfile.db"
-	tags = ["docker.io/username/db"]
-}
-```
-
-Complete list of valid target fields:
-	args, cache-from, cache-to, context, dockerfile, inherits, labels, no-cache, output, platform, pull, secrets, ssh, tags, target
-
-#### HCL variables and functions
-
-Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable), the HCL file format also supports variable block definitions. These can be used to define variables with values provided by the current environment or a default value when unset.
-
-
-
-Example of using interpolation to tag an image with the git sha:
-
-```
-$ cat <<'EOF' > docker-bake.hcl
-variable "TAG" {
-	default = "latest"
-}
-
-group "default" {
-	targets = ["webapp"]
-}
-
-target "webapp" {
-	tags = ["docker.io/username/webapp:${TAG}"]
-}
-EOF
-
-$ docker buildx bake --print webapp
-{
-   "target": {
-      "webapp": {
-         "context": ".",
-         "dockerfile": "Dockerfile",
-         "tags": [
-            "docker.io/username/webapp:latest"
-         ]
-      }
-   }
-}
-
-$ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp
-{
-   "target": {
-      "webapp": {
-         "context": ".",
-         "dockerfile": "Dockerfile",
-         "tags": [
-            "docker.io/username/webapp:985e9e9"
-         ]
-      }
-   }
-}
-```
-
-
-A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hcl.go#L19-L65) provided by [go-cty](https://github.com/zclconf/go-cty/tree/master/cty/function/stdlib) are avaialble for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/hcl2/ext/userfunc) are also supported.
-
-
-
-Example of using the `add` function:
-
-```
-$ cat <<'EOF' > docker-bake.hcl
-variable "TAG" {
-	default = "latest"
-}
-
-group "default" {
-	targets = ["webapp"]
-}
-
-target "webapp" {
-	args = {
-		buildno = "${add(123, 1)}"
-	}
-}
-EOF
-
-$ docker buildx bake --print webapp
-{
-   "target": {
-      "webapp": {
-         "context": ".",
-         "dockerfile": "Dockerfile",
-         "args": {
-            "buildno": "124"
-         }
-      }
-   }
-}
-```
-
-Example of defining an `increment` function:
-
-```
-$ cat <<'EOF' > docker-bake.hcl
-function "increment" {
-	params = [number]
-	result = number + 1
-}
-
-group "default" {
-	targets = ["webapp"]
-}
-
-target "webapp" {
-	args = {
-		buildno = "${increment(123)}"
-	}
-}
-EOF
-
-$ docker buildx bake --print webapp
-{
-   "target": {
-      "webapp": {
-         "context": ".",
-         "dockerfile": "Dockerfile",
-         "args": {
-            "buildno": "124"
-         }
-      }
-   }
-}
-```
-
-Example of only adding tags if a variable is not empty using an `notequal` function:
-
-```
-$ cat <<'EOF' > docker-bake.hcl
-variable "TAG" {default="" }
-
-group "default" {
-	targets = [
-		"webapp",
-	]
-}
-
-target "webapp" {
-	context="."
-	dockerfile="Dockerfile"
-	tags = [
-		"my-image:latest",
-		notequal("",TAG) ? "my-image:${TAG}": "",
-	]
-}
-EOF
-
-$ docker buildx bake --print webapp
-{
-   "target": {
-      "webapp": {
-         "context": ".",
-         "dockerfile": "Dockerfile",
-         "tags": [
-            "my-image:latest"
-         ]
-      }
-   }
-}
-```
-
-
-### `buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`
-
-Imagetools contains commands for working with manifest lists in the registry. These commands are useful for inspecting multi-platform build results.
-
-Create creates a new manifest list based on source manifests. The source manifests can be manifest lists or single platform distribution manifests and must already exist in the registry where the new manifest is created. If only one source is specified create performs a carbon copy.
-
-Options:
-
-| Flag | Description |
-| --- | --- |
-|      --append            | Append to existing manifest
-|      --dry-run           | Show final image instead of pushing
-|  -f, --file stringArray  | Read source descriptor from file
-|  -t, --tag stringArray   | Set reference for new image
-
-#### `--append`
-
-Append appends the new sources to an existing manifest list in the destination.
-
-#### `--dry-run`
-
-Do not push the image, just show it.
-
-#### `-f, --file FILE`
-
-Reads source from files. A source can be a manifest digest, manifest reference or a JSON of OCI descriptor object.
-
-#### `-t, --tag IMAGE`
-
-Name of the image to be created.
-
-Examples:
-
-```
-docker buildx imagetools create --dry-run alpine@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907 sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
-
-docker buildx imagetools create -t tonistiigi/myapp -f image1 -f image2 
-```
-
-
-### `buildx imagetools inspect NAME`
-
-Show details of image in the registry.
-
-Example:
-```
-$ docker buildx imagetools inspect alpine
-Name:      docker.io/library/alpine:latest
-MediaType: application/vnd.docker.distribution.manifest.list.v2+json
-Digest:    sha256:28ef97b8686a0b5399129e9b763d5b7e5ff03576aa5580d6f4182a49c5fe1913
-
-Manifests:
-  Name:      docker.io/library/alpine:latest@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907
-  MediaType: application/vnd.docker.distribution.manifest.v2+json
-  Platform:  linux/amd64
-
-  Name:      docker.io/library/alpine:latest@sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
-  MediaType: application/vnd.docker.distribution.manifest.v2+json
-  Platform:  linux/arm/v6
-
- ...
-```
-
-#### `--raw`
-
-Raw prints the original JSON bytes instead of the formatted output.
-
+[`buildx bake` Reference Docs](docs/reference/buildx_bake.md)
 
 # Setting buildx as default builder in Docker 19.03+
 

bake/bake.go

@@ -2,6 +2,7 @@ package bake
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
@@ -9,7 +10,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/docker/buildx/bake/hclparser"
 	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/docker/pkg/urlutil"
 	hcl "github.com/hashicorp/hcl/v2"
@@ -58,15 +61,12 @@ func ReadLocalFiles(names []string) ([]File, error) {
 	return out, nil
 }
 
-func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
-	var c Config
-	for _, f := range files {
-		cfg, err := ParseFile(f.Data, f.Name)
-		if err != nil {
-			return nil, err
-		}
-		c = mergeConfig(c, *cfg)
+func ReadTargets(ctx context.Context, files []File, targets, overrides []string, defaults map[string]string) (map[string]*Target, error) {
+	c, err := ParseFiles(files, defaults)
+	if err != nil {
+		return nil, err
 	}
+
 	o, err := c.newOverrides(overrides)
 	if err != nil {
 		return nil, err
@@ -86,32 +86,83 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string)
 	return m, nil
 }
 
+func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) {
+	defer func() {
+		err = formatHCLError(err, files)
+	}()
+
+	var c Config
+	var fs []*hcl.File
+	for _, f := range files {
+		cfg, isCompose, composeErr := ParseComposeFile(f.Data, f.Name)
+		if isCompose {
+			if composeErr != nil {
+				return nil, composeErr
+			}
+			c = mergeConfig(c, *cfg)
+			c = dedupeConfig(c)
+		}
+		if !isCompose {
+			hf, isHCL, err := ParseHCLFile(f.Data, f.Name)
+			if isHCL {
+				if err != nil {
+					return nil, err
+				}
+				fs = append(fs, hf)
+			} else if composeErr != nil {
+				return nil, fmt.Errorf("failed to parse %s: parsing yaml: %v, parsing hcl: %w", f.Name, composeErr, err)
+			} else {
+				return nil, err
+			}
+		}
+	}
+
+	if len(fs) > 0 {
+		if err := hclparser.Parse(hcl.MergeFiles(fs), hclparser.Opt{
+			LookupVar: os.LookupEnv,
+			Vars:      defaults,
+		}, &c); err.HasErrors() {
+			return nil, err
+		}
+	}
+	return &c, nil
+}
+
+func dedupeConfig(c Config) Config {
+	c2 := c
+	c2.Targets = make([]*Target, 0, len(c2.Targets))
+	m := map[string]*Target{}
+	for _, t := range c.Targets {
+		if t2, ok := m[t.Name]; ok {
+			t2.Merge(t)
+		} else {
+			m[t.Name] = t
+			c2.Targets = append(c2.Targets, t)
+		}
+	}
+	return c2
+}
+
 func ParseFile(dt []byte, fn string) (*Config, error) {
+	return ParseFiles([]File{{Data: dt, Name: fn}}, nil)
+}
+
+func ParseComposeFile(dt []byte, fn string) (*Config, bool, error) {
 	fnl := strings.ToLower(fn)
 	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
-		return ParseCompose(dt)
+		cfg, err := ParseCompose(dt)
+		return cfg, true, err
 	}
-
 	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
-		return ParseHCL(dt, fn)
+		return nil, false, nil
 	}
-
 	cfg, err := ParseCompose(dt)
-	if err != nil {
-		cfg, err2 := ParseHCL(dt, fn)
-		if err2 != nil {
-			return nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
-		}
-		return cfg, nil
-	}
-	return cfg, nil
+	return cfg, err == nil, err
 }
 
 type Config struct {
-	Variables []*Variable `json:"-" hcl:"variable,block"`
-	Groups    []*Group    `json:"group" hcl:"group,block"`
-	Targets   []*Target   `json:"target" hcl:"target,block"`
-	Remain    hcl.Body    `json:"-" hcl:",remain"`
+	Groups  []*Group  `json:"group" hcl:"group,block"`
+	Targets []*Target `json:"target" hcl:"target,block"`
 }
 
 func mergeConfig(c1, c2 Config) Config {
@@ -157,7 +208,8 @@ func mergeConfig(c1, c2 Config) Config {
 			}
 		}
 		if t1 != nil {
-			t2 = merge(t1, t2)
+			t1.Merge(t2)
+			t2 = t1
 		}
 		c1.Targets = append(c1.Targets, t2)
 	}
@@ -344,23 +396,21 @@ func (c Config) target(name string, visited map[string]struct{}, overrides map[s
 			return nil, err
 		}
 		if t != nil {
-			tt = merge(tt, t)
+			tt.Merge(t)
 		}
 	}
 	t.Inherits = nil
-	tt = merge(merge(defaultTarget(), tt), t)
+	m := defaultTarget()
+	m.Merge(tt)
+	m.Merge(t)
+	tt = m
 	if override, ok := overrides[name]; ok {
-		tt = merge(tt, override)
+		tt.Merge(override)
 	}
 	tt.normalize()
 	return tt, nil
 }
 
-type Variable struct {
-	Name    string `json:"-" hcl:"name,label"`
-	Default string `json:"default,omitempty" hcl:"default,optional"`
-}
-
 type Group struct {
 	Name    string   `json:"-" hcl:"name,label"`
 	Targets []string `json:"targets" hcl:"targets"`
@@ -402,6 +452,61 @@ func (t *Target) normalize() {
 	t.Outputs = removeDupes(t.Outputs)
 }
 
+func (t *Target) Merge(t2 *Target) {
+	if t2.Context != nil {
+		t.Context = t2.Context
+	}
+	if t2.Dockerfile != nil {
+		t.Dockerfile = t2.Dockerfile
+	}
+	if t2.DockerfileInline != nil {
+		t.DockerfileInline = t2.DockerfileInline
+	}
+	for k, v := range t2.Args {
+		if t.Args == nil {
+			t.Args = map[string]string{}
+		}
+		t.Args[k] = v
+	}
+	for k, v := range t2.Labels {
+		if t.Labels == nil {
+			t.Labels = map[string]string{}
+		}
+		t.Labels[k] = v
+	}
+	if t2.Tags != nil { // no merge
+		t.Tags = t2.Tags
+	}
+	if t2.Target != nil {
+		t.Target = t2.Target
+	}
+	if t2.Secrets != nil { // merge
+		t.Secrets = append(t.Secrets, t2.Secrets...)
+	}
+	if t2.SSH != nil { // merge
+		t.SSH = append(t.SSH, t2.SSH...)
+	}
+	if t2.Platforms != nil { // no merge
+		t.Platforms = t2.Platforms
+	}
+	if t2.CacheFrom != nil { // merge
+		t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...)
+	}
+	if t2.CacheTo != nil { // no merge
+		t.CacheTo = t2.CacheTo
+	}
+	if t2.Outputs != nil { // no merge
+		t.Outputs = t2.Outputs
+	}
+	if t2.Pull != nil {
+		t.Pull = t2.Pull
+	}
+	if t2.NoCache != nil {
+		t.NoCache = t2.NoCache
+	}
+	t.Inherits = append(t.Inherits, t2.Inherits...)
+}
+
 func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
 	m2 := make(map[string]build.Options, len(m))
 	for k, v := range m {
@@ -422,6 +527,12 @@ func updateContext(t *build.Inputs, inp *Input) {
 		t.ContextPath = inp.URL
 		return
 	}
+	if strings.HasPrefix(t.ContextPath, "cwd://") {
+		return
+	}
+	if IsRemoteURL(t.ContextPath) {
+		return
+	}
 	st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
 	t.ContextState = &st
 }
@@ -438,7 +549,9 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 	if t.Context != nil {
 		contextPath = *t.Context
 	}
-	contextPath = path.Clean(contextPath)
+	if !strings.HasPrefix(contextPath, "cwd://") && !IsRemoteURL(contextPath) {
+		contextPath = path.Clean(contextPath)
+	}
 	dockerfilePath := "Dockerfile"
 	if t.Dockerfile != nil {
 		dockerfilePath = *t.Dockerfile
@@ -465,6 +578,11 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		bi.DockerfileInline = *t.DockerfileInline
 	}
 	updateContext(&bi, inp)
+	if strings.HasPrefix(bi.ContextPath, "cwd://") {
+		bi.ContextPath = path.Clean(strings.TrimPrefix(bi.ContextPath, "cwd://"))
+	}
+
+	t.Context = &bi.ContextPath
 
 	bo := &build.Options{
 		Inputs:    bi,
@@ -483,13 +601,17 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 
 	bo.Session = append(bo.Session, authprovider.NewDockerAuthProvider(os.Stderr))
 
-	secrets, err := build.ParseSecretSpecs(t.Secrets)
+	secrets, err := buildflags.ParseSecretSpecs(t.Secrets)
 	if err != nil {
 		return nil, err
 	}
 	bo.Session = append(bo.Session, secrets)
 
-	ssh, err := build.ParseSSHSpecs(t.SSH)
+	sshSpecs := t.SSH
+	if len(sshSpecs) == 0 && buildflags.IsGitSSH(contextPath) {
+		sshSpecs = []string{"default"}
+	}
+	ssh, err := buildflags.ParseSSHSpecs(sshSpecs)
 	if err != nil {
 		return nil, err
 	}
@@ -499,19 +621,19 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 		bo.Target = *t.Target
 	}
 
-	cacheImports, err := build.ParseCacheEntry(t.CacheFrom)
+	cacheImports, err := buildflags.ParseCacheEntry(t.CacheFrom)
 	if err != nil {
 		return nil, err
 	}
 	bo.CacheFrom = cacheImports
 
-	cacheExports, err := build.ParseCacheEntry(t.CacheTo)
+	cacheExports, err := buildflags.ParseCacheEntry(t.CacheTo)
 	if err != nil {
 		return nil, err
 	}
 	bo.CacheTo = cacheExports
 
-	outputs, err := build.ParseOutputs(t.Outputs)
+	outputs, err := buildflags.ParseOutputs(t.Outputs)
 	if err != nil {
 		return nil, err
 	}
@@ -524,62 +646,6 @@ func defaultTarget() *Target {
 	return &Target{}
 }
 
-func merge(t1, t2 *Target) *Target {
-	if t2.Context != nil {
-		t1.Context = t2.Context
-	}
-	if t2.Dockerfile != nil {
-		t1.Dockerfile = t2.Dockerfile
-	}
-	if t2.DockerfileInline != nil {
-		t1.DockerfileInline = t2.DockerfileInline
-	}
-	for k, v := range t2.Args {
-		if t1.Args == nil {
-			t1.Args = map[string]string{}
-		}
-		t1.Args[k] = v
-	}
-	for k, v := range t2.Labels {
-		if t1.Labels == nil {
-			t1.Labels = map[string]string{}
-		}
-		t1.Labels[k] = v
-	}
-	if t2.Tags != nil { // no merge
-		t1.Tags = t2.Tags
-	}
-	if t2.Target != nil {
-		t1.Target = t2.Target
-	}
-	if t2.Secrets != nil { // merge
-		t1.Secrets = append(t1.Secrets, t2.Secrets...)
-	}
-	if t2.SSH != nil { // merge
-		t1.SSH = append(t1.SSH, t2.SSH...)
-	}
-	if t2.Platforms != nil { // no merge
-		t1.Platforms = t2.Platforms
-	}
-	if t2.CacheFrom != nil { // no merge
-		t1.CacheFrom = append(t1.CacheFrom, t2.CacheFrom...)
-	}
-	if t2.CacheTo != nil { // no merge
-		t1.CacheTo = t2.CacheTo
-	}
-	if t2.Outputs != nil { // no merge
-		t1.Outputs = t2.Outputs
-	}
-	if t2.Pull != nil {
-		t1.Pull = t2.Pull
-	}
-	if t2.NoCache != nil {
-		t1.NoCache = t2.NoCache
-	}
-	t1.Inherits = append(t1.Inherits, t2.Inherits...)
-	return t1
-}
-
 func removeDupes(s []string) []string {
 	i := 0
 	seen := make(map[string]struct{}, len(s))

bake/bake_test.go

@@ -34,7 +34,7 @@ target "webapp" {
 	ctx := context.TODO()
 
 	t.Run("NoOverrides", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil, nil)
 		require.NoError(t, err)
 		require.Equal(t, 1, len(m))
 
@@ -46,7 +46,7 @@ target "webapp" {
 	})
 
 	t.Run("InvalidTargetOverrides", func(t *testing.T) {
-		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
+		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"}, nil)
 		require.NotNil(t, err)
 		require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
 	})
@@ -63,7 +63,7 @@ target "webapp" {
 				"webapp.args.VAR_FROMENV" + t.Name(),
 				"webapp.args.VAR_INHERITED=override",
 				// not overriding VAR_BOTH on purpose
-			})
+			}, nil)
 			require.NoError(t, err)
 
 			require.Equal(t, "Dockerfile.webapp", *m["webapp"].Dockerfile)
@@ -88,7 +88,7 @@ target "webapp" {
 			m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
 				"webDEP.args.VAR_INHERITED=override",
 				"webDEP.args.VAR_BOTH=override",
-			})
+			}, nil)
 			require.NoError(t, err)
 			require.Equal(t, m["webapp"].Args["VAR_INHERITED"], "override")
 			require.Equal(t, m["webapp"].Args["VAR_BOTH"], "webapp")
@@ -96,23 +96,23 @@ target "webapp" {
 	})
 
 	t.Run("ContextOverride", func(t *testing.T) {
-		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
+		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"}, nil)
 		require.NotNil(t, err)
 
-		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"}, nil)
 		require.NoError(t, err)
 
 		require.Equal(t, "foo", *m["webapp"].Context)
 	})
 
 	t.Run("NoCacheOverride", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"}, nil)
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].NoCache)
 	})
 
 	t.Run("PullOverride", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}, nil)
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].Pull)
 	})
@@ -172,7 +172,7 @@ target "webapp" {
 		}
 		for _, test := range cases {
 			t.Run(test.name, func(t *testing.T) {
-				m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
+				m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides, nil)
 				test.check(t, m, err)
 			})
 		}
@@ -215,7 +215,7 @@ services:
 
 	ctx := context.TODO()
 
-	m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
+	m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 3, len(m))
@@ -226,3 +226,28 @@ services:
 	require.Equal(t, "1", m["webapp"].Args["buildno"])
 	require.Equal(t, "12", m["webapp"].Args["buildno2"])
 }
+
+func TestHCLCwdPrefix(t *testing.T) {
+
+	fp := File{
+		Name: "docker-bake.hc",
+		Data: []byte(
+			`target "app" {
+				context = "cwd://foo"
+				dockerfile = "test"
+			}`),
+	}
+	ctx := context.TODO()
+	m, err := ReadTargets(ctx, []File{fp}, []string{"app"}, nil, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(m))
+	_, ok := m["app"]
+	require.True(t, ok)
+
+	_, err = TargetsToBuildOpt(m, &Input{})
+	require.NoError(t, err)
+
+	require.Equal(t, "test", *m["app"].Dockerfile)
+	require.Equal(t, "foo", *m["app"].Context)
+}

bake/compose.go

@@ -6,22 +6,25 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/docker/cli/cli/compose/loader"
-	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/compose-spec/compose-go/loader"
+	compose "github.com/compose-spec/compose-go/types"
 )
 
-func parseCompose(dt []byte) (*composetypes.Config, error) {
-	parsed, err := loader.ParseYAML([]byte(dt))
+func parseCompose(dt []byte) (*compose.Project, error) {
+	config, err := loader.ParseYAML(dt)
 	if err != nil {
 		return nil, err
 	}
-	return loader.Load(composetypes.ConfigDetails{
-		ConfigFiles: []composetypes.ConfigFile{
+
+	return loader.Load(compose.ConfigDetails{
+		ConfigFiles: []compose.ConfigFile{
 			{
-				Config: parsed,
+				Config: config,
 			},
 		},
 		Environment: envMap(os.Environ()),
+	}, func(options *loader.Options) {
+		options.SkipNormalization = true
 	})
 }
 
@@ -44,7 +47,7 @@ func ParseCompose(dt []byte) (*Config, error) {
 	}
 
 	var c Config
-	var zeroBuildConfig composetypes.BuildConfig
+	var zeroBuildConfig compose.BuildConfig
 	if len(cfg.Services) > 0 {
 		c.Groups = []*Group{}
 		c.Targets = []*Target{}
@@ -53,10 +56,10 @@ func ParseCompose(dt []byte) (*Config, error) {
 
 		for _, s := range cfg.Services {
 
-			if reflect.DeepEqual(s.Build, zeroBuildConfig) {
+			if s.Build == nil || reflect.DeepEqual(s.Build, zeroBuildConfig) {
 				// if not make sure they're setting an image or it's invalid d-c.yml
 				if s.Image == "" {
-					return nil, fmt.Errorf("compose file invalid: service %s has neither an image nor a build context specified. At least one must be provided.", s.Name)
+					return nil, fmt.Errorf("compose file invalid: service %s has neither an image nor a build context specified. At least one must be provided", s.Name)
 				}
 				continue
 			}
@@ -97,7 +100,7 @@ func ParseCompose(dt []byte) (*Config, error) {
 	return &c, nil
 }
 
-func toMap(in composetypes.MappingWithEquals) map[string]string {
+func toMap(in compose.MappingWithEquals) map[string]string {
 	m := map[string]string{}
 	for k, v := range in {
 		if v != nil {

bake/compose_test.go

@@ -9,8 +9,6 @@ import (
 
 func TestParseCompose(t *testing.T) {
 	var dt = []byte(`
-version: "3"
-
 services:
   db:
     build: ./db
@@ -48,8 +46,6 @@ services:
 
 func TestNoBuildOutOfTreeService(t *testing.T) {
 	var dt = []byte(`
-version: "3.7"
-
 services:
     external:
         image: "verycooldb:1337"
@@ -63,8 +59,6 @@ services:
 
 func TestParseComposeTarget(t *testing.T) {
 	var dt = []byte(`
-version: "3.7"
-
 services:
   db:
     build:
@@ -91,8 +85,6 @@ services:
 
 func TestComposeBuildWithoutContext(t *testing.T) {
 	var dt = []byte(`
-version: "3.7"
-
 services:
   db:
     build:
@@ -117,8 +109,6 @@ services:
 
 func TestBogusCompose(t *testing.T) {
 	var dt = []byte(`
-version: "3.7"
-
 services:
   db:
     labels:
@@ -131,5 +121,66 @@ services:
 
 	_, err := ParseCompose(dt)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "has neither an image nor a build context specified. At least one must be provided")
+	require.Contains(t, err.Error(), "has neither an image nor a build context specified: invalid compose project")
+}
+
+func TestAdvancedNetwork(t *testing.T) {
+	var dt = []byte(`
+services:
+  db:
+    networks:
+      - example.com
+    build:
+      context: ./db
+      target: db
+
+networks:
+  example.com:
+    name: example.com
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.5.0.0/24
+          ip_range: 10.5.0.0/24
+          gateway: 10.5.0.254
+`)
+
+	_, err := ParseCompose(dt)
+	require.NoError(t, err)
+}
+
+func TestDependsOnList(t *testing.T) {
+	var dt = []byte(`
+version: "3.8"
+
+services:
+  example-container:
+    image: example/fails:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      other-container:
+        condition: service_healthy
+    networks:
+      default:
+        aliases:
+          - integration-tests
+
+  other-container:
+    image: example/other:latest
+    healthcheck:
+      test: ["CMD", "echo", "success"]
+      retries: 5
+      interval: 5s
+      timeout: 10s
+      start_period: 5s
+
+networks:
+  default:
+    name: test-net
+`)
+
+	_, err := ParseCompose(dt)
+	require.NoError(t, err)
 }

bake/hcl.go

@@ -1,200 +1,42 @@
 package bake
 
 import (
-	"os"
 	"strings"
 
 	hcl "github.com/hashicorp/hcl/v2"
-	"github.com/hashicorp/hcl/v2/ext/userfunc"
-	"github.com/hashicorp/hcl/v2/hclsimple"
-	"github.com/hashicorp/hcl/v2/hclsyntax"
-	"github.com/hashicorp/hcl/v2/json"
+	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/zclconf/go-cty/cty"
-	"github.com/zclconf/go-cty/cty/function"
-	"github.com/zclconf/go-cty/cty/function/stdlib"
 )
 
-// Collection of generally useful functions in cty-using applications, which
-// HCL supports. These functions are available for use in HCL files.
-var (
-	stdlibFunctions = map[string]function.Function{
-		"absolute":               stdlib.AbsoluteFunc,
-		"add":                    stdlib.AddFunc,
-		"and":                    stdlib.AndFunc,
-		"byteslen":               stdlib.BytesLenFunc,
-		"bytesslice":             stdlib.BytesSliceFunc,
-		"chomp":                  stdlib.ChompFunc,
-		"chunklist":              stdlib.ChunklistFunc,
-		"ceil":                   stdlib.CeilFunc,
-		"csvdecode":              stdlib.CSVDecodeFunc,
-		"coalesce":               stdlib.CoalesceFunc,
-		"coalescelist":           stdlib.CoalesceListFunc,
-		"compact":                stdlib.CompactFunc,
-		"concat":                 stdlib.ConcatFunc,
-		"contains":               stdlib.ContainsFunc,
-		"distinct":               stdlib.DistinctFunc,
-		"divide":                 stdlib.DivideFunc,
-		"element":                stdlib.ElementFunc,
-		"equal":                  stdlib.EqualFunc,
-		"flatten":                stdlib.FlattenFunc,
-		"floor":                  stdlib.FloorFunc,
-		"formatdate":             stdlib.FormatDateFunc,
-		"format":                 stdlib.FormatFunc,
-		"formatlist":             stdlib.FormatListFunc,
-		"greaterthan":            stdlib.GreaterThanFunc,
-		"greaterthanorequalto":   stdlib.GreaterThanOrEqualToFunc,
-		"hasindex":               stdlib.HasIndexFunc,
-		"indent":                 stdlib.IndentFunc,
-		"index":                  stdlib.IndexFunc,
-		"int":                    stdlib.IntFunc,
-		"jsondecode":             stdlib.JSONDecodeFunc,
-		"jsonencode":             stdlib.JSONEncodeFunc,
-		"keys":                   stdlib.KeysFunc,
-		"join":                   stdlib.JoinFunc,
-		"length":                 stdlib.LengthFunc,
-		"lessthan":               stdlib.LessThanFunc,
-		"lessthanorequalto":      stdlib.LessThanOrEqualToFunc,
-		"log":                    stdlib.LogFunc,
-		"lookup":                 stdlib.LookupFunc,
-		"lower":                  stdlib.LowerFunc,
-		"max":                    stdlib.MaxFunc,
-		"merge":                  stdlib.MergeFunc,
-		"min":                    stdlib.MinFunc,
-		"modulo":                 stdlib.ModuloFunc,
-		"multiply":               stdlib.MultiplyFunc,
-		"negate":                 stdlib.NegateFunc,
-		"notequal":               stdlib.NotEqualFunc,
-		"not":                    stdlib.NotFunc,
-		"or":                     stdlib.OrFunc,
-		"parseint":               stdlib.ParseIntFunc,
-		"pow":                    stdlib.PowFunc,
-		"range":                  stdlib.RangeFunc,
-		"regexall":               stdlib.RegexAllFunc,
-		"regex":                  stdlib.RegexFunc,
-		"reverse":                stdlib.ReverseFunc,
-		"reverselist":            stdlib.ReverseListFunc,
-		"sethaselement":          stdlib.SetHasElementFunc,
-		"setintersection":        stdlib.SetIntersectionFunc,
-		"setsubtract":            stdlib.SetSubtractFunc,
-		"setsymmetricdifference": stdlib.SetSymmetricDifferenceFunc,
-		"setunion":               stdlib.SetUnionFunc,
-		"signum":                 stdlib.SignumFunc,
-		"slice":                  stdlib.SliceFunc,
-		"sort":                   stdlib.SortFunc,
-		"split":                  stdlib.SplitFunc,
-		"strlen":                 stdlib.StrlenFunc,
-		"substr":                 stdlib.SubstrFunc,
-		"subtract":               stdlib.SubtractFunc,
-		"timeadd":                stdlib.TimeAddFunc,
-		"title":                  stdlib.TitleFunc,
-		"trim":                   stdlib.TrimFunc,
-		"trimprefix":             stdlib.TrimPrefixFunc,
-		"trimspace":              stdlib.TrimSpaceFunc,
-		"trimsuffix":             stdlib.TrimSuffixFunc,
-		"upper":                  stdlib.UpperFunc,
-		"values":                 stdlib.ValuesFunc,
-		"zipmap":                 stdlib.ZipmapFunc,
-	}
-)
-
-// Used in the first pass of decoding instead of the Config struct to disallow
-// interpolation while parsing variable blocks.
-type staticConfig struct {
-	Variables []*Variable `hcl:"variable,block"`
-	Remain    hcl.Body    `hcl:",remain"`
-}
-
-func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
-	if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
-		return parseHCL(dt, fn)
-	}
-	cfg, err := parseHCL(dt, fn+".hcl")
-	if err != nil {
-		cfg2, err2 := parseHCL(dt, fn+".json")
-		if err2 == nil {
-			return cfg2, nil
+func ParseHCLFile(dt []byte, fn string) (*hcl.File, bool, error) {
+	var err error
+	if strings.HasSuffix(fn, ".json") {
+		f, diags := hclparse.NewParser().ParseJSON(dt, fn)
+		if diags.HasErrors() {
+			err = diags
 		}
+		return f, true, err
 	}
-	return cfg, err
-}
-
-func parseHCL(dt []byte, fn string) (_ *Config, err error) {
-	defer func() {
-		err = formatHCLError(dt, err)
-	}()
-
-	// Decode user defined functions, first parsing as hcl and falling back to
-	// json, returning errors based on the file suffix.
-	file, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
-	if hcldiags.HasErrors() {
-		var jsondiags hcl.Diagnostics
-		file, jsondiags = json.Parse(dt, fn)
-		if jsondiags.HasErrors() {
-			fnl := strings.ToLower(fn)
-			if strings.HasSuffix(fnl, ".json") {
-				return nil, jsondiags
-			} else {
-				return nil, hcldiags
-			}
+	if strings.HasSuffix(fn, ".hcl") {
+		f, diags := hclparse.NewParser().ParseHCL(dt, fn)
+		if diags.HasErrors() {
+			err = diags
 		}
+		return f, true, err
 	}
-
-	userFunctions, _, diags := userfunc.DecodeUserFunctions(file.Body, "function", func() *hcl.EvalContext {
-		return &hcl.EvalContext{
-			Functions: stdlibFunctions,
-		}
-	})
+	f, diags := hclparse.NewParser().ParseHCL(dt, fn+".hcl")
 	if diags.HasErrors() {
-		return nil, diags
-	}
-
-	var sc staticConfig
-
-	// Decode only variable blocks without interpolation.
-	if err := hclsimple.Decode(fn, dt, nil, &sc); err != nil {
-		return nil, err
-	}
-
-	// Set all variables to their default value if defined.
-	variables := make(map[string]cty.Value)
-	for _, variable := range sc.Variables {
-		variables[variable.Name] = cty.StringVal(variable.Default)
-	}
-
-	// Override default with values from environment.
-	for _, env := range os.Environ() {
-		parts := strings.SplitN(env, "=", 2)
-		name, value := parts[0], parts[1]
-		if _, ok := variables[name]; ok {
-			variables[name] = cty.StringVal(value)
+		f, diags2 := hclparse.NewParser().ParseJSON(dt, fn+".json")
+		if !diags2.HasErrors() {
+			return f, true, nil
 		}
+		return nil, false, diags
 	}
-
-	functions := make(map[string]function.Function)
-	for k, v := range stdlibFunctions {
-		functions[k] = v
-	}
-	for k, v := range userFunctions {
-		functions[k] = v
-	}
-
-	ctx := &hcl.EvalContext{
-		Variables: variables,
-		Functions: functions,
-	}
-
-	var c Config
-
-	// Decode with variables and functions.
-	if err := hclsimple.Decode(fn, dt, ctx, &c); err != nil {
-		return nil, err
-	}
-	return &c, nil
+	return f, true, nil
 }
 
-func formatHCLError(dt []byte, err error) error {
+func formatHCLError(err error, files []File) error {
 	if err == nil {
 		return nil
 	}
@@ -207,6 +49,13 @@ func formatHCLError(dt []byte, err error) error {
 			continue
 		}
 		if d.Subject != nil {
+			var dt []byte
+			for _, f := range files {
+				if d.Subject.Filename == f.Name {
+					dt = f.Data
+					break
+				}
+			}
 			src := errdefs.Source{
 				Info: &pb.SourceInfo{
 					Filename: d.Subject.Filename,

bake/hcl_test.go

@@ -7,11 +7,9 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestParseHCL(t *testing.T) {
+func TestHCLBasic(t *testing.T) {
 	t.Parallel()
-
-	t.Run("Basic", func(t *testing.T) {
-		dt := []byte(`
+	dt := []byte(`
 		group "default" {
 			targets = ["db", "webapp"]
 		}
@@ -44,32 +42,31 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
-		require.NoError(t, err)
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 4, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "db")
+	require.Equal(t, "./db", *c.Targets[0].Context)
 
-		require.Equal(t, 4, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "db")
-		require.Equal(t, "./db", *c.Targets[0].Context)
+	require.Equal(t, c.Targets[1].Name, "webapp")
+	require.Equal(t, 1, len(c.Targets[1].Args))
+	require.Equal(t, "123", c.Targets[1].Args["buildno"])
 
-		require.Equal(t, c.Targets[1].Name, "webapp")
-		require.Equal(t, 1, len(c.Targets[1].Args))
-		require.Equal(t, "123", c.Targets[1].Args["buildno"])
+	require.Equal(t, c.Targets[2].Name, "cross")
+	require.Equal(t, 2, len(c.Targets[2].Platforms))
+	require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
 
-		require.Equal(t, c.Targets[2].Name, "cross")
-		require.Equal(t, 2, len(c.Targets[2].Platforms))
-		require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
+	require.Equal(t, c.Targets[3].Name, "webapp-plus")
+	require.Equal(t, 1, len(c.Targets[3].Args))
+	require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
+}
 
-		require.Equal(t, c.Targets[3].Name, "webapp-plus")
-		require.Equal(t, 1, len(c.Targets[3].Args))
-		require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
-	})
-
-	t.Run("BasicInJSON", func(t *testing.T) {
-		dt := []byte(`
+func TestHCLBasicInJSON(t *testing.T) {
+	dt := []byte(`
 		{
 			"group": {
 				"default": {
@@ -104,32 +101,32 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.json")
-		require.NoError(t, err)
+	c, err := ParseFile(dt, "docker-bake.json")
+	require.NoError(t, err)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 4, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "db")
-		require.Equal(t, "./db", *c.Targets[0].Context)
+	require.Equal(t, 4, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "db")
+	require.Equal(t, "./db", *c.Targets[0].Context)
 
-		require.Equal(t, c.Targets[1].Name, "webapp")
-		require.Equal(t, 1, len(c.Targets[1].Args))
-		require.Equal(t, "123", c.Targets[1].Args["buildno"])
+	require.Equal(t, c.Targets[1].Name, "webapp")
+	require.Equal(t, 1, len(c.Targets[1].Args))
+	require.Equal(t, "123", c.Targets[1].Args["buildno"])
 
-		require.Equal(t, c.Targets[2].Name, "cross")
-		require.Equal(t, 2, len(c.Targets[2].Platforms))
-		require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
+	require.Equal(t, c.Targets[2].Name, "cross")
+	require.Equal(t, 2, len(c.Targets[2].Platforms))
+	require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
 
-		require.Equal(t, c.Targets[3].Name, "webapp-plus")
-		require.Equal(t, 1, len(c.Targets[3].Args))
-		require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
-	})
+	require.Equal(t, c.Targets[3].Name, "webapp-plus")
+	require.Equal(t, 1, len(c.Targets[3].Args))
+	require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
+}
 
-	t.Run("WithFunctions", func(t *testing.T) {
-		dt := []byte(`
+func TestHCLWithFunctions(t *testing.T) {
+	dt := []byte(`
 		group "default" {
 			targets = ["webapp"]
 		}
@@ -141,20 +138,20 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
-		require.NoError(t, err)
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 1, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "webapp")
-		require.Equal(t, "124", c.Targets[0].Args["buildno"])
-	})
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, "124", c.Targets[0].Args["buildno"])
+}
 
-	t.Run("WithUserDefinedFunctions", func(t *testing.T) {
-		dt := []byte(`
+func TestHCLWithUserDefinedFunctions(t *testing.T) {
+	dt := []byte(`
 		function "increment" {
 			params = [number]
 			result = number + 1
@@ -171,20 +168,20 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
-		require.NoError(t, err)
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 1, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "webapp")
-		require.Equal(t, "124", c.Targets[0].Args["buildno"])
-	})
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, "124", c.Targets[0].Args["buildno"])
+}
 
-	t.Run("WithVariables", func(t *testing.T) {
-		dt := []byte(`
+func TestHCLWithVariables(t *testing.T) {
+	dt := []byte(`
 		variable "BUILD_NUMBER" {
 			default = "123"
 		}
@@ -200,54 +197,426 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
-		require.NoError(t, err)
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 1, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "webapp")
-		require.Equal(t, "123", c.Targets[0].Args["buildno"])
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, "123", c.Targets[0].Args["buildno"])
 
-		os.Setenv("BUILD_NUMBER", "456")
+	os.Setenv("BUILD_NUMBER", "456")
 
-		c, err = ParseHCL(dt, "docker-bake.hcl")
-		require.NoError(t, err)
+	c, err = ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
 
-		require.Equal(t, 1, len(c.Groups))
-		require.Equal(t, "default", c.Groups[0].Name)
-		require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
+	require.Equal(t, 1, len(c.Groups))
+	require.Equal(t, "default", c.Groups[0].Name)
+	require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
 
-		require.Equal(t, 1, len(c.Targets))
-		require.Equal(t, c.Targets[0].Name, "webapp")
-		require.Equal(t, "456", c.Targets[0].Args["buildno"])
-	})
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, "456", c.Targets[0].Args["buildno"])
+}
 
-	t.Run("WithIncorrectVariables", func(t *testing.T) {
-		dt := []byte(`
-		variable "DEFAULT_BUILD_NUMBER" {
-			default = "1"
+func TestHCLWithVariablesInFunctions(t *testing.T) {
+	dt := []byte(`
+		variable "REPO" {
+			default = "user/repo"
 		}
-
-		variable "BUILD_NUMBER" {
-			default = "${DEFAULT_BUILD_NUMBER}"
-		}
-
-		group "default" {
-			targets = ["webapp"]
+		function "tag" {
+			params = [tag]
+			result = ["${REPO}:${tag}"]
 		}
 
 		target "webapp" {
+			tags = tag("v1")
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, []string{"user/repo:v1"}, c.Targets[0].Tags)
+
+	os.Setenv("REPO", "docker/buildx")
+
+	c, err = ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "webapp")
+	require.Equal(t, []string{"docker/buildx:v1"}, c.Targets[0].Tags)
+}
+
+func TestHCLMultiFileSharedVariables(t *testing.T) {
+	dt := []byte(`
+		variable "FOO" {
+			default = "abc"
+		}
+		target "app" {
 			args = {
-				buildno = "${BUILD_NUMBER}"
+				v1 = "pre-${FOO}"
+			}
+		}
+		`)
+	dt2 := []byte(`
+		target "app" {
+			args = {
+				v2 = "${FOO}-post"
 			}
 		}
 		`)
 
-		_, err := ParseHCL(dt, "docker-bake.hcl")
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "docker-bake.hcl:7,17-37: Variables not allowed; Variables may not be used here.")
-	})
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-abc", c.Targets[0].Args["v1"])
+	require.Equal(t, "abc-post", c.Targets[0].Args["v2"])
+
+	os.Setenv("FOO", "def")
+
+	c, err = ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
+	require.Equal(t, "def-post", c.Targets[0].Args["v2"])
+}
+
+func TestHCLVarsWithVars(t *testing.T) {
+	os.Unsetenv("FOO")
+	dt := []byte(`
+		variable "FOO" {
+			default = upper("${BASE}def")
+		}
+		variable "BAR" {
+			default = "-${FOO}-"
+		}
+		target "app" {
+			args = {
+				v1 = "pre-${BAR}"
+			}
+		}
+		`)
+	dt2 := []byte(`
+		variable "BASE" {
+			default = "abc"
+		}
+		target "app" {
+			args = {
+				v2 = "${FOO}-post"
+			}
+		}
+		`)
+
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre--ABCDEF-", c.Targets[0].Args["v1"])
+	require.Equal(t, "ABCDEF-post", c.Targets[0].Args["v2"])
+
+	os.Setenv("BASE", "new")
+
+	c, err = ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre--NEWDEF-", c.Targets[0].Args["v1"])
+	require.Equal(t, "NEWDEF-post", c.Targets[0].Args["v2"])
+}
+
+func TestHCLTypedVariables(t *testing.T) {
+	os.Unsetenv("FOO")
+	dt := []byte(`
+		variable "FOO" {
+			default = 3
+		}
+		variable "IS_FOO" {
+			default = true
+		}
+		target "app" {
+			args = {
+				v1 = FOO > 5 ? "higher" : "lower" 
+				v2 = IS_FOO ? "yes" : "no"
+			}
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "lower", c.Targets[0].Args["v1"])
+	require.Equal(t, "yes", c.Targets[0].Args["v2"])
+
+	os.Setenv("FOO", "5.1")
+	os.Setenv("IS_FOO", "0")
+
+	c, err = ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "higher", c.Targets[0].Args["v1"])
+	require.Equal(t, "no", c.Targets[0].Args["v2"])
+
+	os.Setenv("FOO", "NaN")
+	_, err = ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "failed to parse FOO as number")
+
+	os.Setenv("FOO", "0")
+	os.Setenv("IS_FOO", "maybe")
+
+	_, err = ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "failed to parse IS_FOO as bool")
+}
+
+func TestHCLVariableCycle(t *testing.T) {
+	dt := []byte(`
+		variable "FOO" {
+			default = BAR
+		}
+		variable "FOO2" {
+			default = FOO
+		}
+		variable "BAR" {
+			default = FOO
+		}
+		target "app" {}
+		`)
+
+	_, err := ParseFile(dt, "docker-bake.hcl")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "variable cycle not allowed")
+}
+
+func TestHCLAttrs(t *testing.T) {
+	dt := []byte(`
+		FOO="abc"
+		BAR="attr-${FOO}def"
+		target "app" {
+			args = {
+				"v1": BAR
+			}
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "attr-abcdef", c.Targets[0].Args["v1"])
+
+	// env does not apply if no variable
+	os.Setenv("FOO", "bar")
+	c, err = ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "attr-abcdef", c.Targets[0].Args["v1"])
+	// attr-multifile
+}
+
+func TestHCLAttrsCustomType(t *testing.T) {
+	dt := []byte(`
+		platforms=["linux/arm64", "linux/amd64"]
+		target "app" {
+			platforms = platforms
+			args = {
+				"v1": platforms[0]
+			}
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, []string{"linux/arm64", "linux/amd64"}, c.Targets[0].Platforms)
+	require.Equal(t, "linux/arm64", c.Targets[0].Args["v1"])
+}
+
+func TestHCLMultiFileAttrs(t *testing.T) {
+	os.Unsetenv("FOO")
+	dt := []byte(`
+		variable "FOO" {
+			default = "abc"
+		}
+		target "app" {
+			args = {
+				v1 = "pre-${FOO}"
+			}
+		}
+		`)
+	dt2 := []byte(`
+		FOO="def"
+		`)
+
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
+
+	os.Setenv("FOO", "ghi")
+
+	c, err = ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.hcl"},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-ghi", c.Targets[0].Args["v1"])
+}
+
+func TestJSONAttributes(t *testing.T) {
+	dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`)
+
+	c, err := ParseFile(dt, "docker-bake.json")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-abc-def", c.Targets[0].Args["v1"])
+}
+
+func TestJSONFunctions(t *testing.T) {
+	dt := []byte(`{
+	"FOO": "abc",
+	"function": {
+		"myfunc": {
+			"params": ["inp"],
+			"result": "<${upper(inp)}-${FOO}>"
+		}
+	},
+	"target": {
+		"app": {
+			"args": {
+				"v1": "pre-${myfunc(\"foo\")}"
+			}
+		}
+	}}`)
+
+	c, err := ParseFile(dt, "docker-bake.json")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "pre-<FOO-abc>", c.Targets[0].Args["v1"])
+}
+
+func TestHCLFunctionInAttr(t *testing.T) {
+	dt := []byte(`
+	function "brace" {
+		params = [inp]
+		result = "[${inp}]"
+	}
+	function "myupper" {
+		params = [val]
+		result = "${upper(val)} <> ${brace(v2)}"
+	}
+
+		v1=myupper("foo")
+		v2=lower("BAZ")
+		target "app" {
+			args = {
+				"v1": v1
+			}
+		}
+		`)
+
+	c, err := ParseFile(dt, "docker-bake.hcl")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "FOO <> [baz]", c.Targets[0].Args["v1"])
+}
+
+func TestHCLCombineCompose(t *testing.T) {
+	dt := []byte(`
+		target "app" {
+			context = "dir"
+			args = {
+				v1 = "foo"
+			}
+		}
+		`)
+	dt2 := []byte(`
+version: "3"
+
+services:
+  app:
+    build:
+      dockerfile: Dockerfile-alternate
+      args:
+        v2: "bar"
+`)
+
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+		{Data: dt2, Name: "c2.yml"},
+	}, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "foo", c.Targets[0].Args["v1"])
+	require.Equal(t, "bar", c.Targets[0].Args["v2"])
+	require.Equal(t, "dir", *c.Targets[0].Context)
+	require.Equal(t, "Dockerfile-alternate", *c.Targets[0].Dockerfile)
+}
+
+func TestHCLBuiltinVars(t *testing.T) {
+	dt := []byte(`
+		target "app" {
+			context = BAKE_CMD_CONTEXT
+			dockerfile = "test"
+		}
+		`)
+
+	c, err := ParseFiles([]File{
+		{Data: dt, Name: "c1.hcl"},
+	}, map[string]string{
+		"BAKE_CMD_CONTEXT": "foo",
+	})
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Targets))
+	require.Equal(t, c.Targets[0].Name, "app")
+	require.Equal(t, "foo", *c.Targets[0].Context)
+	require.Equal(t, "test", *c.Targets[0].Dockerfile)
 }

bake/hclparser/expr.go

@@ -0,0 +1,153 @@
+package hclparser
+
+import (
+	"reflect"
+	"unsafe"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/pkg/errors"
+)
+
+func funcCalls(exp hcl.Expression) ([]string, hcl.Diagnostics) {
+	node, ok := exp.(hclsyntax.Node)
+	if !ok {
+		fns, err := jsonFuncCallsRecursive(exp)
+		if err != nil {
+			return nil, hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid expression",
+					Detail:   err.Error(),
+					Subject:  exp.Range().Ptr(),
+					Context:  exp.Range().Ptr(),
+				},
+			}
+		}
+		return fns, nil
+	}
+
+	var funcnames []string
+	hcldiags := hclsyntax.VisitAll(node, func(n hclsyntax.Node) hcl.Diagnostics {
+		if fe, ok := n.(*hclsyntax.FunctionCallExpr); ok {
+			funcnames = append(funcnames, fe.Name)
+		}
+		return nil
+	})
+	if hcldiags.HasErrors() {
+		return nil, hcldiags
+	}
+	return funcnames, nil
+}
+
+func jsonFuncCallsRecursive(exp hcl.Expression) ([]string, error) {
+	je, ok := exp.(jsonExp)
+	if !ok {
+		return nil, errors.Errorf("invalid expression type %T", exp)
+	}
+	m := map[string]struct{}{}
+	for _, e := range elementExpressions(je, exp) {
+		if err := appendJSONFuncCalls(e, m); err != nil {
+			return nil, err
+		}
+	}
+	arr := make([]string, 0, len(m))
+	for n := range m {
+		arr = append(arr, n)
+	}
+	return arr, nil
+}
+
+func appendJSONFuncCalls(exp hcl.Expression, m map[string]struct{}) error {
+	v := reflect.ValueOf(exp)
+	if v.Kind() != reflect.Ptr || v.IsNil() {
+		return errors.Errorf("invalid json expression kind %T %v", exp, v.Kind())
+	}
+	src := v.Elem().FieldByName("src")
+	if src.IsZero() {
+		return errors.Errorf("%v has no property src", v.Elem().Type())
+	}
+	if src.Kind() != reflect.Interface {
+		return errors.Errorf("%v src is not interface: %v", src.Type(), src.Kind())
+	}
+	src = src.Elem()
+	if src.IsNil() {
+		return nil
+	}
+	if src.Kind() == reflect.Ptr {
+		src = src.Elem()
+	}
+	if src.Kind() != reflect.Struct {
+		return errors.Errorf("%v is not struct: %v", src.Type(), src.Kind())
+	}
+
+	// hcl/v2/json/ast#stringVal
+	val := src.FieldByName("Value")
+	if val.IsZero() {
+		return nil
+	}
+	rng := src.FieldByName("SrcRange")
+	if val.IsZero() {
+		return nil
+	}
+	var stringVal struct {
+		Value    string
+		SrcRange hcl.Range
+	}
+
+	if !val.Type().AssignableTo(reflect.ValueOf(stringVal.Value).Type()) {
+		return nil
+	}
+	if !rng.Type().AssignableTo(reflect.ValueOf(stringVal.SrcRange).Type()) {
+		return nil
+	}
+	// reflect.Set does not work for unexported fields
+	stringVal.Value = *(*string)(unsafe.Pointer(val.UnsafeAddr()))
+	stringVal.SrcRange = *(*hcl.Range)(unsafe.Pointer(rng.UnsafeAddr()))
+
+	expr, diags := hclsyntax.ParseExpression([]byte(stringVal.Value), stringVal.SrcRange.Filename, stringVal.SrcRange.Start)
+	if diags.HasErrors() {
+		return nil
+	}
+
+	fns, err := funcCalls(expr)
+	if err != nil {
+		return err
+	}
+
+	for _, fn := range fns {
+		m[fn] = struct{}{}
+	}
+
+	return nil
+}
+
+type jsonExp interface {
+	ExprList() []hcl.Expression
+	ExprMap() []hcl.KeyValuePair
+}
+
+func elementExpressions(je jsonExp, exp hcl.Expression) []hcl.Expression {
+	list := je.ExprList()
+	if len(list) != 0 {
+		exp := make([]hcl.Expression, 0, len(list))
+		for _, e := range list {
+			if je, ok := e.(jsonExp); ok {
+				exp = append(exp, elementExpressions(je, e)...)
+			}
+		}
+		return exp
+	}
+	kvlist := je.ExprMap()
+	if len(kvlist) != 0 {
+		exp := make([]hcl.Expression, 0, len(kvlist)*2)
+		for _, p := range kvlist {
+			exp = append(exp, p.Key)
+			if je, ok := p.Value.(jsonExp); ok {
+				exp = append(exp, elementExpressions(je, p.Value)...)
+			}
+		}
+		return exp
+	}
+	return []hcl.Expression{exp}
+}

bake/hclparser/hclparser.go

@@ -0,0 +1,498 @@
+package hclparser
+
+import (
+	"fmt"
+	"math"
+	"math/big"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/docker/buildx/util/userfunc"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/gohcl"
+	"github.com/pkg/errors"
+	"github.com/zclconf/go-cty/cty"
+)
+
+type Opt struct {
+	LookupVar func(string) (string, bool)
+	Vars      map[string]string
+}
+
+type variable struct {
+	Name    string         `json:"-" hcl:"name,label"`
+	Default *hcl.Attribute `json:"default,omitempty" hcl:"default,optional"`
+	Body    hcl.Body       `json:"-" hcl:",body"`
+}
+
+type functionDef struct {
+	Name     string         `json:"-" hcl:"name,label"`
+	Params   *hcl.Attribute `json:"params,omitempty" hcl:"params"`
+	Variadic *hcl.Attribute `json:"variadic_param,omitempty" hcl:"variadic_params"`
+	Result   *hcl.Attribute `json:"result,omitempty" hcl:"result"`
+}
+
+type inputs struct {
+	Variables []*variable    `hcl:"variable,block"`
+	Functions []*functionDef `hcl:"function,block"`
+
+	Remain hcl.Body `json:"-" hcl:",remain"`
+}
+
+type parser struct {
+	opt Opt
+
+	vars  map[string]*variable
+	attrs map[string]*hcl.Attribute
+	funcs map[string]*functionDef
+
+	ectx *hcl.EvalContext
+
+	progress  map[string]struct{}
+	progressF map[string]struct{}
+	doneF     map[string]struct{}
+}
+
+func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
+	fns, hcldiags := funcCalls(exp)
+	if hcldiags.HasErrors() {
+		return hcldiags
+	}
+
+	for _, fn := range fns {
+		if err := p.resolveFunction(fn); err != nil {
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid expression",
+					Detail:   err.Error(),
+					Subject:  exp.Range().Ptr(),
+					Context:  exp.Range().Ptr(),
+				},
+			}
+		}
+	}
+
+	for _, v := range exp.Variables() {
+		if _, ok := exclude[v.RootName()]; ok {
+			continue
+		}
+		if err := p.resolveValue(v.RootName()); err != nil {
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid expression",
+					Detail:   err.Error(),
+					Subject:  v.SourceRange().Ptr(),
+					Context:  v.SourceRange().Ptr(),
+				},
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *parser) resolveFunction(name string) error {
+	if _, ok := p.doneF[name]; ok {
+		return nil
+	}
+	f, ok := p.funcs[name]
+	if !ok {
+		if _, ok := p.ectx.Functions[name]; ok {
+			return nil
+		}
+		return errors.Errorf("undefined function %s", name)
+	}
+	if _, ok := p.progressF[name]; ok {
+		return errors.Errorf("function cycle not allowed for %s", name)
+	}
+	p.progressF[name] = struct{}{}
+
+	paramExprs, paramsDiags := hcl.ExprList(f.Params.Expr)
+	if paramsDiags.HasErrors() {
+		return paramsDiags
+	}
+	var diags hcl.Diagnostics
+	params := map[string]struct{}{}
+	for _, paramExpr := range paramExprs {
+		param := hcl.ExprAsKeyword(paramExpr)
+		if param == "" {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid param element",
+				Detail:   "Each parameter name must be an identifier.",
+				Subject:  paramExpr.Range().Ptr(),
+			})
+		}
+		params[param] = struct{}{}
+	}
+	var variadic hcl.Expression
+	if f.Variadic != nil {
+		variadic = f.Variadic.Expr
+		param := hcl.ExprAsKeyword(variadic)
+		if param == "" {
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid param element",
+				Detail:   "Each parameter name must be an identifier.",
+				Subject:  f.Variadic.Range.Ptr(),
+			})
+		}
+		params[param] = struct{}{}
+	}
+	if diags.HasErrors() {
+		return diags
+	}
+
+	if diags := p.loadDeps(f.Result.Expr, params); diags.HasErrors() {
+		return diags
+	}
+
+	v, diags := userfunc.NewFunction(f.Params.Expr, variadic, f.Result.Expr, func() *hcl.EvalContext {
+		return p.ectx
+	})
+	if diags.HasErrors() {
+		return diags
+	}
+	p.doneF[name] = struct{}{}
+	p.ectx.Functions[name] = v
+
+	return nil
+}
+
+func (p *parser) resolveValue(name string) (err error) {
+	if _, ok := p.ectx.Variables[name]; ok {
+		return nil
+	}
+	if _, ok := p.progress[name]; ok {
+		return errors.Errorf("variable cycle not allowed for %s", name)
+	}
+	p.progress[name] = struct{}{}
+
+	var v *cty.Value
+	defer func() {
+		if v != nil {
+			p.ectx.Variables[name] = *v
+		}
+	}()
+
+	def, ok := p.attrs[name]
+	if _, builtin := p.opt.Vars[name]; !ok && !builtin {
+		vr, ok := p.vars[name]
+		if !ok {
+			return errors.Errorf("undefined variable %q", name)
+		}
+		def = vr.Default
+	}
+
+	if def == nil {
+		val, ok := p.opt.Vars[name]
+		if !ok {
+			val, _ = p.opt.LookupVar(name)
+		}
+		vv := cty.StringVal(val)
+		v = &vv
+		return
+	}
+
+	if diags := p.loadDeps(def.Expr, nil); diags.HasErrors() {
+		return diags
+	}
+	vv, diags := def.Expr.Value(p.ectx)
+	if diags.HasErrors() {
+		return diags
+	}
+
+	_, isVar := p.vars[name]
+
+	if envv, ok := p.opt.LookupVar(name); ok && isVar {
+		if vv.Type().Equals(cty.Bool) {
+			b, err := strconv.ParseBool(envv)
+			if err != nil {
+				return errors.Wrapf(err, "failed to parse %s as bool", name)
+			}
+			vv := cty.BoolVal(b)
+			v = &vv
+			return nil
+		} else if vv.Type().Equals(cty.String) {
+			vv := cty.StringVal(envv)
+			v = &vv
+			return nil
+		} else if vv.Type().Equals(cty.Number) {
+			n, err := strconv.ParseFloat(envv, 64)
+			if err == nil && (math.IsNaN(n) || math.IsInf(n, 0)) {
+				err = errors.Errorf("invalid number value")
+			}
+			if err != nil {
+				return errors.Wrapf(err, "failed to parse %s as number", name)
+			}
+			vv := cty.NumberVal(big.NewFloat(n))
+			v = &vv
+			return nil
+		} else {
+			// TODO: support lists with csv values
+			return errors.Errorf("unsupported type %s for variable %s", v.Type(), name)
+		}
+	}
+	v = &vv
+	return nil
+}
+
+func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
+	reserved := map[string]struct{}{}
+	schema, _ := gohcl.ImpliedBodySchema(val)
+
+	for _, bs := range schema.Blocks {
+		reserved[bs.Type] = struct{}{}
+	}
+	for k := range opt.Vars {
+		reserved[k] = struct{}{}
+	}
+
+	var defs inputs
+	if err := gohcl.DecodeBody(b, nil, &defs); err != nil {
+		return err
+	}
+
+	if opt.LookupVar == nil {
+		opt.LookupVar = func(string) (string, bool) {
+			return "", false
+		}
+	}
+
+	p := &parser{
+		opt: opt,
+
+		vars:  map[string]*variable{},
+		attrs: map[string]*hcl.Attribute{},
+		funcs: map[string]*functionDef{},
+
+		progress:  map[string]struct{}{},
+		progressF: map[string]struct{}{},
+		doneF:     map[string]struct{}{},
+		ectx: &hcl.EvalContext{
+			Variables: map[string]cty.Value{},
+			Functions: stdlibFunctions,
+		},
+	}
+
+	for _, v := range defs.Variables {
+		// TODO: validate name
+		if _, ok := reserved[v.Name]; ok {
+			continue
+		}
+		p.vars[v.Name] = v
+	}
+	for _, v := range defs.Functions {
+		// TODO: validate name
+		if _, ok := reserved[v.Name]; ok {
+			continue
+		}
+		p.funcs[v.Name] = v
+	}
+
+	attrs, diags := b.JustAttributes()
+	if diags.HasErrors() {
+		for _, d := range diags {
+			if d.Detail != "Blocks are not allowed here." {
+				return diags
+			}
+		}
+	}
+
+	for _, v := range attrs {
+		if _, ok := reserved[v.Name]; ok {
+			continue
+		}
+		p.attrs[v.Name] = v
+	}
+	delete(p.attrs, "function")
+
+	for k := range p.opt.Vars {
+		_ = p.resolveValue(k)
+	}
+
+	for k := range p.attrs {
+		if err := p.resolveValue(k); err != nil {
+			if diags, ok := err.(hcl.Diagnostics); ok {
+				return diags
+			}
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid attribute",
+					Detail:   err.Error(),
+					Subject:  &p.attrs[k].Range,
+					Context:  &p.attrs[k].Range,
+				},
+			}
+		}
+	}
+
+	for k := range p.vars {
+		if err := p.resolveValue(k); err != nil {
+			if diags, ok := err.(hcl.Diagnostics); ok {
+				return diags
+			}
+			r := p.vars[k].Body.MissingItemRange()
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid value",
+					Detail:   err.Error(),
+					Subject:  &r,
+					Context:  &r,
+				},
+			}
+		}
+	}
+
+	for k := range p.funcs {
+		if err := p.resolveFunction(k); err != nil {
+			if diags, ok := err.(hcl.Diagnostics); ok {
+				return diags
+			}
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid function",
+					Detail:   err.Error(),
+					Subject:  &p.funcs[k].Params.Range,
+					Context:  &p.funcs[k].Params.Range,
+				},
+			}
+		}
+	}
+
+	content, _, diags := b.PartialContent(schema)
+	if diags.HasErrors() {
+		return diags
+	}
+
+	for _, a := range content.Attributes {
+		return hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid attribute",
+				Detail:   "global attributes currently not supported",
+				Subject:  &a.Range,
+				Context:  &a.Range,
+			},
+		}
+	}
+
+	m := map[string]map[string][]*hcl.Block{}
+	for _, b := range content.Blocks {
+		if len(b.Labels) == 0 || len(b.Labels) > 1 {
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid block",
+					Detail:   fmt.Sprintf("invalid block label: %v", b.Labels),
+					Subject:  &b.LabelRanges[0],
+					Context:  &b.LabelRanges[0],
+				},
+			}
+		}
+		bm, ok := m[b.Type]
+		if !ok {
+			bm = map[string][]*hcl.Block{}
+			m[b.Type] = bm
+		}
+
+		lbl := b.Labels[0]
+		bm[lbl] = append(bm[lbl], b)
+	}
+
+	vt := reflect.ValueOf(val).Elem().Type()
+	numFields := vt.NumField()
+
+	type value struct {
+		reflect.Value
+		idx int
+	}
+	type field struct {
+		idx    int
+		typ    reflect.Type
+		values map[string]value
+	}
+	types := map[string]field{}
+
+	for i := 0; i < numFields; i++ {
+		tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
+
+		types[tags[0]] = field{
+			idx:    i,
+			typ:    vt.Field(i).Type,
+			values: make(map[string]value),
+		}
+	}
+
+	diags = hcl.Diagnostics{}
+	for _, b := range content.Blocks {
+		v := reflect.ValueOf(val)
+
+		t, ok := types[b.Type]
+		if !ok {
+			continue
+		}
+
+		vv := reflect.New(t.typ.Elem().Elem())
+		diag := gohcl.DecodeBody(b.Body, p.ectx, vv.Interface())
+		if diag.HasErrors() {
+			diags = append(diags, diag...)
+			continue
+		}
+
+		lblIndex := setLabel(vv, b.Labels[0])
+
+		oldValue, exists := t.values[b.Labels[0]]
+		if !exists && lblIndex != -1 {
+			if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice {
+				for i := 0; i < v.Elem().Field(t.idx).Len(); i++ {
+					if b.Labels[0] == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() {
+						exists = true
+						oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i}
+						break
+					}
+				}
+			}
+
+		}
+		if exists {
+			if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
+				m.Call([]reflect.Value{vv})
+			} else {
+				v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv)
+			}
+		} else {
+			slice := v.Elem().Field(t.idx)
+			if slice.IsNil() {
+				slice = reflect.New(t.typ).Elem()
+			}
+			t.values[b.Labels[0]] = value{Value: vv, idx: slice.Len()}
+			v.Elem().Field(t.idx).Set(reflect.Append(slice, vv))
+		}
+	}
+	if diags.HasErrors() {
+		return diags
+	}
+
+	return nil
+}
+
+func setLabel(v reflect.Value, lbl string) int {
+	// cache field index?
+	numFields := v.Elem().Type().NumField()
+	for i := 0; i < numFields; i++ {
+		for _, t := range strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") {
+			if t == "label" {
+				v.Elem().Field(i).Set(reflect.ValueOf(lbl))
+				return i
+			}
+		}
+	}
+	return -1
+}

bake/hclparser/stdlib.go

@@ -0,0 +1,111 @@
+package hclparser
+
+import (
+	"github.com/hashicorp/go-cty-funcs/cidr"
+	"github.com/hashicorp/go-cty-funcs/crypto"
+	"github.com/hashicorp/go-cty-funcs/encoding"
+	"github.com/hashicorp/go-cty-funcs/uuid"
+	"github.com/hashicorp/hcl/v2/ext/tryfunc"
+	"github.com/hashicorp/hcl/v2/ext/typeexpr"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+)
+
+var stdlibFunctions = map[string]function.Function{
+	"absolute":               stdlib.AbsoluteFunc,
+	"add":                    stdlib.AddFunc,
+	"and":                    stdlib.AndFunc,
+	"base64decode":           encoding.Base64DecodeFunc,
+	"base64encode":           encoding.Base64EncodeFunc,
+	"bcrypt":                 crypto.BcryptFunc,
+	"byteslen":               stdlib.BytesLenFunc,
+	"bytesslice":             stdlib.BytesSliceFunc,
+	"can":                    tryfunc.CanFunc,
+	"ceil":                   stdlib.CeilFunc,
+	"chomp":                  stdlib.ChompFunc,
+	"chunklist":              stdlib.ChunklistFunc,
+	"cidrhost":               cidr.HostFunc,
+	"cidrnetmask":            cidr.NetmaskFunc,
+	"cidrsubnet":             cidr.SubnetFunc,
+	"cidrsubnets":            cidr.SubnetsFunc,
+	"csvdecode":              stdlib.CSVDecodeFunc,
+	"coalesce":               stdlib.CoalesceFunc,
+	"coalescelist":           stdlib.CoalesceListFunc,
+	"compact":                stdlib.CompactFunc,
+	"concat":                 stdlib.ConcatFunc,
+	"contains":               stdlib.ContainsFunc,
+	"convert":                typeexpr.ConvertFunc,
+	"distinct":               stdlib.DistinctFunc,
+	"divide":                 stdlib.DivideFunc,
+	"element":                stdlib.ElementFunc,
+	"equal":                  stdlib.EqualFunc,
+	"flatten":                stdlib.FlattenFunc,
+	"floor":                  stdlib.FloorFunc,
+	"formatdate":             stdlib.FormatDateFunc,
+	"format":                 stdlib.FormatFunc,
+	"formatlist":             stdlib.FormatListFunc,
+	"greaterthan":            stdlib.GreaterThanFunc,
+	"greaterthanorequalto":   stdlib.GreaterThanOrEqualToFunc,
+	"hasindex":               stdlib.HasIndexFunc,
+	"indent":                 stdlib.IndentFunc,
+	"index":                  stdlib.IndexFunc,
+	"int":                    stdlib.IntFunc,
+	"jsondecode":             stdlib.JSONDecodeFunc,
+	"jsonencode":             stdlib.JSONEncodeFunc,
+	"keys":                   stdlib.KeysFunc,
+	"join":                   stdlib.JoinFunc,
+	"length":                 stdlib.LengthFunc,
+	"lessthan":               stdlib.LessThanFunc,
+	"lessthanorequalto":      stdlib.LessThanOrEqualToFunc,
+	"log":                    stdlib.LogFunc,
+	"lookup":                 stdlib.LookupFunc,
+	"lower":                  stdlib.LowerFunc,
+	"max":                    stdlib.MaxFunc,
+	"md5":                    crypto.Md5Func,
+	"merge":                  stdlib.MergeFunc,
+	"min":                    stdlib.MinFunc,
+	"modulo":                 stdlib.ModuloFunc,
+	"multiply":               stdlib.MultiplyFunc,
+	"negate":                 stdlib.NegateFunc,
+	"notequal":               stdlib.NotEqualFunc,
+	"not":                    stdlib.NotFunc,
+	"or":                     stdlib.OrFunc,
+	"parseint":               stdlib.ParseIntFunc,
+	"pow":                    stdlib.PowFunc,
+	"range":                  stdlib.RangeFunc,
+	"regexall":               stdlib.RegexAllFunc,
+	"regex":                  stdlib.RegexFunc,
+	"regex_replace":          stdlib.RegexReplaceFunc,
+	"reverse":                stdlib.ReverseFunc,
+	"reverselist":            stdlib.ReverseListFunc,
+	"rsadecrypt":             crypto.RsaDecryptFunc,
+	"sethaselement":          stdlib.SetHasElementFunc,
+	"setintersection":        stdlib.SetIntersectionFunc,
+	"setproduct":             stdlib.SetProductFunc,
+	"setsubtract":            stdlib.SetSubtractFunc,
+	"setsymmetricdifference": stdlib.SetSymmetricDifferenceFunc,
+	"setunion":               stdlib.SetUnionFunc,
+	"sha1":                   crypto.Sha1Func,
+	"sha256":                 crypto.Sha256Func,
+	"sha512":                 crypto.Sha512Func,
+	"signum":                 stdlib.SignumFunc,
+	"slice":                  stdlib.SliceFunc,
+	"sort":                   stdlib.SortFunc,
+	"split":                  stdlib.SplitFunc,
+	"strlen":                 stdlib.StrlenFunc,
+	"substr":                 stdlib.SubstrFunc,
+	"subtract":               stdlib.SubtractFunc,
+	"timeadd":                stdlib.TimeAddFunc,
+	"title":                  stdlib.TitleFunc,
+	"trim":                   stdlib.TrimFunc,
+	"trimprefix":             stdlib.TrimPrefixFunc,
+	"trimspace":              stdlib.TrimSpaceFunc,
+	"trimsuffix":             stdlib.TrimSuffixFunc,
+	"try":                    tryfunc.TryFunc,
+	"upper":                  stdlib.UpperFunc,
+	"urlencode":              encoding.URLEncodeFunc,
+	"uuidv4":                 uuid.V4Func,
+	"uuidv5":                 uuid.V5Func,
+	"values":                 stdlib.ValuesFunc,
+	"zipmap":                 stdlib.ZipmapFunc,
+}

bake/remote.go

@@ -21,7 +21,7 @@ type Input struct {
 }
 
 func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
-	st, filename, ok := detectHttpContext(url)
+	st, filename, ok := detectHTTPContext(url)
 	if !ok {
 		st, ok = detectGitContext(url)
 		if !ok {
@@ -83,7 +83,7 @@ func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, na
 }
 
 func IsRemoteURL(url string) bool {
-	if _, _, ok := detectHttpContext(url); ok {
+	if _, _, ok := detectHTTPContext(url); ok {
 		return true
 	}
 	if _, ok := detectGitContext(url); ok {
@@ -92,7 +92,7 @@ func IsRemoteURL(url string) bool {
 	return false
 }
 
-func detectHttpContext(url string) (*llb.State, string, bool) {
+func detectHTTPContext(url string) (*llb.State, string, bool) {
 	if httpPrefix.MatchString(url) {
 		httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
 		return &httpContext, "context", true

build/build.go

@@ -27,14 +27,18 @@ import (
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
+	gateway "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
+	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/progress/progresswriter"
+	"github.com/moby/buildkit/util/tracing"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -110,6 +114,7 @@ type driverPair struct {
 	driverIndex int
 	platforms   []specs.Platform
 	so          *client.SolveOpt
+	bopts       gateway.BuildOpts
 }
 
 func driverIndexes(m map[string][]driverPair) []int {
@@ -181,6 +186,44 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
 }
 
 func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
+	dps, clients, err := resolveDriversBase(ctx, drivers, auth, opt, pw)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	bopts := make([]gateway.BuildOpts, len(clients))
+
+	span, ctx := tracing.StartSpan(ctx, "load buildkit capabilities", trace.WithSpanKind(trace.SpanKindInternal))
+
+	eg, ctx := errgroup.WithContext(ctx)
+	for i, c := range clients {
+		func(i int, c *client.Client) {
+			eg.Go(func() error {
+				clients[i].Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+					bopts[i] = c.BuildOpts()
+					return nil, nil
+				}, nil)
+				return nil
+			})
+		}(i, c)
+	}
+
+	err = eg.Wait()
+	span.RecordError(err)
+	span.End()
+	if err != nil {
+		return nil, nil, err
+	}
+	for key := range dps {
+		for i, dp := range dps[key] {
+			dps[key][i].bopts = bopts[dp.driverIndex]
+		}
+	}
+
+	return dps, clients, nil
+}
+
+func resolveDriversBase(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
 	availablePlatforms := map[string]int{}
 	for i, d := range drivers {
 		for _, p := range d.Platform {
@@ -245,6 +288,7 @@ func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt ma
 				workers[i] = ww
 				return nil
 			})
+
 		}(i)
 	}
 
@@ -285,7 +329,7 @@ func toRepoOnly(in string) (string, error) {
 	return strings.Join(out, ","), nil
 }
 
-func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
+func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, bopts gateway.BuildOpts, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
 	defers := make([]func(), 0, 2)
 	releaseF := func() {
 		for _, f := range defers {
@@ -322,12 +366,32 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
 		}
 	}
 
+	cacheTo := make([]client.CacheOptionsEntry, 0, len(opt.CacheTo))
+	for _, e := range opt.CacheTo {
+		if e.Type == "gha" {
+			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
+				continue
+			}
+		}
+		cacheTo = append(cacheTo, e)
+	}
+
+	cacheFrom := make([]client.CacheOptionsEntry, 0, len(opt.CacheFrom))
+	for _, e := range opt.CacheFrom {
+		if e.Type == "gha" {
+			if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
+				continue
+			}
+		}
+		cacheFrom = append(cacheFrom, e)
+	}
+
 	so := client.SolveOpt{
 		Frontend:            "dockerfile.v0",
 		FrontendAttrs:       map[string]string{},
 		LocalDirs:           map[string]string{},
-		CacheExports:        opt.CacheTo,
-		CacheImports:        opt.CacheFrom,
+		CacheExports:        cacheTo,
+		CacheImports:        cacheFrom,
 		AllowedEntitlements: opt.Allow,
 	}
 
@@ -399,6 +463,9 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
 			return nil, nil, notSupported(d, driver.OCIExporter)
 		}
 		if e.Type == "docker" {
+			if len(opt.Platforms) > 1 {
+				return nil, nil, errors.Errorf("docker exporter does not currently support exporting manifest lists")
+			}
 			if e.Output == nil {
 				if d.IsMobyDriver() {
 					e.Type = "image"
@@ -465,9 +532,11 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
 
 	// setup networkmode
 	switch opt.NetworkMode {
-	case "host", "none":
+	case "host":
 		so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
 		so.AllowedEntitlements = append(so.AllowedEntitlements, entitlements.EntitlementNetworkHost)
+	case "none":
+		so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
 	case "", "default":
 	default:
 		return nil, nil, errors.Errorf("network mode %q not supported by buildkit", opt.NetworkMode)
@@ -535,7 +604,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 				hasMobyDriver = true
 			}
 			opt.Platforms = dp.platforms
-			so, release, err := toSolveOpt(ctx, d, multiDriver, opt, w, func(name string) (io.WriteCloser, func(), error) {
+			so, release, err := toSolveOpt(ctx, d, multiDriver, opt, dp.bopts, w, func(name string) (io.WriteCloser, func(), error) {
 				return newDockerLoader(ctx, docker, name, w)
 			})
 			if err != nil {
@@ -579,13 +648,25 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
 			dps := m[k]
 			multiDriver := len(m[k]) > 1
 
+			var span trace.Span
+			ctx := ctx
+			if multiTarget {
+				span, ctx = tracing.StartSpan(ctx, k)
+			}
+
 			res := make([]*client.SolveResponse, len(dps))
 			wg := &sync.WaitGroup{}
 			wg.Add(len(dps))
 
 			var pushNames string
 
-			eg.Go(func() error {
+			eg.Go(func() (err error) {
+				defer func() {
+					if span != nil {
+						span.RecordError(err)
+						span.End()
+					}
+				}()
 				pw := progress.WithPrefix(w, "default", false)
 				wg.Wait()
 				select {
@@ -1066,3 +1147,9 @@ func handleLowercaseDockerfile(dir, p string) string {
 	}
 	return p
 }
+
+func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
+	return func(map[string]string) (io.WriteCloser, error) {
+		return wc, nil
+	}
+}

cmd/buildx/main.go

@@ -15,6 +15,11 @@ import (
 	cliflags "github.com/docker/cli/cli/flags"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/util/stack"
+	"github.com/moby/buildkit/util/tracing/detect"
+	"go.opentelemetry.io/otel"
+
+	_ "github.com/moby/buildkit/util/tracing/detect/delegated"
+	_ "github.com/moby/buildkit/util/tracing/env"
 
 	// FIXME: "k8s.io/client-go/plugin/pkg/client/auth/azure" is excluded because of compilation error
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -31,6 +36,10 @@ var experimental string
 func init() {
 	seed.WithTimeAndRand()
 	stack.SetVersionInfo(version.Version, version.Revision)
+
+	detect.ServiceName = "buildx"
+	// do not log tracing errors to stdio
+	otel.SetErrorHandler(skipErrors{})
 }
 
 func main() {
@@ -90,3 +99,7 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+type skipErrors struct{}
+
+func (skipErrors) Handle(err error) {}

commands/bake.go

@@ -9,7 +9,9 @@ import (
 	"github.com/docker/buildx/bake"
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/util/progress"
+	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/pkg/ioutils"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -25,12 +27,28 @@ type bakeOptions struct {
 func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
 	ctx := appcontext.Context()
 
+	ctx, end, err := tracing.TraceCurrentCommand(ctx, "bake")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		end(err)
+	}()
+
 	var url string
+	cmdContext := "cwd://"
 
 	if len(targets) > 0 {
 		if bake.IsRemoteURL(targets[0]) {
 			url = targets[0]
 			targets = targets[1:]
+			if len(targets) > 0 {
+				if bake.IsRemoteURL(targets[0]) {
+					cmdContext = targets[0]
+					targets = targets[1:]
+
+				}
+			}
 		}
 	}
 
@@ -75,6 +93,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 
 	var files []bake.File
 	var inp *bake.Input
+
 	if url != "" {
 		files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
 	} else {
@@ -84,13 +103,21 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 		return err
 	}
 
-	m, err := bake.ReadTargets(ctx, files, targets, overrides)
+	m, err := bake.ReadTargets(ctx, files, targets, overrides, map[string]string{
+		"BAKE_CMD_CONTEXT": cmdContext,
+	})
+	if err != nil {
+		return err
+	}
+
+	// this function can update target context string from the input so call before printOnly check
+	bo, err := bake.TargetsToBuildOpt(m, inp)
 	if err != nil {
 		return err
 	}
 
 	if in.printOnly {
-		dt, err := json.MarshalIndent(map[string]map[string]*bake.Target{"target": m}, "", "   ")
+		dt, err := json.MarshalIndent(map[string]map[string]*bake.Target{"target": m}, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -103,12 +130,25 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 		return nil
 	}
 
-	bo, err := bake.TargetsToBuildOpt(m, inp)
+	resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
 	if err != nil {
 		return err
 	}
 
-	_, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
+	if len(in.metadataFile) > 0 && resp != nil {
+		mdata := map[string]map[string]string{}
+		for k, r := range resp {
+			mdata[k] = r.ExporterResponse
+		}
+		mdatab, err := json.MarshalIndent(mdata, "", "  ")
+		if err != nil {
+			return err
+		}
+		if err := ioutils.AtomicWriteFile(in.metadataFile, mdatab, 0644); err != nil {
+			return err
+		}
+	}
+
 	return err
 }
 

commands/build.go

@@ -2,15 +2,19 @@ package commands
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/buildx/util/progress"
+	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/pkg/ioutils"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/util/appcontext"
@@ -20,6 +24,8 @@ import (
 	"github.com/spf13/pflag"
 )
 
+const defaultTargetName = "default"
+
 type buildOptions struct {
 	commonOptions
 	contextPath    string
@@ -63,15 +69,19 @@ type buildOptions struct {
 }
 
 type commonOptions struct {
-	builder    string
-	noCache    *bool
-	progress   string
-	pull       *bool
+	builder      string
+	noCache      *bool
+	progress     string
+	pull         *bool
+	metadataFile string
+	// golangci-lint#826
+	// nolint:structcheck
 	exportPush bool
+	// nolint:structcheck
 	exportLoad bool
 }
 
-func runBuild(dockerCli command.Cli, in buildOptions) error {
+func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 	if in.squash {
 		return errors.Errorf("squash currently not implemented")
 	}
@@ -81,6 +91,14 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 
 	ctx := appcontext.Context()
 
+	ctx, end, err := tracing.TraceCurrentCommand(ctx, "build")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		end(err)
+	}()
+
 	noCache := false
 	if in.noCache != nil {
 		noCache = *in.noCache
@@ -115,19 +133,23 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 
 	opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(os.Stderr))
 
-	secrets, err := build.ParseSecretSpecs(in.secrets)
+	secrets, err := buildflags.ParseSecretSpecs(in.secrets)
 	if err != nil {
 		return err
 	}
 	opts.Session = append(opts.Session, secrets)
 
-	ssh, err := build.ParseSSHSpecs(in.ssh)
+	sshSpecs := in.ssh
+	if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.contextPath) {
+		sshSpecs = []string{"default"}
+	}
+	ssh, err := buildflags.ParseSSHSpecs(sshSpecs)
 	if err != nil {
 		return err
 	}
 	opts.Session = append(opts.Session, ssh)
 
-	outputs, err := build.ParseOutputs(in.outputs)
+	outputs, err := buildflags.ParseOutputs(in.outputs)
 	if err != nil {
 		return err
 	}
@@ -168,19 +190,19 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 
 	opts.Exports = outputs
 
-	cacheImports, err := build.ParseCacheEntry(in.cacheFrom)
+	cacheImports, err := buildflags.ParseCacheEntry(in.cacheFrom)
 	if err != nil {
 		return err
 	}
 	opts.CacheFrom = cacheImports
 
-	cacheExports, err := build.ParseCacheEntry(in.cacheTo)
+	cacheExports, err := buildflags.ParseCacheEntry(in.cacheTo)
 	if err != nil {
 		return err
 	}
 	opts.CacheTo = cacheExports
 
-	allow, err := build.ParseEntitlements(in.allow)
+	allow, err := buildflags.ParseEntitlements(in.allow)
 	if err != nil {
 		return err
 	}
@@ -192,10 +214,10 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 		contextPathHash = in.contextPath
 	}
 
-	return buildTargets(ctx, dockerCli, map[string]build.Options{"default": opts}, in.progress, contextPathHash, in.builder)
+	return buildTargets(ctx, dockerCli, map[string]build.Options{defaultTargetName: opts}, in.progress, contextPathHash, in.builder, in.metadataFile)
 }
 
-func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode, contextPathHash, instance string) error {
+func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode, contextPathHash, instance string, metadataFile string) error {
 	dis, err := getInstanceOrDefault(ctx, dockerCli, instance, contextPathHash)
 	if err != nil {
 		return err
@@ -205,11 +227,24 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
 	defer cancel()
 	printer := progress.NewPrinter(ctx2, os.Stderr, progressMode)
 
-	_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
+	resp, err := build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
 	err1 := printer.Wait()
 	if err == nil {
 		err = err1
 	}
+	if err != nil {
+		return err
+	}
+
+	if len(metadataFile) > 0 && resp != nil {
+		mdatab, err := json.MarshalIndent(resp[defaultTargetName].ExporterResponse, "", "  ")
+		if err != nil {
+			return err
+		}
+		if err := ioutils.AtomicWriteFile(metadataFile, mdatab, 0644); err != nil {
+			return err
+		}
+	}
 
 	return err
 }
@@ -235,8 +270,12 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.BoolVar(&options.exportLoad, "load", false, "Shorthand for --output=type=docker")
 
 	flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Name and optionally a tag in the 'name:tag' format")
+	flags.SetAnnotation("tag", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t"})
 	flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables")
+	flags.SetAnnotation("build-arg", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg"})
+
 	flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')")
+	flags.SetAnnotation("file", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f"})
 
 	flags.StringArrayVar(&options.labels, "label", []string{}, "Set metadata for an image")
 
@@ -244,6 +283,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.StringArrayVar(&options.cacheTo, "cache-to", []string{}, "Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)")
 
 	flags.StringVar(&options.target, "target", "", "Set the target build stage to build.")
+	flags.SetAnnotation("target", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target"})
 
 	flags.StringSliceVar(&options.allow, "allow", []string{}, "Allow extra privileged entitlement, e.g. network.host, security.insecure")
 
@@ -251,6 +291,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 	flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success")
 	flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build")
 	flags.StringSliceVar(&options.extraHosts, "add-host", []string{}, "Add a custom host-to-IP mapping (host:ip)")
+	flags.SetAnnotation("add-host", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host"})
 	flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
 	flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer")
 	flags.MarkHidden("quiet")
@@ -311,8 +352,15 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) {
 	options.noCache = flags.Bool("no-cache", false, "Do not use cache when building the image")
-	flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
+
+	defaultProgress, ok := os.LookupEnv("BUILDX_PROGRESS_DEFAULT")
+	if !ok {
+		defaultProgress = "auto"
+	}
+	flags.StringVar(&options.progress, "progress", defaultProgress, "Set type of progress output (auto, plain, tty). Use plain to show container output")
+
 	options.pull = flags.Bool("pull", false, "Always attempt to pull a newer version of the image")
+	flags.StringVar(&options.metadataFile, "metadata-file", "", "Write build result metadata to the file")
 }
 
 func listToMap(values []string, defaultEnv bool) map[string]string {

commands/create.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"encoding/csv"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 
@@ -145,7 +146,14 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 
 		if in.driver == "kubernetes" {
 			// naming endpoint to make --append works
-			ep = fmt.Sprintf("%s://%s?deployment=%s", in.driver, in.name, in.nodeName)
+			ep = (&url.URL{
+				Scheme: in.driver,
+				Path:   "/" + in.name,
+				RawQuery: (&url.Values{
+					"deployment": {in.nodeName},
+					"kubeconfig": {os.Getenv("KUBECONFIG")},
+				}).Encode(),
+			}).String()
 		}
 
 		m, err := csvToMap(in.driverOpts)

commands/diskusage.go

@@ -185,8 +185,6 @@ func printSummary(tw *tabwriter.Writer, dus [][]*client.UsageInfo) {
 		}
 	}
 
-	tw = tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
-
 	if shared > 0 {
 		fmt.Fprintf(tw, "Shared:\t%.2f\n", units.Bytes(shared))
 		fmt.Fprintf(tw, "Private:\t%.2f\n", units.Bytes(total-shared))

commands/imagetools/create.go

@@ -118,7 +118,15 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 						return err
 					}
 					srcs[i].Ref = nil
-					srcs[i].Desc = desc
+					if srcs[i].Desc.Digest == "" {
+						srcs[i].Desc = desc
+					} else {
+						var err error
+						srcs[i].Desc, err = mergeDesc(desc, srcs[i].Desc)
+						if err != nil {
+							return err
+						}
+					}
 					return nil
 				})
 			}(i)
@@ -168,7 +176,7 @@ func parseSources(in []string) ([]*src, error) {
 	for i, in := range in {
 		s, err := parseSource(in)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, refereces and descriptors", in)
+			return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, references and descriptors", in)
 		}
 		out[i] = s
 	}
@@ -238,3 +246,19 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
 
 	return cmd
 }
+
+func mergeDesc(d1, d2 ocispec.Descriptor) (ocispec.Descriptor, error) {
+	if d2.Size != 0 && d1.Size != d2.Size {
+		return ocispec.Descriptor{}, errors.Errorf("invalid size mismatch for %s, %d != %d", d1.Digest, d2.Size, d1.Size)
+	}
+	if d2.MediaType != "" {
+		d1.MediaType = d2.MediaType
+	}
+	if len(d2.Annotations) != 0 {
+		d1.Annotations = d2.Annotations // no merge so support removes
+	}
+	if d2.Platform != nil {
+		d1.Platform = d2.Platform // missing items filled in later from image config
+	}
+	return d1, nil
+}

commands/imagetools/inspect.go

@@ -38,7 +38,7 @@ func runInspect(dockerCli command.Cli, in inspectOptions, name string) error {
 	// case images.MediaTypeDockerSchema2Manifest, specs.MediaTypeImageManifest:
 	// TODO: handle distribution manifest and schema1
 	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
-		imagetools.PrintManifestList(dt, desc, name, os.Stdout)
+		return imagetools.PrintManifestList(dt, desc, name, os.Stdout)
 	default:
 		fmt.Printf("%s\n", dt)
 	}

commands/prune.go

@@ -129,7 +129,7 @@ func pruneCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "prune",
-		Short: "Remove build cache ",
+		Short: "Remove build cache",
 		Args:  cli.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			options.builder = rootOpts.builder

commands/rm.go

@@ -11,7 +11,8 @@ import (
 )
 
 type rmOptions struct {
-	builder string
+	builder   string
+	keepState bool
 }
 
 func runRm(dockerCli command.Cli, in rmOptions) error {
@@ -28,7 +29,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		if err != nil {
 			return err
 		}
-		err1 := stop(ctx, dockerCli, ng, true)
+		err1 := rm(ctx, dockerCli, ng, in.keepState)
 		if err := txn.Remove(ng.Name); err != nil {
 			return err
 		}
@@ -40,7 +41,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return err
 	}
 	if ng != nil {
-		err1 := stop(ctx, dockerCli, ng, true)
+		err1 := rm(ctx, dockerCli, ng, in.keepState)
 		if err := txn.Remove(ng.Name); err != nil {
 			return err
 		}
@@ -66,10 +67,13 @@ func rmCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 		},
 	}
 
+	flags := cmd.Flags()
+	flags.BoolVar(&options.keepState, "keep-state", false, "Keep BuildKit state")
+
 	return cmd
 }
 
-func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bool) error {
+func rm(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, keepState bool) error {
 	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
 	if err != nil {
 		return err
@@ -79,34 +83,9 @@ func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bo
 			if err := di.Driver.Stop(ctx, true); err != nil {
 				return err
 			}
-			if rm {
-				if err := di.Driver.Rm(ctx, true); err != nil {
-					return err
-				}
-			}
-		}
-		if di.Err != nil {
-			err = di.Err
-		}
-	}
-	return err
-}
-
-func stopCurrent(ctx context.Context, dockerCli command.Cli, rm bool) error {
-	dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
-	if err != nil {
-		return err
-	}
-	for _, di := range dis {
-		if di.Driver != nil {
-			if err := di.Driver.Stop(ctx, true); err != nil {
+			if err := di.Driver.Rm(ctx, true, !keepState); err != nil {
 				return err
 			}
-			if rm {
-				if err := di.Driver.Rm(ctx, true); err != nil {
-					return err
-				}
-			}
 		}
 		if di.Err != nil {
 			err = di.Err

commands/stop.go

@@ -1,6 +1,9 @@
 package commands
 
 import (
+	"context"
+
+	"github.com/docker/buildx/store"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/moby/buildkit/util/appcontext"
@@ -25,7 +28,7 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 		if err != nil {
 			return err
 		}
-		if err := stop(ctx, dockerCli, ng, false); err != nil {
+		if err := stop(ctx, dockerCli, ng); err != nil {
 			return err
 		}
 		return nil
@@ -36,10 +39,10 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 		return err
 	}
 	if ng != nil {
-		return stop(ctx, dockerCli, ng, false)
+		return stop(ctx, dockerCli, ng)
 	}
 
-	return stopCurrent(ctx, dockerCli, false)
+	return stopCurrent(ctx, dockerCli)
 }
 
 func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
@@ -66,3 +69,39 @@ func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	return cmd
 }
+
+func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup) error {
+	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
+	if err != nil {
+		return err
+	}
+	for _, di := range dis {
+		if di.Driver != nil {
+			if err := di.Driver.Stop(ctx, true); err != nil {
+				return err
+			}
+		}
+		if di.Err != nil {
+			err = di.Err
+		}
+	}
+	return err
+}
+
+func stopCurrent(ctx context.Context, dockerCli command.Cli) error {
+	dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
+	if err != nil {
+		return err
+	}
+	for _, di := range dis {
+		if di.Driver != nil {
+			if err := di.Driver.Stop(ctx, true); err != nil {
+				return err
+			}
+		}
+		if di.Err != nil {
+			err = di.Err
+		}
+	}
+	return err
+}

commands/util.go

@@ -2,8 +2,10 @@ package commands
 
 import (
 	"context"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/driver"
@@ -12,11 +14,13 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/context/docker"
 	"github.com/docker/cli/cli/context/kubernetes"
+	ctxstore "github.com/docker/cli/cli/context/store"
 	dopts "github.com/docker/cli/opts"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 // getStore returns current builder instance store
@@ -192,12 +196,12 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 				contextStore := dockerCli.ContextStore()
 
 				var kcc driver.KubeClientConfig
-				kcc, err = kubernetes.ConfigFromContext(n.Endpoint, contextStore)
+				kcc, err = configFromContext(n.Endpoint, contextStore)
 				if err != nil {
 					// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
 					// try again with name="default".
 					// FIXME: n should retain real context name.
-					kcc, err = kubernetes.ConfigFromContext("default", contextStore)
+					kcc, err = configFromContext("default", contextStore)
 					if err != nil {
 						logrus.Error(err)
 					}
@@ -237,6 +241,21 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 	return dis, nil
 }
 
+func configFromContext(endpointName string, s ctxstore.Reader) (clientcmd.ClientConfig, error) {
+	if strings.HasPrefix(endpointName, "kubernetes://") {
+		u, _ := url.Parse(endpointName)
+
+		if kubeconfig := u.Query().Get("kubeconfig"); kubeconfig != "" {
+			clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+				&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig},
+				&clientcmd.ConfigOverrides{},
+			)
+			return clientConfig, nil
+		}
+	}
+	return kubernetes.ConfigFromContext(endpointName, s)
+}
+
 // clientForEndpoint returns a docker client for an endpoint
 func clientForEndpoint(dockerCli command.Cli, name string) (dockerclient.APIClient, error) {
 	list, err := dockerCli.ContextStore().List()
@@ -403,13 +422,24 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
 		return err
 	}
 
-	// skip when multi drivers
-	if len(ngi.drivers) == 1 {
+	kubernetesDriverCount := 0
+
+	for _, di := range ngi.drivers {
+		if di.info != nil && len(di.info.DynamicNodes) > 0 {
+			kubernetesDriverCount++
+		}
+	}
+
+	isAllKubernetesDrivers := len(ngi.drivers) == kubernetesDriverCount
+
+	if isAllKubernetesDrivers {
+		var drivers []dinfo
+		var dynamicNodes []store.Node
+
 		for _, di := range ngi.drivers {
 			// dynamic nodes are used in Kubernetes driver.
 			// Kubernetes pods are dynamically mapped to BuildKit Nodes.
 			if di.info != nil && len(di.info.DynamicNodes) > 0 {
-				var drivers []dinfo
 				for i := 0; i < len(di.info.DynamicNodes); i++ {
 					// all []dinfo share *build.DriverInfo and *driver.Info
 					diClone := di
@@ -418,14 +448,16 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
 					}
 					drivers = append(drivers, di)
 				}
-				// not append (remove the static nodes in the store)
-				ngi.ng.Nodes = di.info.DynamicNodes
-				ngi.ng.Dynamic = true
-				ngi.drivers = drivers
-				return nil
+				dynamicNodes = append(dynamicNodes, di.info.DynamicNodes...)
 			}
 		}
+
+		// not append (remove the static nodes in the store)
+		ngi.ng.Nodes = dynamicNodes
+		ngi.drivers = drivers
+		ngi.ng.Dynamic = true
 	}
+
 	return nil
 }
 

commands/version.go

@@ -17,7 +17,7 @@ func runVersion(dockerCli command.Cli) error {
 func versionCmd(dockerCli command.Cli) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "version",
-		Short: "Show buildx version information ",
+		Short: "Show buildx version information",
 		Args:  cli.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runVersion(dockerCli)

docs/docsgen/generate.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/buildx/commands"
+	"github.com/docker/cli/cli/command"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const descriptionSourcePath = "docs/reference/"
+
+func generateDocs(opts *options) error {
+	dockerCLI, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+	cmd := &cobra.Command{
+		Use:   "docker [OPTIONS] COMMAND [ARG...]",
+		Short: "The base command for the Docker CLI.",
+	}
+	cmd.AddCommand(commands.NewRootCmd("buildx", true, dockerCLI))
+	return genCmd(cmd, opts.target)
+}
+
+func getMDFilename(cmd *cobra.Command) string {
+	name := cmd.CommandPath()
+	if i := strings.Index(name, " "); i >= 0 {
+		name = name[i+1:]
+	}
+	return strings.ReplaceAll(name, " ", "_") + ".md"
+}
+
+func genCmd(cmd *cobra.Command, dir string) error {
+	for _, c := range cmd.Commands() {
+		if err := genCmd(c, dir); err != nil {
+			return err
+		}
+	}
+	if !cmd.HasParent() {
+		return nil
+	}
+
+	mdFile := getMDFilename(cmd)
+	fullPath := filepath.Join(dir, mdFile)
+
+	content, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return errors.Wrapf(err, "%s does not exist", mdFile)
+		}
+	}
+
+	cs := string(content)
+
+	markerStart := "<!---MARKER_GEN_START-->"
+	markerEnd := "<!---MARKER_GEN_END-->"
+
+	start := strings.Index(cs, markerStart)
+	end := strings.Index(cs, markerEnd)
+
+	if start == -1 {
+		return errors.Errorf("no start marker in %s", mdFile)
+	}
+	if end == -1 {
+		return errors.Errorf("no end marker in %s", mdFile)
+	}
+
+	out, err := cmdOutput(cmd, cs)
+	if err != nil {
+		return err
+	}
+	cont := cs[:start] + markerStart + "\n" + out + "\n" + cs[end:]
+
+	fi, err := os.Stat(fullPath)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(fullPath, []byte(cont), fi.Mode()); err != nil {
+		return errors.Wrapf(err, "failed to write %s", fullPath)
+	}
+	log.Printf("updated %s", fullPath)
+	return nil
+}
+
+func makeLink(txt, link string, f *pflag.Flag, isAnchor bool) string {
+	link = "#" + link
+	annotations, ok := f.Annotations["docs.external.url"]
+	if ok && len(annotations) > 0 {
+		link = annotations[0]
+	} else {
+		if !isAnchor {
+			return txt
+		}
+	}
+
+	return "[" + txt + "](" + link + ")"
+}
+
+func cmdOutput(cmd *cobra.Command, old string) (string, error) {
+	b := &strings.Builder{}
+
+	desc := cmd.Short
+	if cmd.Long != "" {
+		desc = cmd.Long
+	}
+	if desc != "" {
+		fmt.Fprintf(b, "%s\n\n", desc)
+	}
+
+	if len(cmd.Aliases) != 0 {
+		fmt.Fprintf(b, "### Aliases\n\n`%s`", cmd.Name())
+		for _, a := range cmd.Aliases {
+			fmt.Fprintf(b, ", `%s`", a)
+		}
+		fmt.Fprint(b, "\n\n")
+	}
+
+	if len(cmd.Commands()) != 0 {
+		fmt.Fprint(b, "### Subcommands\n\n")
+		fmt.Fprint(b, "| Name | Description |\n")
+		fmt.Fprint(b, "| --- | --- |\n")
+		for _, c := range cmd.Commands() {
+			fmt.Fprintf(b, "| [`%s`](%s) | %s |\n", c.Name(), getMDFilename(c), c.Short)
+		}
+		fmt.Fprint(b, "\n\n")
+	}
+
+	hasFlags := cmd.Flags().HasAvailableFlags()
+
+	cmd.Flags().AddFlagSet(cmd.InheritedFlags())
+
+	if hasFlags {
+		fmt.Fprint(b, "### Options\n\n")
+		fmt.Fprint(b, "| Name | Description |\n")
+		fmt.Fprint(b, "| --- | --- |\n")
+
+		cmd.Flags().VisitAll(func(f *pflag.Flag) {
+			if f.Hidden {
+				return
+			}
+			isLink := strings.Contains(old, "<a name=\""+f.Name+"\"></a>")
+			fmt.Fprint(b, "| ")
+			if f.Shorthand != "" {
+				name := "`-" + f.Shorthand + "`"
+				name = makeLink(name, f.Name, f, isLink)
+				fmt.Fprintf(b, "%s, ", name)
+			}
+			name := "`--" + f.Name
+			if f.Value.Type() != "bool" {
+				name += " " + f.Value.Type()
+			}
+			name += "`"
+			name = makeLink(name, f.Name, f, isLink)
+			fmt.Fprintf(b, "%s | %s |\n", name, f.Usage)
+		})
+		fmt.Fprintln(b, "")
+	}
+
+	return b.String(), nil
+}
+
+type options struct {
+	target string
+}
+
+func parseArgs() (*options, error) {
+	opts := &options{}
+	flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
+	flags.StringVar(&opts.target, "target", descriptionSourcePath, "Docs directory")
+	err := flags.Parse(os.Args[1:])
+	return opts, err
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Printf("error: %+v", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	opts, err := parseArgs()
+	if err != nil {
+		return err
+	}
+	if err := generateDocs(opts); err != nil {
+		return err
+	}
+	return nil
+}

docs/reference/buildx.md

@@ -0,0 +1,31 @@
+# buildx
+
+```
+docker buildx [OPTIONS] COMMAND
+```
+
+<!---MARKER_GEN_START-->
+Build with BuildKit
+
+### Subcommands
+
+| Name | Description |
+| --- | --- |
+| [`bake`](buildx_bake.md) | Build from a file |
+| [`build`](buildx_build.md) | Start a build |
+| [`create`](buildx_create.md) | Create a new builder instance |
+| [`du`](buildx_du.md) | Disk usage |
+| [`imagetools`](buildx_imagetools.md) | Commands to work on images in registry |
+| [`inspect`](buildx_inspect.md) | Inspect current builder instance |
+| [`install`](buildx_install.md) | Install buildx as a 'docker builder' alias |
+| [`ls`](buildx_ls.md) | List builder instances |
+| [`prune`](buildx_prune.md) | Remove build cache |
+| [`rm`](buildx_rm.md) | Remove a builder instance |
+| [`stop`](buildx_stop.md) | Stop builder instance |
+| [`uninstall`](buildx_uninstall.md) | Uninstall the 'docker builder' alias |
+| [`use`](buildx_use.md) | Set the current builder instance |
+| [`version`](buildx_version.md) | Show buildx version information |
+
+
+
+<!---MARKER_GEN_END-->

docs/reference/buildx_bake.md

@@ -0,0 +1,374 @@
+# buildx bake
+
+```
+docker buildx bake [OPTIONS] [TARGET...]
+```
+
+<!---MARKER_GEN_START-->
+Build from a file
+
+### Aliases
+
+`bake`, `f`
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| [`-f`](#file), [`--file stringArray`](#file) | Build definition file |
+| `--load` | Shorthand for --set=*.output=type=docker |
+| `--metadata-file string` | Write build result metadata to the file |
+| [`--no-cache`](#no-cache) | Do not use cache when building the image |
+| [`--print`](#print) | Print the options without building |
+| [`--progress string`](#progress) | Set type of progress output (auto, plain, tty). Use plain to show container output |
+| [`--pull`](#pull) | Always attempt to pull a newer version of the image |
+| `--push` | Shorthand for --set=*.output=type=registry |
+| [`--set stringArray`](#set) | Override target value (eg: targetpattern.key=value) |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Bake is a high-level build command. Each specified target will run in parallel
+as part of the build.
+
+Read [High-level build options](https://github.com/docker/buildx#high-level-build-options) for introduction.
+
+Please note that `buildx bake` command may receive backwards incompatible features in the future if needed. We are looking for feedback on improving the command and extending the functionality further.
+
+## Examples
+
+### <a name="file"></a> Specify a build definition file (-f, --file)
+
+By default, `buildx bake` looks for build definition files in the current directory,
+the following are parsed:
+
+- `docker-compose.yml`
+- `docker-compose.yaml`
+- `docker-bake.json`
+- `docker-bake.override.json`
+- `docker-bake.hcl`
+- `docker-bake.override.hcl`
+
+Use the `-f` / `--file` option to specify the build definition file to use. The
+file can be a Docker Compose, JSON or HCL file. If multiple files are specified
+they are all read and configurations are combined.
+
+The following example uses a Docker Compose file named `docker-compose.dev.yaml`
+as build definition file, and builds all targets in the file:
+
+```console
+$ docker buildx bake -f docker-compose.dev.yaml
+
+[+] Building 66.3s (30/30) FINISHED
+ => [frontend internal] load build definition from Dockerfile  0.1s
+ => => transferring dockerfile: 36B                            0.0s
+ => [backend internal] load build definition from Dockerfile   0.2s
+ => => transferring dockerfile: 3.73kB                         0.0s
+ => [database internal] load build definition from Dockerfile  0.1s
+ => => transferring dockerfile: 5.77kB                         0.0s
+ ...
+```
+
+Pass the names of the targets to build, to build only specific target(s). The
+following example builds the `backend` and `database` targets that are defined
+in the `docker-compose.dev.yaml` file, skipping the build for the `frontend`
+target:
+
+```console
+$ docker buildx bake -f docker-compose.dev.yaml backend database
+
+[+] Building 2.4s (13/13) FINISHED
+ => [backend internal] load build definition from Dockerfile  0.1s
+ => => transferring dockerfile: 81B                           0.0s
+ => [database internal] load build definition from Dockerfile 0.2s
+ => => transferring dockerfile: 36B                           0.0s
+ => [backend internal] load .dockerignore                     0.3s
+ ...
+```
+
+### <a name="no-cache"></a> Do not use cache when building the image (--no-cache)
+
+Same as `build --no-cache`. Do not use cache when building the image.
+
+### <a name="print"></a> Print the options without building (--print)
+
+Prints the resulting options of the targets desired to be built, in a JSON format,
+without starting a build.
+
+```console
+$ docker buildx bake -f docker-bake.hcl --print db
+{
+   "target": {
+      "db": {
+         "context": "./",
+         "dockerfile": "Dockerfile",
+         "tags": [
+            "docker.io/tiborvass/db"
+         ]
+      }
+   }
+}
+```
+
+### <a name="progress"></a> Set type of progress output (--progress)
+
+Same as `build --progress`. Set type of progress output (auto, plain, tty). Use
+plain to show container output (default "auto").
+
+The following example uses `plain` output during the build:
+
+```console
+$ docker buildx bake --progress=plain
+
+#2 [backend internal] load build definition from Dockerfile.test
+#2 sha256:de70cb0bb6ed8044f7b9b1b53b67f624e2ccfb93d96bb48b70c1fba562489618
+#2 ...
+
+#1 [database internal] load build definition from Dockerfile.test
+#1 sha256:453cb50abd941762900a1212657a35fc4aad107f5d180b0ee9d93d6b74481bce
+#1 transferring dockerfile: 36B done
+#1 DONE 0.1s
+...
+```
+
+
+### <a name="pull"></a> Always attempt to pull a newer version of the image (--pull)
+
+Same as `build --pull`.
+
+### <a name="set"></a> Override target configurations from command line (--set)
+
+```
+--set targetpattern.key[.subkey]=value
+```
+
+Override target configurations from command line. The pattern matching syntax is
+defined in https://golang.org/pkg/path/#Match.
+
+**Examples**
+
+```console
+$ docker buildx bake --set target.args.mybuildarg=value
+$ docker buildx bake --set target.platform=linux/arm64
+$ docker buildx bake --set foo*.args.mybuildarg=value # overrides build arg for all targets starting with 'foo'
+$ docker buildx bake --set *.platform=linux/arm64     # overrides platform for all targets
+$ docker buildx bake --set foo*.no-cache              # bypass caching only for targets starting with 'foo'
+```
+
+Complete list of overridable fields:
+args, cache-from, cache-to, context, dockerfile, labels, no-cache, output, platform,
+pull, secrets, ssh, tags, target
+
+### File definition
+
+In addition to compose files, bake supports a JSON and an equivalent HCL file
+format for defining build groups and targets.
+
+A target reflects a single docker build invocation with the same options that
+you would specify for `docker build`. A group is a grouping of targets.
+
+Multiple files can include the same target and final build options will be
+determined by merging them together.
+
+In the case of compose files, each service corresponds to a target.
+
+A group can specify its list of targets with the `targets` option. A target can
+inherit build options by setting the `inherits` option to the list of targets or
+groups to inherit from.
+
+Note: Design of bake command is work in progress, the user experience may change
+based on feedback.
+
+
+**Example HCL definition**
+
+```hcl
+group "default" {
+    targets = ["db", "webapp-dev"]
+}
+
+target "webapp-dev" {
+    dockerfile = "Dockerfile.webapp"
+    tags = ["docker.io/username/webapp"]
+}
+
+target "webapp-release" {
+    inherits = ["webapp-dev"]
+    platforms = ["linux/amd64", "linux/arm64"]
+}
+
+target "db" {
+    dockerfile = "Dockerfile.db"
+    tags = ["docker.io/username/db"]
+}
+```
+
+Complete list of valid target fields:
+
+`args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `inherits`, `labels`,
+`no-cache`, `output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
+
+### HCL variables and functions
+
+Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable),
+the HCL file format also supports variable block definitions. These can be used
+to define variables with values provided by the current environment, or a default
+value when unset.
+
+
+Example of using interpolation to tag an image with the git sha:
+
+```console
+$ cat <<'EOF' > docker-bake.hcl
+variable "TAG" {
+    default = "latest"
+}
+
+group "default" {
+    targets = ["webapp"]
+}
+
+target "webapp" {
+    tags = ["docker.io/username/webapp:${TAG}"]
+}
+EOF
+
+$ docker buildx bake --print webapp
+{
+   "target": {
+      "webapp": {
+         "context": ".",
+         "dockerfile": "Dockerfile",
+         "tags": [
+            "docker.io/username/webapp:latest"
+         ]
+      }
+   }
+}
+
+$ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp
+{
+   "target": {
+      "webapp": {
+         "context": ".",
+         "dockerfile": "Dockerfile",
+         "tags": [
+            "docker.io/username/webapp:985e9e9"
+         ]
+      }
+   }
+}
+```
+
+
+A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hcl.go#L19-L65)
+provided by [go-cty](https://github.com/zclconf/go-cty/tree/main/cty/function/stdlib)
+are available for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/main/ext/userfunc)
+are also supported.
+
+Example of using the `add` function:
+
+```console
+$ cat <<'EOF' > docker-bake.hcl
+variable "TAG" {
+    default = "latest"
+}
+
+group "default" {
+    targets = ["webapp"]
+}
+
+target "webapp" {
+    args = {
+        buildno = "${add(123, 1)}"
+    }
+}
+EOF
+
+$ docker buildx bake --print webapp
+{
+   "target": {
+      "webapp": {
+         "context": ".",
+         "dockerfile": "Dockerfile",
+         "args": {
+            "buildno": "124"
+         }
+      }
+   }
+}
+```
+
+Example of defining an `increment` function:
+
+```console
+$ cat <<'EOF' > docker-bake.hcl
+function "increment" {
+    params = [number]
+    result = number + 1
+}
+
+group "default" {
+    targets = ["webapp"]
+}
+
+target "webapp" {
+    args = {
+        buildno = "${increment(123)}"
+    }
+}
+EOF
+
+$ docker buildx bake --print webapp
+{
+   "target": {
+      "webapp": {
+         "context": ".",
+         "dockerfile": "Dockerfile",
+         "args": {
+            "buildno": "124"
+         }
+      }
+   }
+}
+```
+
+Example of only adding tags if a variable is not empty using an `notequal`
+function:
+
+```console
+$ cat <<'EOF' > docker-bake.hcl
+variable "TAG" {default="" }
+
+group "default" {
+    targets = [
+        "webapp",
+    ]
+}
+
+target "webapp" {
+    context="."
+    dockerfile="Dockerfile"
+    tags = [
+        "my-image:latest",
+        notequal("",TAG) ? "my-image:${TAG}": "",
+    ]
+}
+EOF
+
+$ docker buildx bake --print webapp
+{
+   "target": {
+      "webapp": {
+         "context": ".",
+         "dockerfile": "Dockerfile",
+         "tags": [
+            "my-image:latest"
+         ]
+      }
+   }
+}
+```

docs/reference/buildx_build.md

@@ -0,0 +1,272 @@
+# buildx build
+
+```
+docker buildx build [OPTIONS] PATH | URL | -
+```
+
+<!---MARKER_GEN_START-->
+Start a build
+
+### Aliases
+
+`build`, `b`
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| [`--add-host stringSlice`](https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host) | Add a custom host-to-IP mapping (host:ip) |
+| [`--allow stringSlice`](#allow) | Allow extra privileged entitlement, e.g. network.host, security.insecure |
+| [`--build-arg stringArray`](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg) | Set build-time variables |
+| `--builder string` | Override the configured builder instance |
+| [`--cache-from stringArray`](#cache-from) | External cache sources (eg. user/app:cache, type=local,src=path/to/dir) |
+| [`--cache-to stringArray`](#cache-to) | Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) |
+| [`-f`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f), [`--file string`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f) | Name of the Dockerfile (Default is 'PATH/Dockerfile') |
+| `--iidfile string` | Write the image ID to the file |
+| `--label stringArray` | Set metadata for an image |
+| [`--load`](#load) | Shorthand for --output=type=docker |
+| `--metadata-file string` | Write build result metadata to the file |
+| `--network string` | Set the networking mode for the RUN instructions during build |
+| `--no-cache` | Do not use cache when building the image |
+| [`-o`](#output), [`--output stringArray`](#output) | Output destination (format: type=local,dest=path) |
+| [`--platform stringArray`](#platform) | Set target platform for build |
+| `--progress string` | Set type of progress output (auto, plain, tty). Use plain to show container output |
+| `--pull` | Always attempt to pull a newer version of the image |
+| [`--push`](#push) | Shorthand for --output=type=registry |
+| `--secret stringArray` | Secret file to expose to the build: id=mysecret,src=/local/secret |
+| `--ssh stringArray` | SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]]) |
+| [`-t`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t), [`--tag stringArray`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t) | Name and optionally a tag in the 'name:tag' format |
+| [`--target string`](https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target) | Set the target build stage to build. |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+The `buildx build` command starts a build using BuildKit. This command is similar
+to the UI of `docker build` command and takes the same flags and arguments.
+
+For documentation on most of these flags, refer to the [`docker build`
+documentation](https://docs.docker.com/engine/reference/commandline/build/). In
+here we’ll document a subset of the new flags.
+
+## Examples
+
+### <a name="platform"></a> Set the target platforms for the build (--platform)
+
+```
+--platform=value[,value]
+```
+
+Set the target platform for the build. All `FROM` commands inside the Dockerfile
+without their own `--platform` flag will pull base images for this platform and
+this value will also be the platform of the resulting image. The default value
+will be the current platform of the buildkit daemon.
+
+When using `docker-container` driver with `buildx`, this flag can accept multiple
+values as an input separated by a comma. With multiple values the result will be
+built for all of the specified platforms and joined together into a single manifest
+list.
+
+If the `Dockerfile` needs to invoke the `RUN` command, the builder needs runtime
+support for the specified platform. In a clean setup, you can only execute `RUN`
+commands for your system architecture.
+If your kernel supports [`binfmt_misc`](https://en.wikipedia.org/wiki/Binfmt_misc)
+launchers for secondary architectures, buildx will pick them up automatically.
+Docker desktop releases come with `binfmt_misc` automatically configured for `arm64`
+and `arm` architectures. You can see what runtime platforms your current builder
+instance supports by running `docker buildx inspect --bootstrap`.
+
+Inside a `Dockerfile`, you can access the current platform value through
+`TARGETPLATFORM` build argument. Please refer to the [`docker build`
+documentation](https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope)
+for the full description of automatic platform argument variants .
+
+The formatting for the platform specifier is defined in the [containerd source
+code](https://github.com/containerd/containerd/blob/v1.4.3/platforms/platforms.go#L63).
+
+**Examples**
+
+```console
+$ docker buildx build --platform=linux/arm64 .
+$ docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 .
+$ docker buildx build --platform=darwin .
+```
+
+### <a name="output"></a> Set the export action for the build result (-o, --output)
+
+```
+-o, --output=[PATH,-,type=TYPE[,KEY=VALUE]
+```
+
+Sets the export action for the build result. In `docker build` all builds finish
+by creating a container image and exporting it to `docker images`. `buildx` makes
+this step configurable allowing results to be exported directly to the client,
+oci image tarballs, registry etc.
+
+Buildx with `docker` driver currently only supports local, tarball exporter and
+image exporter. `docker-container` driver supports all the exporters.
+
+If just the path is specified as a value, `buildx` will use the local exporter
+with this path as the destination. If the value is "-", `buildx` will use `tar`
+exporter and write to `stdout`.
+
+**Examples**
+
+```console
+$ docker buildx build -o . .
+$ docker buildx build -o outdir .
+$ docker buildx build -o - - > out.tar
+$ docker buildx build -o type=docker .
+$ docker buildx build -o type=docker,dest=- . > myimage.tar
+$ docker buildx build -t tonistiigi/foo -o type=registry
+```
+
+Supported exported types are:
+
+#### `local`
+
+The `local` export type writes all result files to a directory on the client. The
+new files will be owned by the current user. On multi-platform builds, all results
+will be put in subdirectories by their platform.
+
+Attribute key:
+
+- `dest` - destination directory where files will be written
+
+#### `tar`
+
+The `tar` export type writes all result files as a single tarball on the client.
+On multi-platform builds all results will be put in subdirectories by their platform.
+
+Attribute key:
+
+- `dest` - destination path where tarball will be written. “-” writes to stdout.
+
+#### `oci`
+
+The `oci` export type writes the result image or manifest list as an [OCI image
+layout](https://github.com/opencontainers/image-spec/blob/v1.0.1/image-layout.md)
+tarball on the client.
+
+Attribute key:
+
+- `dest` - destination path where tarball will be written. “-” writes to stdout.
+
+#### `docker`
+
+The `docker` export type writes the single-platform result image as a [Docker image
+specification](https://github.com/docker/docker/blob/v20.10.2/image/spec/v1.2.md)
+tarball on the client. Tarballs created by this exporter are also OCI compatible.
+
+Currently, multi-platform images cannot be exported with the `docker` export type.
+The most common usecase for multi-platform images is to directly push to a registry
+(see [`registry`](#registry)).
+
+Attribute keys:
+
+- `dest` - destination path where tarball will be written. If not specified the
+tar will be loaded automatically to the current docker instance.
+- `context` - name for the docker context where to import the result
+
+#### `image`
+
+The `image` exporter writes the build result as an image or a manifest list. When
+using `docker` driver the image will appear in `docker images`. Optionally, image
+can be automatically pushed to a registry by specifying attributes.
+
+Attribute keys:
+
+- `name` - name (references) for the new image.
+- `push` - boolean to automatically push the image.
+
+#### `registry`
+
+The `registry` exporter is a shortcut for `type=image,push=true`.
+
+
+### <a name="push"></a> Push the build result to a registry (--push)
+
+Shorthand for [`--output=type=registry`](#registry). Will automatically push the
+build result to registry.
+
+### <a name="load"></a> Load the single-platform build result to `docker images` (--load)
+
+Shorthand for [`--output=type=docker`](#docker). Will automatically load the
+single-platform build result to `docker images`.
+
+### <a name="cache-from"></a> Use an external cache source for a build (--cache-from)
+
+```
+--cache-from=[NAME|type=TYPE[,KEY=VALUE]]
+```
+
+Use an external cache source for a build. Supported types are `registry` and `local`.
+The `registry` source can import cache from a cache manifest or (special) image
+configuration on the registry. The `local` source can import cache from local
+files previously exported with `--cache-to`.
+
+If no type is specified, `registry` exporter is used with a specified reference.
+
+`docker` driver currently only supports importing build cache from the registry.
+
+**Examples**
+
+```console
+$ docker buildx build --cache-from=user/app:cache .
+$ docker buildx build --cache-from=user/app .
+$ docker buildx build --cache-from=type=registry,ref=user/app .
+$ docker buildx build --cache-from=type=local,src=path/to/cache .
+```
+
+### <a name="cache-to"></a> Export build cache to an external cache destination (--cache-to)
+
+```
+--cache-to=[NAME|type=TYPE[,KEY=VALUE]]
+```
+
+Export build cache to an external cache destination. Supported types are `registry`,
+`local` and `inline`. Registry exports build cache to a cache manifest in the
+registry, local exports cache to a local directory on the client and inline writes
+the cache metadata into the image configuration.
+
+`docker` driver currently only supports exporting inline cache metadata to image
+configuration. Alternatively, `--build-arg BUILDKIT_INLINE_CACHE=1` can be used
+to trigger inline cache exporter.
+
+Attribute key:
+
+- `mode` - Specifies how many layers are exported with the cache. “min” on only
+exports layers already in the final build stage, “max” exports layers for
+all stages. Metadata is always exported for the whole build.
+
+**Examples**
+
+```console
+$ docker buildx build --cache-to=user/app:cache .
+$ docker buildx build --cache-to=type=inline .
+$ docker buildx build --cache-to=type=registry,ref=user/app .
+$ docker buildx build --cache-to=type=local,dest=path/to/cache .
+```
+
+### <a name="allow"></a> Allow extra privileged entitlement (--allow)
+
+```
+--allow=ENTITLEMENT
+```
+
+Allow extra privileged entitlement. List of entitlements:
+
+- `network.host` - Allows executions with host networking.
+- `security.insecure` - Allows executions without sandbox. See
+[related Dockerfile extensions](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---securityinsecuresandbox).
+
+For entitlements to be enabled, the `buildkitd` daemon also needs to allow them
+with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_create.md#--buildkitd-flags-flags))
+
+**Examples**
+
+```console
+$ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
+$ docker buildx build --allow security.insecure .
+```

docs/reference/buildx_create.md

@@ -0,0 +1,189 @@
+# buildx create
+
+```
+docker buildx create [OPTIONS] [CONTEXT|ENDPOINT]
+```
+
+<!---MARKER_GEN_START-->
+Create a new builder instance
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| [`--append`](#append) | Append a node to builder instead of changing it |
+| `--builder string` | Override the configured builder instance |
+| [`--buildkitd-flags string`](#buildkitd-flags) | Flags for buildkitd daemon |
+| [`--config string`](#config) | BuildKit config file |
+| [`--driver string`](#driver) | Driver to use (available: []) |
+| [`--driver-opt stringArray`](#driver-opt) | Options for the driver |
+| [`--leave`](#leave) | Remove a node from builder instead of changing it |
+| [`--name string`](#name) | Builder instance name |
+| [`--node string`](#node) | Create/modify node with given name |
+| [`--platform stringArray`](#platform) | Fixed platforms for current node |
+| [`--use`](#use) | Set the current builder instance |
+
+
+<!---MARKER_GEN_END-->
+
+
+## Description
+
+Create makes a new builder instance pointing to a docker context or endpoint,
+where context is the name of a context from `docker context ls` and endpoint is
+the address for docker socket (eg. `DOCKER_HOST` value).
+
+By default, the current Docker configuration is used for determining the
+context/endpoint value.
+
+Builder instances are isolated environments where builds can be invoked. All
+Docker contexts also get the default builder instance.
+
+## Examples
+
+### <a name="append"></a> Append a new node to an existing builder (--append)
+
+The `--append` flag changes the action of the command to append a new node to an
+existing builder specified by `--name`. Buildx will choose an appropriate node
+for a build based on the platforms it supports.
+
+**Examples**
+
+```console
+$ docker buildx create mycontext1
+eager_beaver
+
+$ docker buildx create --name eager_beaver --append mycontext2
+eager_beaver
+```
+
+### <a name="buildkitd-flags"></a> Specify options for the buildkitd daemon (--buildkitd-flags)
+
+```
+--buildkitd-flags FLAGS
+```
+
+Adds flags when starting the buildkitd daemon. They take precedence over the
+configuration file specified by [`--config`](#--config-file). See `buildkitd --help`
+for the available flags.
+
+**Example**
+
+```
+--buildkitd-flags '--debug --debugaddr 0.0.0.0:6666'
+```
+
+### <a name="config"></a> Specify a configuration file for the buildkitd daemon (--config)
+
+```
+--config FILE
+```
+
+Specifies the configuration file for the buildkitd daemon to use. The configuration
+can be overridden by [`--buildkitd-flags`](#--buildkitd-flags-flags).
+See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
+
+### <a name="driver"></a> Set the builder driver to use (--driver)
+
+```
+--driver DRIVER
+```
+
+Sets the builder driver to be used. There are two available drivers, each have
+their own specificities.
+
+- `docker` - Uses the builder that is built into the docker daemon. With this
+  driver, the [`--load`](buildx_build.md#--load) flag is implied by default on
+  `buildx build`. However, building multi-platform images or exporting cache is
+  not currently supported.
+- `docker-container` - Uses a buildkit container that will be spawned via docker.
+  With this driver, both building multi-platform images and exporting cache are
+  supported. However, images built will not automatically appear in `docker images`
+  (see [`build --load`](buildx_build.md#--load)).
+- `kubernetes` - Uses a kubernetes pods. With this driver, you can spin up pods
+  with defined buildkit container image to build your images.
+
+
+### <a name="driver-opt"></a> Set additional driver-specific options (--driver-opt)
+
+```
+--driver-opt OPTIONS
+```
+
+Passes additional driver-specific options. Details for each driver:
+
+- `docker` - No driver options
+- `docker-container`
+    - `image=IMAGE` - Sets the container image to be used for running buildkit.
+    - `network=NETMODE` - Sets the network mode for running the buildkit container.
+    - Example:
+
+      ```console
+      --driver docker-container --driver-opt image=moby/buildkit:master,network=host
+      ```
+- `kubernetes`
+    - `image=IMAGE` - Sets the container image to be used for running buildkit.
+    - `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
+    - `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
+    - `requests.cpu` - Sets the request CPU value specified in units of Kubernetes CPU. Example `requests.cpu=100m`, `requests.cpu=2`
+    - `requests.memory` - Sets the request memory value specified in bytes or with a valid suffix. Example `requests.memory=500Mi`, `requests.memory=4G`
+    - `limits.cpu` - Sets the limit CPU value specified in units of Kubernetes CPU. Example `limits.cpu=100m`, `limits.cpu=2`
+    - `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G`
+    - `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
+    - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
+    - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
+
+### <a name="leave"></a> Remove a node from a builder (--leave)
+
+The `--leave` flag changes the action of the command to remove a node from a
+builder. The builder needs to be specified with `--name` and node that is removed
+is set with `--node`.
+
+**Examples**
+
+```console
+$ docker buildx create --name mybuilder --node mybuilder0 --leave
+```
+
+### <a name="name"></a> Specify the name of the builder (--name)
+
+```
+--name NAME
+```
+
+The `--name` flag specifies the name of the builder to be created or modified.
+If none is specified, one will be automatically generated.
+
+### <a name="node"></a> Specify the name of the node (--node)
+
+```
+--node NODE
+```
+
+The `--node` flag specifies the name of the node to be created or modified. If
+none is specified, it is the name of the builder it belongs to, with an index
+number suffix.
+
+### <a name="platform"></a> Set the platforms supported by the node
+
+```
+--platform PLATFORMS
+```
+
+The `--platform` flag sets the platforms supported by the node. It expects a
+comma-separated list of platforms of the form OS/architecture/variant. The node
+will also automatically detect the platforms it supports, but manual values take
+priority over the detected ones and can be used when multiple nodes support
+building for the same platform.
+
+**Examples**
+
+```console
+$ docker buildx create --platform linux/amd64
+$ docker buildx create --platform linux/arm64,linux/arm/v8
+```
+
+### <a name="use"></a> Automatically switch to the newly created builder
+
+The `--use` flag automatically switches the current builder to the newly created
+one. Equivalent to running `docker buildx use $(docker buildx create ...)`.

docs/reference/buildx_du.md

@@ -0,0 +1,19 @@
+# buildx du
+
+```
+docker buildx du
+```
+
+<!---MARKER_GEN_START-->
+Disk usage
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| `--filter filter` | Provide filter values |
+| `--verbose` | Provide a more verbose output |
+
+
+<!---MARKER_GEN_END-->

docs/reference/buildx_imagetools.md

@@ -0,0 +1,24 @@
+# buildx imagetools
+
+```
+docker buildx imagetools [OPTIONS] COMMAND
+```
+
+<!---MARKER_GEN_START-->
+Commands to work on images in registry
+
+### Subcommands
+
+| Name | Description |
+| --- | --- |
+| [`create`](buildx_imagetools_create.md) | Create a new image based on source images |
+| [`inspect`](buildx_imagetools_inspect.md) | Show details of image in the registry |
+
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Imagetools contains commands for working with manifest lists in the registry.
+These commands are useful for inspecting multi-platform build results.

docs/reference/buildx_imagetools_create.md

@@ -0,0 +1,80 @@
+# buildx imagetools create
+
+```
+docker buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]
+```
+
+<!---MARKER_GEN_START-->
+Create a new image based on source images
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| [`--append`](#append) | Append to existing manifest |
+| `--builder string` | Override the configured builder instance |
+| [`--dry-run`](#dry-run) | Show final image instead of pushing |
+| [`-f`](#file), [`--file stringArray`](#file) | Read source descriptor from file |
+| [`-t`](#tag), [`--tag stringArray`](#tag) | Set reference for new image |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Imagetools contains commands for working with manifest lists in the registry.
+These commands are useful for inspecting multi-platform build results.
+
+Create a new manifest list based on source manifests. The source manifests can
+be manifest lists or single platform distribution manifests and must already
+exist in the registry where the new manifest is created. If only one source is
+specified, create performs a carbon copy.
+
+## Examples
+
+### <a name="append"></a> Append new sources to an existing manifest list (--append)
+
+Use the `--append` flag to append the new sources to an existing manifest list
+in the destination.
+
+### <a name="dry-run"></a> Show final image instead of pushing (--dry-run)
+
+Use the `--dry-run` flag to not push the image, just show it.
+
+### <a name="file"></a> Read source descriptor from a file (-f, --file)
+
+```
+-f FILE or --file FILE
+```
+
+Reads source from files. A source can be a manifest digest, manifest reference,
+or a JSON of OCI descriptor object.
+
+In order to define annotations or additional platform properties like `os.version` and
+`os.features` you need to add them in the OCI descriptor object encoded in JSON.
+
+```
+docker buildx imagetools inspect --raw alpine | jq '.manifests[0] | .platform."os.version"="10.1"' > descr.json
+docker buildx imagetools create -f descr.json myuser/image
+```
+
+The descriptor in the file is merged with existing descriptor in the registry if it exists.
+
+The supported fields for the descriptor are defined in [OCI spec](https://github.com/opencontainers/image-spec/blob/master/descriptor.md#properties) .
+
+
+### <a name="tag"></a> Set reference for new image  (-t, --tag)
+
+```
+-t IMAGE or --tag IMAGE
+```
+
+Use the `-t` or `--tag` flag to set the name of the image to be created.
+
+**Examples**
+
+```console
+$ docker buildx imagetools create --dry-run alpine@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907 sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
+
+$ docker buildx imagetools create -t tonistiigi/myapp -f image1 -f image2
+```

docs/reference/buildx_imagetools_inspect.md

@@ -0,0 +1,47 @@
+# buildx imagetools inspect
+
+```
+docker buildx imagetools inspect [OPTIONS] NAME
+```
+
+<!---MARKER_GEN_START-->
+Show details of image in the registry
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| [`--raw`](#raw) | Show original JSON manifest |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Show details of image in the registry.
+
+Example:
+
+```console
+$ docker buildx imagetools inspect alpine
+
+Name:      docker.io/library/alpine:latest
+MediaType: application/vnd.docker.distribution.manifest.list.v2+json
+Digest:    sha256:28ef97b8686a0b5399129e9b763d5b7e5ff03576aa5580d6f4182a49c5fe1913
+
+Manifests:
+  Name:      docker.io/library/alpine:latest@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907
+  MediaType: application/vnd.docker.distribution.manifest.v2+json
+  Platform:  linux/amd64
+
+  Name:      docker.io/library/alpine:latest@sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
+  MediaType: application/vnd.docker.distribution.manifest.v2+json
+  Platform:  linux/arm/v6
+ ...
+```
+
+### <a name="raw"></a> Show original, unformatted JSON manifest (--raw)
+
+Use the `--raw` option to print the original JSON bytes instead of the formatted
+output.

docs/reference/buildx_inspect.md

@@ -0,0 +1,58 @@
+# buildx inspect
+
+```
+docker buildx inspect [NAME]
+```
+
+<!---MARKER_GEN_START-->
+Inspect current builder instance
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| [`--bootstrap`](#bootstrap) | Ensure builder has booted before inspecting |
+| `--builder string` | Override the configured builder instance |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Shows information about the current or specified builder.
+
+## Examples
+
+### Get information about a builder instance
+
+By default, `inspect` shows information about the current builder. Specify the
+name of the builder to inspect to get information about that builder.
+The following example shows information about a builder instance named
+`elated_tesla`:
+
+```console
+$ docker buildx inspect elated_tesla
+
+Name:   elated_tesla
+Driver: docker-container
+
+Nodes:
+Name:      elated_tesla0
+Endpoint:  unix:///var/run/docker.sock
+Status:    running
+Platforms: linux/amd64
+
+Name:      elated_tesla1
+Endpoint:  ssh://ubuntu@1.2.3.4
+Status:    running
+Platforms: linux/arm64, linux/arm/v7, linux/arm/v6
+```
+
+### <a name="bootstrap"></a> Ensure that the builder is running before inspecting (--bootstrap)
+
+Use the `--bootstrap` option to ensure that the builder is running before
+inspecting it. If the driver is `docker-container`, then `--bootstrap` starts
+the buildkit container and waits until it is operational. Bootstrapping is
+automatically done during build, and therefore not necessary. The same BuildKit
+container is used during the lifetime of the associated builder node (as
+displayed in `buildx ls`).

docs/reference/buildx_install.md

@@ -0,0 +1,11 @@
+# buildx install
+
+```
+docker buildx install
+```
+
+<!---MARKER_GEN_START-->
+Install buildx as a 'docker builder' alias
+
+
+<!---MARKER_GEN_END-->

docs/reference/buildx_ls.md

@@ -0,0 +1,31 @@
+# buildx ls
+
+```
+docker buildx ls
+```
+
+<!---MARKER_GEN_START-->
+List builder instances
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Lists all builder instances and the nodes for each instance
+
+**Example**
+
+```console
+$ docker buildx ls
+
+NAME/NODE       DRIVER/ENDPOINT             STATUS  PLATFORMS
+elated_tesla *  docker-container
+  elated_tesla0 unix:///var/run/docker.sock running linux/amd64
+  elated_tesla1 ssh://ubuntu@1.2.3.4        running linux/arm64, linux/arm/v7, linux/arm/v6
+default         docker
+  default       default                     running linux/amd64
+```
+
+Each builder has one or more nodes associated with it. The current builder's
+name is marked with a `*`.

docs/reference/buildx_prune.md

@@ -0,0 +1,23 @@
+# buildx prune
+
+```
+docker buildx prune
+```
+
+<!---MARKER_GEN_START-->
+Remove build cache
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `-a`, `--all` | Remove all unused images, not just dangling ones |
+| `--builder string` | Override the configured builder instance |
+| `--filter filter` | Provide filter values (e.g. 'until=24h') |
+| `-f`, `--force` | Do not prompt for confirmation |
+| `--keep-storage bytes` | Amount of disk space to keep for cache |
+| `--verbose` | Provide a more verbose output |
+
+
+<!---MARKER_GEN_END-->
+

docs/reference/buildx_rm.md

@@ -0,0 +1,30 @@
+# buildx rm
+
+```
+docker buildx rm [NAME]
+```
+
+<!---MARKER_GEN_START-->
+Remove a builder instance
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| [`--keep-state`](#keep-state) | Keep BuildKit state |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Removes the specified or current builder. It is a no-op attempting to remove the
+default builder.
+
+## Examples
+
+### <a name="keep-state"></a> Keep BuildKit state (--keep-state)
+
+Keep BuildKit state, so it can be reused by a new builder with the same name.
+Currently, only supported by the [`docker-container` driver](buildx_create.md#driver).

docs/reference/buildx_stop.md

@@ -0,0 +1,16 @@
+# buildx stop
+
+```
+docker buildx stop [NAME]
+```
+
+<!---MARKER_GEN_START-->
+Stop builder instance
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Stops the specified or current builder. This will not prevent buildx build to
+restart the builder. The implementation of stop depends on the driver.

docs/reference/buildx_uninstall.md

@@ -0,0 +1,11 @@
+# buildx uninstall
+
+```
+docker buildx uninstall
+```
+
+<!---MARKER_GEN_START-->
+Uninstall the 'docker builder' alias
+
+
+<!---MARKER_GEN_END-->

docs/reference/buildx_use.md

@@ -0,0 +1,25 @@
+# buildx use
+
+```
+docker buildx use [OPTIONS] NAME
+```
+
+<!---MARKER_GEN_START-->
+Set the current builder instance
+
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| `--default` | Set builder as default for current context |
+| `--global` | Builder persists context changes |
+
+
+<!---MARKER_GEN_END-->
+
+## Description
+
+Switches the current builder instance. Build commands invoked after this command
+will run on a specified builder. Alternatively, a context name can be used to
+switch to the default builder of that context.

docs/reference/buildx_version.md

@@ -0,0 +1,21 @@
+# buildx version
+
+```
+docker buildx version
+```
+
+<!---MARKER_GEN_START-->
+Show buildx version information
+
+
+<!---MARKER_GEN_END-->
+
+## Examples
+
+### View version information
+
+
+```console
+$ docker buildx version
+github.com/docker/buildx v0.5.1-docker 11057da37336192bfc57d81e02359ba7ba848e4a
+```

driver/bkimage/bkimage.go

@@ -2,5 +2,5 @@ package bkimage
 
 const (
 	DefaultImage         = "moby/buildkit:buildx-stable-1" // TODO: make this verified
-	DefaultRootlessImage = "moby/buildkit:v0.6.2-rootless"
+	DefaultRootlessImage = DefaultImage + "-rootless"
 )

driver/docker-container/driver.go

@@ -17,13 +17,18 @@ import (
 	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/util/appdefaults"
+	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/pkg/errors"
 )
 
+const volumeStateSuffix = "_state"
+
 type Driver struct {
 	driver.InitConfig
 	factory driver.Factory
@@ -102,6 +107,13 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		hc := &container.HostConfig{
 			Privileged: true,
 			UsernsMode: "host",
+			Mounts: []mount.Mount{
+				{
+					Type:   mount.TypeVolume,
+					Source: d.Name + volumeStateSuffix,
+					Target: appdefaults.Root,
+				},
+			},
 		}
 		if d.netMode != "" {
 			hc.NetworkMode = container.NetworkMode(d.netMode)
@@ -225,7 +237,7 @@ func (d *Driver) start(ctx context.Context, l progress.SubLogger) error {
 }
 
 func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
-	container, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
+	ctn, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
 	if err != nil {
 		if dockerclient.IsErrNotFound(err) {
 			return &driver.Info{
@@ -235,7 +247,7 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
 		return nil, err
 	}
 
-	if container.State.Running {
+	if ctn.State.Running {
 		return &driver.Info{
 			Status: driver.Running,
 		}, nil
@@ -257,16 +269,30 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	info, err := d.Info(ctx)
 	if err != nil {
 		return err
 	}
 	if info.Status != driver.Inactive {
-		return d.DockerAPI.ContainerRemove(ctx, d.Name, dockertypes.ContainerRemoveOptions{
+		container, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
+		if err != nil {
+			return err
+		}
+		if err := d.DockerAPI.ContainerRemove(ctx, d.Name, dockertypes.ContainerRemoveOptions{
 			RemoveVolumes: true,
-			Force:         true,
-		})
+			Force:         force,
+		}); err != nil {
+			return err
+		}
+		for _, v := range container.Mounts {
+			if v.Name == d.Name+volumeStateSuffix {
+				if rmVolume {
+					return d.DockerAPI.VolumeRemove(ctx, d.Name+volumeStateSuffix, false)
+				}
+			}
+		}
+
 	}
 	return nil
 }
@@ -279,9 +305,16 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 
 	conn = demuxConn(conn)
 
+	exp, err := detect.Exporter()
+	if err != nil {
+		return nil, err
+	}
+
+	td, _ := exp.(client.TracerDelegate)
+
 	return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
 		return conn, nil
-	}))
+	}), client.WithTracerDelegate(td))
 }
 
 func (d *Driver) Factory() driver.Factory {

driver/docker/driver.go

@@ -33,7 +33,7 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	return nil
 }
 

driver/driver.go

@@ -54,7 +54,7 @@ type Driver interface {
 	Bootstrap(context.Context, progress.Logger) error
 	Info(context.Context) (*Info, error)
 	Stop(ctx context.Context, force bool) error
-	Rm(ctx context.Context, force bool) error
+	Rm(ctx context.Context, force bool, rmVolume bool) error
 	Client(ctx context.Context) (*client.Client, error)
 	Features() map[Feature]bool
 	IsMobyDriver() bool
@@ -78,7 +78,7 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, er
 			}
 		}
 
-		c, err := d.Client(context.TODO())
+		c, err := d.Client(ctx)
 		if err != nil {
 			if errors.Cause(err) == ErrNotRunning && try <= 2 {
 				continue

driver/kubernetes/driver.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/buildx/util/progress"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -142,7 +143,7 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
 	}
@@ -169,9 +170,17 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	exp, err := detect.Exporter()
+	if err != nil {
+		return nil, err
+	}
+
+	td, _ := exp.(client.TracerDelegate)
+
 	return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
 		return conn, nil
-	}))
+	}), client.WithTracerDelegate(td))
 }
 
 func (d *Driver) Factory() driver.Factory {

driver/kubernetes/factory.go

@@ -85,6 +85,14 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 			if err != nil {
 				return nil, err
 			}
+		case "requests.cpu":
+			deploymentOpt.RequestsCPU = v
+		case "requests.memory":
+			deploymentOpt.RequestsMemory = v
+		case "limits.cpu":
+			deploymentOpt.LimitsCPU = v
+		case "limits.memory":
+			deploymentOpt.LimitsMemory = v
 		case "rootless":
 			deploymentOpt.Rootless, err = strconv.ParseBool(v)
 			if err != nil {

driver/kubernetes/manifest/manifest.go

@@ -7,18 +7,23 @@ import (
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 type DeploymentOpt struct {
-	Namespace     string
-	Name          string
-	Image         string
-	Replicas      int
-	BuildkitFlags []string
-	Rootless      bool
-	NodeSelector  map[string]string
-	Platforms     []v1.Platform
+	Namespace      string
+	Name           string
+	Image          string
+	Replicas       int
+	BuildkitFlags  []string
+	Rootless       bool
+	NodeSelector   map[string]string
+	RequestsCPU    string
+	RequestsMemory string
+	LimitsCPU      string
+	LimitsMemory   string
+	Platforms      []v1.Platform
 }
 
 const (
@@ -76,6 +81,10 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 									},
 								},
 							},
+							Resources: corev1.ResourceRequirements{
+								Requests: corev1.ResourceList{},
+								Limits:   corev1.ResourceList{},
+							},
 						},
 					},
 				},
@@ -92,6 +101,38 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 		d.Spec.Template.Spec.NodeSelector = opt.NodeSelector
 	}
 
+	if opt.RequestsCPU != "" {
+		reqCPU, err := resource.ParseQuantity(opt.RequestsCPU)
+		if err != nil {
+			return nil, err
+		}
+		d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = reqCPU
+	}
+
+	if opt.RequestsMemory != "" {
+		reqMemory, err := resource.ParseQuantity(opt.RequestsMemory)
+		if err != nil {
+			return nil, err
+		}
+		d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = reqMemory
+	}
+
+	if opt.LimitsCPU != "" {
+		limCPU, err := resource.ParseQuantity(opt.LimitsCPU)
+		if err != nil {
+			return nil, err
+		}
+		d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = limCPU
+	}
+
+	if opt.LimitsMemory != "" {
+		limMemory, err := resource.ParseQuantity(opt.LimitsMemory)
+		if err != nil {
+			return nil, err
+		}
+		d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = limMemory
+	}
+
 	return d, nil
 }
 

go.mod

@@ -9,60 +9,58 @@ require (
 	github.com/bugsnag/panicwrap v1.2.0 // indirect
 	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
 	github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e // indirect
-	github.com/containerd/console v1.0.1
-	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
+	github.com/compose-spec/compose-go v0.0.0-20210706130854-69459d4976b5
+	github.com/containerd/console v1.0.2
+	github.com/containerd/containerd v1.5.2
 	github.com/denisenkom/go-mssqldb v0.0.0-20190315220205-a8ed825ac853 // indirect
-	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
+	github.com/docker/cli v20.10.7+incompatible
 	github.com/docker/compose-on-kubernetes v0.4.19-0.20190128150448-356b2919c496 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/docker v20.10.7+incompatible
+	github.com/docker/docker-credential-helpers v0.6.4-0.20210125172408-38bea2ce277a // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
-	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4 // indirect
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
 	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
 	github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
 	github.com/fvbommel/sortorder v1.0.1 // indirect
 	github.com/gofrs/flock v0.7.3
-	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/gofrs/uuid v3.3.0+incompatible // indirect
 	github.com/google/certificate-transparency-go v1.0.21 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
-	github.com/hashicorp/hcl/v2 v2.6.0
+	github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840
+	github.com/hashicorp/hcl/v2 v2.8.2
 	github.com/jinzhu/gorm v1.9.2 // indirect
 	github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
 	github.com/jinzhu/now v1.0.0 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/lib/pq v1.10.0 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
-	github.com/miekg/pkcs11 v0.0.0-20190322140431-074fd7a1ed19 // indirect
-	github.com/moby/buildkit v0.8.1-0.20201205083753-0af7b1b9c693
+	github.com/moby/buildkit v0.8.2-0.20210702160134-1a7543a10527
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.7.1 // indirect
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.0.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.7.0
 	github.com/theupdateframework/notary v0.6.1 // indirect
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
-	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	github.com/zclconf/go-cty v1.4.0
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	github.com/zclconf/go-cty v1.7.1
+	go.opentelemetry.io/otel v1.0.0-RC1
+	go.opentelemetry.io/otel/trace v1.0.0-RC1
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
 	gopkg.in/fatih/pool.v2 v2.0.0 // indirect
 	gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
-	k8s.io/api v0.19.0
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v0.19.0
+	k8s.io/api v0.20.6
+	k8s.io/apimachinery v0.20.6
+	k8s.io/client-go v0.20.6
 )
 
 replace (
-	// protobuf: corresponds to containerd (through buildkit)
-	github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
-	github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
-
-	// genproto: corresponds to containerd (through buildkit)
-	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
+	github.com/docker/cli => github.com/docker/cli v20.10.3-0.20210702143511-f782d1355eff+incompatible
+	github.com/docker/docker => github.com/docker/docker v20.10.3-0.20210609100121-ef4d47340142+incompatible
 )

gometalinter.json

@@ -1,13 +0,0 @@
-{
-  "Vendor": true,
-  "Deadline": "8m",
-  "Exclude": [".*.pb.go"],
-  "DisableAll": true,
-  "Enable": [
-    "gofmt",
-    "goimports",
-    "ineffassign",
-    "vet",
-    "deadcode"
-  ]
-}

hack/cross

@@ -3,7 +3,7 @@
 . $(dirname $0)/util
 set -e
 
-: ${TARGETPLATFORM=linux/amd64,linux/arm/v7,linux/arm64,darwin/amd64,windows/amd64,linux/ppc64le,linux/s390x}
+: ${TARGETPLATFORM=linux/amd64,linux/arm/v7,linux/arm64,darwin/amd64,windows/amd64,linux/ppc64le,linux/s390x,linux/riscv64}
 : ${EXPORT_LOCAL=}
 
 importCacheFlags=""

hack/dockerfiles/docs.Dockerfile

@@ -0,0 +1,29 @@
+# syntax = docker/dockerfile:1.2
+
+FROM golang:1.16-alpine AS docsgen
+WORKDIR /src
+RUN --mount=target=. \
+  --mount=target=/root/.cache,type=cache \
+  go build -mod=vendor -o /out/docsgen ./docs/docsgen
+
+FROM alpine AS gen
+RUN apk add --no-cache rsync git
+WORKDIR /src
+COPY --from=docsgen /out/docsgen /usr/bin
+RUN --mount=target=/context \
+  --mount=target=.,type=tmpfs,readwrite  \
+  rsync -a /context/. . && \
+  docsgen && \
+  mkdir /out && cp -r docs/reference /out
+
+FROM scratch AS update
+COPY --from=gen /out /out
+
+FROM gen AS validate
+RUN --mount=target=/context \
+  --mount=target=.,type=tmpfs,readwrite  \
+  rsync -a /context/. . && \
+  git add -A && \
+  rm -rf docs/reference/* && \
+  cp -rf /out/* ./docs/ && \
+  ./hack/validate-docs check

hack/dockerfiles/lint.Dockerfile

@@ -1,12 +1,10 @@
-# syntax=docker/dockerfile:1.0-experimental
+# syntax=docker/dockerfile:1.2
 
-FROM golang:1.13-alpine
-RUN apk add --no-cache git yamllint
-RUN go get -u gopkg.in/alecthomas/gometalinter.v1 \
-  && mv /go/bin/gometalinter.v1 /go/bin/gometalinter \
-  && gometalinter --install
+FROM golang:1.16-alpine
+RUN apk add --no-cache gcc musl-dev yamllint
+RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.36.0
 WORKDIR /go/src/github.com/docker/buildx
-RUN --mount=target=/go/src/github.com/docker/buildx \
-  gometalinter --config=gometalinter.json ./...
-RUN --mount=target=/go/src/github.com/docker/buildx \
-  yamllint -c .yamllint.yml --strict .
+RUN --mount=target=/go/src/github.com/docker/buildx --mount=target=/root/.cache,type=cache \
+  golangci-lint run
+RUN --mount=target=/go/src/github.com/docker/buildx --mount=target=/root/.cache,type=cache \
+  yamllint -c .yamllint.yml --strict .

hack/dockerfiles/vendor.Dockerfile

@@ -1,5 +1,6 @@
-# syntax = docker/dockerfile:1.0-experimental
-FROM golang:1.13-alpine AS vendored
+# syntax = docker/dockerfile:1.2
+
+FROM golang:1.16-alpine AS vendored
 RUN  apk add --no-cache git rsync
 WORKDIR /src
 RUN --mount=target=/context \

hack/release

@@ -1,22 +1,12 @@
 #!/usr/bin/env bash
 
-TAG=$1
-OUT=$2
+OUT=${1:-release-out}
 
 . $(dirname $0)/util
 set -eu -o pipefail
 
 : ${PLATFORMS=linux/amd64}
 
-usage() {
-  echo "usage: ./hack/release <tag> <out>"
-  exit 1
-}
-
-if [ -z "$TAG" ] || [ -z "$OUT" ]; then
-  usage
-fi
-
 importCacheFlags=""
 if [[ -n "$cacheRefFrom" ]] && [[ "$cacheType" = "local" ]]; then
   for ref in $cacheRefFrom; do
@@ -29,3 +19,10 @@ buildxCmd build $importCacheFlags \
   --platform "$PLATFORMS" \
   --output "type=local,dest=$OUT" \
   $currentcontext
+
+# wrap binaries
+{ set +x; } 2>/dev/null
+if [[ $PLATFORMS =~ "," ]]; then
+  mv -f ./$OUT/**/* ./$OUT/
+  find ./$OUT -type d -empty -delete
+fi

hack/update-docs

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+. $(dirname $0)/util
+set -eu
+
+output=$(mktemp -d -t buildx-output.XXXXXXXXXX)
+
+buildxCmd build \
+  --target "update" \
+  --output "type=local,dest=$output" \
+  --file "./hack/dockerfiles/docs.Dockerfile" \
+  .
+
+rm -rf ./docs/reference/*
+cp -R "$output"/out/* ./docs/
+rm -rf $output

hack/validate-docs

@@ -0,0 +1,29 @@
+#!/usr/bin/env sh
+set -eu
+
+case ${1:-} in
+  '')
+    . $(dirname $0)/util
+    buildxCmd build \
+      --target validate \
+      --file ./hack/dockerfiles/docs.Dockerfile \
+      .
+    ;;
+  check)
+    status="$(git status --porcelain -- docs/reference 2>/dev/null)"
+    diffs=$(echo "$status" | grep -v '^[RAD] ' || true)
+    if [ "$diffs" ]; then
+      {
+        set +x
+        echo 'The result of ./hack/update-docs differs'
+        echo
+        echo "$diffs"
+        echo
+        echo 'Please vendor your package with ./hack/update-docs'
+        echo
+      } >&2
+      exit 1
+    fi
+    echo 'Congratulations! All docs changes are done the right way.'
+    ;;
+esac

store/store.go

@@ -2,13 +2,13 @@ package store
 
 import (
 	"encoding/json"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"sort"
 
+	"github.com/docker/docker/pkg/ioutils"
 	"github.com/gofrs/flock"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -95,7 +95,7 @@ func (t *Txn) Save(ng *NodeGroup) error {
 	if err != nil {
 		return err
 	}
-	return atomicWriteFile(filepath.Join(t.s.root, "instances", name), dt, 0600)
+	return ioutils.AtomicWriteFile(filepath.Join(t.s.root, "instances", name), dt, 0600)
 }
 
 func (t *Txn) Remove(name string) error {
@@ -116,14 +116,14 @@ func (t *Txn) SetCurrent(key, name string, global, def bool) error {
 	if err != nil {
 		return err
 	}
-	if err := atomicWriteFile(filepath.Join(t.s.root, "current"), dt, 0600); err != nil {
+	if err := ioutils.AtomicWriteFile(filepath.Join(t.s.root, "current"), dt, 0600); err != nil {
 		return err
 	}
 
 	h := toHash(key)
 
 	if def {
-		if err := atomicWriteFile(filepath.Join(t.s.root, "defaults", h), []byte(name), 0600); err != nil {
+		if err := ioutils.AtomicWriteFile(filepath.Join(t.s.root, "defaults", h), []byte(name), 0600); err != nil {
 			return err
 		}
 	} else {
@@ -137,7 +137,7 @@ func (t *Txn) reset(key string) error {
 	if err != nil {
 		return err
 	}
-	if err := atomicWriteFile(filepath.Join(t.s.root, "current"), dt, 0600); err != nil {
+	if err := ioutils.AtomicWriteFile(filepath.Join(t.s.root, "current"), dt, 0600); err != nil {
 		return err
 	}
 	return nil
@@ -202,35 +202,6 @@ type current struct {
 
 var namePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9\.\-_]*$`)
 
-func atomicWriteFile(filename string, data []byte, perm os.FileMode) error {
-	f, err := ioutil.TempFile(filepath.Dir(filename), ".tmp-"+filepath.Base(filename))
-	if err != nil {
-		return err
-	}
-	err = os.Chmod(f.Name(), perm)
-	if err != nil {
-		f.Close()
-		return err
-	}
-	n, err := f.Write(data)
-	if err == nil && n < len(data) {
-		f.Close()
-		return io.ErrShortWrite
-	}
-	if err != nil {
-		f.Close()
-		return err
-	}
-	if err := f.Sync(); err != nil {
-		f.Close()
-		return err
-	}
-	if err := f.Close(); err != nil {
-		return err
-	}
-	return os.Rename(f.Name(), filename)
-}
-
 func toHash(in string) string {
 	return digest.FromBytes([]byte(in)).Hex()[:20]
 }

util/buildflags/cache.go

@@ -1,7 +1,8 @@
-package build
+package buildflags
 
 import (
 	"encoding/csv"
+	"os"
 	"strings"
 
 	"github.com/moby/buildkit/client"
@@ -45,6 +46,9 @@ func ParseCacheEntry(in []string) ([]client.CacheOptionsEntry, error) {
 		if im.Type == "" {
 			return nil, errors.Errorf("type required form> %q", in)
 		}
+		if !addGithubToken(&im) {
+			continue
+		}
 		imports = append(imports, im)
 	}
 	return imports, nil
@@ -58,3 +62,20 @@ func isRefOnlyFormat(in []string) bool {
 	}
 	return true
 }
+
+func addGithubToken(ci *client.CacheOptionsEntry) bool {
+	if ci.Type != "gha" {
+		return true
+	}
+	if _, ok := ci.Attrs["token"]; !ok {
+		if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
+			ci.Attrs["token"] = v
+		}
+	}
+	if _, ok := ci.Attrs["url"]; !ok {
+		if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
+			ci.Attrs["url"] = v
+		}
+	}
+	return ci.Attrs["token"] != "" && ci.Attrs["url"] != ""
+}

util/buildflags/entitlements.go

@@ -1,4 +1,4 @@
-package build
+package buildflags
 
 import (
 	"github.com/moby/buildkit/util/entitlements"

util/buildflags/output.go

@@ -1,4 +1,4 @@
-package build
+package buildflags
 
 import (
 	"encoding/csv"

util/buildflags/secrets.go

@@ -1,4 +1,4 @@
-package build
+package buildflags
 
 import (
 	"encoding/csv"
@@ -34,6 +34,7 @@ func parseSecret(value string) (*secretsprovider.Source, error) {
 
 	fs := secretsprovider.Source{}
 
+	var typ string
 	for _, field := range fields {
 		parts := strings.SplitN(field, "=", 2)
 		key := strings.ToLower(parts[0])
@@ -45,16 +46,23 @@ func parseSecret(value string) (*secretsprovider.Source, error) {
 		value := parts[1]
 		switch key {
 		case "type":
-			if value != "file" {
+			if value != "file" && value != "env" {
 				return nil, errors.Errorf("unsupported secret type %q", value)
 			}
+			typ = value
 		case "id":
 			fs.ID = value
 		case "source", "src":
 			fs.FilePath = value
+		case "env":
+			fs.Env = value
 		default:
 			return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
 		}
 	}
+	if typ == "env" && fs.Env == "" {
+		fs.Env = fs.FilePath
+		fs.FilePath = ""
+	}
 	return &fs, nil
 }

util/buildflags/ssh.go

@@ -1,10 +1,11 @@
-package build
+package buildflags
 
 import (
 	"strings"
 
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
+	"github.com/moby/buildkit/util/gitutil"
 )
 
 func ParseSSHSpecs(sl []string) (session.Attachable, error) {
@@ -29,3 +30,9 @@ func parseSSH(value string) (*sshprovider.AgentConfig, error) {
 	}
 	return &cfg, nil
 }
+
+// IsGitSSH returns true if the given repo URL is accessed over ssh
+func IsGitSSH(url string) bool {
+	_, gitProtocol := gitutil.ParseProtocol(url)
+	return gitProtocol == gitutil.SSHProtocol
+}

util/imagetools/create.go

@@ -47,13 +47,16 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
 
 				switch mt {
 				case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+					p := descs[i].Platform
 					if descs[i].Platform == nil {
-						p, err := r.loadPlatform(ctx, in, dt)
-						if err != nil {
+						p = &ocispec.Platform{}
+					}
+					if p.OS == "" || p.Architecture == "" {
+						if err := r.loadPlatform(ctx, p, in, dt); err != nil {
 							return err
 						}
-						descs[i].Platform = p
 					}
+					descs[i].Platform = p
 				case images.MediaTypeDockerSchema1Manifest:
 					return errors.Errorf("schema1 manifests are not allowed in manifest lists")
 				}
@@ -166,24 +169,35 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D
 	return err
 }
 
-func (r *Resolver) loadPlatform(ctx context.Context, in string, dt []byte) (*ocispec.Platform, error) {
+func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error {
 	var manifest ocispec.Manifest
 	if err := json.Unmarshal(dt, &manifest); err != nil {
-		return nil, errors.WithStack(err)
+		return errors.WithStack(err)
 	}
 
 	dt, err := r.GetDescriptor(ctx, in, manifest.Config)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	var p ocispec.Platform
 	if err := json.Unmarshal(dt, &p); err != nil {
-		return nil, errors.WithStack(err)
+		return errors.WithStack(err)
 	}
 
 	p = platforms.Normalize(p)
-	return &p, nil
+
+	if p2.Architecture == "" {
+		p2.Architecture = p.Architecture
+		if p2.Variant == "" {
+			p2.Variant = p.Variant
+		}
+	}
+	if p2.OS == "" {
+		p2.OS = p.OS
+	}
+
+	return nil
 }
 
 func detectMediaType(dt []byte) (string, error) {

util/progress/printer.go

@@ -2,6 +2,8 @@ package progress
 
 import (
 	"context"
+	"io"
+	"io/ioutil"
 	"os"
 
 	"github.com/containerd/console"
@@ -9,6 +11,13 @@ import (
 	"github.com/moby/buildkit/util/progress/progressui"
 )
 
+const (
+	PrinterModeAuto  = "auto"
+	PrinterModeTty   = "tty"
+	PrinterModePlain = "plain"
+	PrinterModeQuiet = "quiet"
+)
+
 type Printer struct {
 	status chan *client.SolveStatus
 	done   <-chan struct{}
@@ -34,17 +43,23 @@ func NewPrinter(ctx context.Context, out console.File, mode string) *Printer {
 		done:   doneCh,
 	}
 
-	if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && mode == "auto" {
+	if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && mode == PrinterModeAuto {
 		mode = v
 	}
 
 	go func() {
 		var c console.Console
-		if cons, err := console.ConsoleFromFile(out); err == nil && (mode == "auto" || mode == "tty") {
-			c = cons
+		var w io.Writer = out
+		switch mode {
+		case PrinterModeQuiet:
+			w = ioutil.Discard
+		case PrinterModeAuto, PrinterModeTty:
+			if cons, err := console.ConsoleFromFile(out); err == nil {
+				c = cons
+			}
 		}
 		// not using shared context to not disrupt display but let is finish reporting errors
-		pw.err = progressui.DisplaySolveStatus(ctx, "", c, out, statusCh)
+		pw.err = progressui.DisplaySolveStatus(ctx, "", c, w, statusCh)
 		close(doneCh)
 	}()
 	return pw

util/progress/reset.go

@@ -55,6 +55,6 @@ type pw struct {
 	status chan *client.SolveStatus
 }
 
-func (p *pw) Status() chan *client.SolveStatus {
-	return p.status
+func (w *pw) Status() chan *client.SolveStatus {
+	return w.status
 }

util/tracing/trace.go

@@ -0,0 +1,30 @@
+package tracing
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/moby/buildkit/util/tracing/detect"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func TraceCurrentCommand(ctx context.Context, name string) (context.Context, func(error), error) {
+	tp, err := detect.TracerProvider()
+	if err != nil {
+		return context.Background(), nil, err
+	}
+	ctx, span := tp.Tracer("").Start(ctx, name, trace.WithAttributes(
+		attribute.String("command", strings.Join(os.Args, " ")),
+	))
+
+	return ctx, func(err error) {
+		if err != nil {
+			span.RecordError(err)
+		}
+		span.End()
+
+		detect.Shutdown(context.TODO())
+	}, nil
+}

util/userfunc/README.md

@@ -25,4 +25,4 @@ inclusion in a `hcl.EvalContext`. It also returns a new `cty.Body` that
 contains the remainder of the content from the given body, allowing for
 further processing of remaining content.
 
-For more information, see [the godoc reference](http://godoc.org/github.com/hashicorp/hcl/v2/ext/userfunc).
+For more information, see [the godoc reference](https://pkg.go.dev/github.com/hashicorp/hcl/v2/ext/userfunc?tab=doc).

util/userfunc/decode.go

@@ -0,0 +1,79 @@
+package userfunc
+
+import (
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty/function"
+)
+
+var funcBodySchema = &hcl.BodySchema{
+	Attributes: []hcl.AttributeSchema{
+		{
+			Name:     "params",
+			Required: true,
+		},
+		{
+			Name:     "variadic_param",
+			Required: false,
+		},
+		{
+			Name:     "result",
+			Required: true,
+		},
+	},
+}
+
+func decodeUserFunctions(body hcl.Body, blockType string, contextFunc ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) {
+	schema := &hcl.BodySchema{
+		Blocks: []hcl.BlockHeaderSchema{
+			{
+				Type:       blockType,
+				LabelNames: []string{"name"},
+			},
+		},
+	}
+
+	content, remain, diags := body.PartialContent(schema)
+	if diags.HasErrors() {
+		return nil, remain, diags
+	}
+
+	// first call to getBaseCtx will populate context, and then the same
+	// context will be used for all subsequent calls. It's assumed that
+	// all functions in a given body should see an identical context.
+	var baseCtx *hcl.EvalContext
+	getBaseCtx := func() *hcl.EvalContext {
+		if baseCtx == nil {
+			if contextFunc != nil {
+				baseCtx = contextFunc()
+			}
+		}
+		// baseCtx might still be nil here, and that's okay
+		return baseCtx
+	}
+
+	funcs = make(map[string]function.Function)
+
+	for _, block := range content.Blocks {
+		name := block.Labels[0]
+		funcContent, funcDiags := block.Body.Content(funcBodySchema)
+		diags = append(diags, funcDiags...)
+		if funcDiags.HasErrors() {
+			continue
+		}
+
+		paramsExpr := funcContent.Attributes["params"].Expr
+		resultExpr := funcContent.Attributes["result"].Expr
+		var varParamExpr hcl.Expression
+		if funcContent.Attributes["variadic_param"] != nil {
+			varParamExpr = funcContent.Attributes["variadic_param"].Expr
+		}
+		f, funcDiags := NewFunction(paramsExpr, varParamExpr, resultExpr, getBaseCtx)
+		if funcDiags.HasErrors() {
+			diags = append(diags, funcDiags...)
+			continue
+		}
+		funcs[name] = f
+	}
+
+	return funcs, remain, diags
+}

util/userfunc/public.go

@@ -0,0 +1,125 @@
+package userfunc
+
+import (
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+)
+
+// A ContextFunc is a callback used to produce the base EvalContext for
+// running a particular set of functions.
+//
+// This is a function rather than an EvalContext directly to allow functions
+// to be decoded before their context is complete. This will be true, for
+// example, for applications that wish to allow functions to refer to themselves.
+//
+// The simplest use of a ContextFunc is to give user functions access to the
+// same global variables and functions available elsewhere in an application's
+// configuration language, but more complex applications may use different
+// contexts to support lexical scoping depending on where in a configuration
+// structure a function declaration is found, etc.
+type ContextFunc func() *hcl.EvalContext
+
+// DecodeUserFunctions looks for blocks of the given type in the given body
+// and, for each one found, interprets it as a custom function definition.
+//
+// On success, the result is a mapping of function names to implementations,
+// along with a new body that represents the remaining content of the given
+// body which can be used for further processing.
+//
+// The result expression of each function is parsed during decoding but not
+// evaluated until the function is called.
+//
+// If the given ContextFunc is non-nil, it will be called to obtain the
+// context in which the function result expressions will be evaluated. If nil,
+// or if it returns nil, the result expression will have access only to
+// variables named after the declared parameters. A non-nil context turns
+// the returned functions into closures, bound to the given context.
+//
+// If the returned diagnostics set has errors then the function map and
+// remain body may be nil or incomplete.
+func DecodeUserFunctions(body hcl.Body, blockType string, context ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) {
+	return decodeUserFunctions(body, blockType, context)
+}
+
+// NewFunction creates a new function instance from preparsed HCL expressions.
+func NewFunction(paramsExpr, varParamExpr, resultExpr hcl.Expression, getBaseCtx func() *hcl.EvalContext) (function.Function, hcl.Diagnostics) {
+	var params []string
+	var varParam string
+
+	paramExprs, paramsDiags := hcl.ExprList(paramsExpr)
+	if paramsDiags.HasErrors() {
+		return function.Function{}, paramsDiags
+	}
+	for _, paramExpr := range paramExprs {
+		param := hcl.ExprAsKeyword(paramExpr)
+		if param == "" {
+			return function.Function{}, hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid param element",
+				Detail:   "Each parameter name must be an identifier.",
+				Subject:  paramExpr.Range().Ptr(),
+			}}
+		}
+		params = append(params, param)
+	}
+
+	if varParamExpr != nil {
+		varParam = hcl.ExprAsKeyword(varParamExpr)
+		if varParam == "" {
+			return function.Function{}, hcl.Diagnostics{{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid variadic_param",
+				Detail:   "The variadic parameter name must be an identifier.",
+				Subject:  varParamExpr.Range().Ptr(),
+			}}
+		}
+	}
+
+	spec := &function.Spec{}
+	for _, paramName := range params {
+		spec.Params = append(spec.Params, function.Parameter{
+			Name: paramName,
+			Type: cty.DynamicPseudoType,
+		})
+	}
+	if varParamExpr != nil {
+		spec.VarParam = &function.Parameter{
+			Name: varParam,
+			Type: cty.DynamicPseudoType,
+		}
+	}
+	impl := func(args []cty.Value) (cty.Value, error) {
+		ctx := getBaseCtx()
+		ctx = ctx.NewChild()
+		ctx.Variables = make(map[string]cty.Value)
+
+		// The cty function machinery guarantees that we have at least
+		// enough args to fill all of our params.
+		for i, paramName := range params {
+			ctx.Variables[paramName] = args[i]
+		}
+		if spec.VarParam != nil {
+			varArgs := args[len(params):]
+			ctx.Variables[varParam] = cty.TupleVal(varArgs)
+		}
+
+		result, diags := resultExpr.Value(ctx)
+		if diags.HasErrors() {
+			// Smuggle the diagnostics out via the error channel, since
+			// a diagnostics sequence implements error. Caller can
+			// type-assert this to recover the individual diagnostics
+			// if desired.
+			return cty.DynamicVal, diags
+		}
+		return result, nil
+	}
+	spec.Type = func(args []cty.Value) (cty.Type, error) {
+		val, err := impl(args)
+		return val.Type(), err
+	}
+	spec.Impl = func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+		return impl(args)
+	}
+	return function.New(spec), nil
+}

vendor/github.com/Microsoft/go-winio/CODEOWNERS

@@ -0,0 +1 @@
+  * @microsoft/containerplat

vendor/github.com/Microsoft/go-winio/fileinfo.go

@@ -5,21 +5,14 @@ package winio
 import (
 	"os"
 	"runtime"
-	"syscall"
 	"unsafe"
-)
 
-//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
-//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
-
-const (
-	fileBasicInfo = 0
-	fileIDInfo    = 0x12
+	"golang.org/x/sys/windows"
 )
 
 // FileBasicInfo contains file access time and file attributes information.
 type FileBasicInfo struct {
-	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
+	CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
 	FileAttributes                                          uint32
 	pad                                                     uint32 // padding
 }
@@ -27,7 +20,7 @@ type FileBasicInfo struct {
 // GetFileBasicInfo retrieves times and attributes for a file.
 func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
 	bi := &FileBasicInfo{}
-	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+	if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
 		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
 	}
 	runtime.KeepAlive(f)
@@ -36,13 +29,32 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
 
 // SetFileBasicInfo sets times and attributes for a file.
 func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
-	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+	if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
 		return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
 	}
 	runtime.KeepAlive(f)
 	return nil
 }
 
+// FileStandardInfo contains extended information for the file.
+// FILE_STANDARD_INFO in WinBase.h
+// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info
+type FileStandardInfo struct {
+	AllocationSize, EndOfFile int64
+	NumberOfLinks             uint32
+	DeletePending, Directory  bool
+}
+
+// GetFileStandardInfo retrieves ended information for the file.
+func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
+	si := &FileStandardInfo{}
+	if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return si, nil
+}
+
 // FileIDInfo contains the volume serial number and file ID for a file. This pair should be
 // unique on a system.
 type FileIDInfo struct {
@@ -53,7 +65,7 @@ type FileIDInfo struct {
 // GetFileID retrieves the unique (volume, file ID) pair for a file.
 func GetFileID(f *os.File) (*FileIDInfo, error) {
 	fileID := &FileIDInfo{}
-	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
+	if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
 		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
 	}
 	runtime.KeepAlive(f)

vendor/github.com/Microsoft/go-winio/go.mod

@@ -3,7 +3,7 @@ module github.com/Microsoft/go-winio
 go 1.12
 
 require (
-	github.com/pkg/errors v0.8.1
-	github.com/sirupsen/logrus v1.4.1
-	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.7.0
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
 )

vendor/github.com/Microsoft/go-winio/go.sum

@@ -1,18 +1,14 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/Microsoft/go-winio/hvsock.go

@@ -1,3 +1,5 @@
+// +build windows
+
 package winio
 
 import (

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -429,10 +429,10 @@ type PipeConfig struct {
 	// when the pipe is in message mode.
 	MessageMode bool
 
-	// InputBufferSize specifies the size the input buffer, in bytes.
+	// InputBufferSize specifies the size of the input buffer, in bytes.
 	InputBufferSize int32
 
-	// OutputBufferSize specifies the size the input buffer, in bytes.
+	// OutputBufferSize specifies the size of the output buffer, in bytes.
 	OutputBufferSize int32
 }
 

vendor/github.com/Microsoft/go-winio/pkg/guid/guid.go

@@ -1,3 +1,5 @@
+// +build windows
+
 // Package guid provides a GUID type. The backing structure for a GUID is
 // identical to that used by the golang.org/x/sys/windows GUID type.
 // There are two main binary encodings used for a GUID, the big-endian encoding,

vendor/github.com/Microsoft/go-winio/pkg/security/grantvmgroupaccess.go

@@ -0,0 +1,161 @@
+// +build windows
+
+package security
+
+import (
+	"os"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+)
+
+type (
+	accessMask          uint32
+	accessMode          uint32
+	desiredAccess       uint32
+	inheritMode         uint32
+	objectType          uint32
+	shareMode           uint32
+	securityInformation uint32
+	trusteeForm         uint32
+	trusteeType         uint32
+
+	explicitAccess struct {
+		accessPermissions accessMask
+		accessMode        accessMode
+		inheritance       inheritMode
+		trustee           trustee
+	}
+
+	trustee struct {
+		multipleTrustee          *trustee
+		multipleTrusteeOperation int32
+		trusteeForm              trusteeForm
+		trusteeType              trusteeType
+		name                     uintptr
+	}
+)
+
+const (
+	accessMaskDesiredPermission accessMask = 1 << 31 // GENERIC_READ
+
+	accessModeGrant accessMode = 1
+
+	desiredAccessReadControl desiredAccess = 0x20000
+	desiredAccessWriteDac    desiredAccess = 0x40000
+
+	gvmga = "GrantVmGroupAccess:"
+
+	inheritModeNoInheritance                  inheritMode = 0x0
+	inheritModeSubContainersAndObjectsInherit inheritMode = 0x3
+
+	objectTypeFileObject objectType = 0x1
+
+	securityInformationDACL securityInformation = 0x4
+
+	shareModeRead  shareMode = 0x1
+	shareModeWrite shareMode = 0x2
+
+	sidVmGroup = "S-1-5-83-0"
+
+	trusteeFormIsSid trusteeForm = 0
+
+	trusteeTypeWellKnownGroup trusteeType = 5
+)
+
+// GrantVMGroupAccess sets the DACL for a specified file or directory to
+// include Grant ACE entries for the VM Group SID. This is a golang re-
+// implementation of the same function in vmcompute, just not exported in
+// RS5. Which kind of sucks. Sucks a lot :/
+func GrantVmGroupAccess(name string) error {
+	// Stat (to determine if `name` is a directory).
+	s, err := os.Stat(name)
+	if err != nil {
+		return errors.Wrapf(err, "%s os.Stat %s", gvmga, name)
+	}
+
+	// Get a handle to the file/directory. Must defer Close on success.
+	fd, err := createFile(name, s.IsDir())
+	if err != nil {
+		return err // Already wrapped
+	}
+	defer syscall.CloseHandle(fd)
+
+	// Get the current DACL and Security Descriptor. Must defer LocalFree on success.
+	ot := objectTypeFileObject
+	si := securityInformationDACL
+	sd := uintptr(0)
+	origDACL := uintptr(0)
+	if err := getSecurityInfo(fd, uint32(ot), uint32(si), nil, nil, &origDACL, nil, &sd); err != nil {
+		return errors.Wrapf(err, "%s GetSecurityInfo %s", gvmga, name)
+	}
+	defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd)))
+
+	// Generate a new DACL which is the current DACL with the required ACEs added.
+	// Must defer LocalFree on success.
+	newDACL, err := generateDACLWithAcesAdded(name, s.IsDir(), origDACL)
+	if err != nil {
+		return err // Already wrapped
+	}
+	defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL)))
+
+	// And finally use SetSecurityInfo to apply the updated DACL.
+	if err := setSecurityInfo(fd, uint32(ot), uint32(si), uintptr(0), uintptr(0), newDACL, uintptr(0)); err != nil {
+		return errors.Wrapf(err, "%s SetSecurityInfo %s", gvmga, name)
+	}
+
+	return nil
+}
+
+// createFile is a helper function to call [Nt]CreateFile to get a handle to
+// the file or directory.
+func createFile(name string, isDir bool) (syscall.Handle, error) {
+	namep := syscall.StringToUTF16(name)
+	da := uint32(desiredAccessReadControl | desiredAccessWriteDac)
+	sm := uint32(shareModeRead | shareModeWrite)
+	fa := uint32(syscall.FILE_ATTRIBUTE_NORMAL)
+	if isDir {
+		fa = uint32(fa | syscall.FILE_FLAG_BACKUP_SEMANTICS)
+	}
+	fd, err := syscall.CreateFile(&namep[0], da, sm, nil, syscall.OPEN_EXISTING, fa, 0)
+	if err != nil {
+		return 0, errors.Wrapf(err, "%s syscall.CreateFile %s", gvmga, name)
+	}
+	return fd, nil
+}
+
+// generateDACLWithAcesAdded generates a new DACL with the two needed ACEs added.
+// The caller is responsible for LocalFree of the returned DACL on success.
+func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintptr, error) {
+	// Generate pointers to the SIDs based on the string SIDs
+	sid, err := syscall.StringToSid(sidVmGroup)
+	if err != nil {
+		return 0, errors.Wrapf(err, "%s syscall.StringToSid %s %s", gvmga, name, sidVmGroup)
+	}
+
+	inheritance := inheritModeNoInheritance
+	if isDir {
+		inheritance = inheritModeSubContainersAndObjectsInherit
+	}
+
+	eaArray := []explicitAccess{
+		explicitAccess{
+			accessPermissions: accessMaskDesiredPermission,
+			accessMode:        accessModeGrant,
+			inheritance:       inheritance,
+			trustee: trustee{
+				trusteeForm: trusteeFormIsSid,
+				trusteeType: trusteeTypeWellKnownGroup,
+				name:        uintptr(unsafe.Pointer(sid)),
+			},
+		},
+	}
+
+	modifiedDACL := uintptr(0)
+	if err := setEntriesInAcl(uintptr(uint32(1)), uintptr(unsafe.Pointer(&eaArray[0])), origDACL, &modifiedDACL); err != nil {
+		return 0, errors.Wrapf(err, "%s SetEntriesInAcl %s", gvmga, name)
+	}
+
+	return modifiedDACL, nil
+}

vendor/github.com/Microsoft/go-winio/pkg/security/syscall_windows.go

@@ -0,0 +1,7 @@
+package security
+
+//go:generate go run mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go
+
+//sys getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (err error) [failretval!=0] = advapi32.GetSecurityInfo
+//sys setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (err error) [failretval!=0] = advapi32.SetSecurityInfo
+//sys setEntriesInAcl(count uintptr, pListOfEEs uintptr, oldAcl uintptr, newAcl *uintptr) (err error) [failretval!=0] = advapi32.SetEntriesInAclW

vendor/github.com/Microsoft/go-winio/pkg/security/zsyscall_windows.go

@@ -0,0 +1,70 @@
+// Code generated by 'go generate'; DO NOT EDIT.
+
+package security
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return errERROR_EINVAL
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
+
+	procGetSecurityInfo  = modadvapi32.NewProc("GetSecurityInfo")
+	procSetEntriesInAclW = modadvapi32.NewProc("SetEntriesInAclW")
+	procSetSecurityInfo  = modadvapi32.NewProc("SetSecurityInfo")
+)
+
+func getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall9(procGetSecurityInfo.Addr(), 8, uintptr(handle), uintptr(objectType), uintptr(si), uintptr(unsafe.Pointer(ppsidOwner)), uintptr(unsafe.Pointer(ppsidGroup)), uintptr(unsafe.Pointer(ppDacl)), uintptr(unsafe.Pointer(ppSacl)), uintptr(unsafe.Pointer(ppSecurityDescriptor)), 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func setEntriesInAcl(count uintptr, pListOfEEs uintptr, oldAcl uintptr, newAcl *uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall6(procSetEntriesInAclW.Addr(), 4, uintptr(count), uintptr(pListOfEEs), uintptr(oldAcl), uintptr(unsafe.Pointer(newAcl)), 0, 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall9(procSetSecurityInfo.Addr(), 7, uintptr(handle), uintptr(objectType), uintptr(si), uintptr(psidOwner), uintptr(psidGroup), uintptr(pDacl), uintptr(pSacl), 0, 0)
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}

vendor/github.com/Microsoft/go-winio/syscall.go

@@ -1,3 +1,3 @@
 package winio
 
-//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go
+//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go

vendor/github.com/Microsoft/go-winio/vhd/vhd.go

@@ -2,150 +2,322 @@
 
 package vhd
 
-import "syscall"
+import (
+	"fmt"
+	"syscall"
 
-//go:generate go run mksyscall_windows.go -output zvhd.go vhd.go
+	"github.com/Microsoft/go-winio/pkg/guid"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
 
-//sys createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.CreateVirtualDisk
-//sys openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.OpenVirtualDisk
-//sys detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) [failretval != 0] = VirtDisk.DetachVirtualDisk
+//go:generate go run mksyscall_windows.go -output zvhd_windows.go vhd.go
 
-type virtualStorageType struct {
-	DeviceID uint32
-	VendorID [16]byte
-}
+//sys createVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = virtdisk.CreateVirtualDisk
+//sys openVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *OpenVirtualDiskParameters, handle *syscall.Handle) (err error) [failretval != 0] = virtdisk.OpenVirtualDisk
+//sys attachVirtualDisk(handle syscall.Handle, securityDescriptor *uintptr, attachVirtualDiskFlag uint32, providerSpecificFlags uint32, parameters *AttachVirtualDiskParameters, overlapped *syscall.Overlapped) (err error) [failretval != 0] = virtdisk.AttachVirtualDisk
+//sys detachVirtualDisk(handle syscall.Handle, detachVirtualDiskFlags uint32, providerSpecificFlags uint32) (err error) [failretval != 0] = virtdisk.DetachVirtualDisk
+//sys getVirtualDiskPhysicalPath(handle syscall.Handle, diskPathSizeInBytes *uint32, buffer *uint16) (err error) [failretval != 0] = virtdisk.GetVirtualDiskPhysicalPath
 
 type (
-	createVirtualDiskFlag uint32
-	VirtualDiskAccessMask uint32
+	CreateVirtualDiskFlag uint32
 	VirtualDiskFlag       uint32
+	AttachVirtualDiskFlag uint32
+	DetachVirtualDiskFlag uint32
+	VirtualDiskAccessMask uint32
 )
 
-const (
-	// Flags for creating a VHD (not exported)
-	createVirtualDiskFlagNone                        createVirtualDiskFlag = 0
-	createVirtualDiskFlagFullPhysicalAllocation      createVirtualDiskFlag = 1
-	createVirtualDiskFlagPreventWritesToSourceDisk   createVirtualDiskFlag = 2
-	createVirtualDiskFlagDoNotCopyMetadataFromParent createVirtualDiskFlag = 4
+type VirtualStorageType struct {
+	DeviceID uint32
+	VendorID guid.GUID
+}
 
-	// Access Mask for opening a VHD
-	VirtualDiskAccessNone     VirtualDiskAccessMask = 0
-	VirtualDiskAccessAttachRO VirtualDiskAccessMask = 65536
-	VirtualDiskAccessAttachRW VirtualDiskAccessMask = 131072
-	VirtualDiskAccessDetach   VirtualDiskAccessMask = 262144
-	VirtualDiskAccessGetInfo  VirtualDiskAccessMask = 524288
-	VirtualDiskAccessCreate   VirtualDiskAccessMask = 1048576
-	VirtualDiskAccessMetaOps  VirtualDiskAccessMask = 2097152
-	VirtualDiskAccessRead     VirtualDiskAccessMask = 851968
-	VirtualDiskAccessAll      VirtualDiskAccessMask = 4128768
-	VirtualDiskAccessWritable VirtualDiskAccessMask = 3276800
-
-	// Flags for opening a VHD
-	OpenVirtualDiskFlagNone                        VirtualDiskFlag = 0
-	OpenVirtualDiskFlagNoParents                   VirtualDiskFlag = 0x1
-	OpenVirtualDiskFlagBlankFile                   VirtualDiskFlag = 0x2
-	OpenVirtualDiskFlagBootDrive                   VirtualDiskFlag = 0x4
-	OpenVirtualDiskFlagCachedIO                    VirtualDiskFlag = 0x8
-	OpenVirtualDiskFlagCustomDiffChain             VirtualDiskFlag = 0x10
-	OpenVirtualDiskFlagParentCachedIO              VirtualDiskFlag = 0x20
-	OpenVirtualDiskFlagVhdSetFileOnly              VirtualDiskFlag = 0x40
-	OpenVirtualDiskFlagIgnoreRelativeParentLocator VirtualDiskFlag = 0x80
-	OpenVirtualDiskFlagNoWriteHardening            VirtualDiskFlag = 0x100
-)
-
-type createVersion2 struct {
-	UniqueID                 [16]byte // GUID
+type CreateVersion2 struct {
+	UniqueID                 guid.GUID
 	MaximumSize              uint64
 	BlockSizeInBytes         uint32
 	SectorSizeInBytes        uint32
+	PhysicalSectorSizeInByte uint32
 	ParentPath               *uint16 // string
 	SourcePath               *uint16 // string
 	OpenFlags                uint32
-	ParentVirtualStorageType virtualStorageType
-	SourceVirtualStorageType virtualStorageType
-	ResiliencyGUID           [16]byte // GUID
+	ParentVirtualStorageType VirtualStorageType
+	SourceVirtualStorageType VirtualStorageType
+	ResiliencyGUID           guid.GUID
 }
 
-type createVirtualDiskParameters struct {
+type CreateVirtualDiskParameters struct {
 	Version  uint32 // Must always be set to 2
-	Version2 createVersion2
+	Version2 CreateVersion2
 }
 
-type openVersion2 struct {
-	GetInfoOnly    int32    // bool but 4-byte aligned
-	ReadOnly       int32    // bool but 4-byte aligned
-	ResiliencyGUID [16]byte // GUID
+type OpenVersion2 struct {
+	GetInfoOnly    bool
+	ReadOnly       bool
+	ResiliencyGUID guid.GUID
 }
 
-type openVirtualDiskParameters struct {
+type OpenVirtualDiskParameters struct {
 	Version  uint32 // Must always be set to 2
-	Version2 openVersion2
+	Version2 OpenVersion2
 }
 
-// CreateVhdx will create a simple vhdx file at the given path using default values.
+type AttachVersion2 struct {
+	RestrictedOffset uint64
+	RestrictedLength uint64
+}
+
+type AttachVirtualDiskParameters struct {
+	Version  uint32 // Must always be set to 2
+	Version2 AttachVersion2
+}
+
+const (
+	VIRTUAL_STORAGE_TYPE_DEVICE_VHDX = 0x3
+
+	// Access Mask for opening a VHD
+	VirtualDiskAccessNone     VirtualDiskAccessMask = 0x00000000
+	VirtualDiskAccessAttachRO VirtualDiskAccessMask = 0x00010000
+	VirtualDiskAccessAttachRW VirtualDiskAccessMask = 0x00020000
+	VirtualDiskAccessDetach   VirtualDiskAccessMask = 0x00040000
+	VirtualDiskAccessGetInfo  VirtualDiskAccessMask = 0x00080000
+	VirtualDiskAccessCreate   VirtualDiskAccessMask = 0x00100000
+	VirtualDiskAccessMetaOps  VirtualDiskAccessMask = 0x00200000
+	VirtualDiskAccessRead     VirtualDiskAccessMask = 0x000d0000
+	VirtualDiskAccessAll      VirtualDiskAccessMask = 0x003f0000
+	VirtualDiskAccessWritable VirtualDiskAccessMask = 0x00320000
+
+	// Flags for creating a VHD
+	CreateVirtualDiskFlagNone                              CreateVirtualDiskFlag = 0x0
+	CreateVirtualDiskFlagFullPhysicalAllocation            CreateVirtualDiskFlag = 0x1
+	CreateVirtualDiskFlagPreventWritesToSourceDisk         CreateVirtualDiskFlag = 0x2
+	CreateVirtualDiskFlagDoNotCopyMetadataFromParent       CreateVirtualDiskFlag = 0x4
+	CreateVirtualDiskFlagCreateBackingStorage              CreateVirtualDiskFlag = 0x8
+	CreateVirtualDiskFlagUseChangeTrackingSourceLimit      CreateVirtualDiskFlag = 0x10
+	CreateVirtualDiskFlagPreserveParentChangeTrackingState CreateVirtualDiskFlag = 0x20
+	CreateVirtualDiskFlagVhdSetUseOriginalBackingStorage   CreateVirtualDiskFlag = 0x40
+	CreateVirtualDiskFlagSparseFile                        CreateVirtualDiskFlag = 0x80
+	CreateVirtualDiskFlagPmemCompatible                    CreateVirtualDiskFlag = 0x100
+	CreateVirtualDiskFlagSupportCompressedVolumes          CreateVirtualDiskFlag = 0x200
+
+	// Flags for opening a VHD
+	OpenVirtualDiskFlagNone                        VirtualDiskFlag = 0x00000000
+	OpenVirtualDiskFlagNoParents                   VirtualDiskFlag = 0x00000001
+	OpenVirtualDiskFlagBlankFile                   VirtualDiskFlag = 0x00000002
+	OpenVirtualDiskFlagBootDrive                   VirtualDiskFlag = 0x00000004
+	OpenVirtualDiskFlagCachedIO                    VirtualDiskFlag = 0x00000008
+	OpenVirtualDiskFlagCustomDiffChain             VirtualDiskFlag = 0x00000010
+	OpenVirtualDiskFlagParentCachedIO              VirtualDiskFlag = 0x00000020
+	OpenVirtualDiskFlagVhdsetFileOnly              VirtualDiskFlag = 0x00000040
+	OpenVirtualDiskFlagIgnoreRelativeParentLocator VirtualDiskFlag = 0x00000080
+	OpenVirtualDiskFlagNoWriteHardening            VirtualDiskFlag = 0x00000100
+	OpenVirtualDiskFlagSupportCompressedVolumes    VirtualDiskFlag = 0x00000200
+
+	// Flags for attaching a VHD
+	AttachVirtualDiskFlagNone                          AttachVirtualDiskFlag = 0x00000000
+	AttachVirtualDiskFlagReadOnly                      AttachVirtualDiskFlag = 0x00000001
+	AttachVirtualDiskFlagNoDriveLetter                 AttachVirtualDiskFlag = 0x00000002
+	AttachVirtualDiskFlagPermanentLifetime             AttachVirtualDiskFlag = 0x00000004
+	AttachVirtualDiskFlagNoLocalHost                   AttachVirtualDiskFlag = 0x00000008
+	AttachVirtualDiskFlagNoSecurityDescriptor          AttachVirtualDiskFlag = 0x00000010
+	AttachVirtualDiskFlagBypassDefaultEncryptionPolicy AttachVirtualDiskFlag = 0x00000020
+	AttachVirtualDiskFlagNonPnp                        AttachVirtualDiskFlag = 0x00000040
+	AttachVirtualDiskFlagRestrictedRange               AttachVirtualDiskFlag = 0x00000080
+	AttachVirtualDiskFlagSinglePartition               AttachVirtualDiskFlag = 0x00000100
+	AttachVirtualDiskFlagRegisterVolume                AttachVirtualDiskFlag = 0x00000200
+
+	// Flags for detaching a VHD
+	DetachVirtualDiskFlagNone DetachVirtualDiskFlag = 0x0
+)
+
+// CreateVhdx is a helper function to create a simple vhdx file at the given path using
+// default values.
 func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
-	var (
-		defaultType virtualStorageType
-		handle      syscall.Handle
-	)
-
-	parameters := createVirtualDiskParameters{
+	params := CreateVirtualDiskParameters{
 		Version: 2,
-		Version2: createVersion2{
+		Version2: CreateVersion2{
 			MaximumSize:      uint64(maxSizeInGb) * 1024 * 1024 * 1024,
 			BlockSizeInBytes: blockSizeInMb * 1024 * 1024,
 		},
 	}
 
-	if err := createVirtualDisk(
-		&defaultType,
-		path,
-		uint32(VirtualDiskAccessNone),
-		nil,
-		uint32(createVirtualDiskFlagNone),
-		0,
-		&parameters,
-		nil,
-		&handle); err != nil {
+	handle, err := CreateVirtualDisk(path, VirtualDiskAccessNone, CreateVirtualDiskFlagNone, &params)
+	if err != nil {
 		return err
 	}
 
 	if err := syscall.CloseHandle(handle); err != nil {
 		return err
 	}
-
 	return nil
 }
 
-// DetachVhd detaches a mounted container layer vhd found at `path`.
+// DetachVirtualDisk detaches a virtual hard disk by handle.
+func DetachVirtualDisk(handle syscall.Handle) (err error) {
+	if err := detachVirtualDisk(handle, 0, 0); err != nil {
+		return errors.Wrap(err, "failed to detach virtual disk")
+	}
+	return nil
+}
+
+// DetachVhd detaches a vhd found at `path`.
 func DetachVhd(path string) error {
 	handle, err := OpenVirtualDisk(
 		path,
 		VirtualDiskAccessNone,
-		OpenVirtualDiskFlagCachedIO|OpenVirtualDiskFlagIgnoreRelativeParentLocator)
-
+		OpenVirtualDiskFlagCachedIO|OpenVirtualDiskFlagIgnoreRelativeParentLocator,
+	)
 	if err != nil {
 		return err
 	}
 	defer syscall.CloseHandle(handle)
-	return detachVirtualDisk(handle, 0, 0)
+	return DetachVirtualDisk(handle)
+}
+
+// AttachVirtualDisk attaches a virtual hard disk for use.
+func AttachVirtualDisk(handle syscall.Handle, attachVirtualDiskFlag AttachVirtualDiskFlag, parameters *AttachVirtualDiskParameters) (err error) {
+	// Supports both version 1 and 2 of the attach parameters as version 2 wasn't present in RS5.
+	if err := attachVirtualDisk(
+		handle,
+		nil,
+		uint32(attachVirtualDiskFlag),
+		0,
+		parameters,
+		nil,
+	); err != nil {
+		return errors.Wrap(err, "failed to attach virtual disk")
+	}
+	return nil
+}
+
+// AttachVhd attaches a virtual hard disk at `path` for use. Attaches using version 2
+// of the ATTACH_VIRTUAL_DISK_PARAMETERS.
+func AttachVhd(path string) (err error) {
+	handle, err := OpenVirtualDisk(
+		path,
+		VirtualDiskAccessNone,
+		OpenVirtualDiskFlagCachedIO|OpenVirtualDiskFlagIgnoreRelativeParentLocator,
+	)
+	if err != nil {
+		return err
+	}
+
+	defer syscall.CloseHandle(handle)
+	params := AttachVirtualDiskParameters{Version: 2}
+	if err := AttachVirtualDisk(
+		handle,
+		AttachVirtualDiskFlagNone,
+		&params,
+	); err != nil {
+		return errors.Wrap(err, "failed to attach virtual disk")
+	}
+	return nil
 }
 
 // OpenVirtualDisk obtains a handle to a VHD opened with supplied access mask and flags.
-func OpenVirtualDisk(path string, accessMask VirtualDiskAccessMask, flag VirtualDiskFlag) (syscall.Handle, error) {
-	var (
-		defaultType virtualStorageType
-		handle      syscall.Handle
-	)
-	parameters := openVirtualDiskParameters{Version: 2}
-	if err := openVirtualDisk(
-		&defaultType,
-		path,
-		uint32(accessMask),
-		uint32(flag),
+func OpenVirtualDisk(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag) (syscall.Handle, error) {
+	parameters := OpenVirtualDiskParameters{Version: 2}
+	handle, err := OpenVirtualDiskWithParameters(
+		vhdPath,
+		virtualDiskAccessMask,
+		openVirtualDiskFlags,
 		&parameters,
-		&handle); err != nil {
+	)
+	if err != nil {
 		return 0, err
 	}
 	return handle, nil
 }
+
+// OpenVirtualDiskWithParameters obtains a handle to a VHD opened with supplied access mask, flags and parameters.
+func OpenVirtualDiskWithParameters(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag, parameters *OpenVirtualDiskParameters) (syscall.Handle, error) {
+	var (
+		handle      syscall.Handle
+		defaultType VirtualStorageType
+	)
+	if parameters.Version != 2 {
+		return handle, fmt.Errorf("only version 2 VHDs are supported, found version: %d", parameters.Version)
+	}
+	if err := openVirtualDisk(
+		&defaultType,
+		vhdPath,
+		uint32(virtualDiskAccessMask),
+		uint32(openVirtualDiskFlags),
+		parameters,
+		&handle,
+	); err != nil {
+		return 0, errors.Wrap(err, "failed to open virtual disk")
+	}
+	return handle, nil
+}
+
+// CreateVirtualDisk creates a virtual harddisk and returns a handle to the disk.
+func CreateVirtualDisk(path string, virtualDiskAccessMask VirtualDiskAccessMask, createVirtualDiskFlags CreateVirtualDiskFlag, parameters *CreateVirtualDiskParameters) (syscall.Handle, error) {
+	var (
+		handle      syscall.Handle
+		defaultType VirtualStorageType
+	)
+	if parameters.Version != 2 {
+		return handle, fmt.Errorf("only version 2 VHDs are supported, found version: %d", parameters.Version)
+	}
+
+	if err := createVirtualDisk(
+		&defaultType,
+		path,
+		uint32(virtualDiskAccessMask),
+		nil,
+		uint32(createVirtualDiskFlags),
+		0,
+		parameters,
+		nil,
+		&handle,
+	); err != nil {
+		return handle, errors.Wrap(err, "failed to create virtual disk")
+	}
+	return handle, nil
+}
+
+// GetVirtualDiskPhysicalPath takes a handle to a virtual hard disk and returns the physical
+// path of the disk on the machine. This path is in the form \\.\PhysicalDriveX where X is an integer
+// that represents the particular enumeration of the physical disk on the caller's system.
+func GetVirtualDiskPhysicalPath(handle syscall.Handle) (_ string, err error) {
+	var (
+		diskPathSizeInBytes uint32 = 256 * 2 // max path length 256 wide chars
+		diskPhysicalPathBuf [256]uint16
+	)
+	if err := getVirtualDiskPhysicalPath(
+		handle,
+		&diskPathSizeInBytes,
+		&diskPhysicalPathBuf[0],
+	); err != nil {
+		return "", errors.Wrap(err, "failed to get disk physical path")
+	}
+	return windows.UTF16ToString(diskPhysicalPathBuf[:]), nil
+}
+
+// CreateDiffVhd is a helper function to create a differencing virtual disk.
+func CreateDiffVhd(diffVhdPath, baseVhdPath string, blockSizeInMB uint32) error {
+	// Setting `ParentPath` is how to signal to create a differencing disk.
+	createParams := &CreateVirtualDiskParameters{
+		Version: 2,
+		Version2: CreateVersion2{
+			ParentPath:       windows.StringToUTF16Ptr(baseVhdPath),
+			BlockSizeInBytes: blockSizeInMB * 1024 * 1024,
+			OpenFlags:        uint32(OpenVirtualDiskFlagCachedIO),
+		},
+	}
+
+	vhdHandle, err := CreateVirtualDisk(
+		diffVhdPath,
+		VirtualDiskAccessNone,
+		CreateVirtualDiskFlagNone,
+		createParams,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create differencing vhd: %s", err)
+	}
+	if err := syscall.CloseHandle(vhdHandle); err != nil {
+		return fmt.Errorf("failed to close differencing vhd handle: %s", err)
+	}
+	return nil
+}

vendor/github.com/Microsoft/go-winio/vhd/zvhd.go

@@ -1,99 +0,0 @@
-// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
-
-package vhd
-
-import (
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-var _ unsafe.Pointer
-
-// Do the interface allocations only once for common
-// Errno values.
-const (
-	errnoERROR_IO_PENDING = 997
-)
-
-var (
-	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
-)
-
-// errnoErr returns common boxed Errno values, to prevent
-// allocations at runtime.
-func errnoErr(e syscall.Errno) error {
-	switch e {
-	case 0:
-		return nil
-	case errnoERROR_IO_PENDING:
-		return errERROR_IO_PENDING
-	}
-	// TODO: add more here, after collecting data on the common
-	// error values see on Windows. (perhaps when running
-	// all.bat?)
-	return e
-}
-
-var (
-	modVirtDisk = windows.NewLazySystemDLL("VirtDisk.dll")
-
-	procCreateVirtualDisk = modVirtDisk.NewProc("CreateVirtualDisk")
-	procOpenVirtualDisk   = modVirtDisk.NewProc("OpenVirtualDisk")
-	procDetachVirtualDisk = modVirtDisk.NewProc("DetachVirtualDisk")
-)
-
-func createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return
-	}
-	return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, flags, providerSpecificFlags, parameters, o, handle)
-}
-
-func _createVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
-	r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(flags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(handle)))
-	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return
-	}
-	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, flags, parameters, handle)
-}
-
-func _openVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
-	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(flags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
-	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) {
-	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(flags), uintptr(providerSpecificFlags))
-	if r1 != 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}

vendor/github.com/Microsoft/go-winio/vhd/zvhd_windows.go

@@ -0,0 +1,106 @@
+// Code generated by 'go generate'; DO NOT EDIT.
+
+package vhd
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return errERROR_EINVAL
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modvirtdisk = windows.NewLazySystemDLL("virtdisk.dll")
+
+	procAttachVirtualDisk          = modvirtdisk.NewProc("AttachVirtualDisk")
+	procCreateVirtualDisk          = modvirtdisk.NewProc("CreateVirtualDisk")
+	procDetachVirtualDisk          = modvirtdisk.NewProc("DetachVirtualDisk")
+	procGetVirtualDiskPhysicalPath = modvirtdisk.NewProc("GetVirtualDiskPhysicalPath")
+	procOpenVirtualDisk            = modvirtdisk.NewProc("OpenVirtualDisk")
+)
+
+func attachVirtualDisk(handle syscall.Handle, securityDescriptor *uintptr, attachVirtualDiskFlag uint32, providerSpecificFlags uint32, parameters *AttachVirtualDiskParameters, overlapped *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall6(procAttachVirtualDisk.Addr(), 6, uintptr(handle), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(attachVirtualDiskFlag), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(overlapped)))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func createVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, createVirtualDiskFlags, providerSpecificFlags, parameters, overlapped, handle)
+}
+
+func _createVirtualDisk(virtualStorageType *VirtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(createVirtualDiskFlags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func detachVirtualDisk(handle syscall.Handle, detachVirtualDiskFlags uint32, providerSpecificFlags uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(detachVirtualDiskFlags), uintptr(providerSpecificFlags))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func getVirtualDiskPhysicalPath(handle syscall.Handle, diskPathSizeInBytes *uint32, buffer *uint16) (err error) {
+	r1, _, e1 := syscall.Syscall(procGetVirtualDiskPhysicalPath.Addr(), 3, uintptr(handle), uintptr(unsafe.Pointer(diskPathSizeInBytes)), uintptr(unsafe.Pointer(buffer)))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func openVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *OpenVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, openVirtualDiskFlags, parameters, handle)
+}
+
+func _openVirtualDisk(virtualStorageType *VirtualStorageType, path *uint16, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *OpenVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(openVirtualDiskFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}

vendor/github.com/Microsoft/go-winio/zsyscall_windows.go

@@ -19,6 +19,7 @@ const (
 
 var (
 	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+	errERROR_EINVAL     error = syscall.EINVAL
 )
 
 // errnoErr returns common boxed Errno values, to prevent
@@ -26,7 +27,7 @@ var (
 func errnoErr(e syscall.Errno) error {
 	switch e {
 	case 0:
-		return nil
+		return errERROR_EINVAL
 	case errnoERROR_IO_PENDING:
 		return errERROR_IO_PENDING
 	}
@@ -37,243 +38,62 @@ func errnoErr(e syscall.Errno) error {
 }
 
 var (
-	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
-	modws2_32   = windows.NewLazySystemDLL("ws2_32.dll")
-	modntdll    = windows.NewLazySystemDLL("ntdll.dll")
 	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
+	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
+	modntdll    = windows.NewLazySystemDLL("ntdll.dll")
+	modws2_32   = windows.NewLazySystemDLL("ws2_32.dll")
 
-	procCancelIoEx                                           = modkernel32.NewProc("CancelIoEx")
-	procCreateIoCompletionPort                               = modkernel32.NewProc("CreateIoCompletionPort")
-	procGetQueuedCompletionStatus                            = modkernel32.NewProc("GetQueuedCompletionStatus")
-	procSetFileCompletionNotificationModes                   = modkernel32.NewProc("SetFileCompletionNotificationModes")
-	procWSAGetOverlappedResult                               = modws2_32.NewProc("WSAGetOverlappedResult")
-	procConnectNamedPipe                                     = modkernel32.NewProc("ConnectNamedPipe")
-	procCreateNamedPipeW                                     = modkernel32.NewProc("CreateNamedPipeW")
-	procCreateFileW                                          = modkernel32.NewProc("CreateFileW")
-	procGetNamedPipeInfo                                     = modkernel32.NewProc("GetNamedPipeInfo")
-	procGetNamedPipeHandleStateW                             = modkernel32.NewProc("GetNamedPipeHandleStateW")
-	procLocalAlloc                                           = modkernel32.NewProc("LocalAlloc")
-	procNtCreateNamedPipeFile                                = modntdll.NewProc("NtCreateNamedPipeFile")
-	procRtlNtStatusToDosErrorNoTeb                           = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
-	procRtlDosPathNameToNtPathName_U                         = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
-	procRtlDefaultNpAcl                                      = modntdll.NewProc("RtlDefaultNpAcl")
-	procLookupAccountNameW                                   = modadvapi32.NewProc("LookupAccountNameW")
+	procAdjustTokenPrivileges                                = modadvapi32.NewProc("AdjustTokenPrivileges")
+	procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
 	procConvertSidToStringSidW                               = modadvapi32.NewProc("ConvertSidToStringSidW")
 	procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
-	procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
-	procLocalFree                                            = modkernel32.NewProc("LocalFree")
 	procGetSecurityDescriptorLength                          = modadvapi32.NewProc("GetSecurityDescriptorLength")
-	procGetFileInformationByHandleEx                         = modkernel32.NewProc("GetFileInformationByHandleEx")
-	procSetFileInformationByHandle                           = modkernel32.NewProc("SetFileInformationByHandle")
-	procAdjustTokenPrivileges                                = modadvapi32.NewProc("AdjustTokenPrivileges")
 	procImpersonateSelf                                      = modadvapi32.NewProc("ImpersonateSelf")
-	procRevertToSelf                                         = modadvapi32.NewProc("RevertToSelf")
-	procOpenThreadToken                                      = modadvapi32.NewProc("OpenThreadToken")
-	procGetCurrentThread                                     = modkernel32.NewProc("GetCurrentThread")
-	procLookupPrivilegeValueW                                = modadvapi32.NewProc("LookupPrivilegeValueW")
-	procLookupPrivilegeNameW                                 = modadvapi32.NewProc("LookupPrivilegeNameW")
+	procLookupAccountNameW                                   = modadvapi32.NewProc("LookupAccountNameW")
 	procLookupPrivilegeDisplayNameW                          = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
+	procLookupPrivilegeNameW                                 = modadvapi32.NewProc("LookupPrivilegeNameW")
+	procLookupPrivilegeValueW                                = modadvapi32.NewProc("LookupPrivilegeValueW")
+	procOpenThreadToken                                      = modadvapi32.NewProc("OpenThreadToken")
+	procRevertToSelf                                         = modadvapi32.NewProc("RevertToSelf")
 	procBackupRead                                           = modkernel32.NewProc("BackupRead")
 	procBackupWrite                                          = modkernel32.NewProc("BackupWrite")
+	procCancelIoEx                                           = modkernel32.NewProc("CancelIoEx")
+	procConnectNamedPipe                                     = modkernel32.NewProc("ConnectNamedPipe")
+	procCreateFileW                                          = modkernel32.NewProc("CreateFileW")
+	procCreateIoCompletionPort                               = modkernel32.NewProc("CreateIoCompletionPort")
+	procCreateNamedPipeW                                     = modkernel32.NewProc("CreateNamedPipeW")
+	procGetCurrentThread                                     = modkernel32.NewProc("GetCurrentThread")
+	procGetNamedPipeHandleStateW                             = modkernel32.NewProc("GetNamedPipeHandleStateW")
+	procGetNamedPipeInfo                                     = modkernel32.NewProc("GetNamedPipeInfo")
+	procGetQueuedCompletionStatus                            = modkernel32.NewProc("GetQueuedCompletionStatus")
+	procLocalAlloc                                           = modkernel32.NewProc("LocalAlloc")
+	procLocalFree                                            = modkernel32.NewProc("LocalFree")
+	procSetFileCompletionNotificationModes                   = modkernel32.NewProc("SetFileCompletionNotificationModes")
+	procNtCreateNamedPipeFile                                = modntdll.NewProc("NtCreateNamedPipeFile")
+	procRtlDefaultNpAcl                                      = modntdll.NewProc("RtlDefaultNpAcl")
+	procRtlDosPathNameToNtPathName_U                         = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
+	procRtlNtStatusToDosErrorNoTeb                           = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
+	procWSAGetOverlappedResult                               = modws2_32.NewProc("WSAGetOverlappedResult")
 	procbind                                                 = modws2_32.NewProc("bind")
 )
 
-func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) {
-	r1, _, e1 := syscall.Syscall(procCancelIoEx.Addr(), 2, uintptr(file), uintptr(unsafe.Pointer(o)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0)
-	newport = syscall.Handle(r0)
-	if newport == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) {
-	r1, _, e1 := syscall.Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(h), uintptr(flags), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) {
+func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
 	var _p0 uint32
-	if wait {
+	if releaseAll {
 		_p0 = 1
-	} else {
-		_p0 = 0
 	}
-	r1, _, e1 := syscall.Syscall6(procWSAGetOverlappedResult.Addr(), 5, uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)), 0)
+	r0, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize)))
+	success = r0 != 0
+	if true {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procConvertSecurityDescriptorToStringSecurityDescriptorW.Addr(), 5, uintptr(unsafe.Pointer(sd)), uintptr(revision), uintptr(secInfo), uintptr(unsafe.Pointer(sddl)), uintptr(unsafe.Pointer(sddlSize)), 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) {
-	r1, _, e1 := syscall.Syscall(procConnectNamedPipe.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(o)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(name)
-	if err != nil {
-		return
-	}
-	return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa)
-}
-
-func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall9(procCreateNamedPipeW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)), 0)
-	handle = syscall.Handle(r0)
-	if handle == syscall.InvalidHandle {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(name)
-	if err != nil {
-		return
-	}
-	return _createFile(_p0, access, mode, sa, createmode, attrs, templatefile)
-}
-
-func _createFile(name *uint16, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0)
-	handle = syscall.Handle(r0)
-	if handle == syscall.InvalidHandle {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procGetNamedPipeInfo.Addr(), 5, uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
-	r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func localAlloc(uFlags uint32, length uint32) (ptr uintptr) {
-	r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(uFlags), uintptr(length), 0)
-	ptr = uintptr(r0)
-	return
-}
-
-func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
-	r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
-	status = ntstatus(r0)
-	return
-}
-
-func rtlNtStatusToDosError(status ntstatus) (winerr error) {
-	r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
-	if r0 != 0 {
-		winerr = syscall.Errno(r0)
-	}
-	return
-}
-
-func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
-	r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
-	status = ntstatus(r0)
-	return
-}
-
-func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
-	r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
-	status = ntstatus(r0)
-	return
-}
-
-func lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(accountName)
-	if err != nil {
-		return
-	}
-	return _lookupAccountName(systemName, _p0, sid, sidSize, refDomain, refDomainSize, sidNameUse)
-}
-
-func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall9(procLookupAccountNameW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
@@ -281,11 +101,7 @@ func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidS
 func convertSidToStringSid(sid *byte, str **uint16) (err error) {
 	r1, _, e1 := syscall.Syscall(procConvertSidToStringSidW.Addr(), 2, uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(str)), 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
@@ -302,126 +118,73 @@ func convertStringSecurityDescriptorToSecurityDescriptor(str string, revision ui
 func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision uint32, sd *uintptr, size *uint32) (err error) {
 	r1, _, e1 := syscall.Syscall6(procConvertStringSecurityDescriptorToSecurityDescriptorW.Addr(), 4, uintptr(unsafe.Pointer(str)), uintptr(revision), uintptr(unsafe.Pointer(sd)), uintptr(unsafe.Pointer(size)), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procConvertSecurityDescriptorToStringSecurityDescriptorW.Addr(), 5, uintptr(unsafe.Pointer(sd)), uintptr(revision), uintptr(secInfo), uintptr(unsafe.Pointer(sddl)), uintptr(unsafe.Pointer(sddlSize)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func localFree(mem uintptr) {
-	syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0)
-	return
-}
-
 func getSecurityDescriptorLength(sd uintptr) (len uint32) {
 	r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
 	len = uint32(r0)
 	return
 }
 
-func getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
-	var _p0 uint32
-	if releaseAll {
-		_p0 = 1
-	} else {
-		_p0 = 0
-	}
-	r0, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize)))
-	success = r0 != 0
-	if true {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
 func impersonateSelf(level uint32) (err error) {
 	r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(level), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func revertToSelf() (err error) {
-	r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0)
+func lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(accountName)
+	if err != nil {
+		return
+	}
+	return _lookupAccountName(systemName, _p0, sid, sidSize, refDomain, refDomainSize, sidNameUse)
+}
+
+func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall9(procLookupAccountNameW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) {
-	var _p0 uint32
-	if openAsSelf {
-		_p0 = 1
-	} else {
-		_p0 = 0
+func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(systemName)
+	if err != nil {
+		return
 	}
-	r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0)
+	return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageId)
+}
+
+func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeDisplayNameW.Addr(), 5, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)), 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func getCurrentThread() (h syscall.Handle) {
-	r0, _, _ := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0)
-	h = syscall.Handle(r0)
+func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(systemName)
+	if err != nil {
+		return
+	}
+	return _lookupPrivilegeName(_p0, luid, buffer, size)
+}
+
+func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeNameW.Addr(), 4, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
 	return
 }
 
@@ -442,53 +205,27 @@ func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err err
 func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) {
 	r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid)))
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(systemName)
-	if err != nil {
-		return
+func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) {
+	var _p0 uint32
+	if openAsSelf {
+		_p0 = 1
 	}
-	return _lookupPrivilegeName(_p0, luid, buffer, size)
-}
-
-func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeNameW.Addr(), 4, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), 0, 0)
+	r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
 
-func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(systemName)
-	if err != nil {
-		return
-	}
-	return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageId)
-}
-
-func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procLookupPrivilegeDisplayNameW.Addr(), 5, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)), 0)
+func revertToSelf() (err error) {
+	r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
@@ -501,22 +238,14 @@ func backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, proce
 	var _p1 uint32
 	if abort {
 		_p1 = 1
-	} else {
-		_p1 = 0
 	}
 	var _p2 uint32
 	if processSecurity {
 		_p2 = 1
-	} else {
-		_p2 = 0
 	}
 	r1, _, e1 := syscall.Syscall9(procBackupRead.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }
@@ -529,22 +258,162 @@ func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, p
 	var _p1 uint32
 	if abort {
 		_p1 = 1
-	} else {
-		_p1 = 0
 	}
 	var _p2 uint32
 	if processSecurity {
 		_p2 = 1
-	} else {
-		_p2 = 0
 	}
 	r1, _, e1 := syscall.Syscall9(procBackupWrite.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0)
 	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall(procCancelIoEx.Addr(), 2, uintptr(file), uintptr(unsafe.Pointer(o)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall(procConnectNamedPipe.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(o)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _createFile(_p0, access, mode, sa, createmode, attrs, templatefile)
+}
+
+func _createFile(name *uint16, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0)
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0)
+	newport = syscall.Handle(r0)
+	if newport == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return
+	}
+	return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa)
+}
+
+func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
+	r0, _, e1 := syscall.Syscall9(procCreateNamedPipeW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)), 0)
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func getCurrentThread() (h syscall.Handle) {
+	r0, _, _ := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0)
+	h = syscall.Handle(r0)
+	return
+}
+
+func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
+	r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetNamedPipeInfo.Addr(), 5, uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func localAlloc(uFlags uint32, length uint32) (ptr uintptr) {
+	r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(uFlags), uintptr(length), 0)
+	ptr = uintptr(r0)
+	return
+}
+
+func localFree(mem uintptr) {
+	syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0)
+	return
+}
+
+func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(h), uintptr(flags), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
+	r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
+	status = ntstatus(r0)
+	return
+}
+
+func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
+	r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
+	status = ntstatus(r0)
+	return
+}
+
+func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
+	r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
+	status = ntstatus(r0)
+	return
+}
+
+func rtlNtStatusToDosError(status ntstatus) (winerr error) {
+	r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
+	if r0 != 0 {
+		winerr = syscall.Errno(r0)
+	}
+	return
+}
+
+func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) {
+	var _p0 uint32
+	if wait {
+		_p0 = 1
+	}
+	r1, _, e1 := syscall.Syscall6(procWSAGetOverlappedResult.Addr(), 5, uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)), 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
 	}
 	return
 }
@@ -552,11 +421,7 @@ func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, p
 func bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) {
 	r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
 	if r1 == socketError {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
+		err = errnoErr(e1)
 	}
 	return
 }

vendor/github.com/Microsoft/hcsshim/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

vendor/github.com/Microsoft/hcsshim/.gitignore

@@ -1 +1,3 @@
 *.exe
+.idea
+.vscode

vendor/github.com/Microsoft/hcsshim/.gometalinter.json

@@ -1,17 +0,0 @@
-{
-    "Vendor": true,
-    "Deadline": "2m",
-    "Sort": [
-        "linter",
-        "severity",
-        "path",
-        "line"
-    ],
-    "Skip": [
-        "internal\\schema2"
-    ],
-    "EnableGC": true,
-    "Enable": [
-        "gofmt"
-    ]
-}

vendor/github.com/Microsoft/hcsshim/CODEOWNERS

@@ -1,3 +1,3 @@
-* @microsoft/containerplat
-
+* @microsoft/containerplat
+
 /hcn/* @nagiesek