Merge pull request #1851 from crazy-max/test-flow

test: update workflow

.github/CONTRIBUTING.md

@@ -116,6 +116,43 @@ commit automatically with `git commit -s`.
 
 ### Run the unit- and integration-tests
 
+Running tests:
+
+```bash
+make test
+```
+
+This runs all unit and integration tests, in a containerized environment.
+Locally, every package can be tested separately with standard Go tools, but
+integration tests are skipped if local user doesn't have enough permissions or
+worker binaries are not installed.
+
+```bash
+# run unit tests only
+make test-unit
+
+# run integration tests only
+make test-integration
+
+# test a specific package
+TESTPKGS=./bake make test
+
+# run all integration tests with a specific worker
+TESTFLAGS="--run=//worker=docker-container -v" make test
+```
+
+> **Note**
+>
+> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant
+> images in a docker volume if you are repeatedly calling `make test`. This
+> helps to avoid rate limiting on the remote registry side.
+
+> **Note**
+>
+> If you are working behind a proxy, you can set some of or all
+> `HTTP_PROXY=http://ip:port`, `HTTPS_PROXY=http://ip:port`, `NO_PROXY=http://ip:port`
+> for the test framework to specify the proxy build args.
+
 To enter a demo container environment and experiment, you may run:
 
 ```

.github/workflows/build.yml

@@ -23,14 +23,18 @@ env:
   BUILDKIT_IMAGE: "moby/buildkit:latest"
   REPO_SLUG: "docker/buildx-bin"
   DESTDIR: "./bin"
+  TEST_CACHE_SCOPE: "test"
 
 jobs:
-  test:
+  prepare-test:
     runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -39,20 +43,89 @@ jobs:
           driver-opts: image=${{ env.BUILDKIT_IMAGE }}
           buildkitd-flags: --debug
       -
-        name: Test
+        name: Build
         uses: docker/bake-action@v3
         with:
-          targets: test
+          targets: integration-test-base
           set: |
-            *.cache-from=type=gha,scope=test
+            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
-            *.cache-to=type=gha,scope=test
+            *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
+
+  test:
+    runs-on: ubuntu-22.04
+    needs:
+      - prepare-test
+    env:
+      TESTFLAGS: "-v --parallel=6 --timeout=30m"
+      TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
+      GOTESTSUM_FORMAT: "standard-verbose"
+      TEST_IMAGE_BUILD: "0"
+      TEST_IMAGE_ID: "buildx-tests"
+    strategy:
+      fail-fast: false
+      matrix:
+        worker:
+          - docker
+          - docker-container
+          - remote
+        pkg:
+          - ./tests
+        include:
+          - pkg: ./...
+            skip-integration-tests: 1
+    steps:
       -
-        name: Upload coverage
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: ${{ env.BUILDX_VERSION }}
+          driver-opts: image=${{ env.BUILDKIT_IMAGE }}
+          buildkitd-flags: --debug
+      -
+        name: Build test image
+        uses: docker/bake-action@v3
+        with:
+          targets: integration-test
+          set: |
+            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
+            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
+      -
+        name: Test
+        run: |
+          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
+          ./hack/test
+        env:
+          TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
+          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
+          TESTPKGS: "${{ matrix.pkg }}"
+          SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
+      -
+        name: Send to Codecov
+        if: always()
         uses: codecov/codecov-action@v3
         with:
-          directory: ${{ env.DESTDIR }}/coverage
+          directory: ./bin/testreports
+      -
+        name: Generate annotations
+        if: always()
+        uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9
+        with:
+          directory: ./bin/testreports
+      -
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-reports
+          path: ./bin/testreports
 
-  prepare:
+  prepare-binaries:
     runs-on: ubuntu-22.04
     outputs:
       matrix: ${{ steps.platforms.outputs.matrix }}
@@ -73,11 +146,11 @@ jobs:
   binaries:
     runs-on: ubuntu-22.04
     needs:
-      - prepare
+      - prepare-binaries
     strategy:
       fail-fast: false
       matrix:
-        platform: ${{ fromJson(needs.prepare.outputs.matrix) }}
+        platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }}
     steps:
       -
         name: Prepare
@@ -115,6 +188,8 @@ jobs:
 
   bin-image:
     runs-on: ubuntu-22.04
+    needs:
+      - test
     if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }}
     steps:
       -
@@ -166,6 +241,7 @@ jobs:
   release:
     runs-on: ubuntu-22.04
     needs:
+      - test
       - binaries
     steps:
       -

.github/workflows/test.yml

@@ -1,81 +0,0 @@
-name: test
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'master'
-      - 'v[0-9]*'
-    tags:
-      - 'v*'
-  pull_request:
-    paths-ignore:
-      - '.github/releases.json'
-      - 'README.md'
-      - 'docs/**'
-
-env:
-  SETUP_BUILDX_VERSION: "latest"
-  SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest"
-  TESTFLAGS: "-v --parallel=6 --timeout=30m"
-  TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
-  GOTESTSUM_FORMAT: "standard-verbose"
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        worker:
-          - docker
-          - docker-container
-          - remote
-        pkg:
-          - ./tests
-        typ:
-          - integration
-        # already run as part of build.yml
-        # include:
-        #   - pkg: ./...
-        #     skip-integration-tests: 1
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Expose GitHub Runtime
-        uses: crazy-max/ghaction-github-runtime@v2
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: ${{ env.SETUP_BUILDX_VERSION }}
-          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
-          buildkitd-flags: --debug
-      -
-        name: Test
-        run: |
-          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
-          ./hack/test ${{ matrix.typ }}
-        env:
-          TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
-          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
-          TESTPKGS: "${{ matrix.pkg }}"
-          SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
-      -
-        name: Generate annotations
-        if: always()
-        uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9
-        with:
-          directory: ./bin/testreports
-      -
-        name: Upload test reports
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-reports
-          path: ./bin/testreports

Makefile

@@ -45,7 +45,15 @@ lint:
 
 .PHONY: test
 test:
-	$(BUILDX_CMD) bake test
+	./hack/test
+
+.PHONY: test-unit
+test-unit:
+	TESTPKGS=./... SKIP_INTEGRATION_TESTS=1 ./hack/test
+
+.PHONY: test
+test-integration:
+	TESTPKGS=./tests ./hack/test
 
 .PHONY: validate-vendor
 validate-vendor:

hack/test

@@ -3,41 +3,18 @@
 set -eu -o pipefail
 
 : "${BUILDX_CMD=docker buildx}"
-: "${CACHE_FROM=}"
-: "${CACHE_TO=}"
 
-: ${TEST_INTEGRATION=}
+: "${TEST_IMAGE_BUILD=1}"
-: ${TEST_REPORT_SUFFIX=}
+: "${TEST_IMAGE_ID=buildx-tests}"
-: ${TEST_KEEP_CACHE=}
-: ${TEST_DOCKERD=}
-: ${TEST_BUILDKIT_IMAGE=}
 
-if [ -n "$CACHE_FROM" ]; then
+: "${TEST_REPORT_SUFFIX=}"
-  for cfrom in $CACHE_FROM; do
+: "${TEST_KEEP_CACHE=}"
-    setFlags+=(--set "*.cache-from=$cfrom")
+: "${TEST_DOCKERD=}"
-  done
+: "${TEST_BUILDKIT_IMAGE=}"
+
+if [ "$TEST_IMAGE_BUILD" = "1" ]; then
+  ${BUILDX_CMD} bake integration-test --set "*.output=type=docker,name=$TEST_IMAGE_ID"
 fi
-if [ -n "$CACHE_TO" ]; then
-  for cto in $CACHE_TO; do
-    setFlags+=(--set "*.cache-to=$cto")
-  done
-fi
-
-if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi
-
-while test $# -gt 0; do
-  case "$1" in
-    integration)
-      TEST_INTEGRATION=1
-      ;;
-    *)
-      echo "unknown arg $1"
-      ;;
-  esac
-  shift
-done
-
-iid="buildx-tests"
 
 testReportsDir="$(pwd)/bin/testreports"
 mkdir -p "$testReportsDir"
@@ -45,18 +22,13 @@ testReportsVol="-v $testReportsDir:/testreports"
 gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
 gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
 
-${BUILDX_CMD} bake integration-test "${setFlags[@]}" --set "*.output=type=docker,name=$iid"
-
 cacheVolume="buildx-test-cache"
 if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
   docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine
 fi
-
-if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
-  docker start -a -i $cid
-fi
-
 if [ "$TEST_KEEP_CACHE" != "1" ]; then
-  docker rm -v $cacheVolume
+  trap 'docker rm -v $cacheVolume' EXIT
 fi
+
+cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+docker start -a -i $cid