tests: run remote worker on tcp and container endpoints

.github/workflows/build.yml

@@ -30,54 +30,53 @@ env:
   GOTESTSUM_VERSION: "v1.9.0"  # same as one in Dockerfile
 
 jobs:
-  prepare-test-integration:
-    runs-on: ubuntu-22.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=${{ env.BUILDKIT_IMAGE }}
-          buildkitd-flags: --debug
-      -
-        name: Build
-        uses: docker/bake-action@v4
-        with:
-          targets: integration-test-base
-          set: |
-            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
-            *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
-
   test-integration:
     runs-on: ubuntu-22.04
-    needs:
-      - prepare-test-integration
     env:
       TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
       TEST_IMAGE_BUILD: "0"
       TEST_IMAGE_ID: "buildx-tests"
     strategy:
       fail-fast: false
       matrix:
+        buildkit:
+          - master
+          - latest
+          - buildx-stable-1
+          - v0.13.0
+          - v0.12.5
+          - v0.11.6
         worker:
-          - docker
-          - docker\+containerd  # same as docker, but with containerd snapshotter
           - docker-container
-          - remote
+          - remote+docker-container
+          - remote+tcp
         pkg:
           - ./tests
+        include:
+          - worker: docker
+            pkg: ./tests
+          - worker: docker+containerd  # same as docker, but with containerd snapshotter
+            pkg: ./tests
     steps:
       -
         name: Prepare
         run: |
-          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.buildkit }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+          if [ -n "${{ matrix.buildkit }}" ]; then
+            echo "TEST_BUILDKIT_TAG=${{ matrix.buildkit }}" >> $GITHUB_ENV
+          fi
+          testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
+          case "${{ matrix.worker }}" in
+            docker | docker+containerd | docker@* | docker+containerd@* | remote*)
+              echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
+              ;;
+            *)
+              echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
+              ;;
+          esac
+          if [[ "${{ matrix.worker }}" == "docker"* ]]; then
+            echo "TEST_DOCKERD=1" >> $GITHUB_ENV
+          fi
       -
         name: Checkout
         uses: actions/checkout@v4
@@ -99,16 +98,13 @@ jobs:
         with:
           targets: integration-test
           set: |
-            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
             *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
       -
         name: Test
         run: |
           ./hack/test
         env:
           TEST_REPORT_SUFFIX: "-${{ env.TESTREPORTS_NAME }}"
-          TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}"
-          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"
       -
         name: Send to Codecov
@@ -355,27 +351,3 @@ jobs:
         with:
           draft: true
           files: ${{ env.DESTDIR }}/*
-
-  buildkit-edge:
-    runs-on: ubuntu-22.04
-    continue-on-error: true
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=moby/buildkit:master
-          buildkitd-flags: --debug
-      -
-        # Just run a bake target to check eveything runs fine
-        name: Build
-        uses: docker/bake-action@v4
-        with:
-          targets: binaries

Dockerfile

@@ -7,14 +7,14 @@ ARG XX_VERSION=1.4.0
 ARG DOCKER_VERSION=25.0.2
 ARG GOTESTSUM_VERSION=v1.9.0
 ARG REGISTRY_VERSION=2.8.0
-ARG BUILDKIT_VERSION=v0.12.5
+ARG UNDOCK_VERSION=0.7.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest
 FROM moby/moby-bin:$DOCKER_VERSION AS docker-engine
 FROM dockereng/cli-bin:$DOCKER_VERSION AS docker-cli
 FROM registry:$REGISTRY_VERSION AS registry
-FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
+FROM crazymax/undock:$UNDOCK_VERSION AS undock
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -90,8 +90,7 @@ COPY --link --from=gotestsum /out/gotestsum /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
 COPY --link --from=docker-cli / /usr/bin/
-COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
-COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
+COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
 
 FROM integration-test-base AS integration-test

hack/test

@@ -10,6 +10,7 @@ set -eu -o pipefail
 : "${TEST_REPORT_SUFFIX=}"
 : "${TEST_KEEP_CACHE=}"
 : "${TEST_DOCKERD=}"
+: "${TEST_BUILDKIT_TAG=}"
 : "${TEST_BUILDKIT_IMAGE=}"
 
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
@@ -24,11 +25,11 @@ gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_S
 
 cacheVolume="buildx-test-cache"
 if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
-  docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine
+  docker create -v /root/.cache -v /root/.cache/registry -v /root/.cache/undock -v /go/pkg/mod --name "$cacheVolume" alpine
 fi
 if [ "$TEST_KEEP_CACHE" != "1" ]; then
   trap 'docker rm -v $cacheVolume' EXIT
 fi
 
-cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e TEST_BUILDKIT_TAG -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
 docker start -a -i $cid

tests/bake.go

@@ -5,7 +5,6 @@
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/containerd/continuity/fs/fstest"
@@ -139,7 +138,7 @@
 	addr := gitutil.GitServeHTTP(git, t)
 
 	out, err := bakeCmd(sb, withDir(dir), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest))
 	require.NoError(t, err, out)
 
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }
@@ -181,7 +180,7 @@
 	addr := gitutil.GitServeHTTP(git, t)
 
 	out, err := bakeCmd(sb, withDir(dirSrc), withArgs(addr, "--file", "cwd://local-docker-bake.hcl", "--set", "*.output=type=local,dest="+dirDest))
 	require.NoError(t, err, out)
 
 	require.FileExists(t, filepath.Join(dirDest, "bar"))
 }
@@ -250,7 +249,7 @@
 	addr := gitutil.GitServeHTTP(git, t)
 
 	out, err := bakeCmd(sb, withDir(dirSrc), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest))
 	require.NoError(t, err, out)
 
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }
@@ -290,7 +289,7 @@
 	addrSrc := gitutil.GitServeHTTP(gitSrc, t)
 
 	out, err := bakeCmd(sb, withDir("/tmp"), withArgs(addrSpec, addrSrc, "--set", "*.output=type=local,dest="+dirDest))
 	require.NoError(t, err, out)
 
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }
@@ -324,7 +323,7 @@
 	addr := gitutil.GitServeHTTP(git, t)
 
 	out, err := bakeCmd(sb, withDir("/tmp"), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest))
 	require.NoError(t, err, out)
 
 	require.FileExists(t, filepath.Join(dirDest, "super-cool.txt"))
 }
@@ -370,7 +369,7 @@
 		withArgs(addr, "--set", "*.output=type=local,dest="+dirDest),
 	)
 	require.Error(t, err, out)
 	require.Contains(t, out, "outside of the working directory, please set BAKE_ALLOW_REMOTE_FS_ACCESS")
 
 	out, err = bakeCmd(
 		sb,
@@ -641,9 +640,10 @@
 }
 
 func testBakeMultiExporters(t *testing.T, sb integration.Sandbox) {
-	if sb.Name() != "docker" {
-		t.Skip("skipping test for non-docker workers")
+	if !isDockerContainerWorker(sb) {
+		t.Skip("only testing with docker-container worker")
 	}
+	skipNoCompatBuildKit(t, sb, ">= 0.13.0-0", "multi exporters")
 
 	registry, err := sb.NewRegistry()
 	if errors.Is(err, integration.ErrRequirements) {
@@ -654,30 +654,12 @@
 	targetReg := registry + "/buildx/registry:latest"
 	targetStore := "buildx:local-" + identity.NewID()
 
-	var builderName string
 	t.Cleanup(func() {
-		if builderName == "" {
-			return
-		}
-
 		cmd := dockerCmd(sb, withArgs("image", "rm", targetStore))
 		cmd.Stderr = os.Stderr
 		require.NoError(t, cmd.Run())
-
-		out, err := rmCmd(sb, withArgs(builderName))
-		require.NoError(t, err, out)
 	})
 
-	// TODO: use stable buildkit image when v0.13.0 released
-	out, err := createCmd(sb, withArgs(
-		"--driver", "docker-container",
-		"--buildkitd-flags=--allow-insecure-entitlement=network.host",
-		"--driver-opt", "network=host",
-		"--driver-opt", "image=moby/buildkit:v0.13.0-rc3",
-	))
-	require.NoError(t, err, out)
-	builderName = strings.TrimSpace(out)
-
 	dockerfile := []byte(`
 FROM scratch
 COPY foo /foo
@@ -699,7 +681,6 @@
 		"--set", fmt.Sprintf("*.output=type=oci,dest=%s/result", dir),
 	}
 	cmd := buildxCmd(sb, withDir(dir), withArgs("bake"), withArgs(outputs...))
-	cmd.Env = append(cmd.Env, "BUILDX_BUILDER="+builderName)
 	outb, err := cmd.CombinedOutput()
 	require.NoError(t, err, string(outb))
 
@@ -722,9 +703,10 @@
 }
 
 func testBakeLoadPush(t *testing.T, sb integration.Sandbox) {
-	if sb.Name() != "docker" {
-		t.Skip("skipping test for non-docker workers")
+	if !isDockerContainerWorker(sb) {
+		t.Skip("only testing with docker-container worker")
 	}
+	skipNoCompatBuildKit(t, sb, ">= 0.13.0-0", "multi exporters")
 
 	registry, err := sb.NewRegistry()
 	if errors.Is(err, integration.ErrRequirements) {
@@ -734,30 +716,12 @@
 
 	target := registry + "/buildx/registry:" + identity.NewID()
 
-	var builderName string
 	t.Cleanup(func() {
-		if builderName == "" {
-			return
-		}
-
 		cmd := dockerCmd(sb, withArgs("image", "rm", target))
 		cmd.Stderr = os.Stderr
 		require.NoError(t, cmd.Run())
-
-		out, err := rmCmd(sb, withArgs(builderName))
-		require.NoError(t, err, out)
 	})
 
-	// TODO: use stable buildkit image when v0.13.0 released
-	out, err := createCmd(sb, withArgs(
-		"--driver", "docker-container",
-		"--buildkitd-flags=--allow-insecure-entitlement=network.host",
-		"--driver-opt", "network=host",
-		"--driver-opt", "image=moby/buildkit:v0.13.0-rc3",
-	))
-	require.NoError(t, err, out)
-	builderName = strings.TrimSpace(out)
-
 	dockerfile := []byte(`
 FROM scratch
 COPY foo /foo
@@ -774,7 +738,6 @@
 	)
 
 	cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--push", "--load", fmt.Sprintf("--set=*.tags=%s", target)))
-	cmd.Env = append(cmd.Env, "BUILDX_BUILDER="+builderName)
 	outb, err := cmd.CombinedOutput()
 	require.NoError(t, err, string(outb))