From 59f87c881f3361dd18f08b9a59bec21bda9cd6d8 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:59:41 +0100 Subject: [PATCH] tests: run remote worker on tcp and container endpoints Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/build.yml | 19 ++++++-- Dockerfile | 4 -- docker-bake.hcl | 4 -- tests/workers/docker.go | 3 +- tests/workers/remote.go | 90 ++++++++++++++++++++++++++++++------- 5 files changed, 90 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81c430516a62..f08824dfebdc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,13 +48,14 @@ jobs: - v0.11.6 worker: - docker-container - - remote + - remote+docker-container + - remote+tcp pkg: - ./tests include: - worker: docker pkg: ./tests - - worker: docker\+containerd # same as docker, but with containerd snapshotter + - worker: docker+containerd # same as docker, but with containerd snapshotter pkg: ./tests steps: - @@ -64,6 +65,18 @@ jobs: if [ -n "${{ matrix.buildkit }}" ]; then echo "TEST_BUILDKIT_TAG=${{ matrix.buildkit }}" >> $GITHUB_ENV fi + testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" + case "${{ matrix.worker }}" in + docker | docker+containerd | docker@* | docker+containerd@* | remote*) + echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV + ;; + *) + echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV + ;; + esac + if [[ "${{ matrix.worker }}" == "docker"* ]]; then + echo "TEST_DOCKERD=1" >> $GITHUB_ENV + fi - name: Checkout uses: actions/checkout@v4 @@ -92,8 +105,6 @@ jobs: ./hack/test env: TEST_REPORT_SUFFIX: "-${{ env.TESTREPORTS_NAME }}" - TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}" - TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" TESTPKGS: "${{ matrix.pkg }}" - name: Send to Codecov diff --git a/Dockerfile b/Dockerfile index 3cdb1e21bf64..d6c13962f9f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ ARG XX_VERSION=1.4.0 ARG DOCKER_VERSION=25.0.2 ARG GOTESTSUM_VERSION=v1.9.0 ARG REGISTRY_VERSION=2.8.0 -ARG BUILDKIT_VERSION=v0.13.0 ARG UNDOCK_VERSION=0.7.0 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx @@ -15,7 +14,6 @@ FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest FROM moby/moby-bin:$DOCKER_VERSION AS docker-engine FROM dockereng/cli-bin:$DOCKER_VERSION AS docker-cli FROM registry:$REGISTRY_VERSION AS registry -FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit FROM crazymax/undock:$UNDOCK_VERSION AS undock FROM golatest AS gobase @@ -92,8 +90,6 @@ COPY --link --from=gotestsum /out/gotestsum /usr/bin/ COPY --link --from=registry /bin/registry /usr/bin/ COPY --link --from=docker-engine / /usr/bin/ COPY --link --from=docker-cli / /usr/bin/ -COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/ -COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/ COPY --link --from=undock /usr/local/bin/undock /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ diff --git a/docker-bake.hcl b/docker-bake.hcl index 084fdd6450c3..435d8df70ab4 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -180,9 +180,6 @@ variable "HTTPS_PROXY" { variable "NO_PROXY" { default = "" } -variable "TEST_BUILDKIT_TAG" { - default = null -} target "integration-test-base" { inherits = ["_common"] @@ -190,7 +187,6 @@ target "integration-test-base" { HTTP_PROXY = HTTP_PROXY HTTPS_PROXY = HTTPS_PROXY NO_PROXY = NO_PROXY - BUILDKIT_VERSION = TEST_BUILDKIT_TAG } target = "integration-test-base" output = ["type=cacheonly"] diff --git a/tests/workers/docker.go b/tests/workers/docker.go index cd6d6084b8af..37c3a4d8c017 100644 --- a/tests/workers/docker.go +++ b/tests/workers/docker.go @@ -24,7 +24,8 @@ func InitDockerWorker() { type dockerWorker struct { id string containerdSnapshotter bool - unsupported []string + + unsupported []string } func (c dockerWorker) Name() string { diff --git a/tests/workers/remote.go b/tests/workers/remote.go index 4ded704f4f3f..73c0af685ccb 100644 --- a/tests/workers/remote.go +++ b/tests/workers/remote.go @@ -2,24 +2,41 @@ package workers import ( "context" + "fmt" + "net" "os" "os/exec" + "sync" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/testutil/integration" - bkworkers "github.com/moby/buildkit/util/testutil/workers" "github.com/pkg/errors" ) +var protos = []string{ + "docker-container", + "tcp", +} + func InitRemoteWorker() { - integration.Register(&remoteWorker{ - id: "remote", - }) + for _, p := range protos { + integration.Register(&remoteWorker{ + id: "remote+" + p, + proto: p, + }) + } } type remoteWorker struct { - id string + id string + proto string + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once } func (w remoteWorker) Name() string { @@ -35,32 +52,71 @@ func (w remoteWorker) NetNSDetached() bool { } func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (b integration.Backend, cl func() error, err error) { - oci := bkworkers.OCI{ID: w.id} - bk, bkclose, err := oci.New(ctx, cfg) - if err != nil { - return bk, cl, err + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr } + bkCtnName := "buildkit-integration-" + identity.NewID() name := "integration-remote-" + identity.NewID() + envs := append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + ) + + // random host port for buildkit container + l, _ := net.Listen("tcp", ":0") //nolint:gosec + _ = l.Close() + bkPort := l.Addr().(*net.TCPAddr).Port + + // create buildkit container + bkCtnCmd := exec.Command("docker", "run", + "-d", "--rm", + "--privileged", + "-p", fmt.Sprintf("%d:1234", bkPort), + "--name="+bkCtnName, + "moby/buildkit:buildx-stable-1", + "--addr=tcp://0.0.0.0:1234", + ) + bkCtnCmd.Env = envs + if err := bkCtnCmd.Run(); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create buildkit container %s", bkCtnName) + } + + // create builder + var endpoint string + switch w.proto { + case "docker-container": + endpoint = fmt.Sprintf("docker-container://%s", bkCtnName) + case "tcp": + endpoint = fmt.Sprintf("tcp://localhost:%d", bkPort) + default: + return nil, nil, errors.Errorf("unsupported protocol %s", w.proto) + } cmd := exec.Command("buildx", "create", "--bootstrap", "--name="+name, "--driver=remote", - bk.Address(), + endpoint, ) - cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name) + cmd.Env = envs if err := cmd.Run(); err != nil { return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name) } cl = func() error { - var err error - if err1 := bkclose(); err == nil { - err = err1 + cmd := exec.Command("docker", "container", "rm", "-f", name) + cmd.Env = envs + if err1 := cmd.Run(); err1 != nil { + err = errors.Wrapf(err1, "failed to remove buildkit container %s", bkCtnName) } - cmd := exec.Command("buildx", "rm", "-f", name) - if err1 := cmd.Run(); err == nil { - err = err1 + cmd = exec.Command("buildx", "rm", "-f", name) + cmd.Env = envs + if err1 := cmd.Run(); err1 != nil { + err = errors.Wrapf(err1, "failed to remove buildx instance %s", name) } return err }