diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03fcb6bf48..76f745b94d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,16 @@ variables: DOCKER_REPOSITORY: mendersoftware/mender-convert + MULTIPLATFORM_BUILD: "true" + MULTIPLATFORM_PLATFORMS: "linux/amd64,linux/arm64" S3_BUCKET_NAME: mender-convert-images # These variables are present elsewhere in the repository too. Make sure to # search for and change them too. MENDER_ARTIFACT_VERSION: - value: "master" + value: "3.11.2" description: |- - Version of mender-artifact for the builder image - 'master' or a specific version + Version of mender-artifact for the builder image - a specific version MENDER_CLIENT_VERSION: value: "latest" description: |- @@ -71,13 +73,12 @@ variables: include: - project: 'Northern.tech/Mender/mendertesting' - file: '.gitlab-ci-check-commits.yml' - - project: 'Northern.tech/Mender/mendertesting' - file: '.gitlab-ci-check-license.yml' - - project: 'Northern.tech/Mender/mendertesting' - file: '.gitlab-ci-github-status-updates.yml' - - project: 'Northern.tech/Mender/mendertesting' - file: '.gitlab-ci-check-shell-format.yml' + file: + - '.gitlab-ci-check-commits.yml' + - '.gitlab-ci-check-license.yml' + - '.gitlab-ci-github-status-updates.yml' + - '.gitlab-ci-check-shell-format.yml' + - '.gitlab-ci-check-docker-build.yml' stages: - test @@ -106,25 +107,17 @@ test:check-shell-formatting: -not -path "scripts/linkbot/*" -not -wholename "scripts/test/*") -build: - stage: build - needs: [] - image: docker:git - services: - - docker:20.10.21-dind - script: - - IMAGE_NAME=$DOCKER_REPOSITORY:pr ./docker-build --build-arg MENDER_ARTIFACT_VERSION=${MENDER_ARTIFACT_VERSION} - - docker save $DOCKER_REPOSITORY:pr > image.tar - artifacts: - expire_in: 2w - paths: - - image.tar +build:docker-multiplatform: + variables: + DOCKER_BUILDKIT: 1 + GITLAB_REGISTRY_TAG: '${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID}' + EXTRA_DOCKER_ARGS: "--build-arg MENDER_ARTIFACT_VERSION=${MENDER_ARTIFACT_VERSION}" .template_convert_raspbian: &convert_raspbian stage: convert needs: - - job: build - artifacts: true + - job: build:docker-multiplatform + artifacts: false image: ubuntu:22.04 services: - docker:dind @@ -133,8 +126,8 @@ build: before_script: - apt update && apt install -yy bash wget xz-utils awscli docker.io curl - - export IMAGE_NAME=$DOCKER_REPOSITORY:pr - - docker load -i image.tar + - docker pull ${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID} + - export IMAGE_NAME=${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID} - mkdir -p input - cd input @@ -185,8 +178,8 @@ convert_raspbian_raspberrypi4: .template_test_acceptance: &test_acceptance stage: test_acceptance needs: - - job: build - artifacts: true + - job: build:docker-multiplatform + artifacts: false image: ubuntu:${ACC_TESTS_UBUNTU_DISTRO} services: - docker:dind @@ -204,8 +197,8 @@ convert_raspbian_raspberrypi4: - python3 -m pip install -U pip - pip3 install --ignore-installed -r tests/requirements_py3.txt # Load image under test - - export IMAGE_NAME=$DOCKER_REPOSITORY:pr - - docker load -i image.tar + - docker pull ${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID} + - export IMAGE_NAME=${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID} # Set mender-image-tests submodule to correct version - git submodule update --init --recursive # Get mender-artifact for the tests @@ -268,8 +261,8 @@ convert_raspbian_raspberrypi4: test_acceptance_prebuilt_raspberrypi3: <<: *test_acceptance_prebuilt_raspberrypi needs: - - job: build - artifacts: true + - job: build:docker-multiplatform + artifacts: false - job: convert_raspbian_raspberrypi3 artifacts: true variables: @@ -278,8 +271,8 @@ test_acceptance_prebuilt_raspberrypi3: test_acceptance_prebuilt_raspberrypi4: <<: *test_acceptance_prebuilt_raspberrypi needs: - - job: build - artifacts: true + - job: build:docker-multiplatform + artifacts: false - job: convert_raspbian_raspberrypi4 artifacts: true variables: @@ -370,27 +363,6 @@ test_acceptance_ubuntu_raspberrypi3: --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME} - done -.template:publish:docker-image: - stage: publish - tags: - - hetzner-amd-beefy - image: docker - services: - - name: docker:20.10.21-dind - alias: docker - before_script: - - docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD - - docker load -i image.tar - script: - - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}} - - - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$IMAGE_VERSION - - docker push $DOCKER_REPOSITORY:$IMAGE_VERSION - - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$IMAGE_VERSION)" >> publish.env - artifacts: - reports: - dotenv: publish.env - publish:s3:manual: when: manual extends: .template:publish:s3 @@ -400,14 +372,25 @@ publish:s3:automatic: - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"' extends: .template:publish:s3 -publish:docker-image:manual: - when: manual - extends: .template:publish:docker-image +# excludes non multiplatform build job +build:docker: + rules: + - when: never + +# excludes non multiplatform build job +publish:image: + rules: + - when: never -publish:docker-image:automatic: +# excludes non multiplatform build job +publish:image:mender: rules: - - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+|[0-9]+\.[0-9]+\.([0-9]+|x))$/' - extends: .template:publish:docker-image + - when: never + +# excludes non multiplatform build job +publish:image:saas: + rules: + - when: never #################### # Hardware testing # @@ -459,10 +442,9 @@ test:hardware:acceptance: variables: RASPBERRYPI_PLATFORM: raspberrypi4 stage: test_acceptance - needs: [] needs: - - job: build - artifacts: true + - job: build:docker-multiplatform + artifacts: false - job: test:hardware:convert artifacts: true - job: test:hardware:flash diff --git a/Dockerfile b/Dockerfile index 6f39b02cb7..a2380f43ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,21 @@ -# Build pxz in separate image to avoid big image size -FROM ubuntu:20.04 AS build -RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ +# Cross-compile pxz (Parallel LZMA compression) in separate image +FROM --platform=$BUILDPLATFORM debian:12 AS build +ARG TARGETARCH +RUN dpkg --add-architecture ${TARGETARCH} && \ + apt-get update && \ + env DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes \ build-essential \ + gcc-aarch64-linux-gnu \ git \ - liblzma-dev - -# Parallel xz (LZMA) compression + liblzma-dev:${TARGETARCH} RUN git clone https://github.com/jnovy/pxz.git /root/pxz -RUN cd /root/pxz && make - -FROM ubuntu:20.04 +WORKDIR /root/pxz +RUN if [ "$TARGETARCH" = "arm64" ]; then CC=aarch64-linux-gnu-gcc; else CC=cc; fi; env CC=$CC make -ARG MENDER_ARTIFACT_VERSION=master +FROM ubuntu:24.04 +ARG TARGETARCH +ARG MENDER_ARTIFACT_VERSION +RUN if [ "$MENDER_ARTIFACT_VERSION" = "" ]; then echo "MENDER_ARTIFACT_VERSION must be set!" 1>&2; exit 1; fi RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ # For 'ar' command to unpack .deb @@ -58,6 +62,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ # GRUB command line tools, primarily grub-probe grub-common \ # to be able to run package installations on foreign architectures + binfmt-support \ qemu-user-static COPY --from=build /root/pxz/pxz /usr/bin/pxz @@ -66,8 +71,9 @@ COPY --from=build /root/pxz/pxz /usr/bin/pxz RUN echo "Defaults secure_path=\"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:$PATH\"" > /etc/sudoers.d/secure_path_override RUN chmod 0440 /etc/sudoers.d/secure_path_override -RUN wget -q -O /usr/bin/mender-artifact https://downloads.mender.io/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact \ - && chmod +x /usr/bin/mender-artifact +RUN deb_filename=mender-artifact_${MENDER_ARTIFACT_VERSION}-1%2Bubuntu%2Bnoble_${TARGETARCH}.deb && \ + wget "https://downloads.mender.io/repos/debian/pool/main/m/mender-artifact/${deb_filename}" \ + --output-document=/mender-artifact.deb && apt install /mender-artifact.deb && rm /mender-artifact.deb WORKDIR / diff --git a/docker-build b/docker-build index fd7cf116f4..33d1080046 100755 --- a/docker-build +++ b/docker-build @@ -17,7 +17,7 @@ set -e GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \ - git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)" + git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null | head -n1)" if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2 @@ -25,4 +25,11 @@ fi IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"} -eval docker build . -t "${IMAGE_NAME}" "$*" +MENDER_ARTIFACT_VERSION=$(sed -n '/MENDER_ARTIFACT_VERSION:/{n;s/.*value: "\(.*\)".*/\1/p;}' .gitlab-ci.yml) +if [ -z "$MENDER_ARTIFACT_VERSION" ]; then + echo "Could not parse MENDER_ARTIFACT_VERSION from .gitlab-ci.yml" 1>&2 + exit 1 +fi + +echo "Building ${IMAGE_NAME} for local development" 1>&2 +docker build -t "${IMAGE_NAME}" --build-arg MENDER_ARTIFACT_VERSION="${MENDER_ARTIFACT_VERSION}" .