From f55de747b13b6f8fc706b7ffcdda1a729e5b2c92 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Aug 2024 16:46:36 +0200 Subject: [PATCH] [CI] Add steps in pipeline to run unit tests with MacOS ARM (#1207) Add new steps in the CI pipeline to run unit tests in MacOS ARM. Updated environment variables used for defining the agent images used in the Buildkite steps to use consitent namings. --- .buildkite/pipeline.yml | 67 +++++++++++++++-------- .buildkite/scripts/pre-install-command.sh | 50 +++++++++++++++-- .buildkite/scripts/run-tests.sh | 12 ++-- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f716c34fb..36462b9af 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,10 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - SETUP_MAGE_VERSION: '1.14.0' + SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + SETUP_MAGE_VERSION: '1.15.0' DOCKER_REGISTRY: 'docker.elastic.co' DOCKER_IMG: "${DOCKER_REGISTRY}/package-registry/package-registry" DOCKER_IMG_PR: "${DOCKER_REGISTRY}/observability-ci/package-registry" + # Agent images used in pipeline steps + LINUX_GOLANG_AGENT_IMAGE: "golang:${SETUP_GOLANG_VERSION}" + WINDOWS_AGENT_IMAGE: "family/core-windows-2022" + MACOS_ARM_AGENT_IMAGE: "generic-13-ventura-arm" steps: - label: ":golangci-lint: Checks formatting / linting" @@ -12,7 +17,7 @@ steps: command: - ".buildkite/scripts/lint.sh" agents: - image: "golang:${SETUP_GOLANG_VERSION}" + image: "${LINUX_GOLANG_AGENT_IMAGE}" cpu: "8" memory: "4G" @@ -21,30 +26,43 @@ steps: command: - ".buildkite/scripts/build.sh" agents: - image: "golang:${SETUP_GOLANG_VERSION}" + image: "${LINUX_GOLANG_AGENT_IMAGE}" cpu: "8" memory: "4G" - - label: ":linux: Test on Linux" - key: test-linux - command: - - ".buildkite/scripts/run-tests.sh" - agents: - image: "golang:${SETUP_GOLANG_VERSION}" - cpu: "8" - memory: "4G" - artifact_paths: - - "tests-report-linux.xml" + - group: ":go: Run Unit tests" + key: unit-tests + steps: + - label: ":linux: Test on Linux" + key: test-linux + command: + - ".buildkite/scripts/run-tests.sh" + agents: + image: "${LINUX_GOLANG_AGENT_IMAGE}" + cpu: "8" + memory: "4G" + artifact_paths: + - "tests-report-linux.xml" - - label: ":windows: Test on Windows" - key: test-win - command: - - ".buildkite/scripts/run-tests.ps1" - agents: - provider: "gcp" - image: "family/core-windows-2022" - artifact_paths: - - "tests-report-win.xml" + - label: ":windows: Test on Windows" + key: test-win + command: + - ".buildkite/scripts/run-tests.ps1" + agents: + provider: "gcp" + image: "${WINDOWS_AGENT_IMAGE}" + artifact_paths: + - "tests-report-win.xml" + + - label: ":macos: Test on Macos ARM" + key: test-macos-arm + command: + - ".buildkite/scripts/run-tests.sh" + agents: + provider: "orka" + imagePrefix: "${MACOS_ARM_AGENT_IMAGE}" + artifact_paths: + - "tests-report-darwin.xml" - label: ":junit: Junit annotate" plugins: @@ -52,6 +70,7 @@ steps: artifacts: "tests-report-*.xml" fail-build-on-error: true report-skipped: true + always-annotate: true agents: provider: "gcp" #junit plugin requires docker depends_on: @@ -59,6 +78,8 @@ steps: allow_failure: true - step: "test-win" allow_failure: true + - step: "test-macos-arm" + allow_failure: true - label: "Publish docker image" key: "publish" @@ -70,6 +91,8 @@ steps: allow_failure: false - step: "test-win" allow_failure: false + - step: "test-macos-arm" + allow_failure: false - step: "build" allow_failure: false - step: "lint" diff --git a/.buildkite/scripts/pre-install-command.sh b/.buildkite/scripts/pre-install-command.sh index 4b767aa0a..a3dc3e8f3 100755 --- a/.buildkite/scripts/pre-install-command.sh +++ b/.buildkite/scripts/pre-install-command.sh @@ -3,6 +3,27 @@ source .buildkite/scripts/tooling.sh set -euo pipefail +platform_type="$(uname)" +hw_type="$(uname -m)" +platform_type_lowercase="$(echo "${platform_type}" | tr '[:upper:]' '[:lower:]')" + +check_platform_architecture() { + case "${hw_type}" in + "x86_64") + arch_type="amd64" + ;; + "aarch64") + arch_type="arm64" + ;; + "arm64") + arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + create_bin_folder() { mkdir -p "${WORKSPACE}/bin" } @@ -12,16 +33,37 @@ add_bin_path(){ export PATH="${WORKSPACE}/bin:${PATH}" } -with_mage() { +with_go() { create_bin_folder - retry 5 curl -sL -o "${WORKSPACE}/bin/mage.tar.gz" "https://github.com/magefile/mage/releases/download/v${SETUP_MAGE_VERSION}/mage_${SETUP_MAGE_VERSION}_Linux-64bit.tar.gz" + check_platform_architecture + + echo "--- Install Golang" + echo "GVM ${SETUP_GVM_VERSION} (platform ${platform_type_lowercase} arch ${arch_type}" + retry 5 curl -sL -o "${WORKSPACE}/bin/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}" + + chmod +x "${WORKSPACE}/bin/gvm" + eval "$(gvm "$(cat .go-version)")" + go version + which go + PATH="${PATH}:$(go env GOPATH)/bin" + export PATH +} + +with_mage() { + check_platform_architecture + + if [[ "${platform_type_lowercase}" == "darwin" ]]; then + # MacOS ARM VM images do not have golang installed by default + with_go + fi - tar -xvf "${WORKSPACE}/bin/mage.tar.gz" -C "${WORKSPACE}/bin" - chmod +x "${WORKSPACE}/bin/mage" + echo "--- Install mage" + go install "github.com/magefile/mage@v${SETUP_MAGE_VERSION}" mage --version } with_go_junit_report() { + echo "--- Install go-junit-report" go install github.com/jstemmer/go-junit-report/v2@latest } diff --git a/.buildkite/scripts/run-tests.sh b/.buildkite/scripts/run-tests.sh index 2a9de355b..5fe33e83d 100755 --- a/.buildkite/scripts/run-tests.sh +++ b/.buildkite/scripts/run-tests.sh @@ -7,15 +7,19 @@ add_bin_path with_mage with_go_junit_report +platform_type="$(uname | tr '[:upper:]' '[:lower:]')" +tests_report_txt_file="tests-report-${platform_type}.txt" +tests_report_xml_file="tests-report-${platform_type}.xml" +echo "--- Run Unit tests" set +e -mage -debug test > tests-report-linux.txt +mage -debug test > "${tests_report_txt_file}" exit_code=$? set -e # Buildkite collapse logs under --- symbols # need to change --- to anything else or switch off collapsing (note: not available at the moment of this commit) -awk '{gsub("---", "----"); print }' tests-report-linux.txt +awk '{gsub("---", "----"); print }' "${tests_report_txt_file}" -# Create Junit report for junit annotation plugin -go-junit-report > tests-report-linux.xml < tests-report-linux.txt +echo "Create Junit report for junit annotation plugin ${tests_report_xml_file}" +go-junit-report > "${tests_report_xml_file}" < "${tests_report_txt_file}" exit $exit_code