From 119c3fc99203cf6fb63dda7c3bb307c8841f179a Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Wed, 4 Oct 2023 11:09:29 +0300 Subject: [PATCH] AVRO-3653: [CI] Linux ARM64 on GitHub actions (#1964) * AVRO-3653: [CI] Use Github Actions for testing on Linux ARM64 Migrate the Java build from .travis.yaml Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: [CI] Add jobs for all SDKs Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: [CI] Override env.GITHUB_WORKFLOW This is a workaround for https://github.com/uraimo/run-on-arch-action/pull/100 Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Add ASLv2 for the new workflow file Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Git push should trigger only if it is on master branch Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Can't use ubuntu-20.04-xl Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3563: Rust - build only, no tests Python: change the default wait time. Just for debugging purposes Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Cast the env vars values from string to int Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Build only apache-avro Rust crate Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Format the Python code Fixes the CI for Python Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Install .NET 7.0 Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Install libyaml-dev for the Ruby job Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Run the Rust SDK tests Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Use tox instead of tox-wheel Port the improvement from AVRO-3696 Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Update Maven to 3.9.1 Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Python Tox 4.x uses TOX_OVERRIDE https://github.com/tox-dev/tox/discussions/2681 Tox 4.x does not support Python 3.6 Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Try to debug the issue with Snappy code in Java Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Allow passing BENCH_MAX_WRITE_SECONDS and BENCH_MAX_READ_SECONDS to tox Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Exclude Snappy codec tests for ARM64 Install python3-distutils for the Python SDK Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: $(MSBuildThisFileDirectory) already has trailing '/' Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Use Ubuntu 22.04 for the ARM64 VMs Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Try with python3.7-distutils Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Exclude TestNettyServer. It fails too often Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra Signed-off-by: Martin Tzvetanov Grigorov --------- Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-arm64.yml | 300 +++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 .github/workflows/test-arm64.yml diff --git a/.github/workflows/test-arm64.yml b/.github/workflows/test-arm64.yml new file mode 100644 index 00000000000..44c45993995 --- /dev/null +++ b/.github/workflows/test-arm64.yml @@ -0,0 +1,300 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI on Linux ARM64 + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + java: + name: Java on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Temurin JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + + - name: Cache Local Maven Repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build + run: | + set -x + cd lang/java + export MAVEN_VERSION="3.9.4" + wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz + tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz + export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION + export PATH="$M2_HOME/bin:$PATH" + java -version + mvn -version + #MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test + ./build.sh clean test + + c: + name: C on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y cmake liblzma-dev libsnappy-dev libjansson-dev zlib1g-dev pkg-config + + - name: Build + run: | + set -x + cd lang/c + ./build.sh clean test + + cplusplus: + name: C++ on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y gcc g++ libboost-all-dev cmake + + - name: Build + run: | + set -x + cd lang/c++ + ./build.sh clean test + + csharp: + name: C# on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Nuget + uses: actions/cache@v3 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget libzstd-dev libicu-dev + wget https://dot.net/v1/dotnet-install.sh + bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1 + bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0 + bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0 + bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0 + + - name: Build + run: | + set -x + export PATH=$HOME/.dotnet:$PATH + dotnet --list-sdks + cd lang/csharp + ./build.sh clean test + + python: + name: Python on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y python3 python3-dev python3-pip git libbz2-dev libjansson-dev liblzma-dev libsnappy-dev libzstd-dev + python3 -m pip install --upgrade pip setuptools tox + + - name: Build + run: | + set -x + cd lang/py + ./build.sh clean test + + ruby: + name: Ruby on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache gems + uses: actions/cache@v3 + with: + path: .gem + key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} + restore-keys: | + ${{ runner.os }}-gems- + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y ruby-dev bundler libsnappy-dev libyaml-dev + + - name: Build + run: | + set -x + cd lang/ruby + ./build.sh clean test + + rust: + name: Rust on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Cargo + uses: actions/cache@v3 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: cargo-cache1- + + - name: Cache Rust dependencies + uses: actions/cache@v3 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: lang/rust/target + key: ${{ runner.os }}-target-cache1-stable- + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: stable + components: rustfmt + targets: x86_64-unknown-linux-gnu + + - name: Build + run: | + set -x + cd lang/rust + ./build.sh test + + perl: + name: Perl on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y libjansson-dev libcompress-raw-zlib-perl libcpan-uploader-perl libencode-perl libio-string-perl libjson-xs-perl libmodule-install-perl libmodule-install-readmefrompod-perl libobject-tiny-perl libperl-critic-perl libsnappy-dev libtest-exception-perl libtest-pod-perl cpanminus make gcc wget + cpanm Error::Simple + cpanm Regexp::Common + cpanm Try::Tiny + cpanm Compress::Zstd + cpanm Module::Install::Repository + cpanm inc::Module::Install + + - name: Build + run: | + set -x + cd lang/perl + ./build.sh clean test + + php: + name: PHP on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget php php-xml php-mbstring php-curl php-gmp php-bz2 unzip libtidy-dev libpq5 + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + php -r "if (hash_file('sha384', 'composer-setup.php') === file_get_contents('https://composer.github.io/installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" + php composer-setup.php --version=2.2.5 + php -r "unlink('composer-setup.php');" + sudo mv composer.phar /usr/local/bin/composer + + - name: Build + run: | + set -x + cd lang/php + composer --version + ./build.sh clean test + + javascript: + name: JavaScript on Linux ARM64 + runs-on: ["self-hosted", "asf-arm"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Npm + uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -q -y wget tar xz-utils + + - name: Build + run: | + set -x + cd lang/js + #export PATH=/node-v${{ env.NODE_VERSION }}-linux-arm64/bin:$PATH + ./build.sh clean test