From fe47482f694283a5874df1f42f4f9e532c077fef Mon Sep 17 00:00:00 2001 From: Anway De Date: Thu, 27 Jun 2024 13:44:42 +0000 Subject: [PATCH] build: update workflow file and build matrix script --- .github/workflows/builds.yml | 125 +++++++++++++++++------- contrib/build.sh | 180 +++++++++++++++++++++++++---------- 2 files changed, 222 insertions(+), 83 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index e84955d684..508c4a1d2f 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1,49 +1,93 @@ -# Builds for different compilers, architectures and targets +# Builds for all compilers, architectures and targets name: Builds on: workflow_call: inputs: - targets: - type: string - description: Build targets - default: all,fdctl - machines: - type: string - description: Build architectures - default: linux_gcc_x86_64,linux_clang_x86_64 gcc: type: string - description: GCC compiler versions - default: gcc-8.5.0,gcc-12.4.0 + description: GCC versions to use + default: all clang: type: string - description: Clang compiler versions - default: clang-17.0.6 + description: Clang versions to use + default: all + machine: + type: string + description: Machines to build for + default: all + verbose: + type: boolean + description: Show error outputs + default: false + exit_on_err: + type: boolean + description: Exit on the first error + default: false workflow_dispatch: inputs: - targets: - type: string - description: Build targets (comma-separated) - default: all,fdctl - machines: - type: string - description: Build architectures (comma-separated) - default: linux_gcc_x86_64,linux_clang_x86_64 gcc: type: string - description: GCC compiler versions (comma-separated) - default: gcc-8.5.0,gcc-12.4.0 + description: GCC versions to use (comma-separated | none | all) + default: all clang: type: string - description: Clang compiler versions (comma-separated) - default: clang-17.0.6 + description: Clang versions to use (comma-separated | none | all) + default: all + machine: + type: string + description: Machines to build for (comma-separated | all) + default: all + verbose: + type: boolean + description: Show error outputs + default: false + exit_on_err: + type: boolean + description: Exit on the first error + default: false concurrency: group: builds_${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - build: + build_gcc: + runs-on: X64 + if: ${{ inputs.gcc != 'none' }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: dtolnay/rust-toolchain@1.73.0 + + - name: Build command line args + run: | + ARGS="" + # verbose + if [ "${{ inputs.verbose }}" == "true" ]; then + ARGS="$ARGS --verbose" + fi + # exit-on-err + if [ "${{ inputs.exit_on_err }}" == "true" ]; then + ARGS="$ARGS --exit-on-err" + fi + # machine + if [ "${{ inputs.machine }}" != "all" ]; then + ARGS="$ARGS --machines ${{ inputs.machine }}" + fi + # gcc + if [ "${{ inputs.gcc }}" != "all" ]; then + ARGS="$ARGS --gcc-versions ${{ inputs.gcc }}" + fi + echo "BUILD_ARGS=$ARGS" >> $GITHUB_ENV + + - name: Run gcc builds + run: | + contrib/build.sh --no-rust --no-clang ${{ env.BUILD_ARGS }} + + build_clang: runs-on: 512G + if: ${{ inputs.clang != 'none' }} steps: - uses: actions/checkout@v4 with: @@ -51,12 +95,27 @@ jobs: - uses: dtolnay/rust-toolchain@1.73.0 - - uses: ./.github/actions/deps + - name: Build command line args + run: | + ARGS="" + # verbose + if [ "${{ inputs.verbose }}" == "true" ]; then + ARGS="$ARGS --verbose" + fi + # exit-on-err + if [ "${{ inputs.exit_on_err }}" == "true" ]; then + ARGS="$ARGS --exit-on-err" + fi + # machine + if [ "${{ inputs.machine }}" != "all" ]; then + ARGS="$ARGS --machines ${{ inputs.machine }}" + fi + # clang + if [ "${{ inputs.clang }}" != "all" ]; then + ARGS="$ARGS --clang-versions ${{ inputs.clang }}" + fi + echo "ARGS=$ARGS" >> $GITHUB_ENV - - name: Start build + - name: Run clang builds run: | - contrib/build.sh --no-deps --no-rust \ - --targets ${{ inputs.targets }} \ - --machines ${{ inputs.machines }} \ - --gcc-versions ${{ inputs.gcc || 'none --no-gcc' }} \ - --clang-versions ${{ inputs.clang || 'none --no-clang' }} + contrib/build.sh --no-rust --no-gcc ${{ env.BUILD_ARGS }} diff --git a/contrib/build.sh b/contrib/build.sh index f1e6e62b0b..4ccb043408 100755 --- a/contrib/build.sh +++ b/contrib/build.sh @@ -1,16 +1,32 @@ #!/bin/bash -set -e +inf() { + printf "[INFO] $@" +} + +err() { + printf "[ERROR] $@" +} elapsed() { - start=$1 - stop=$2 - diff=$(($stop-$start)) - min=$((10#$diff / 60)) - sec=$((10#$diff % 60)) + local start=$1 + local stop=$2 + local diff=$(($stop-$start)) + local min=$((10#$diff / 60)) + local sec=$((10#$diff % 60)) printf "%d:%02d" "$min" "$sec"; } +finish() { + local CODE=$1 + local STOP=$(date +%s) + inf "Total Elapsed Time: " + elapsed $START $STOP + echo + rm -f $LOG_FILE + exit $CODE +} + GCC=() CLANG=() TARGETS=() @@ -32,6 +48,12 @@ while [[ $# -gt 0 ]]; do "--no-rust") NO_RUST=1 ;; + "--exit-on-err") + EXIT_ON_ERR=1 + ;; + "--verbose") + VERBOSE=1 + ;; "--targets") IFS=',' read -r -a TARGETS <<< "$1" shift 1 @@ -62,21 +84,23 @@ BUILD_TARGETS=( all ) EXTRA_TARGETS=( asm ppp seccomp-policies ) BINARY_TARGETS=( fdctl fddev ) -NONBINARY_TARGETS=( ${BUILD_TARGETS[@]} ${EXTRA_TARGETS[@]} ${TEST_TARGETS[@]} ) +OTHER_TARGETS=( ${BUILD_TARGETS[@]} ${EXTRA_TARGETS[@]} ${TEST_TARGETS[@]} ) declare -A CUSTOM_TARGETS=() -CUSTOM_TARGETS+=( ["linux_gcc_minimal"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["linux_gcc_noarch64"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["linux_gcc_noarch128"]="${NONBINARY_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_gcc_minimal"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_gcc_noarch64"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_gcc_noarch128"]="${OTHER_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["macos_clang_m1"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["linux_clang_minimal"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["linux_clang_noarch64"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["linux_clang_noarch128"]="${NONBINARY_TARGETS[@]}" ) -CUSTOM_TARGETS+=( ["freebsd_clang_noarch128"]="${NONBINARY_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["macos_clang_m1"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_clang_minimal"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_clang_noarch64"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["linux_clang_noarch128"]="${OTHER_TARGETS[@]}" ) +CUSTOM_TARGETS+=( ["freebsd_clang_noarch128"]="${OTHER_TARGETS[@]}" ) FAIL=0 +LOG_FILE=$(mktemp) +START=$(date +%s) if [[ $NO_GCC -ne 1 ]]; then if [[ ${#GCC[@]} -eq 0 ]]; then @@ -104,35 +128,49 @@ if [[ ${#MACHINES[@]} -eq 0 ]]; then done fi +echo "*************************" echo "Starting Build Matrix..." -echo "========================" -echo "Machines: [ ${MACHINES[@]} ]" -echo "Clang: [ ${CLANG[@]} ]" -echo "Gcc: [ ${GCC[@]} ]" -echo "Targets: [ ${TARGETS[@]} ]" +echo "*************************" +echo "machines=[ ${MACHINES[@]} ]" +echo "clang=[ ${CLANG[@]} ]" +echo "gcc=[ ${GCC[@]} ]" +echo "targets=[ ${TARGETS[@]} ]" echo -echo "Setting up build environment..." +inf "Setting up build environment...\n" cd $FD_REPO_DIR if [[ $NO_RUST -ne 1 ]]; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y > /dev/null 2>&1 fi +inf "Installing packages and fetching repositories...\n" +FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev check fetch > /dev/null 2>&1 make -j distclean > /dev/null 2>&1 -set +e - if [[ $NO_GCC -ne 1 ]]; then - START=$(date +%s) - echo "Starting gcc builds..." + GCC_START=$(date +%s) + inf "Starting gcc builds...\n\n" for compiler in "${GCC[@]}"; do if [[ ! -f /opt/gcc/$compiler/activate ]]; then - echo "Environment activate script not found at /opt/gcc/$compiler... exiting." - exit 2 + err "Environment activate script not found at /opt/gcc/$compiler... exiting.\n" + finish 2 fi source /opt/gcc/$compiler/activate if [[ $NO_DEPS -ne 1 ]]; then - ./deps.sh nuke - FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install > /dev/null 2>&1 + start=$(date +%s) + inf "Installing dependencies with $compiler...\n" + find opt -mindepth 1 -maxdepth 1 ! -name 'git' -print0 | xargs -0 rm -rf > /dev/null 2>&1 + ./deps.sh +dev install > $LOG_FILE 2>&1 + if [[ $? -ne 0 ]]; then + err "Failed to install deps with $compiler... exiting.\n" + if [[ $VERBOSE -eq 1 ]]; then + cat $LOG_FILE + fi + finish 3 + fi + stop=$(date +%s) + inf "Elapsed Time: " + elapsed $start $stop + echo fi for machine in "${MACHINES[@]}"; do MACHINE="${machine%.mk}" @@ -147,46 +185,74 @@ if [[ $NO_GCC -ne 1 ]]; then TARGETS+=( "${BINARY_TARGETS[@]}" ) fi fi - echo "Starting builds for $MACHINE with $compiler..." + inf "Starting builds for $MACHINE with $compiler...\n" FAILED=() start=$(date +%s) + >$LOG_FILE for target in "${TARGETS[@]}"; do - MACHINE=${MACHINE} CC=gcc make -j $target + MACHINE=${MACHINE} CC=gcc make -j $target >> $LOG_FILE 2>&1 if [[ $? -ne 0 ]]; then FAILED+=( $target ) FAIL=1 fi done stop=$(date +%s) - printf "Done... Elapsed Time: " + inf "Done... Elapsed Time: " elapsed $start $stop echo if [[ ${#FAILED[@]} -gt 0 ]]; then - echo "Failed Targets: ${FAILED[@]}" + err "Failed Targets: " + echo "${FAILED[@]}" + inf "To reproduce, run:\n" + echo " source /opt/gcc/$compiler/activate" + echo " ./deps.sh nuke" + echo " FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install" + echo " make -j distclean" + echo " MACHINE=${MACHINE} CC=gcc make -j ${FAILED[@]}" + if [[ $VERBOSE -eq 1 ]]; then + err "Failure Logs:\n" + cat $LOG_FILE + fi + if [[ $EXIT_ON_ERR -eq 1 ]]; then + finish 1 + fi fi make -j distclean > /dev/null 2>&1 echo fi done done - STOP=$(date +%s) - printf "Done with gcc builds... Total Elapsed Time: " - elapsed $START $STOP + GCC_STOP=$(date +%s) + inf "Done with gcc builds in " + elapsed $GCC_START $GCC_STOP echo fi if [[ $NO_CLANG -ne 1 ]]; then - START=$(date +%s) - echo "Starting clang builds..." + CLANG_START=$(date +%s) + inf "Starting clang builds...\n\n" for compiler in "${CLANG[@]}"; do if [[ ! -f /opt/clang/$compiler/activate ]]; then - echo "Environment activate script not found at /opt/clang/$compiler... exiting." - exit 2 + err "Environment activate script not found at /opt/clang/$compiler... exiting.\n" + finish 2 fi source /opt/clang/$compiler/activate if [[ $NO_DEPS -ne 1 ]]; then - ./deps.sh nuke - FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install > /dev/null 2>&1 + start=$(date +%s) + inf "Installing dependencies with $compiler...\n" + find opt -mindepth 1 -maxdepth 1 ! -name 'git' -print0 | xargs -0 rm -rf > /dev/null 2>&1 + ./deps.sh +dev install > $LOG_FILE 2>&1 + if [[ $? -ne 0 ]]; then + err "Failed to install deps with $compiler... exiting.\n" + if [[ $VERBOSE -eq 1 ]]; then + cat $LOG_FILE + fi + finish 3 + fi + stop=$(date +%s) + inf "Elapsed Time: " + elapsed $start $stop + echo fi for machine in "${MACHINES[@]}"; do MACHINE="${machine%.mk}" @@ -201,32 +267,46 @@ if [[ $NO_CLANG -ne 1 ]]; then TARGETS+=( "${BINARY_TARGETS[@]}" ) fi fi - echo "Starting builds for $MACHINE with $compiler..." + inf "Starting builds for $MACHINE with $compiler...\n" FAILED=() start=$(date +%s) + >$LOG_FILE for target in "${TARGETS[@]}"; do - MACHINE=${MACHINE} CC=clang make -j $target + MACHINE=${MACHINE} CC=gcc make -j $target >> $LOG_FILE 2>&1 if [[ $? -ne 0 ]]; then FAILED+=( $target ) FAIL=1 fi done stop=$(date +%s) - printf "Done... Elapsed Time: " + inf "Done... Elapsed Time: " elapsed $start $stop echo if [[ ${#FAILED[@]} -gt 0 ]]; then - echo "Failed Targets: ${FAILED[@]}" + err "Failed Targets: " + echo "${FAILED[@]}" + echo " source /opt/clang/$compiler/activate" + echo " ./deps.sh nuke" + echo " FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install" + echo " make -j distclean" + echo " MACHINE=${MACHINE} CC=clang make -j ${FAILED[@]}" + if [[ $VERBOSE -eq 1 ]]; then + err "Failure Logs:\n" + cat $LOG_FILE + fi + if [[ $EXIT_ON_ERR -eq 1 ]]; then + finish 1 + fi fi make -j distclean > /dev/null 2>&1 echo fi done done - STOP=$(date +%s) - printf "Done with clang builds... Total Elapsed Time: " - elapsed $START $STOP + CLANG_STOP=$(date +%s) + inf "Done with clang builds in " + elapsed $CLANG_START $CLANG_STOP echo fi -exit $FAIL +finish $FAIL