From ae93bbb28c65b42b06272fa56e82f1bcb2a72a2d Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Sun, 11 Aug 2024 13:19:48 -0400 Subject: [PATCH 1/7] CI: Add an example manifest for a PR test build flow This manifest shows how a ci-job.yml could be constructed to test one or more PRs for each of the sub-components. Signed-off-by: Bill Mills --- ci/ci-job-example.yml | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 ci/ci-job-example.yml diff --git a/ci/ci-job-example.yml b/ci/ci-job-example.yml new file mode 100644 index 0000000..20d9ab8 --- /dev/null +++ b/ci/ci-job-example.yml @@ -0,0 +1,44 @@ +# Example CI job for testing PRs +# This shows multiple PRs at the same time +# however normally only one would be done per job + +header: + version: 9 + # We include the base job here + includes: + - genericarm64.yml + +# override the source of the layers +# kas will not pull directly from the SHA of the PR commit +# the repo and branch of the PR can be found from th gh command +# gh pr status --json "headRefName,headRepository,headRepositoryOwner" #49 +# The info below is from meta-openamp #49 +repos: + layers/poky: + url: https://git.yoctoproject.org/git/poky + refspec: yocto-5.0.1 + layers/meta-openamp: + url: https://github.com/wmamills/meta-openamp.git + refspec: wam-update-open-amp-lic-md5 + +# override the commit of these source repos +# for libmetal and open-amp BRANCH must be blank +# the recipe .inc file will add nobranch=1 +# for openamp-system-reference we explicitly add nobranch=1 here +# NOTE: any override has higher priority than the base var name +# therefore these work even if the base recipe has = and not ?= +# The commits below are: +# libmetal #302 +# open-amp #605 +# openamp-system-reference #45 +# (They were valid at the time but may not be when you try this) +local_conf_header: + autorev: | + PREFERRED_PROVIDER_libmetal = "libmetal-dev" + SRCREV:pn-libmetal-dev = "c58d3721bc070a19e1ec1d145e0be2f979072f05" + BRANCH:pn-libmetal-dev = "" + PREFERRED_PROVIDER_open-amp = "open-amp-dev" + SRCREV:pn-open-amp-dev = "fb7fc82e65f14ee8cb6e340426a35d80fa1f8091" + BRANCH:pn-open-amp-dev = "" + OPENAMP_SYS_REF_SRCREV = "65b35eabaa5ebe8dd3806e54494c65677c6c19e2" + OPENAMP_SYS_REF_BRANCH = "none;nobranch=1" From 47dd05dfa0deac62848580bcab666a6c890d57d1 Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Tue, 13 Aug 2024 16:35:58 -0400 Subject: [PATCH 2/7] scripts: add a copy of venv-wrapper Add a copy of the venv-wrapper utility from: https://github.com/wmamills/cloudbuild/blob/main/venv-wrapper Signed-off-by: Bill Mills --- scripts/venv-wrapper | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 scripts/venv-wrapper diff --git a/scripts/venv-wrapper b/scripts/venv-wrapper new file mode 100755 index 0000000..96b08cb --- /dev/null +++ b/scripts/venv-wrapper @@ -0,0 +1,111 @@ +#!/bin/bash + +VENV_BASE=~/opt/venv +BIN_DIR=~/.local/bin + +# Allow override of above settings +if [ -r ~/.prjinfo/venv-wrapper.conf ]; then + . ~/.prjinfo/venv-wrapper.conf +fi + +ME=$0 +MY_NAME=$(basename $ME) +ME_FULL=$(readlink -f $ME) + +do_wrapper_cmd() { + SUB_CMD=$1 + MY_NAME=$2 + shift 2 + + case $SUB_CMD in + "install"|"create"|"purge"|"run") + do_$SUB_CMD "$@" + ;; + *) + echo "Unknown command $SUB_CMD" + echo "usage: venv-wrapper install|create|purge|run []" + ;; + esac +} + +# create and install to ~/.local/bin +do_install() { + mkdir -p $BIN_DIR + + case $MY_NAME in + "self"|"self-link") + ln -fs $ME_FULL $BIN_DIR/venv-wrapper + return + ;; + "self-copy") + rm -f $BIN_DIR/venv-wrapper + cp -p $ME_FULL $BIN_DIR/venv-wrapper + return + ;; + esac + + do_create "$@" + if [ ! -x $BIN_DIR/venv-wrapper ]; then + ln -s $ME_FULL $BIN_DIR/venv-wrapper + fi + + if [ -e $BIN_DIR/$MY_NAME ]; then + RL=$(readlink $BIN_DIR/$MY_NAME) + if [ x"$RL" != x"venv-wrapper" ]; then + echo "$BIN_DIR/$MY_NAME already exists and is not a symlink to venv-wrapper" + exit 2 + fi + else + ln -s venv-wrapper $BIN_DIR/$MY_NAME + fi +} + +# create the venv and do pip install of any extra args +do_create() { + mkdir -p $VENV_BASE + python3 -m venv $VENV_BASE/$MY_NAME + if [ -n "$1" ]; then + (do_install_pip "$@") + fi +} + +# do the pip install step in a sub-shell +do_install_pip() { + # in a sub-shell so env changed will not persist + . $VENV_BASE/$MY_NAME/bin/activate + pip3 install "$@" +} + +# create and install to ~/.local/bin +do_purge() { + mkdir -p $BIN_DIR + + # delete the venv + if [ -r $VENV_BASE/$MY_NAME/bin/activate ]; then + rm -rf $VENV_BASE/$MY_NAME + fi + + # delete the symlink + if [ -e $BIN_DIR/$MY_NAME ]; then + RL=$(readlink $BIN_DIR/$MY_NAME) + if [ x"$RL" == x"venv-wrapper" ]; then + rm -f $BIN_DIR/$MY_NAME + else + echo "$BIN_DIR/$MY_NAME already exists and is not a symlink to venv-wrapper" + exit 2 + fi + fi + + hash -r +} + +do_run() { + . $VENV_BASE/$MY_NAME/bin/activate + $VENV_BASE/$MY_NAME/bin/$MY_NAME "$@" +} + +if [ "$MY_NAME" == "venv-wrapper" ]; then + do_wrapper_cmd "$@" +else + do_run "$@" +fi From 2b9932e4ac8b177e35c0783c3ed067f814e8a20f Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Tue, 13 Aug 2024 10:33:30 -0400 Subject: [PATCH 3/7] build-host-setup: quiet the apt-get commands Be less verbose for the apt-get commands. Signed-off-by: Bill Mills --- scripts/build-host-setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build-host-setup.sh b/scripts/build-host-setup.sh index 7d28052..425104f 100755 --- a/scripts/build-host-setup.sh +++ b/scripts/build-host-setup.sh @@ -1,16 +1,16 @@ #!/bin/bash # setup host for building -sudo apt-get update +sudo apt-get -qq update if [ -r ~/.aws/credentials ]; then - sudo apt-get install -y s3fs + sudo apt-get install -qqy s3fs mkdir -p ~/net/openamp-builds s3fs -o profile=openamp-builds-rw openamp-builds ~/net/openamp-builds fi # Yocto/OE requirements -sudo apt-get install -y gawk wget git-core diffstat unzip texinfo \ +sudo apt-get install -qqy gawk wget git-core diffstat unzip texinfo \ build-essential chrpath socat cpio python3 python3-pip \ python3-pexpect xz-utils debianutils iputils-ping curl git \ zstd libssl-dev lz4 From 76b8454f046b1adbf43d5e402c53398020f43fcb Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Tue, 13 Aug 2024 10:31:49 -0400 Subject: [PATCH 4/7] build-host-setup: use venv-wrapper for kas For better compatibility with python 3.12 in Ubuntu 24.04, use a venv to install kas. The venv will work on prior versions of python as well so this method is better than --break-system-packages. Here we use the venv-wrapper utility to install kas. Signed-off-by: Bill Mills --- scripts/build-host-setup.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/build-host-setup.sh b/scripts/build-host-setup.sh index 425104f..3b53fac 100755 --- a/scripts/build-host-setup.sh +++ b/scripts/build-host-setup.sh @@ -1,5 +1,8 @@ #!/bin/bash +ME=$0 +MY_DIR=$(dirname $ME) + # setup host for building sudo apt-get -qq update @@ -13,14 +16,10 @@ fi sudo apt-get install -qqy gawk wget git-core diffstat unzip texinfo \ build-essential chrpath socat cpio python3 python3-pip \ python3-pexpect xz-utils debianutils iputils-ping curl git \ - zstd libssl-dev lz4 + zstd libssl-dev lz4 python3-pip python3-venv # we need kas if ! which kas; then - pip3 install kas - if ! which kas; then - # fix up for this time, assume .bashrc will get it next time - PATH=~/.local/bin:$PATH - fi + $MY_DIR/venv-wrapper install self-copy + $MY_DIR/venv-wrapper install kas kas fi - From 735b721a002178961b6d96d17ad6e77ff2ea80a6 Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Tue, 3 Oct 2023 08:12:41 -0400 Subject: [PATCH 5/7] github: add initial build workflow * Only trigger manually for now. * Display result sizes and the build stats * Save the images and packages * This Yocto version needs Ubuntu 22.04 and has errors on 24.04 (so we use a runner label that selects that) Signed-off-by: Bill Mills --- .github/workflows/build.yml | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..308e57e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,56 @@ +# Initial workflow for OE Builds + +name: build + +# Controls when the action will run. +on: + # No merge or pr triggers yet + + # Allow workflow to be manually run from the Actions tab + workflow_dispatch: + +jobs: + # This workflow contains a single job called "build" + build-x86: + # The type of runner that the job will run on + runs-on: + labels: big-job-x86-ubuntu-22.04 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v4 + + - name: Setup Build Host + run: | + ./scripts/build-host-setup.sh + + - name: check PATH + run: echo "$PATH" + + - name: Use Kas to build + run: "kas build ci/genericarm64.yml" + + - name: show file stats + run: | + du -sh build/sstate-cache/ || true + du -sh build/downloads/ || true + du -sh build/tmp/deploy/images/ || true + du -sh build/tmp/deploy/ipk || true + cat build/tmp/buildstats/20*/build_stats || true + + - name: After stats + run: echo After stats + + - name: Archive Images + uses: actions/upload-artifact@v4 + with: + name: images + path: | + build/tmp/deploy/images + + - name: Archive Packages + uses: actions/upload-artifact@v4 + with: + name: packages + path: | + build/tmp/deploy/ipk From fb17cc272cb029a5a27563e4cec4656da6576a4f Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Sat, 17 Aug 2024 22:15:36 -0400 Subject: [PATCH 6/7] github: add dispatch arguments Add target and quick_test arguments for manual dispatch. Signed-off-by: Bill Mills --- .github/workflows/build.yml | 49 +++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 308e57e..e3f953e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,23 @@ on: # Allow workflow to be manually run from the Actions tab workflow_dispatch: + inputs: + target: + description: 'Which target' + required: true + default: 'genericarm64' + type: choice + options: + - 'ci/genericarm64.yml' + - 'ci/genericarm64-ly.yml' + - 'ci/generic-armv7a.yml' + - 'ci/qemuarm64.yml' + - 'for-ref/stm32mp157c-dk2.yml' + - 'for-ref/xilinx-vendor.sh' + quick_test: + description: 'Do only a quick test build' + type: boolean + default: off jobs: # This workflow contains a single job called "build" @@ -24,11 +41,32 @@ jobs: run: | ./scripts/build-host-setup.sh - - name: check PATH - run: echo "$PATH" + - name: Info + run: | + echo "path: $PATH" + echo "target: ${{ inputs.target }}" + echo "quick_test: ${{ inputs.quick_test }}" - - name: Use Kas to build - run: "kas build ci/genericarm64.yml" + - name: Do build + run: | + TARGET="${{ inputs.target }}" + if ${{ inputs.quick_test }} ; then + QUICK_TEST_ARG=":mixins/quick-test.yml" + else + QUICK_TEST_ARG="" + fi + case $TARGET in + *.yml) + kas build ${TARGET}${QUICK_TEST_ARG} + ;; + *.sh) + ./$TARGET + ;; + *) + echo "Unknown target type" + exit 2 + ;; + esac - name: show file stats run: | @@ -38,9 +76,6 @@ jobs: du -sh build/tmp/deploy/ipk || true cat build/tmp/buildstats/20*/build_stats || true - - name: After stats - run: echo After stats - - name: Archive Images uses: actions/upload-artifact@v4 with: From c6c8aa58a07dba86612796082b9e53026cf73d6a Mon Sep 17 00:00:00 2001 From: Bill Mills Date: Sat, 17 Aug 2024 22:41:43 -0400 Subject: [PATCH 7/7] github: add push support Run on any push to a branch named wip-test-* Find target based on push content. If one of ci/ci-job.{sh,yml} is present, run that. If not fallback to genericarm64 Signed-off-by: Bill Mills --- .github/workflows/build.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3f953e..fe3d2c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,10 @@ name: build # Controls when the action will run. on: - # No merge or pr triggers yet + # push to any branch named wip-test-* + push: + branches: + - 'wip-test-*' # Allow workflow to be manually run from the Actions tab workflow_dispatch: @@ -50,11 +53,25 @@ jobs: - name: Do build run: | TARGET="${{ inputs.target }}" - if ${{ inputs.quick_test }} ; then + + # if no target specified (ex. push to wip-test-*), find one + if [ -z "$TARGET" ]; then + for t in ci/ci-job.sh ci/ci-job.yml ci/genericarm64.yml; do + if [ -r "$t" ]; then + TARGET="$t" + break + fi + done + fi + + # if this a "quick test"? + if [ x"${{ inputs.quick_test }}" == x"true" ]; then QUICK_TEST_ARG=":mixins/quick-test.yml" else QUICK_TEST_ARG="" fi + + # now handle kas vs shell script case $TARGET in *.yml) kas build ${TARGET}${QUICK_TEST_ARG}