From 4f48a6f77a2bbdd37c4e2bde1f7ade87090ab410 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 08:34:07 +0200 Subject: [PATCH 01/11] Add a workflow to concretize on top of the current nightlies on CVMFS --- .../workflows/concretize-current-nightly.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/concretize-current-nightly.yaml diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml new file mode 100644 index 00000000..cb315b93 --- /dev/null +++ b/.github/workflows/concretize-current-nightly.yaml @@ -0,0 +1,78 @@ +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + concretize-current-nightly: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.image }}-${{ matrix.build_type }} + cancel-in-progress: true + strategy: + matrix: + image: [alma9, ubuntu22] + build_type: [release, nightly] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Start container + run: | + name=$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]') + if [ "${{ matrix.image }}" = "alma9" ]; then + docker run --name container --privileged -v ${GITHUB_WORKSPACE}:/key4hep-spack -v ~/.cache/ccache:/root/.cache/ccache -d ghcr.io/key4hep/key4hep-images/alma9-cvmfs tail -f /dev/null + elif [ "${{ matrix.image }}" = "ubuntu22" ]; then + docker run --name container --privileged -v ${GITHUB_WORKSPACE}:/key4hep-spack -v ~/.cache/ccache:/root/.cache/ccache -d ghcr.io/key4hep/key4hep-images/ubuntu22-cvmfs tail -f /dev/null + else + echo "Unknown image" + exit 1 + fi + + - name: Setup environment and concretize + run: | + + cat <<'EOF' > ${GITHUB_WORKSPACE}/script_container.sh + set -e + + cd / + + if [ "${{ matrix.image }}" = "alma9" ]; then + os=almalinux9 + elif [ "${{ matrix.image }}" = "ubuntu22" ]; then + os=ubuntu22 + else + echo "Unknown image" + exit 1 + fi + + rel=$(find /cvmfs/sw-nightlies.hsf.org/key4hep/releases/ -maxdepth 3 -type f -wholename "*$os*opt*/.scratch" | sort -r | head -1) + rel=$(dirname $rel) + + git clone https://github.com/spack/spack -q + cd /spack + git checkout $(cat $rel/.spack-commit) + source $rel/.cherry-pick + source spack/share/spack/setup-env.sh + + if [ "${{ matrix.build_type }}" = "release" ]; then + env=key4hep-release + elif [ "${{ matrix.build_type }}" = "nightly" ]; then + env=key4hep-nightly-opt + pip3 install pyyaml + python3 /key4hep-spack/scripts/fetch_nightly_versions.py --path /key4hep-spack/environments/key4hep-common/packages.yaml --extra-path /key4hep-spack/environments/key4hep-nightly-opt/packages.yaml "" + else + echo "Unknown build type" + exit 1 + fi + cd /key4hep-spack/environments/${env} + spack env activate . + spack concretize + + EOF + + chmod +x ${GITHUB_WORKSPACE}/script_container.sh + + # cat ${GITHUB_WORKSPACE}/script_container.sh + + docker exec container /bin/bash -c "/mount.sh && /key4hep-spack/script_container.sh" From 81b7ca7c43ac76ba2bfbb833bde01bb9edac61ad Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 08:37:34 +0200 Subject: [PATCH 02/11] Fix the path for cherry picking --- .github/workflows/concretize-current-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index cb315b93..17fef923 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -52,7 +52,7 @@ jobs: git clone https://github.com/spack/spack -q cd /spack git checkout $(cat $rel/.spack-commit) - source $rel/.cherry-pick + source $/key4hep-spack/.cherry-pick source spack/share/spack/setup-env.sh if [ "${{ matrix.build_type }}" = "release" ]; then From 5b05109007a0e16b70659c60855092121b522010 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 08:39:01 +0200 Subject: [PATCH 03/11] Fix typo --- .github/workflows/concretize-current-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index 17fef923..e8893b84 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -52,7 +52,7 @@ jobs: git clone https://github.com/spack/spack -q cd /spack git checkout $(cat $rel/.spack-commit) - source $/key4hep-spack/.cherry-pick + source /key4hep-spack/.cherry-pick source spack/share/spack/setup-env.sh if [ "${{ matrix.build_type }}" = "release" ]; then From 1de6c4aee8a205e9e4728cf879216c2764f9ca2a Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 08:47:57 +0200 Subject: [PATCH 04/11] Add upstream, only run for nightlies --- .github/workflows/concretize-current-nightly.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index e8893b84..395dc74a 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -1,3 +1,6 @@ +# Concretize using the current nightlies as upstream to check if packages are changed or not +# with the new changes + on: push: branches: @@ -13,7 +16,7 @@ jobs: strategy: matrix: image: [alma9, ubuntu22] - build_type: [release, nightly] + build_type: [nightly] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -51,7 +54,7 @@ jobs: git clone https://github.com/spack/spack -q cd /spack - git checkout $(cat $rel/.spack-commit) + git checkout $(cat /key4hep-spack/.latest-commit) source /key4hep-spack/.cherry-pick source spack/share/spack/setup-env.sh @@ -67,6 +70,7 @@ jobs: fi cd /key4hep-spack/environments/${env} spack env activate . + spack config add "upstreams:nightly-from-scratch:install_tree: $rel" spack concretize EOF From 72e47c8aa5265c601724283f6f680592d2ffb285 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 08:50:08 +0200 Subject: [PATCH 05/11] Fix path --- .github/workflows/concretize-current-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index 395dc74a..d1512d38 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -56,7 +56,7 @@ jobs: cd /spack git checkout $(cat /key4hep-spack/.latest-commit) source /key4hep-spack/.cherry-pick - source spack/share/spack/setup-env.sh + source /spack/share/spack/setup-env.sh if [ "${{ matrix.build_type }}" = "release" ]; then env=key4hep-release From 31c88af706431620f743fa4b0c244e59ae70c801 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 10:55:43 +0200 Subject: [PATCH 06/11] Set GITHUB_TOKEN --- .github/workflows/concretize-current-nightly.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index d1512d38..091835ed 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -20,6 +20,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: GH_TOKEN + shell: bash + run: echo "GITHUB_TOKEN=${GH_TOKEN}" >> $GITHUB_ENV + - name: Start container run: | name=$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]') From 47e5824a573868a9ea54f7527823d6c70cb68fa3 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 11:05:12 +0200 Subject: [PATCH 07/11] Actually read the secret --- .github/workflows/concretize-current-nightly.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index 091835ed..a9b782d2 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -22,6 +22,8 @@ jobs: - uses: actions/checkout@v4 - name: GH_TOKEN shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "GITHUB_TOKEN=${GH_TOKEN}" >> $GITHUB_ENV - name: Start container From 7b6540a20d7f0b46db9d3167500c57c53a76f429 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 11:15:30 +0200 Subject: [PATCH 08/11] Fix token usage --- .github/workflows/concretize-current-nightly.yaml | 10 ++++------ .github/workflows/concretize.yaml | 4 ++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index a9b782d2..b66d1574 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -20,12 +20,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: GH_TOKEN - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: echo "GITHUB_TOKEN=${GH_TOKEN}" >> $GITHUB_ENV - - name: Start container run: | name=$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]') @@ -39,8 +33,12 @@ jobs: fi - name: Setup environment and concretize + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + export GITHUB_TOKEN=${GH_TOKEN} + cat <<'EOF' > ${GITHUB_WORKSPACE}/script_container.sh set -e diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml index 64f88286..1c59d478 100644 --- a/.github/workflows/concretize.yaml +++ b/.github/workflows/concretize.yaml @@ -30,8 +30,12 @@ jobs: fi - name: Setup environment and concretize + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + export GITHUB_TOKEN=${GH_TOKEN} + cat <<'EOF' > ${GITHUB_WORKSPACE}/script_container.sh set -e From 52e9fece37b740a332f74346dbf23420b696cd95 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 11:19:30 +0200 Subject: [PATCH 09/11] Move cd / --- .github/workflows/concretize-current-nightly.yaml | 2 +- .github/workflows/concretize.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index b66d1574..d4669ac0 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -42,7 +42,6 @@ jobs: cat <<'EOF' > ${GITHUB_WORKSPACE}/script_container.sh set -e - cd / if [ "${{ matrix.image }}" = "alma9" ]; then os=almalinux9 @@ -56,6 +55,7 @@ jobs: rel=$(find /cvmfs/sw-nightlies.hsf.org/key4hep/releases/ -maxdepth 3 -type f -wholename "*$os*opt*/.scratch" | sort -r | head -1) rel=$(dirname $rel) + cd / git clone https://github.com/spack/spack -q cd /spack git checkout $(cat /key4hep-spack/.latest-commit) diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml index 1c59d478..423b5629 100644 --- a/.github/workflows/concretize.yaml +++ b/.github/workflows/concretize.yaml @@ -39,10 +39,8 @@ jobs: cat <<'EOF' > ${GITHUB_WORKSPACE}/script_container.sh set -e - cd / - ls -lah / - ls -lah /key4hep-spack + cd / git clone https://github.com/spack/spack -q source spack/share/spack/setup-env.sh From 42b1d3a4e86979c4d3291c9736af2063ef7fb623 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 11:31:33 +0200 Subject: [PATCH 10/11] Print the list of packages that would be installed --- .github/workflows/concretize-current-nightly.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index d4669ac0..85e0f08d 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -75,7 +75,11 @@ jobs: cd /key4hep-spack/environments/${env} spack env activate . spack config add "upstreams:nightly-from-scratch:install_tree: $rel" - spack concretize + spack concretize >> conc.txt + cat conc.txt + echo "=========================================" + echo "List of changes:" + cat conc.txt | grep -E " ^" EOF From 19f6ff7279b5fb7ab6d0f5a3b225c6c2f4c5894d Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 7 Oct 2024 13:48:31 +0200 Subject: [PATCH 11/11] Fix the list of changes --- .github/workflows/concretize-current-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concretize-current-nightly.yaml b/.github/workflows/concretize-current-nightly.yaml index 85e0f08d..14f7199d 100644 --- a/.github/workflows/concretize-current-nightly.yaml +++ b/.github/workflows/concretize-current-nightly.yaml @@ -79,7 +79,7 @@ jobs: cat conc.txt echo "=========================================" echo "List of changes:" - cat conc.txt | grep -E " ^" + cat conc.txt | grep -E " -" EOF