Merge pull request #196 from Dasharo/develop

v2.1.0
Dasharo · Dec 9, 2024 · 9f8383f · 9f8383f
2 parents b822e35 + 0795508
commit 9f8383f
Show file tree

Hide file tree

Showing 32 changed files with 1,386 additions and 538 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,47 @@
+---
+name: Build DTS
+on:
+  workflow_call:
+    inputs:
+      cacheless:
+        type: boolean
+        required: true
+
+jobs:
+  build:
+    runs-on:
+      labels: dts-builder
+    env:
+      FAILED_ATTEMPTS: 0
+    steps:
+      - name: Checkout meta-dts repo
+        uses: actions/checkout@v4
+        with:
+          path: "meta-dts"
+      - name: Prepare cache-less build configuration
+        if: ${{ inputs.cacheless }}
+        shell: bash
+        run: |
+          sed -i '/cache.yml/d' meta-dts/kas.yml
+      - name: Build DTS image
+        shell: bash
+        id: build_image
+        run: |
+          for attempt in {1..5}; do
+            if kas-container build meta-dts/kas.yml; then
+              echo "Build command succeeded on attempt $attempt"
+              break
+            else
+              echo "Build command failed on attempt $attempt"
+              echo "FAILED_ATTEMPTS=$attempt" >> $GITHUB_ENV
+              if [ $attempt -lt 5 ]; then
+                sleep 5
+              else
+                exit 1
+              fi
+            fi
+          done
+      - name: Report build command
+        if: always()
+        run: |
+          echo "$FAILED_ATTEMPTS build attempts failed"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,41 +8,11 @@ on:
 
 jobs:
   build:
-    name: Build system image
-    runs-on:
-      labels: dts-builder
-    steps:
-      - name: Checkout meta-dts repo
-        uses: actions/checkout@v2
-        with:
-          path: "meta-dts"
-      - name: Build DTS image
-        shell: bash
-        id: build_image
-        run: |
-          for attempt in {1..5}; do
-            if kas-container build meta-dts/kas.yml; then
-              echo "Build command succeeded on attempt $attempt"
-              break
-            else
-              echo "Build command failed on attempt $attempt"
-              if [ $attempt -lt 5 ]; then
-                sleep 5
-              fi
-            fi
-          done
-        continue-on-error: true
-      - name: Report build command
-        run: |
-          if [ ${{ steps.build_image.outcome }} == 'failure' ]; then
-            echo "All build attempts failed."
-            exit 1
-          else
-            echo "At least one build attempt succeeded."
-          fi
+    uses: ./.github/workflows/build.yml
+    with:
+      cacheless: false
   deploy-images:
     name: Deploy DTS artifacts on boot.dasharo.com and GitHub Release
-    if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
     needs: build
     runs-on:
       labels: dts-builder
@@ -102,6 +72,13 @@ jobs:
           # do not release iso image until issue is fixed
           # see: https://github.com/dasharo/dasharo-issues/issues/288
           # scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.iso [email protected]:boot/dts/${DTS_VER}/
+      - name: Deploy manifest on boot.dasharo.com
+        shell: bash
+        run: |
+          DTS_VER="${{ env.DTS_VER }}"
+          cd build/tmp/deploy/images/genericx86-64/
+          cp dts-base-image-genericx86-64.manifest dts-base-image-${DTS_VER}.manifest
+          scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.manifest [email protected]:boot/dts/${DTS_VER}/
       - name: Deploy sha256 on boot.dasharo.com
         shell: bash
         run: |
@@ -111,13 +88,15 @@ jobs:
           sha256sum dts-base-image-${DTS_VER}.cpio.gz > dts-base-image-${DTS_VER}.cpio.gz.sha256
           sha256sum dts-base-image-${DTS_VER}.wic.gz > dts-base-image-${DTS_VER}.wic.gz.sha256
           sha256sum dts-base-image-${DTS_VER}.wic.bmap > dts-base-image-${DTS_VER}.wic.bmap.sha256
+          sha256sum dts-base-image-${DTS_VER}.manifest > dts-base-image-${DTS_VER}.manifest.sha256
           # do not release iso image until issue is fixed
           # see: https://github.com/dasharo/dasharo-issues/issues/288
           # sha256sum dts-base-image-${DTS_VER}.iso > dts-base-image-${DTS_VER}.iso.sha256
           scp -i ~/.ssh/dts-ci-key bzImage-${DTS_VER}.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.cpio.gz.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.gz.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.bmap.sha256 [email protected]:boot/dts/${DTS_VER}/
+          scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.manifest.sha256 [email protected]:boot/dts/${DTS_VER}/
           # do not release iso image until issue is fixed
           # see: https://github.com/dasharo/dasharo-issues/issues/288
           # scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.iso.sha256 [email protected]:boot/dts/${DTS_VER}/
@@ -152,4 +131,4 @@ jobs:
           rm -rf ~/.ssh/dts-ci-key
           rm -rf dts-release-cicd-pipeline
           rm -f ~/.ssh/gitea_dts_release_cicd
-          rm -rf build
+          rm -rf build meta-dts
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -7,41 +7,11 @@ on:
 
 jobs:
   build:
-    name: Build system image
-    runs-on:
-      labels: dts-builder
-    steps:
-      - name: Checkout meta-dts repo
-        uses: actions/checkout@v2
-        with:
-          path: "meta-dts"
-      - name: Build DTS image
-        shell: bash
-        id: build_image
-        run: |
-          for attempt in {1..5}; do
-            if kas-container build meta-dts/kas.yml; then
-              echo "Build command succeeded on attempt $attempt"
-              break
-            else
-              echo "Build command failed on attempt $attempt"
-              if [ $attempt -lt 5 ]; then
-                sleep 5
-              fi
-            fi
-          done
-        continue-on-error: true
-      - name: Report build command
-        run: |
-          if [ ${{ steps.build_image.outcome }} == 'failure' ]; then
-            echo "All build attempts failed."
-            exit 1
-          else
-            echo "At least one build attempt succeeded."
-          fi
+    uses: ./.github/workflows/build.yml
+    with:
+      cacheless: false
   deploy-images:
     name: Deploy DTS artifacts on boot.dasharo.com
-    if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
     needs: build
     runs-on:
       labels: dts-builder
@@ -97,6 +67,13 @@ jobs:
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.gz [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.bmap [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.iso [email protected]:boot/dts/${DTS_VER}/
+      - name: Deploy manifest on boot.dasharo.com
+        shell: bash
+        run: |
+          DTS_VER="${{ env.DTS_VER }}"
+          cd build/tmp/deploy/images/genericx86-64/
+          cp dts-base-image-genericx86-64.manifest dts-base-image-${DTS_VER}.manifest
+          scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.manifest [email protected]:boot/dts/${DTS_VER}/
       - name: Deploy sha256 on boot.dasharo.com
         shell: bash
         run: |
@@ -107,11 +84,13 @@ jobs:
           sha256sum dts-base-image-${DTS_VER}.wic.gz > dts-base-image-${DTS_VER}.wic.gz.sha256
           sha256sum dts-base-image-${DTS_VER}.wic.bmap > dts-base-image-${DTS_VER}.wic.bmap.sha256
           sha256sum dts-base-image-${DTS_VER}.iso > dts-base-image-${DTS_VER}.iso.sha256
+          sha256sum dts-base-image-${DTS_VER}.manifest > dts-base-image-${DTS_VER}.manifest.sha256
           scp -i ~/.ssh/dts-ci-key bzImage-${DTS_VER}.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.cpio.gz.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.gz.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.wic.bmap.sha256 [email protected]:boot/dts/${DTS_VER}/
           scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.iso.sha256 [email protected]:boot/dts/${DTS_VER}/
+          scp -i ~/.ssh/dts-ci-key dts-base-image-${DTS_VER}.manifest.sha256 [email protected]:boot/dts/${DTS_VER}/
       - name: Update iPXE menu
         shell: bash
         run: |
@@ -143,4 +122,4 @@ jobs:
           rm -rf ~/.ssh/dts-ci-key
           rm -rf dts-release-cicd-pipeline
           rm -f ~/.ssh/gitea_dts_release_cicd
-          rm -rf build
+          rm -rf build meta-dts
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,117 @@
+---
+name: Run DTS tests
+on:
+  pull_request:
+    branches:
+      - 'main'
+jobs:
+  build-dts:
+    uses: ./.github/workflows/build.yml
+    with:
+      cacheless: false
+  run-tests:
+    name: Run DTS tests
+    needs: build-dts
+    runs-on:
+      labels: dts-builder
+    steps:
+      - name: Checkout OSFV repo
+        uses: actions/checkout@v4
+        with:
+          repository: 'Dasharo/open-source-firmware-validation'
+          path: 'open-source-firmware-validation'
+          submodules: 'recursive'
+          ref: 'develop'
+      - name: Run IPXE server
+        shell: bash
+        run: |
+          cd open-source-firmware-validation
+          ./scripts/ci/ipxe-run.sh ../build/tmp/deploy/images/genericx86-64/dts-base-image-genericx86-64.cpio.gz ../build/tmp/deploy/images/genericx86-64/bzImage &
+      - name: Install requirements
+        shell: bash
+        run: |
+          cd open-source-firmware-validation
+          python3 -m virtualenv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+      - name: Run QEMU
+        shell: bash
+        id: run_qemu
+        run: |
+          cd open-source-firmware-validation/scripts/ci
+          mkdir qemu-data
+          touch qemu-data/hdd.qcow2
+          ./qemu-run.sh nographic os &
+      - name: Create directory for logs
+        shell: bash
+        id: log_dirs
+        run: |
+          timestamp=$(date -u +%Y-%m-%dT%H:%M:%S%Z)
+          directory="/tmp/dts-test-ci-${timestamp}"
+          mkdir $directory
+          echo "directory=$directory" >> "$GITHUB_OUTPUT"
+      - name: Run tests
+        shell: bash
+        env:
+          LOG_DIR: ${{ steps.log_dirs.outputs.directory }}
+        run: |
+          cd open-source-firmware-validation
+          source venv/bin/activate
+          # This file is already present on dts-builder.
+          # It contains credentials for DPP subscriptions.
+          # It has the following form:
+          # DPP_PASSWORD="..."
+          # <SUBSCRIPTION_TYPE>_DOWNLOADS="download key"
+          # <SUBSCRIPTION_TYPE>_LOGS="logs key"
+          # (...)
+          source ~/.secrets/dpp-keys
+          ip_addr=$(ip -o -4 addr list eno2 | awk '{print $4}' | cut -d/ -f1)
+
+          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$MSI_DOWNLOAD \
+          -v dpp_logs_key:$MSI_LOGS -v boot_dts_from_ipxe_shell:True \
+          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
+          -i "msi_comm" -i "msi_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_msi.log | grep "| PASS |\|| FAIL |"
+
+          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$MSI_HEADS_DOWNLOAD \
+          -v dpp_logs_key:$MSI_HEADS_LOGS -v boot_dts_from_ipxe_shell:True \
+          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
+          -i "msi_heads" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_msi_heads.log | grep "| PASS |\|| FAIL |"
+
+          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$OPTIPLEX_DOWNLOAD \
+          -v dpp_logs_key:$OPTIPLEX_LOGS -v boot_dts_from_ipxe_shell:True \
+          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
+          -i "optiplex_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_optiplex.log | grep "| PASS |\|| FAIL |"
+
+          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$NOVACUSTOM_HEADS_DOWNLOAD \
+          -v dpp_logs_key:$NOVACUSTOM_HEADS_LOGS -v boot_dts_from_ipxe_shell:True \
+          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
+          -i "novacustom_heads" -i "novacustom_comm" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_nc_heads.log | grep "| PASS |\|| FAIL |"
+
+          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$PCENGINES_DOWNLOAD \
+          -v dpp_logs_key:$PCENGINES_LOGS -v boot_dts_from_ipxe_shell:True \
+          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
+          -i "pcengines_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_pcengines.log | grep "| PASS |\|| FAIL |"
+
+      - name: Copy log
+        shell: bash
+        if: ${{ !cancelled() }}
+        env:
+          LOG_DIR: ${{ steps.log_dirs.outputs.directory }}
+        run: |
+          cp open-source-firmware-validation/log.html $LOG_DIR/log.html
+  cleanup:
+    name: Cleanup
+    if: always()
+    needs: run-tests
+    runs-on:
+      labels: dts-builder
+    steps:
+      - name: Cleanup after tests
+        shell: bash
+        run: |
+          rm -rf $GITHUB_WORKSPACE; mkdir $GITHUB_WORKSPACE
diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
@@ -7,52 +7,11 @@ on:
 
 jobs:
   build:
-    name: Build system image without using cache
-    runs-on:
-      labels: dts-builder
-    steps:
-      - name: Prepare SSH key
-        shell: bash
-        env:
-          SSH_KEY: ${{secrets.SSH_KEY}}
-        run: |
-          echo -e ${SSH_KEY} > ~/.ssh/dts-ci-key
-          chmod 600 ~/.ssh/dts-ci-key
-      - name: Checkout meta-dts repo
-        uses: actions/checkout@v2
-        with:
-          path: "meta-dts"
-      - name: Prepare cache-less build configuration
-        shell: bash
-        run: |
-          sed -i '/cache.yml/d' meta-dts/kas.yml
-      - name: Build DTS image
-        shell: bash
-        id: build_image
-        run: |
-          for attempt in {1..5}; do
-            if kas-container build meta-dts/kas.yml; then
-              echo "Build command succeeded on attempt $attempt"
-              break
-            else
-              echo "Build command failed on attempt $attempt"
-              if [ $attempt -lt 5 ]; then
-                sleep 5
-              fi
-            fi
-          done
-        continue-on-error: true
-      - name: Report build command
-        run: |
-          if [ ${{ steps.build_image.outcome }} == 'failure' ]; then
-            echo "All build attempts failed."
-            exit 1
-          else
-            echo "At least one build attempt succeeded."
-          fi
+    uses: ./.github/workflows/build.yml
+    with:
+      cacheless: true
   deploy-cache:
     name: Deploy cache on cache.dasharo.com
-    if: always()
     needs: build
     runs-on:
       labels: dts-builder

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 default_stages: [pre-commit]
 
 default_install_hook_types: [pre-commit, commit-msg]
-exclude: .*\.(patch)|(diff)$
+exclude: .*\.(patch)|(diff)$|CHANGELOG.md
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks