CI improvements

.github/workflows/build.yml

@@ -62,12 +62,16 @@ jobs:
           cp configs/config.${{ matrix.vendor }}_${{ matrix.model }} .config
           make olddefconfig
           make
+      - name: Create patched binary with pre-enabled serial output
+        run: |
+          cp build/coreboot.rom build/coreboot_serial_enabled.rom
+          build/util/cbfstool/cbfstool build/coreboot_serial_enabled.rom write -r SMMSTORE -f ci/smmstore-serial-enabled
       - name: Save artifacts
         uses: actions/upload-artifact@v2
         with:
           name: "dasharo-${{ matrix.vendor }}-${{ matrix.model }}-${{ matrix.build }}"
           path: |
-            build/coreboot.rom
+            build/coreboot_*.rom
           retention-days: 30
   build_protectli:
     environment: Protectli
@@ -97,3 +101,57 @@ jobs:
           path: |
             build/coreboot.rom
           retention-days: 30
+  test_msi:
+    runs-on: self-hosted
+    strategy:
+      matrix:
+        vendor: [ msi ]
+        model: [ ms7d25_ddr4, ms7d25_ddr5, ms7e06_ddr5 ] # no ms7e06_ddr4 in lab yet
+    needs: [ build_msi ]
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Fetch CI script
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: ci
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout test repository
+        uses: actions/checkout@v4
+        with:
+          repository: Dasharo/open-source-firmware-validation
+          ref: msi_fixes
+          path: validation
+          submodules: true
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r validation/requirements.txt
+
+      - name: Fetch latest firmware
+        uses: actions/download-artifact@v3
+        with:
+          name: dasharo-${{ matrix.vendor }}-${{ matrix.model }}-${{ matrix.build }}
+          path: firmware
+
+      - name: Flash firmware
+        shell: bash
+        run: |
+          fw_file="firmware/coreboot_serial_enabled.rom"
+          ./ci/ci.sh -r validation -v "${{ matrix.vendor }}" -m "${{ matrix.model }}" -f "$fw_file" flash
+
+      - name: Run tests
+        shell: bash
+        run: ./ci/ci.sh -r validation -v "${{ matrix.vendor }}" -m "${{ matrix.model }}" test
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-${{ matrix.vendor }}-${{ matrix.model }}
+          path: test-results-${{ matrix.vendor }}-${{ matrix.model }}
+        if: always()

ci/ci.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir="$(dirname "${BASH_SOURCE[0]}")"
+
+vendor=""
+model=""
+
+usage() {
+    echo "Usage: ci.sh -v vendor -m model -f firmware command" 1>&2
+    exit 1
+}
+
+while getopts "r:v:m:f:" o; do
+    case "${o}" in
+        r) root="${OPTARG}" ;;
+        v) vendor="${OPTARG}" ;;
+        m) model="${OPTARG}" ;;
+        f) fw_file="${OPTARG}" ;;
+        *) usage ;;
+    esac
+done
+shift $((OPTIND-1))
+
+[ -n "$vendor" ] || usage
+[ -n "$model" ] || usage
+[ $# -eq 1 ] || usage
+
+command="$1"
+[[ "$command" =~ ^(flash|test)$ ]] || usage
+shift 1
+
+options=(
+    -L TRACE
+)
+
+compatibility_tests=()
+performance_tests=()
+security_tests=()
+stability_tests=()
+
+case "$vendor" in
+    msi)
+        case "$model" in
+            ms7d25_ddr4)
+                device_ip="192.168.10.183"
+                rte_ip="192.168.10.199"
+                pikvm_ip="192.168.10.16"
+                config="msi-pro-z690-a-wifi-ddr4"
+                ;;
+            ms7d25_ddr5)
+                device_ip="192.168.10.93"
+                rte_ip="192.168.10.188"
+                pikvm_ip="192.168.10.45"
+                config="msi-pro-z690-a-ddr5"
+                ;;
+            ms7e06_ddr5)
+                device_ip="192.168.10.10"
+                rte_ip="192.168.10.127"
+                pikvm_ip="192.168.10.226"
+                config="msi-pro-z790-p"
+                ;;
+            *)
+                echo unknown board: $model
+                exit 1
+                ;;
+        esac
+        ;;
+    *)
+        echo unknown vendor: $vendor
+        exit 1
+        ;;
+esac
+
+options+=(
+    -v device_ip:$device_ip
+    -v rte_ip:$rte_ip
+    -v pikvm_ip:$pikvm_ip
+    -v config:$config
+)
+
+if [ "$command" == "flash" ]; then
+    "$script_dir/scp.py" "$rte_ip" "$fw_file" /tmp/dasharo.rom
+    "$script_dir/ssh.py" "$rte_ip" /home/root/flash.sh /tmp/dasharo.rom
+    exit 0
+fi
+
+case "$vendor" in
+    msi)
+        case "$model" in
+            ms7d25_ddr4 | ms7d25_ddr5 | ms7e06_ddr5)
+                compatibility_tests+=(
+                    custom-boot-menu-key
+                    esp-scanning
+                    # Z690A DDR5 with current memmory modules do not support XMP profiles
+                    # memory-profile
+                    power-after-fail
+                )
+
+                security_tests+=(
+                    me-neuter
+                    option-rom
+                    rebar
+                )
+                ;;
+            *)
+                echo unknown board: $model
+                exit 1
+                ;;
+        esac
+        ;;
+    *)
+        echo unknown vendor: $vendor
+        exit 1
+        ;;
+esac
+
+mkdir -p "test-results-$vendor-$model"
+
+run_robot() {
+    local category="$1"
+    local test="$2"
+
+    # We want to run all tests, even if some fail
+    robot -l "test-results-$vendor-$model/dasharo-${category}.log.html" \
+          -r "test-results-$vendor-$model/dasharo-${category}.html" \
+          -o "test-results-$vendor-$model/dasharo-${category}.xml" \
+          ${options[@]} "$root/dasharo-${category}/$test.robot" || true
+}
+
+if [ "$command" == "test" ]; then
+    for test in "${compatibility_tests[@]}"; do
+        run_robot compatibility "$test"
+    done
+
+    for test in "${performance_tests[@]}"; do
+        run_robot performance "$test"
+    done
+
+    for test in "${security_tests[@]}"; do
+        run_robot security "$test"
+    done
+
+    for test in "${stability_tests[@]}"; do
+        run_robot stability "$test"
+    done
+fi

ci/scp.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import paramiko
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("host")
+    parser.add_argument("local")
+    parser.add_argument("remote")
+    args = parser.parse_args()
+
+    client = paramiko.client.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(args.host, username="root", password="meta-rte")
+
+    ftp = client.open_sftp()
+    ftp.put(args.local, args.remote)
+    ftp.close()
+
+if __name__ == '__main__':
+    main()

ci/ssh.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser, REMAINDER
+from select import select
+import paramiko
+import sys
+import os
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("host")
+    parser.add_argument("command")
+    parser.add_argument("args", nargs=REMAINDER)
+    args = parser.parse_args()
+
+    client = paramiko.client.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(args.host, username="root", password="meta-rte")
+
+    cmd = f"{args.command} "
+    for arg in args.args:
+        cmd += f'\'{arg}\' '
+
+    transport = client.get_transport()
+    channel = transport.open_session()
+    channel.exec_command(cmd)
+
+    while True:
+        if channel.exit_status_ready():
+            break
+        rl, _wl, _xl = select([channel], [], [])
+        if len(rl) > 0:
+            if channel.recv_ready():
+                data = channel.recv(1024)
+                os.write(sys.stdout.fileno(), data)
+
+            if channel.recv_stderr_ready():
+                data = channel.recv_stderr(1024)
+                os.write(sys.stderr.fileno(), data)
+
+if __name__ == '__main__':
+    main()