IPC-based verification framework (#45)

* test: Add IPC verification framework

* test: Remove broken inline asm

* docker: Deploy containers per branch for testing

.github/workflows/build-docker.yml

@@ -27,6 +27,6 @@ jobs:
           context: .
           file: util/container/Dockerfile
           push: true
-          tags: ghcr.io/pulp-platform/snitch_cluster:latest
+          tags: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
           build-args: |-
             SNITCH_LLVM_VERSION=latest

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     name: Build documentation
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster
+      image: ghcr.io/pulp-platform/snitch_cluster:ipc-verification
     steps:
       - uses: actions/checkout@v2
       - name: Build docs
@@ -29,7 +29,7 @@ jobs:
     name: Simulate SW on Snitch Cluster w/ Verilator
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster
+      image: ghcr.io/pulp-platform/snitch_cluster:ipc-verification
     steps:
       - uses: actions/checkout@v2
         with:
@@ -43,8 +43,8 @@ jobs:
       - name: Run Tests
         working-directory: target/snitch_cluster
         run: |-
-          ./sw/tests/run.py sw/tests/passing-apps.list --simulator verilator
-          ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator
+          ./sw/tests/run.py sw/tests/run.yaml --simulator verilator
+          ./sw/apps/run.py sw/apps/run.yaml --simulator verilator
 
   ############################################
   # Build SW on Snitch Cluster w/ Banshee #
@@ -54,7 +54,7 @@ jobs:
     name: Simulate SW on Snitch Cluster w/ Banshee
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster
+      image: ghcr.io/pulp-platform/snitch_cluster:ipc-verification
     steps:
       - uses: actions/checkout@v2
         with:
@@ -67,5 +67,5 @@ jobs:
           SNITCH_LOG: info
         working-directory: target/snitch_cluster
         run: |-
-          ./sw/tests/run.py sw/tests/banshee-apps.list --simulator banshee
-          ./sw/apps/run.py sw/apps/passing-apps.list --simulator banshee
+          ./sw/tests/run.py sw/tests/run.yaml --simulator banshee
+          ./sw/apps/run.py sw/apps/run.yaml --simulator banshee

.gitlab-ci.yml

@@ -93,8 +93,8 @@ snitch-cluster-vlt:
   script:
     - cd target/snitch_cluster
     - $VERILATOR make bin/snitch_cluster.vlt
-    - $VERILATOR ./sw/tests/run.py sw/tests/passing-apps.list --simulator verilator
-    - $VERILATOR ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator
+    - $VERILATOR ./sw/tests/run.py sw/tests/run.yaml --simulator verilator
+    - $VERILATOR ./sw/apps/run.py sw/apps/run.yaml --simulator verilator
   # yamllint enable rule:line-length
 
 # VCS
@@ -103,17 +103,17 @@ snitch-cluster-vcs:
   script:
     - cd target/snitch_cluster
     - $VCS make bin/snitch_cluster.vcs
-    - $VCS ./sw/tests/run.py sw/tests/passing-apps.list --simulator vcs
-    - $VCS ./sw/apps/run.py sw/apps/passing-apps.list --simulator vcs
+    - $VCS ./sw/tests/run.py sw/tests/run.yaml --simulator vcs
+    - $VCS ./sw/apps/run.py sw/apps/run.yaml --simulator vcs
 
 # Questa
 snitch-cluster-vsim:
   needs: [snitch-cluster-sw]
   script:
     - cd target/snitch_cluster
     - $QUESTA make bin/snitch_cluster.vsim
-    - $QUESTA ./sw/tests/run.py sw/tests/passing-apps.list --simulator vsim
-    - $QUESTA ./sw/apps/run.py sw/apps/passing-apps.list --simulator vsim
+    - $QUESTA ./sw/tests/run.py sw/tests/run.yaml --simulator vsim
+    - $QUESTA ./sw/apps/run.py sw/apps/run.yaml --simulator vsim
 
 # Banshee
 snitch-cluster-banshee:
@@ -129,5 +129,5 @@ snitch-cluster-banshee:
     - cd banshee
     - cargo install --debug --path .
     - cd ../target/snitch_cluster
-    - ./sw/tests/run.py sw/tests/banshee-apps.list --simulator banshee
-    - ./sw/apps/run.py sw/apps/passing-apps.list --simulator banshee
+    - ./sw/tests/run.py sw/tests/run.yaml --simulator banshee
+    - ./sw/apps/run.py sw/apps/run.yaml --simulator banshee

python-requirements.txt

@@ -18,6 +18,7 @@ pyyaml
 pytablewriter
 termcolor
 pandas
+pyelftools
 
 -r docs/requirements.txt
 -r sw/dnn/requirements.txt

sw/blas/axpy/Makefile

@@ -15,12 +15,15 @@ APP     ?= axpy
 SRCS    ?= $(SRC_DIR)/main.c
 INCDIRS += $(DATA_DIR) $(SRC_DIR)
 
-$(DATA_DIR)/data.h: $(DATA_DIR)/datagen.py
+DATAGEN_PY = $(DATA_DIR)/datagen.py
+DATA_H     = $(DATA_DIR)/data.h
+
+$(DATA_H): $(DATAGEN_PY)
 	$< $(LENGTH) > $@
 
 .PHONY: clean-data clean
 
 clean-data:
-	rm -f $(DATA_DIR)/data.h
+	rm -f $(DATA_H)
 
 clean: clean-data

sw/blas/axpy/data/datagen.py

@@ -8,27 +8,24 @@
 import sys
 import argparse
 import numpy as np
+import os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
+from data_utils import format_scalar_definition, format_vector_definition, \
+                       format_vector_declaration, format_ifdef_wrapper  # noqa: E402
 
 MIN = -1000
 MAX = +1000
 
+# Aligns data to the size of a beat to avoid misaligned transfers
+BEAT_ALIGNMENT = 64
+# AXI splits bursts crossing 4KB address boundaries. To minimize
+# the occurrence of these splits the data should be aligned to 4KB
+BURST_ALIGNMENT = 4096
 
-def format_vector_definition(id, vector):
-    s = f'double {id}[{len(vector)}] = ' + '{\n'
-    for el in vector:
-        s += f'\t{el},\n'
-    s += '};'
-    return s
-
-
-def format_vector_declaration(id, vector):
-    s = f'double {id}[{len(vector)}];'
-    return s
 
-
-def format_scalar_definition(id, scalar, typ):
-    s = f'{typ} {id} = {scalar};'
-    return s
+def golden_model(a, x, y):
+    return a*x + y
 
 
 def main():
@@ -38,23 +35,29 @@ def main():
         'length',
         type=int,
         help='Vector length')
+    parser.add_argument(
+        '--section',
+        type=str,
+        help='Section to store vectors in')
     args = parser.parse_args()
     length = args.length
+    section = args.section
 
     # Randomly generate inputs
     a = np.random.uniform(MIN, MAX, 1)
     x = np.random.uniform(MIN, MAX, length)
     y = np.random.uniform(MIN, MAX, length)
     z = np.zeros(length)
-    g = a*x + y
+    g = golden_model(a, x, y)
 
     # Format header file
-    l_str = format_scalar_definition('l', length, 'uint32_t')
-    a_str = format_scalar_definition('a', a[0], 'double')
-    x_str = format_vector_definition('x', x)
-    y_str = format_vector_definition('y', y)
-    z_str = format_vector_declaration('z', z)
-    g_str = format_vector_definition('g', g)
+    l_str = format_scalar_definition('const uint32_t', 'l', length)
+    a_str = format_scalar_definition('const double', 'a', a[0])
+    x_str = format_vector_definition('double', 'x', x, alignment=BURST_ALIGNMENT, section=section)
+    y_str = format_vector_definition('double', 'y', y, alignment=BURST_ALIGNMENT, section=section)
+    z_str = format_vector_declaration('double', 'z', z, alignment=BURST_ALIGNMENT, section=section)
+    g_str = format_vector_definition('double', 'g', g)
+    g_str = format_ifdef_wrapper('BIST', g_str)
     f_str = '\n\n'.join([l_str, a_str, x_str, y_str, z_str, g_str])
     f_str += '\n'
 

sw/blas/axpy/src/axpy.h

@@ -19,7 +19,11 @@ inline void axpy(uint32_t l, double a, double* x, double* y, double* z) {
 
 #else
 
-    snrt_ssr_loop_1d(SNRT_SSR_DM_ALL, frac, sizeof(double));
+    // TODO(colluca): revert once Banshee supports SNRT_SSR_DM_ALL
+    // snrt_ssr_loop_1d(SNRT_SSR_DM_ALL, frac, sizeof(double));
+    snrt_ssr_loop_1d(SNRT_SSR_DM0, frac, sizeof(double));
+    snrt_ssr_loop_1d(SNRT_SSR_DM1, frac, sizeof(double));
+    snrt_ssr_loop_1d(SNRT_SSR_DM2, frac, sizeof(double));
 
     snrt_ssr_read(SNRT_SSR_DM0, SNRT_SSR_1D, x + offset);
     snrt_ssr_read(SNRT_SSR_DM1, SNRT_SSR_1D, y + offset);

sw/blas/axpy/src/main.c

@@ -9,7 +9,6 @@
 #include "data.h"
 
 int main() {
-    uint32_t nerr = 0;
     double *local_x, *local_y, *local_z;
 
     // Allocate space in TCDM
@@ -35,12 +34,24 @@ int main() {
 
     snrt_cluster_hw_barrier();
 
-    // Check computation is correct
+    // Copy data out of TCDM
     if (snrt_is_dm_core()) {
+        size_t size = l * sizeof(double);
+        snrt_dma_start_1d(z, local_z, size);
+    }
+
+#ifdef BIST
+    uint32_t nerr = l;
+
+    // Check computation is correct
+    if (snrt_global_core_idx() == 0) {
         for (int i = 0; i < l; i++) {
-            if (local_z[i] != g[i]) nerr++;
+            if (local_z[i] == g[i]) nerr--;
         }
     }
 
     return nerr;
+#endif
+
+    return 0;
 }

sw/blas/axpy/verify.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <[email protected]>
+
+import sys
+from pathlib import Path
+import numpy as np
+from data.datagen import golden_model
+
+sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
+import verification  # noqa: E402
+from elf import Elf  # noqa: E402
+from data_utils import bytes_to_doubles  # noqa: E402
+
+
+ERR_THRESHOLD = 1E-10
+
+
+def main():
+    # Run simulation and get outputs
+    args = verification.parse_args()
+    raw_results = verification.simulate(sim_bin=args.sim_bin,
+                                        snitch_bin=args.snitch_bin,
+                                        log=args.log,
+                                        output_uids=['z'])
+    z_actual = np.array(bytes_to_doubles(raw_results['z']))
+
+    # Extract input operands from ELF file
+    elf = Elf(args.snitch_bin)
+    a = np.array(bytes_to_doubles(elf.get_symbol_contents('a')))
+    x = np.array(bytes_to_doubles(elf.get_symbol_contents('x')))
+    y = np.array(bytes_to_doubles(elf.get_symbol_contents('y')))
+
+    # Verify results
+    z_golden = golden_model(a, x, y)
+    relative_err = np.absolute((z_golden - z_actual) / z_golden)
+    fail = np.any(relative_err > ERR_THRESHOLD)
+
+    return int(fail)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sw/tests/non_null_exitcode.c

@@ -1,9 +1,11 @@
 // Copyright 2023 ETH Zurich and University of Bologna.
 // Licensed under the Apache License, Version 2.0, see LICENSE for details.
 // SPDX-License-Identifier: Apache-2.0
-
-// Simply returns a return code different from 0.
+//
+// Luca Colagrande <[email protected]>
+//
+// Simply returns an exit code different from 0.
 // Should be used as a test to check that the simulator or whoever
 // is running the program actually captures an error when it occurs.
 
-int main() { return 1; }
+int main() { return 14; }

target/common/common.mk

@@ -152,15 +152,15 @@ define QUESTASIM
 	@echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@
 	@echo '${VSIM} +permissive ${VSIM_FLAGS} -work ${MKFILE_DIR}/${VSIM_BUILDDIR} -c \
 				-ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \
-				$1 +permissive-off ++$$binary' >> $@
+				$1 +permissive-off ++$$binary ++$$2' >> $@
 	@chmod +x $@
 	@echo "#!/bin/bash" > [email protected]
 	@echo 'binary=$$(pwd)/$$1' >> $@.gui
 	@echo 'cd ${MKFILE_DIR}' >> $@.gui
 	@echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@.gui
 	@echo '${VSIM} +permissive ${VSIM_FLAGS} -work ${MKFILE_DIR}/${VSIM_BUILDDIR} \
 				-ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \
-				$1 +permissive-off ++$$binary' >> $@.gui
+				$1 +permissive-off ++$$binary ++$$2' >> $@.gui
 	@chmod +x $@.gui
 endef