Add Litmus tests

.gitmodules

@@ -6,3 +6,6 @@
 	path = sw/deps/cva6-sdk
 	url = https://github.com/pulp-platform/cva6-sdk.git
 	ignore = dirty
+[submodule "sw/deps/litmus-tests"]
+	path = sw/deps/litmus-tests
+	url = https://github.com/pulp-platform/CHERI-Litmus.git

cheshire.mk

@@ -10,11 +10,13 @@ BENDER ?= bender
 
 # Caution: Questasim requires this to point to the *actual* compiler install path
 CXX_PATH := $(shell which $(CXX))
+QUESTA   ?= questa-2023.4
 
 VLOG_ARGS ?= -suppress 2583 -suppress 13314 -timescale 1ns/1ps
 
 # Common Bender flags for Cheshire RTL
 CHS_BENDER_RTL_FLAGS ?= -t rtl -t cva6 -t cv64a6_imafdcsclic_sv39
+NUM_CORES            ?= 1
 
 # Define used paths (prefixed to avoid name conflicts)
 CHS_ROOT      ?= $(shell $(BENDER) path cheshire)
@@ -61,7 +63,7 @@ chs-clean-deps:
 ######################
 
 CHS_NONFREE_REMOTE ?= [email protected]:pulp-restricted/cheshire-nonfree.git
-CHS_NONFREE_COMMIT ?= fd3526f
+CHS_NONFREE_COMMIT ?= 1deb6804931b6ded1ec282b0766d0501ff8ce734
 
 CHS_PHONY += chs-nonfree-init
 chs-nonfree-init:
@@ -85,12 +87,15 @@ $(CHS_ROOT)/hw/regs/cheshire_reg_pkg.sv $(CHS_ROOT)/hw/regs/cheshire_reg_top.sv:
 	$(REGTOOL) -r $< --outdir $(dir $@)
 
 # CLINT
-CLINTCORES ?= 1
+CLINTCORES ?= $(NUM_CORES)
 include $(CLINTROOT)/clint.mk
 $(CLINTROOT)/.generated:
 	flock -x $@ $(MAKE) clint && touch $@
 
 # OpenTitan peripherals
+$(CHS_ROOT)/hw/rv_plic.cfg.hjson: $(CHS_ROOT)/util/gen_pliccfg.py
+	$(CHS_ROOT)/util/gen_pliccfg.py --num-cores $(NUM_CORES) > $@
+
 include $(OTPROOT)/otp.mk
 $(OTPROOT)/.generated: $(CHS_ROOT)/hw/rv_plic.cfg.hjson
 	flock -x $@ sh -c "cp $< $(dir $@)/src/rv_plic/; $(MAKE) -j1 otp" && touch $@
@@ -168,6 +173,66 @@ CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/s25fs512s.v
 CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/24FC1025.v
 CHS_SIM_ALL += $(CHS_ROOT)/target/sim/vsim/compile.cheshire_soc.tcl
 
+################
+# Litmus tests #
+################
+LITMUS_NCORES     ?= 2
+LITMUS_DIR        := $(CHS_SW_DIR)/deps/litmus-tests
+LITMUS_BIN_DIR    := $(LITMUS_DIR)/binaries
+LITMUS_WORK_DIR   := $(CHS_ROOT)/work-litmus
+LITMUS_SIMLOG_DIR := $(LITMUS_WORK_DIR)/simlogs
+LITMUS_TEST_LIST  := $(LITMUS_WORK_DIR)/litmus-tests.list
+LITMUS_RESULTS    := $(LITMUS_WORK_DIR)/compare.log
+
+$(LITMUS_DIR)/.git:
+	cd $(CHS_ROOT) && git submodule update --init --recursive $(LITMUS_DIR)
+
+.PHONY: chs-build-litmus-tests
+chs-build-litmus-tests: $(LITMUS_DIR)/.git
+	cd $(LITMUS_DIR)/frontend; ./make.sh
+	cd $(LITMUS_DIR)/binaries; ./make-riscv.sh ../tests/ cheshire $(LITMUS_NCORES)
+
+$(LITMUS_WORK_DIR):
+	mkdir -p $(LITMUS_WORK_DIR)
+
+$(LITMUS_SIMLOG_DIR):
+	mkdir -p $(LITMUS_SIMLOG_DIR)
+
+$(LITMUS_TEST_LIST): $(LITMUS_WORK_DIR) $(LITMUS_SIMLOG_DIR)
+	@echo Generating $@ ...
+	@LITMUS_ROOT=$(LITMUS_DIR) LITMUS_WORK=$(LITMUS_WORK_DIR) $(CHS_ROOT)/util/litmus create_list
+
+$(LITMUS_SIMLOG_DIR)/%.log: $(LITMUS_BIN_DIR)/%.elf $(CHS_SIM_ALL)
+	@echo "Running test $(notdir $<) (Log file: $@)"
+	@cd target/sim/vsim && $(QUESTA) vsim -c -do "set PRELMODE 1; set BOOTMODE 0; set BINARY $<; source start.cheshire_soc.tcl; run -all" > $@ 2>&1
+	@echo "Finished test $<"
+
+.PHONY: chs-run-litmus-tests
+chs-run-litmus-tests: $(LITMUS_TEST_LIST)
+	$(eval LITMUS_TESTS_ELF = $(shell xargs printf '\n%s' < $(LITMUS_TEST_LIST)))
+	@echo Running $(words $(LITMUS_TESTS_ELF)) tests
+	@$(MAKE) $(addprefix $(LITMUS_SIMLOG_DIR)/, $(LITMUS_TESTS_ELF:.elf=.log))
+	@echo "Finished running litmus tests"
+
+.PHONY: chs-check-litmus-tests
+chs-check-litmus-tests:
+	$(eval export LITMUS_ROOT=$(LITMUS_DIR))
+	$(eval export LITMUS_WORK=$(LITMUS_WORK_DIR))
+	@echo "Parsing UART output from simulation logs.."
+	@$(CHS_ROOT)/util/litmus parse_uart
+	@echo "Patching UART logs.."
+	@$(CHS_ROOT)/util/litmus patch_uart
+	@echo "Concatenating logs in a single file.."
+	@$(CHS_ROOT)/util/litmus combine_logs
+	@echo "Comparing logs with reference model.."
+	@$(CHS_ROOT)/util/litmus check > $(LITMUS_RESULTS)
+	@echo "Done! Check '$(LITMUS_RESULTS)' file"
+
+.PHONY: chs-clean-litmus-tests
+chs-clean-litmus-tests:
+	rm -rf $(LITMUS_WORK_DIR)
+	cd $(LITMUS_DIR)/binaries; rm *.elf *.dump
+
 ###########
 # DRAMSys #
 ###########

hw/bootrom/cheshire_bootrom.S

@@ -5,10 +5,15 @@
 // Nicole Narr <[email protected]>
 // Christopher Reinwardt <[email protected]>
 // Paul Scheffler <[email protected]>
+// Enrico Zelioli <[email protected]>
 
-// TODO: Avoid hardcoding in addresses and offsets
+#include <regs/cheshire.h>
+#include <regs/axi_llc.h>
 
-#include "smp.h"
+// The hart that non-SMP tests should run on
+#ifndef NONSMP_HART
+#define NONSMP_HART 0
+#endif
 
 .section .text._start
 
@@ -47,7 +52,11 @@ _start:
     li x31, 0
 
     // Pause SMP harts
-    smp_pause(t0, t1)
+    li   t1, 0x8
+    csrw mie, t1
+    li   t0, NONSMP_HART
+    csrr t1, mhartid
+    bne  t0, t1, _wait_for_ipi
 
     // Init stack and global pointer with safe, linked values
     la sp, __stack_pointer$
@@ -57,56 +66,101 @@ _start:
     .option pop
 
     // If LLC present: Wait for end of BIST, then extend stack and set to all SPM
-    la t0, __base_regs
-    lw t0, 80(t0)   // regs.HW_FEATURES
-    andi t0, t0, 2  // regs.HW_FEATURES.llc
+    la   t0, __base_regs
+    lw   t0, CHESHIRE_HW_FEATURES_REG_OFFSET(t0)
+    andi t0, t0, 2  // HW_FEATURES.llc
     beqz t0, _prom_check_run
-    la t0, __base_llc
+    la   t0, __base_llc
 _wait_llc_bist:
-    lw t1, 72(t0)   // llc.BIST_STATUS_DONE_BIT
+    lw   t1, AXI_LLC_BIST_STATUS_REG_OFFSET(t0) // Check BIST status done bit
     beqz t1, _wait_llc_bist
-    li t1, -1
-    sw t1, 0(t0)    // llc.CFG_SPM_LOW
-    sw t1, 4(t0)    // llc.CFG_SPM_HIGH
-    li t1, 1
-    sw t1, 16(t0)   // llc.CFG_COMMIT
+    li   t1, -1
+    sw   t1, AXI_LLC_CFG_SPM_LOW_REG_OFFSET(t0)
+    sw   t1, AXI_LLC_CFG_SPM_HIGH_REG_OFFSET(t0)
+    li   t1, 1
+    sw   t1, AXI_LLC_COMMIT_CFG_REG_OFFSET(t0)
     // Correct stack to start at end of SPM
-    la t0, __base_regs
-    la sp, __base_spm
-    lw t0, 84(t0)   // regs.LLC_SIZE
-    add sp, sp, t0
+    la   t0, __base_regs
+    la   sp, __base_spm
+    lw   t0, CHESHIRE_LLC_SIZE_REG_OFFSET(t0)
+    add  sp, sp, t0
     addi sp, sp, -8
 
 // Enter Platform ROM if present.
 _prom_check_run:
     // Note that we have internal access to SPM here *if and only if* there is an LLC.
     la t0, __base_regs
-    lw t0, 72(t0)   // regs.PLATFORM_ROM
+    lw t0, CHESHIRE_PLATFORM_ROM_REG_OFFSET(t0)
     beqz t0, _boot
     jalr t0
 
+// Move to next stage of booting
+// 1. Write the address of next stage boot loader in Cheshire's scratch registers
+// 2. Resume execution of all other harts
 .global boot_next_stage
 boot_next_stage:
-    // Non-SMP hart: Write boot address into global scratch registers
-    la t0, __base_regs
-    sw a0, 16(t0)   // regs.SCRATCH[4]
+
+    // Non-SMP hart: write boot address into global scratch registers
+    la   t0, __base_regs
+    sw   a0, CHESHIRE_SCRATCH_4_REG_OFFSET(t0)
     srli a0, a0, 32
-    sw a0, 20(t0)   // regs.SCRATCH[5]
+    sw   a0, CHESHIRE_SCRATCH_5_REG_OFFSET(t0)
     fence
-    // Resume SMP harts
-    smp_resume(t0, t1, t2)
+
+    // Resume SMP harts: set CLINT IPI registers
+    // NOTE: this will cause CLINT to send IPIs to all cores, therefore also the
+    // non-smp hart will receive one. The following instructions make sure that
+    // all harts will wait until the IPI is received (WFI with global ie disabled),
+    // then clear the IPI in the CLINT and wait until all other harts are done with it.
+    la   t0, __base_clint
+    la   t2, __base_regs
+    lw   t2, CHESHIRE_NUM_INT_HARTS_REG_OFFSET(t2)
+    slli t2, t2, 2
+    add  t2, t0, t2  // t2 = CLINT_BASE + (n_harts * 4)
+1:
+    li   t1, 1
+    sw   t1, 0(t0)
+    addi t0, t0, 4
+    blt  t0, t2, 1b
+
+// Stall hart until IPI is raised
+_wait_for_ipi:
+
+    // Wait until this hart receives IPI
+    wfi
+    csrr t1, mip
+    andi t1, t1, 0x8
+    beqz t1, _wait_for_ipi
+
+    // Clear CLINT IPI register for this hart
+    la   t0, __base_clint
+    csrr t1, mhartid
+    slli t1, t1, 2
+    add  t1, t1, t0
+    sw   zero, 0(t1)   // *(CLINT_BASE + hart_id * 4) = 0
+
+    la   t2, __base_regs
+    lw   t2, CHESHIRE_NUM_INT_HARTS_REG_OFFSET(t2)
+    slli t2, t2, 2
+    add  t2, t0, t2    // t2 = CLINT_BASE + (n_harts * 4)
+
+    // Wait until *all* CLINT IPI registers are cleared
+1:
+    lw   t1, 0(t0)
+    bnez t1, 1b
+    addi t0, t0, 4
+    blt  t0, t2, 1b
+
+    // Jump to next stage
     // Load boot address from global scratch registers
-    la t0, __base_regs
-    lwu t1, 20(t0)  // regs.SCRATCH[5]
+    la   t0, __base_regs
+    lwu  t1, CHESHIRE_SCRATCH_5_REG_OFFSET(t0)
     slli t1, t1, 32
-    lwu t0, 16(t0)  // regs.SCRATCH[4]
-    or t0, t0, t1
-    // Store hartid to a0
-    csrr a0, mhartid
-    // Jump to boot address
-    jalr ra, 0(t0)
-    // We should never get here
-    ret
+    lwu  t0, CHESHIRE_SCRATCH_4_REG_OFFSET(t0)
+    or   t0, t0, t1
+    csrr a0, mhartid  // Store hartid to a0
+    jalr ra, 0(t0)    // Jump to boot address
+    ret               // We should never get here
 
 // Reset regs, full fence, then jump to main
 _boot:
@@ -120,9 +174,10 @@ _boot:
 .global _exit
 _exit:
     // Save the return value to scratch register 2 and wait forever
+    // Set bit 0 to signal that the execution is done.
     slli a0, a0, 1
     ori  a0, a0, 1
     la t0, __base_regs
-    sw a0, 8(t0)     // regs.SCRATCH[2]
+    sw a0, CHESHIRE_SCRATCH_2_REG_OFFSET(t0)
 1:  wfi
     j 1b

hw/rv_plic.cfg.hjson

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 // Paul Scheffler <[email protected]>
+// Enrico Zelioli <[email protected]>
+// AUTOMATICALLY GENERATED by gen_pliccfg.py; edit the script instead.
 
 {
     instance_name: "rv_plic",

sw/boot/zsl.c

@@ -14,6 +14,7 @@
 #include "gpt.h"
 #include "dif/uart.h"
 #include "printf.h"
+#include "smp.h"
 
 // Type for firmware payload
 typedef int (*payload_t)(uint64_t, uint64_t, uint64_t);
@@ -51,40 +52,48 @@ static inline void load_part_or_spin(void *priv, const uint64_t *pguid, void *co
 }
 
 int main(void) {
-    // Get system parameters
-    uint32_t bootmode = *reg32(&__base_regs, CHESHIRE_BOOT_MODE_REG_OFFSET);
-    uint32_t rtc_freq = *reg32(&__base_regs, CHESHIRE_RTC_FREQ_REG_OFFSET);
-    uint64_t core_freq = clint_get_core_freq(rtc_freq, 2500);
-    rgp = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_3_REG_OFFSET);
-    uint32_t read = *reg32(&__base_regs, CHESHIRE_SCRATCH_0_REG_OFFSET);
-    void *priv = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_1_REG_OFFSET);
 
-    // Initialize UART
-    uart_init(&__base_uart, core_freq, __BOOT_BAUDRATE);
+    uint64_t hart_id = get_mhartid();
 
-    // Print boot-critical cat, and also parameters
-    printf(" /\\___/\\       Boot mode:       %d\r\n"
-           "( o   o )      Real-time clock: %d Hz\r\n"
-           "(  =^=  )      System clock:    %d Hz\r\n"
-           "(        )     Read global ptr: 0x%08x\r\n"
-           "(    P    )    Read pointer:    0x%08x\r\n"
-           "(  U # L   )   Read argument:   0x%08x\r\n"
-           "(    P      )\r\n"
-           "(           ))))))))))\r\n\r\n",
-           bootmode, rtc_freq, core_freq, rgp, read, priv);
+    if (hart_id == 0) {
+        // Get system parameters
+        uint32_t bootmode = *reg32(&__base_regs, CHESHIRE_BOOT_MODE_REG_OFFSET);
+        uint32_t rtc_freq = *reg32(&__base_regs, CHESHIRE_RTC_FREQ_REG_OFFSET);
+        uint64_t core_freq = clint_get_core_freq(rtc_freq, 2500);
+        rgp = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_3_REG_OFFSET);
+        uint32_t read = *reg32(&__base_regs, CHESHIRE_SCRATCH_0_REG_OFFSET);
+        void *priv = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_1_REG_OFFSET);
 
-    // If this is a GPT disk boot, load payload and device tree
-    if (read & 1) {
-        rread = (gpt_read_t)(void *)(uintptr_t)(read & ~1);
-        load_part_or_spin(priv, __BOOT_DTB_TYPE_GUID, __BOOT_ZSL_DTB, "device tree", 64);
-        load_part_or_spin(priv, __BOOT_FW_TYPE_GUID, __BOOT_ZSL_FW, "firmware", 8192);
+        // Initialize UART
+        uart_init(&__base_uart, core_freq, __BOOT_BAUDRATE);
+
+        // Print boot-critical cat, and also parameters
+        printf(" /\\___/\\       Boot mode:       %d\r\n"
+               "( o   o )      Real-time clock: %d Hz\r\n"
+               "(  =^=  )      System clock:    %d Hz\r\n"
+               "(        )     Read global ptr: 0x%08x\r\n"
+               "(    P    )    Read pointer:    0x%08x\r\n"
+               "(  U # L   )   Read argument:   0x%08x\r\n"
+               "(    P      )\r\n"
+               "(           ))))))))))\r\n\r\n",
+               bootmode, rtc_freq, core_freq, rgp, read, priv);
+
+        // If this is a GPT disk boot, load payload and device tree
+        if (read & 1) {
+            rread = (gpt_read_t)(void *)(uintptr_t)(read & ~1);
+            load_part_or_spin(priv, __BOOT_DTB_TYPE_GUID, __BOOT_ZSL_DTB, "device tree", 64);
+            load_part_or_spin(priv, __BOOT_FW_TYPE_GUID, __BOOT_ZSL_FW, "firmware", 8192);
+        }
+
+        // Launch payload
+        printf("[ZSL] Launch firmware at %lx with device tree at %lx\r\n", __BOOT_ZSL_FW,
+               __BOOT_ZSL_DTB);
+        smp_resume();
     }
 
-    // Launch payload
     payload_t fw = __BOOT_ZSL_FW;
-    printf("[ZSL] Launch firmware at %lx with device tree at %lx\r\n", fw, __BOOT_ZSL_DTB);
     fencei();
-    return fw(0, (uintptr_t)__BOOT_ZSL_DTB, 0);
+    return fw(hart_id, (uintptr_t)__BOOT_ZSL_DTB, 0);
 }
 
 // On trap, report relevant CSRs and spin