Skip to content

Commit

Permalink
Add Litmus tests
Browse files Browse the repository at this point in the history
  • Loading branch information
ezelioli committed Dec 2, 2024
1 parent 9e08280 commit f0141b1
Show file tree
Hide file tree
Showing 3 changed files with 179 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@
path = sw/deps/cva6-sdk
url = https://github.com/pulp-platform/cva6-sdk.git
ignore = dirty
[submodule "sw/deps/litmus-tests"]
path = sw/deps/litmus-tests
url = https://github.com/pulp-platform/CHERI-Litmus.git
60 changes: 60 additions & 0 deletions cheshire.mk
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ BENDER ?= bender

# Caution: Questasim requires this to point to the *actual* compiler install path
CXX_PATH := $(shell which $(CXX))
QUESTA := questa-2023.4

VLOG_ARGS ?= -suppress 2583 -suppress 13314 -timescale 1ns/1ps

Expand Down Expand Up @@ -168,6 +169,65 @@ CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/s25fs512s.v
CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/24FC1025.v
CHS_SIM_ALL += $(CHS_ROOT)/target/sim/vsim/compile.cheshire_soc.tcl

################
# Litmus tests #
################
LITMUS_NCORES ?= 2
LITMUS_DIR := $(CHS_SW_DIR)/deps/litmus-tests
LITMUS_BIN_DIR := $(LITMUS_DIR)/binaries
LITMUS_WORK_DIR := $(CHS_ROOT)/work-litmus
LITMUS_SIMLOG_DIR := $(LITMUS_WORK_DIR)/simlogs
LITMUS_TEST_LIST := $(LITMUS_WORK_DIR)/litmus-tests.list
LITMUS_RESULTS := $(LITMUS_WORK_DIR)/compare.log

$(LITMUS_DIR)/.git:
cd $(CHS_ROOT) && git submodule update --init --recursive $(LITMUS_DIR)

.PHONY: chs-build-litmus-tests
chs-build-litmus-tests: $(LITMUS_DIR)/.git
cd $(LITMUS_DIR)/frontend; ./make.sh
cd $(LITMUS_DIR)/binaries; ./make-riscv.sh ../tests/ cheshire $(LITMUS_NCORES)

$(LITMUS_WORK_DIR):
mkdir -p $(LITMUS_WORK_DIR)

$(LITMUS_SIMLOG_DIR):
mkdir -p $(LITMUS_SIMLOG_DIR)

$(LITMUS_TEST_LIST): $(LITMUS_WORK_DIR) $(LITMUS_SIMLOG_DIR)
@echo Generating $@ ...
@LITMUS_ROOT=$(LITMUS_DIR) LITMUS_WORK=$(LITMUS_WORK_DIR) $(CHS_ROOT)/util/litmus create_list

$(LITMUS_SIMLOG_DIR)/%.log: $(LITMUS_BIN_DIR)/%.elf $(CHS_SIM_ALL)
@echo "Running test $(notdir $<) (Log file: $@)"
@cd target/sim/vsim && $(QUESTA) vsim -c -do "set PRELMODE 1; set BOOTMODE 0; set BINARY $<; source start.cheshire_soc.tcl; run -all" > $@ 2>&1
@echo "Finished test $<"

.PHONY: chs-run-litmus-tests
chs-run-litmus-tests: $(LITMUS_TEST_LIST)
$(eval LITMUS_TESTS_ELF = $(shell xargs printf '\n%s' < $(LITMUS_TEST_LIST)))
@echo Running $(words $(LITMUS_TESTS_ELF)) tests
@$(MAKE) $(addprefix $(LITMUS_SIMLOG_DIR)/, $(LITMUS_TESTS_ELF:.elf=.log))
@echo "Finished running litmus tests"

.PHONY: chs-check-litmus-tests
chs-check-litmus-tests:
$(eval export LITMUS_ROOT=$(LITMUS_DIR))
$(eval export LITMUS_WORK=$(LITMUS_WORK_DIR))
@echo "Parsing UART output from simulation logs.."
@$(CHS_ROOT)/util/litmus parse_uart
@echo "Patching UART logs.."
@$(CHS_ROOT)/util/litmus patch_uart
@echo "Concatenating logs in a single file.."
@$(CHS_ROOT)/util/litmus combine_logs
@echo "Comparing logs with reference model.."
@$(CHS_ROOT)/util/litmus check > $(LITMUS_RESULTS)
@echo "Done! Check '$(LITMUS_RESULTS)' file"

chs-clean-litmus-tests:
rm -rf $(LITMUS_WORK_DIR)
cd $(LITMUS_DIR)/binaries; rm *.elf *.dump

###########
# DRAMSys #
###########
Expand Down
116 changes: 116 additions & 0 deletions util/litmus
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/bin/bash

########################################################
## Post-processing of Litmus tests simulation output. ##
## Call any of the defined utility functions. ##
########################################################

USAGE="USAGE: $0 <command>\nFor a list of commands available, run: $0 --help\n"
COMMANDS="show_vars create_list parse_uart patch_uart combine_logs check cleanup_simlogs"

CWD=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CHS_ROOT=$(cd ${CWD}/..; pwd)

## Set helper variables to default values if not specified
set_vars () {
[ -z ${LITMUS_ROOT} ] && LITMUS_ROOT=${CHS_ROOT}/sw/tests/riscv-litmus-tests
[ -z ${LITMUS_WORK} ] && LITMUS_WORK=${CHS_ROOT}/work-litmus
[ -z ${LITMUS_LIST} ] && LITMUS_LIST=${LITMUS_WORK}/litmus-tests.list
[ -z ${LITMUS_SIMLOGS} ] && LITMUS_SIMLOGS=${LITMUS_WORK}/simlogs
[ -z ${LITMUS_UART} ] && LITMUS_UART=${LITMUS_WORK}/uart
[ -z ${LITMUS_LOGS} ] && LITMUS_LOGS=${LITMUS_WORK}/logs
[ -z ${LITMUS_LOG} ] && LITMUS_LOG=${LITMUS_WORK}/litmus.log
}

## Print all helper variables (for debug purposes)
litmus_show_vars () {
echo "CHS_ROOT = ${CHS_ROOT}"
echo "LITMUS_WORK = ${LITMUS_WORK}"
echo "LITMUS_SIMLOGS = ${LITMUS_SIMLOGS}"
echo "LITMUS_UART = ${LITMUS_UART}"
echo "LITMUS_LOGS = ${LITMUS_LOGS}"
}

## Write all names of litmus tests binaries in `${LITMUS_LIST}` file
litmus_create_list () {
[ -f ${LITMUS_LIST} ] && rm ${LITMUS_LIST}
touch ${LITMUS_LIST}
for f in $(find ${LITMUS_ROOT}/binaries/ -name "*.elf"); do
# f=$(echo $f | sed 's/\[/\\\[/g') # add backslashes before brackets
f=$(basename $f)
echo $f >> ${LITMUS_LIST}
done
}

## Extract UART log from simulation transcripts
litmus_parse_uart () {
mkdir -p ${LITMUS_UART}
for file in $(ls ${LITMUS_SIMLOGS}/*.log); do
# Extract test name from file path
IFS='/' read -ra filename <<< "${file}"
filename=${filename[-1]} # remove basename
testname=${filename::-4} # remove ".log" at the end
sed -n 's/^# \[UART\] \(.*\)/\1/p' < ${file} > ${LITMUS_UART}/${filename}.uart.log
done
}

## Patch the UART output with header and trailer
litmus_patch_uart () {
mkdir -p ${LITMUS_LOGS}
for file in $(ls ${LITMUS_UART}/*.uart.log); do
# Extract test name from file path
IFS='/' read -ra filename <<< "${file}"
filename=${filename[-1]} # remove basename
testname=${filename::-9} # remove ".uart.log" at the end
outfile="${LITMUS_LOGS}/${testname}.litmus.log"
echo "Test $(basename ${testname} .log) Allowed" > ${outfile}
echo "Histogram" >> ${outfile}
cat ${file} >> ${outfile}
echo "" >> ${outfile}
done
}

## Combine all log files in `${LITMUS_LOGS}` directory within a single log file
litmus_combine_logs () {
[ -f ${LITMUS_LOG} ] && rm ${LITMUS_LOG}
for file in $(ls ${LITMUS_LOGS}/*); do
cat ${file} >> ${LITMUS_LOG}
done
}

## Compare the Litmus tests logs with the reference model
litmus_check () {
cd ${LITMUS_ROOT} && MCMP7=/home/nwistoff/.opam/centos/bin/mcompare7 LITMUS_LOG=${LITMUS_LOG} ./ci/compare_model.sh
}

## Clean up incomplete simulation log files in `${LITMUS_WORK}` directory.
## This can be useful in case some simulations failed (e.g. due to insufficient disk space).
litmus_cleanup_simlogs () {
for file in $(ls ${LITMUS_WORK}/*.log); do
grep -e \$finish ${file} > /tmp/null
[ $? == 0 ] || rm ${file}
done
}

######################
## Parse parameters ##
######################

if [ $# -lt 1 ]; then
printf "${USAGE}"
exit 1
fi

case $1 in
-h | --help)
printf "Commands available:\n"
for cmd in ${COMMANDS}; do echo "- ${cmd}"; done
;;
*)
CMD="litmus_$1"
;;
esac

## Run specified command
set_vars
eval ${CMD}

0 comments on commit f0141b1

Please sign in to comment.