[Draft] ✨ 🐛 Bug fixes and vstart CSR support

.github/workflows/ci.yml

@@ -235,6 +235,7 @@ jobs:
   compile-ara:
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["tc-verilator", "tc-isa-sim"]

.gitmodules

@@ -21,9 +21,6 @@
 [submodule "hardware/deps/common_verification"]
 	path = hardware/deps/common_verification
 	url = https://github.com/pulp-platform/common_verification.git
-[submodule "hardware/deps/cva6"]
-	path = hardware/deps/cva6
-	url = https://github.com/pulp-platform/cva6.git
 [submodule "toolchain/newlib"]
 	path = toolchain/newlib
 	url = https://sourceware.org/git/newlib-cygwin.git
@@ -32,3 +29,9 @@
 	path = toolchain/riscv-llvm
 	url = https://github.com/llvm/llvm-project.git
 	ignore = dirty
+[submodule "hardware/deps/apb"]
+	path = hardware/deps/apb
+	url = https://github.com/pulp-platform/apb.git
+[submodule "hardware/deps/cva6"]
+	path = hardware/deps/cva6
+	url = [email protected]:MaistoV/cva6_fork.git

Bender.lock

@@ -1,36 +1,62 @@
 ---
 packages:
+  apb:
+    revision: 77ddf073f194d44b9119949d2421be59789e69ae
+    version: 0.2.4
+    source:
+      Git: "https://github.com/pulp-platform/apb.git"
+    dependencies:
+      - common_cells
   axi:
-    revision: 442ff3375710513623f95944d66cc2bd09b2f155
-    version: 0.29.1
+    revision: 9251564ed67e3e71adf46dbeba62ef4435d2524c
+    version: 0.31.1
     source:
       Git: "https://github.com/pulp-platform/axi.git"
     dependencies:
       - common_cells
       - common_verification
   common_cells:
-    revision: 015917ff33e5f944e866814f72f2074fb0f4220f
-    version: 1.22.1
+    revision: 53b0b58af2db5bd3c850a7038fae170ed78326bb
+    version: 1.31.1
     source:
       Git: "https://github.com/pulp-platform/common_cells.git"
     dependencies:
       - common_verification
       - tech_cells_generic
   common_verification:
-    revision: 6fc76fb013315af9fabbb90b431863d498df2d6d
-    version: 0.2.0
+    revision: 9c07fa860593b2caabd9b5681740c25fac04b878
+    version: 0.2.3
     source:
       Git: "https://github.com/pulp-platform/common_verification.git"
     dependencies: []
   cva6:
-    revision: 3245e44ec49c1cdcd19eb298cd81f0672eaf81ca
+    revision: 5e2e520696aa63545b91fca38ce340314291be5c
     version: ~
     source:
-      Git: "https://github.com/pulp-platform/cva6.git"
-    dependencies: []
+      Git: "https://github.com/MaistoV/cva6_fork.git"
+    dependencies:
+      - axi
+      - common_cells
+      - fpnew
+      - tech_cells_generic
+  fpnew:
+    revision: 3116391bf66660f806b45e212b9949c528b4e270
+    version: 0.7.0
+    source:
+      Git: "https://github.com/openhwgroup/cvfpu.git"
+    dependencies:
+      - common_cells
+      - fpu_div_sqrt_mvp
+  fpu_div_sqrt_mvp:
+    revision: 86e1f558b3c95e91577c41b2fc452c86b04e85ac
+    version: 1.0.4
+    source:
+      Git: "https://github.com/pulp-platform/fpu_div_sqrt_mvp.git"
+    dependencies:
+      - common_cells
   tech_cells_generic:
-    revision: 203038f857158ae4634c47ce0281f402cc2a1344
-    version: 0.2.4
+    revision: 298b7297d220ba2601d0f24f684f97ff32f61123
+    version: 0.2.12
     source:
       Git: "https://github.com/pulp-platform/tech_cells_generic.git"
     dependencies:

Bender.yml

@@ -10,8 +10,9 @@ package:
 dependencies:
   axi:                { git: "https://github.com/pulp-platform/axi.git",                version: 0.29.1 }
   common_cells:       { git: "https://github.com/pulp-platform/common_cells.git",       version: 1.22.1 }
-  cva6:               { git: "https://github.com/pulp-platform/cva6.git",               rev: acc_port   }
+  cva6:               { git: "https://github.com/MaistoV/cva6_fork.git",              rev: ara_cheshire }
   tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.1  }
+  apb:                { git: "https://github.com/pulp-platform/apb.git",                version: 0.2.4  }
 
 workspace:
   checkout_dir: "hardware/deps"
@@ -27,7 +28,6 @@ sources:
 
     # Sources
     # Level 1
-    - hardware/src/axi_to_mem.sv
     - hardware/src/ctrl_registers.sv
     - hardware/src/cva6_accel_first_pass_decoder.sv
     - hardware/src/ara_dispatcher.sv

apps/.gitignore

@@ -1,2 +1,4 @@
 bin
 common/link.ld
+*.o*
+data.S*

apps/Makefile

@@ -21,6 +21,12 @@ APPS_DIR := $(ROOT_DIR)
 COMMON_DIR := $(ROOT_DIR)/common
 TESTS_DIR := $(ROOT_DIR)/riscv-tests/isa
 
+# Build environment for Linux
+LINUX ?= 0
+ifeq ($(LINUX), 1)
+include $(COMMON_DIR)/linux.mk
+endif
+
 # This will overwrite the ROOT_DIR variable from the included makefile
 include $(COMMON_DIR)/runtime.mk
 include $(COMMON_DIR)/riscv_tests.mk
@@ -33,9 +39,9 @@ BINARIES := $(filter-out bin/benchmarks, $(addprefix bin/,$(APPS)))
 CVA6_EXTENSIONS := rv64ui rv64uc rv64um rv64uf rv64ud rv64si
 # Atomics are messy, since there is currently no memory region capable of handling them
 # CVA6_EXTENSIONS := rv64ua
-CVA6_BINARIES := $(addprefix bin/, $(cva6_tests))
+CVA6_BINARIES := $(addsuffix $(IS_LINUX_EXTENSION), $(addprefix bin/, $(cva6_tests)))
 ARA_EXTENSIONS := rv64uv
-ARA_BINARIES := $(addprefix bin/, $(ara_tests))
+ARA_BINARIES := $(addsuffix $(IS_LINUX_EXTENSION), $(addprefix bin/, $(ara_tests)))
 
 # FFT requires special treatment because of its header files
 ifeq ($(ENV_DEFINES),)
@@ -95,14 +101,18 @@ endef
 $(foreach app,$(APPS),$(eval $(call app_compile_template_spike,$(app))))
 
 define app_compile_template
-bin/$1: $1/data.S.o $(addsuffix .o, $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
+bin/$1: $1/data.S.o$$(IS_LINUX_EXTENSION) $(addsuffix .o$$(IS_LINUX_EXTENSION), $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude $(RISCV_CCFLAGS) -o $$@ $$(addsuffix .o, $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS) -T$$(CURDIR)/common/link.ld
-	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
-	$$(RISCV_STRIP) $$@ -S --strip-unneeded
+	$$(RISCV_CC) $(RISCV_CCFLAGS) -o $$@$$(IS_LINUX_EXTENSION) $$(addsuffix .o$$(IS_LINUX_EXTENSION), $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS) $$(LD_FLAGS)
+	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@$$(IS_LINUX_EXTENSION) > $$@$$(IS_LINUX_EXTENSION).dump
+	# Don't strip symbols for Linux build since need them for debug
+	if [ "$$(IS_LINUX_EXTENSION)" == "" ]; then \
+		$$(RISCV_STRIP) $$@$$(IS_LINUX_EXTENSION) -S --strip-unneeded; \
+	fi
 endef
 $(foreach app,$(APPS),$(eval $(call app_compile_template,$(app))))
 
+
 # Make the RISC-V tests
 riscv_tests: $(CVA6_BINARIES) $(ARA_BINARIES)
 
@@ -111,7 +121,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_GCC) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC) $$(LD_FLAGS)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -121,7 +131,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM) $$(LD_FLAGS)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -169,13 +179,14 @@ benchmarks_clean:
 
 .PHONY: clean
 clean: riscv_tests_spike_clean benchmarks_clean
+	rm -vf bin/*
 	rm -vf $(BINARIES)
 	rm -vf $(CVA6_BINARIES)
 	rm -vf $(ARA_BINARIES)
 	rm -vf $(addsuffix .dump,$(BINARIES))
 	rm -vf $(addsuffix .dump,$(CVA6_BINARIES))
 	rm -vf $(addsuffix .dump,$(ARA_BINARIES))
-	rm -vf $(addsuffix /main.c.o,$(APPS))
+	rm -vf $(addsuffix /main.c.o$(IS_LINUX_EXTENSION),$(APPS))
 	rm -vf $(RUNTIME_GCC)
 	rm -vf $(RUNTIME_LLVM)
 	rm -vf $(RUNTIME_SPIKE)

apps/common/linux.mk

@@ -0,0 +1,47 @@
+IS_LINUX_EXTENSION := .linux
+
+CVA6_SDK ?= /usr/scratch/fenga3/vmaisto/cva6-sdk_fork_backup
+ROOTFS_DEST ?= $(CVA6_SDK)/rootfs/ara/apps/bin
+cp_to_rootfs: 
+	mkdir -p $(ROOTFS_DEST)
+	@echo "[Copying binaries to rootfs directory $(ROOTFS_DEST)]"
+	cp -v bin/*.linux $(ROOTFS_DEST)
+
+# Set the runtime variables to empty, the Linux libs will takcare of that 
+LD_FLAGS     := 
+RUNTIME_GCC   ?= common/util-gcc.c.o
+RUNTIME_LLVM  ?= common/util-llvm.c.o
+
+
+# Override
+INSTALL_DIR             ?= $(ARA_DIR)/install
+GCC_INSTALL_DIR         ?= $(CVA6_SDK)/buildroot/output/host/
+LLVM_INSTALL_DIR        ?= $(INSTALL_DIR)/riscv-llvm
+
+RISCV_XLEN    ?= 64
+RISCV_ARCH    ?= rv$(RISCV_XLEN)gcv
+RISCV_ABI     ?= lp64d
+RISCV_TARGET  ?= riscv$(RISCV_XLEN)-buildroot-linux-gnu-
+
+# Don't use LLVM
+RISCV_PREFIX  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)
+RISCV_CC      ?= $(RISCV_PREFIX)gcc
+RISCV_CXX     ?= $(RISCV_PREFIX)g++
+RISCV_OBJDUMP ?= $(RISCV_PREFIX)objdump
+RISCV_OBJCOPY ?= $(RISCV_PREFIX)objcopy
+RISCV_AS      ?= $(RISCV_PREFIX)as
+RISCV_AR      ?= $(RISCV_PREFIX)ar
+RISCV_LD      ?= $(RISCV_PREFIX)ld
+RISCV_STRIP   ?= $(RISCV_PREFIX)strip
+
+# Override flags
+# LLVM_FLAGS     ?= -march=rv64gcv_zfh_zvfh0p1 -mabi=$(RISCV_ABI) -mno-relax -fuse-ld=lld
+LLVM_FLAGS     ?= -march=rv64gcv -mabi=$(RISCV_ABI)
+LLVM_V_FLAGS   ?= #+no-optimized-zero-stride-load
+# RISCV_FLAGS    ?= $(LLVM_FLAGS) $(LLVM_V_FLAGS) -mcmodel=medany -I$(CURDIR)/common -std=gnu99 -O3 -ffast-math -fno-common -fno-builtin-printf $(DEFINES) $(RISCV_WARNINGS)
+RISCV_FLAGS    ?= -g $(LLVM_FLAGS) $(LLVM_V_FLAGS) -I$(CURDIR)/common -std=gnu99 -O0 $(DEFINES) $(RISCV_WARNINGS)
+RISCV_CCFLAGS  ?= $(RISCV_FLAGS) #-ffunction-sections -fdata-sections
+RISCV_CXXFLAGS ?= $(RISCV_FLAGS) -ffunction-sections -fdata-sections
+RISCV_LDFLAGS  ?= #-static -nostartfiles -lm -Wl,--gc-sections
+
+RISCV_OBJDUMP_FLAGS ?= -S

apps/common/printf.h

@@ -29,6 +29,11 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
+
+#ifdef __linux__
+  #include <stdio.h>
+#else // ! __linux__
+
 #ifndef _PRINTF_H_
 #define _PRINTF_H_
 
@@ -100,4 +105,5 @@ int fctprintf(void (*out)(char character, void *arg), void *arg,
 }
 #endif
 
+#endif // __linux__
 #endif // _PRINTF_H_

apps/common/runtime.h

@@ -7,13 +7,19 @@
   asm volatile(                                                                \
       "csrs mstatus, %[bits];" ::[bits] "r"(0x00000600 & (0x00000600 >> 1)))
 
-extern int64_t event_trigger;
-extern int64_t timer;
-// SoC-level CSR
-extern uint64_t hw_cnt_en_reg;
+// SoC-level CSR, put in memory for Linux build
+#ifdef __linux__
+  int64_t event_trigger;
+  int64_t timer;
+  uint64_t hw_cnt_en_reg;
+#else // ! __linux__
+  extern int64_t event_trigger;
+  extern int64_t timer;
+  extern uint64_t hw_cnt_en_reg;
+#endif // __linux__
 
 // Return the current value of the cycle counter
-inline int64_t get_cycle_count() {
+int64_t get_cycle_count() {
   int64_t cycle_count;
   // The fence is needed to be sure that Ara is idle, and it is not performing
   // the last vector stores when we read mcycle with stop_timer()
@@ -31,26 +37,26 @@ inline int64_t get_cycle_count() {
 #define HW_CNT_READY hw_cnt_en_reg = 1;
 #define HW_CNT_NOT_READY hw_cnt_en_reg = 0;
 // Start and stop the counter
-inline void start_timer() { timer = -get_cycle_count(); }
-inline void stop_timer() { timer += get_cycle_count(); }
+void start_timer() { timer = -get_cycle_count(); }
+void stop_timer() { timer += get_cycle_count(); }
 
 // Get the value of the timer
-inline int64_t get_timer() { return timer; }
+int64_t get_timer() { return timer; }
 #else
 #define HW_CNT_READY ;
 #define HW_CNT_NOT_READY ;
 // Start and stop the counter
-inline void start_timer() {
+void start_timer() {
   while (0)
     ;
 }
-inline void stop_timer() {
+void stop_timer() {
   while (0)
     ;
 }
 
 // Get the value of the timer
-inline int64_t get_timer() { return 0; }
+int64_t get_timer() { return 0; }
 #endif
 
 #endif // _RUNTIME_H_

apps/common/runtime.mk

@@ -42,7 +42,7 @@ ISA_SIM_MOD_INSTALL_DIR ?= $(INSTALL_DIR)/riscv-isa-sim-mod
 RISCV_XLEN    ?= 64
 RISCV_ARCH    ?= rv$(RISCV_XLEN)gcv
 RISCV_ABI     ?= lp64d
-RISCV_TARGET  ?= riscv$(RISCV_XLEN)-unknown-elf
+RISCV_TARGET  ?= riscv$(RISCV_XLEN)-unknown-elf-
 
 # Use LLVM
 RISCV_PREFIX  ?= $(LLVM_INSTALL_DIR)/bin/
@@ -56,7 +56,9 @@ RISCV_LD      ?= $(RISCV_PREFIX)ld.lld
 RISCV_STRIP   ?= $(RISCV_PREFIX)llvm-strip
 
 # Use gcc to compile scalar riscv-tests
-RISCV_CC_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)-gcc
+RISCV_CC_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)gcc
+RISCV_OBJCOPY_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)objcopy
+RISCV_OBJDUMP_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)objdump
 
 # Benchmark with spike
 spike_env_dir ?= $(ARA_DIR)/apps/riscv-tests
@@ -109,6 +111,8 @@ RUNTIME_GCC   ?= common/crt0-gcc.S.o common/printf-gcc.c.o common/string-gcc.c.o
 RUNTIME_LLVM  ?= common/crt0-llvm.S.o common/printf-llvm.c.o common/string-llvm.c.o common/serial-llvm.c.o common/util-llvm.c.o
 RUNTIME_SPIKE ?= $(spike_env_dir)/benchmarks/common/crt.S.o.spike $(spike_env_dir)/benchmarks/common/syscalls.c.o.spike common/util.c.o.spike
 
+LD_FLAGS ?= -T$(CURDIR)/common/link.ld
+
 .INTERMEDIATE: $(RUNTIME_GCC) $(RUNTIME_LLVM)
 
 %-gcc.S.o: %.S
@@ -123,10 +127,10 @@ RUNTIME_SPIKE ?= $(spike_env_dir)/benchmarks/common/crt.S.o.spike $(spike_env_di
 %-llvm.c.o: %.c
 	$(RISCV_CC) $(RISCV_CCFLAGS) -c $< -o $@
 
-%.S.o: %.S
+%.S.o$(IS_LINUX_EXTENSION): %.S
 	$(RISCV_CC) $(RISCV_CCFLAGS) -c $< -o $@
 
-%.c.o: %.c
+%.c.o$(IS_LINUX_EXTENSION): %.c
 	$(RISCV_CC) $(RISCV_CCFLAGS) -c $< -o $@
 
 %.S.o.spike: %.S patch-spike-crt0