Parameterize number of FPUs per core (#6)

* spatz_cluster: Parameterize the number of FPUs and IPUs

* tc-llvm: Compile LLVM with a generic gcc

* spatz: Parameterize DRAM start address and length

* sw: Allow externally-defined platforms

* sw: Fix platform's `bootdata` struct

* sw: Increase the length multiplier

* sw: Define number of FPUs

* sw: Clean up the runtime

* cfg: Propagate defines to the compilation flow

* spatz: Fix mismatch in the VRF

* hw: Clean up the bootrom

* hw: Flatten FPU count configuration

* ci: Update python dependencies

* sw: Initialize TLS

* bender: Install bender with cargo

* memora: Makefile changes do not trigger recompilation

* cfg: Switch DMA and compute cores

.gitlab/.gitlab-ci.yml

@@ -107,6 +107,7 @@ snRuntime-test-vsim:
     - export PYTHON3=$(which python3.6)
     - ${PYTHON3} -m pip install --user jsonref
     - ${PYTHON3} -m pip install --user jsonschema
+    - ${PYTHON3} -m pip install --user jstyleson
     - ${PYTHON3} -m pip install --user dataclasses
     - ${PYTHON3} -m pip install --user mako
     # Compile the binaries
@@ -134,6 +135,7 @@ snRuntime-test-vcs:
     - export PYTHON3=$(which python3.6)
     - ${PYTHON3} -m pip install --user jsonref
     - ${PYTHON3} -m pip install --user jsonschema
+    - ${PYTHON3} -m pip install --user jstyleson
     - ${PYTHON3} -m pip install --user dataclasses
     - ${PYTHON3} -m pip install --user mako
     # Compile the binaries
@@ -162,6 +164,7 @@ snRuntime-test-vlt:
     - export PYTHON3=$(which python3.6)
     - ${PYTHON3} -m pip install --user jsonref
     - ${PYTHON3} -m pip install --user jsonschema
+    - ${PYTHON3} -m pip install --user jstyleson
     - ${PYTHON3} -m pip install --user dataclasses
     - ${PYTHON3} -m pip install --user mako
     # Compile the binaries
@@ -191,6 +194,7 @@ riscvTests-test-vlt:
     - export PYTHON3=$(which python3.6)
     - ${PYTHON3} -m pip install --user jsonref
     - ${PYTHON3} -m pip install --user jsonschema
+    - ${PYTHON3} -m pip install --user jstyleson
     - ${PYTHON3} -m pip install --user dataclasses
     - ${PYTHON3} -m pip install --user mako
     # Compile the binaries
@@ -220,6 +224,7 @@ spatzBenchmarks-test-vlt:
     - export PYTHON3=$(which python3.6)
     - ${PYTHON3} -m pip install --user jsonref
     - ${PYTHON3} -m pip install --user jsonschema
+    - ${PYTHON3} -m pip install --user jstyleson
     - ${PYTHON3} -m pip install --user dataclasses
     - ${PYTHON3} -m pip install --user mako
     # Compile the binaries

Makefile

@@ -81,8 +81,8 @@ tc-llvm: sw/toolchain/llvm-project
 	cd sw/toolchain/llvm-project && mkdir -p build && cd build; \
 	$(CMAKE) \
 		-DCMAKE_INSTALL_PREFIX=$(LLVM_INSTALL_DIR) \
-		-DCMAKE_CXX_COMPILER=g++-8.2.0 \
-		-DCMAKE_C_COMPILER=gcc-8.2.0 \
+		-DCMAKE_CXX_COMPILER=${CXX} \
+		-DCMAKE_C_COMPILER=${CC} \
 		-DLLVM_OPTIMIZED_TABLEGEN=True \
 		-DLLVM_ENABLE_PROJECTS="clang;lld" \
 		-DLLVM_TARGETS_TO_BUILD="RISCV" \
@@ -107,18 +107,18 @@ tc-riscv-isa-sim: sw/toolchain/riscv-isa-sim sw/toolchain/dtc
 
 bender: check-bender
 check-bender:
-	@if [ -x $(BENDER_INSTALL_DIR)/bender ]; then \
+	@if [ -x $(BENDER_INSTALL_DIR)/bin/bender ]; then \
 		req="bender $(BENDER_VERSION)"; \
-		current="$$($(BENDER_INSTALL_DIR)/bender --version)"; \
+		current="$$($(BENDER_INSTALL_DIR)/bin/bender --version)"; \
 		if [ "$$(printf '%s\n' "$${req}" "$${current}" | sort -V | head -n1)" != "$${req}" ]; then \
 			rm -rf $(BENDER_INSTALL_DIR); \
 		fi \
 	fi
-	@$(MAKE) -C $(ROOT_DIR) $(BENDER_INSTALL_DIR)/bender
+	@$(MAKE) -C $(ROOT_DIR) $(BENDER_INSTALL_DIR)/bin/bender
 
-$(BENDER_INSTALL_DIR)/bender:
+$(BENDER_INSTALL_DIR)/bin/bender:
 	mkdir -p $(BENDER_INSTALL_DIR) && cd $(BENDER_INSTALL_DIR) && \
-	curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh -s -- $(BENDER_VERSION)
+	cargo install bender --version $(BENDER_VERSION) --root $(BENDER_INSTALL_DIR) --locked
 
 ###############
 #  Verilator  #

Memora.yml

@@ -7,29 +7,25 @@ cache_root_dir: /usr/scratch2/larain1/matheusd/gitlabci/buildcache/spatz
 artifacts:
   tc-riscv-gcc:
     inputs:
-      - Makefile
       - sw/toolchain/riscv-gnu-toolchain.version
     outputs:
       - install/riscv-gcc
 
   tc-llvm:
     inputs:
-      - Makefile
       - sw/toolchain/llvm-project.version
     outputs:
       - install/llvm
 
   riscv-isa-sim:
     inputs:
-      - Makefile
       - sw/toolchain/riscv-isa-sim.version
       - sw/toolchain/riscv-opcodes.version
     outputs:
       - install/riscv-isa-sim
 
   verilator:
     inputs:
-      - Makefile
       - sw/toolchain/verilator.version
     outputs:
       - install/verilator

hw/ip/snitch_test/src/tb_lib.hh

@@ -117,12 +117,12 @@ extern GlobalMemory MEM;
 
 // The boot data generated along with the system RTL.
 struct BootData {
-    uint32_t boot_addr;
-    uint32_t core_count;
-    uint32_t hartid_base;
-    uint32_t tcdm_start;
-    uint32_t tcdm_size;
-    uint32_t tcdm_offset;
+    uint64_t boot_addr;
+    uint64_t core_count;
+    uint64_t hartid_base;
+    uint64_t tcdm_start;
+    uint64_t tcdm_size;
+    uint64_t tcdm_offset;
     uint64_t global_mem_start;
     uint64_t global_mem_end;
 };

hw/ip/spatz/src/vregfile.sv

@@ -99,11 +99,9 @@ module vregfile import spatz_pkg::*; #(
       );
 
       always_latch begin
-`ifndef SYNTHESIS
         if (!rst_ni)
           mem[vreg][b] <= '0;
         else
-`endif
           if (clk_latch)
             mem[vreg][b] <= wdata_q[b*8 +: 8];
       end

hw/ip/spatz_cc/src/spatz_cc.sv

@@ -70,7 +70,7 @@ module spatz_cc
     parameter int                          unsigned        NumSpatzOutstandingLoads = 0,
     // Enable V Extension
     parameter bit                                          RVV                      = 1,
-    // Spatz paramaters
+    // Spatz parameters
     parameter int                          unsigned        NumSpatzFPUs             = 4,
     parameter int                          unsigned        NumSpatzIPUs             = 1,
     /// Add isochronous clock-domain crossings e.g., make it possible to operate

hw/system/spatz_cluster/Makefile

@@ -11,7 +11,15 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MKFILE_DIR  := $(dir $(MKFILE_PATH))
 
 # Configuration file
-SPATZ_CLUSTER_CFG ?= $(SPATZ_CLUSTER_DIR)/cfg/spatz_cluster.default.hjson
+SPATZ_CLUSTER_CFG         ?= $(SPATZ_CLUSTER_DIR)/cfg/spatz_cluster.default.hjson
+SPATZ_CLUSTER_CFG_DEFINES += -DMEM_DRAM_ORIGIN=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['dram']['address'])")
+SPATZ_CLUSTER_CFG_DEFINES += -DMEM_DRAM_SIZE=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['dram']['length'])")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_BASE_HARTID=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['cluster']['cluster_base_hartid'])")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_CLUSTER_CORE_NUM=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(len(jstyleson.load(f)['cluster']['cores']))")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_TCDM_START_ADDR=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['cluster']['cluster_base_addr'])")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_CLUSTER_OFFSET=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['cluster']['cluster_base_offset'])")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_TCDM_SIZE=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['cluster']['tcdm']['size'] * 1024)")
+SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_NFPU_PER_CORE=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG)'); print(jstyleson.load(f)['cluster']['n_fpu'])")
 
 # Include Makefrag
 include $(ROOT)/util/Makefrag
@@ -62,10 +70,10 @@ VLT_COBJ += $(VLT_BUILDDIR)/vlt/verilated_vcd_c.o
 # Prerequisites #
 #################
 
-test/bootrom.elf test/bootrom.dump test/bootrom.bin: src/generated/spatz_cluster_wrapper.sv test/bootrom.S test/bootrom.ld Makefile
-	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-gcc -mabi=ilp32 -march=rv32imaf -static -nostartfiles -Ttest/bootrom.ld test/bootrom.S test/bootdata_bootrom.cc -I../../ip/snitch_test/src -I../../../sw/snRuntime/include -o test/bootrom.elf
-	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-objdump -D test/bootrom.elf > test/bootrom.dump
-	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-objcopy -O binary test/bootrom.elf test/bootrom.bin
+test/bootrom.elf test/bootrom.dump test/bootrom.bin: src/generated/spatz_cluster_wrapper.sv test/bootrom.S test/bootrom.ld test/bootdata_bootrom.cc Makefile
+	${GCC_INSTALL_DIR}/bin/riscv32-unknown-elf-gcc -mabi=ilp32 -march=rv32imaf -static -nostartfiles -Ttest/bootrom.ld test/bootrom.S test/bootdata_bootrom.cc -I../../ip/snitch_test/src -I../../../sw/snRuntime/include -I../../../sw/toolchain/riscv-opcodes/ -o test/bootrom.elf
+	${GCC_INSTALL_DIR}/bin/riscv32-unknown-elf-objdump -D test/bootrom.elf > test/bootrom.dump
+	${GCC_INSTALL_DIR}/bin/riscv32-unknown-elf-objcopy -O binary test/bootrom.elf test/bootrom.bin
 
 .PHONY: bootrom
 bootrom: $(CURDIR)/src/generated/bootrom.sv
@@ -167,13 +175,13 @@ lint/tmp/files: ${BENDER}
 ## Build SW into sw/build with the LLVM toolchain
 sw: clean.sw
 	mkdir -p sw/build
-	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} .. && make
+	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} ${SPATZ_CLUSTER_CFG_DEFINES} .. && make
 
 # VSIM
 ## Build SW into sw/build with the LLVM toolchain (including tests) for Questasim simulator
 sw.vsim: clean.sw bin/spatz_cluster.vsim
 	mkdir -p sw/build
-	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vsim -DBUILD_TESTS=ON .. && make -j8
+	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vsim -DBUILD_TESTS=ON ${SPATZ_CLUSTER_CFG_DEFINES} .. && make -j8
 
 ## Build SW and run all tests with Questasim simulator
 sw.test.vsim: sw.vsim
@@ -183,7 +191,7 @@ sw.test.vsim: sw.vsim
 ## Build SW into sw/build with the LLVM toolchain (including tests) for VCS simulator
 sw.vcs: clean.sw bin/spatz_cluster.vcs
 	mkdir -p sw/build
-	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vcs -DBUILD_TESTS=ON .. && make -j8
+	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vcs -DBUILD_TESTS=ON ${SPATZ_CLUSTER_CFG_DEFINES} .. && make -j8
 
 ## Build SW and run all tests with VCS simulator
 sw.test.vcs: sw.vcs
@@ -193,7 +201,7 @@ sw.test.vcs: sw.vcs
 ## Build SW into sw/build with the LLVM toolchain (including tests) for Verilator simulator
 sw.vlt: clean.sw bin/spatz_cluster.vlt
 	mkdir -p sw/build
-	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vlt -DBUILD_TESTS=ON .. && make -j8
+	cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vlt -DBUILD_TESTS=ON ${SPATZ_CLUSTER_CFG_DEFINES} .. && make -j8
 
 ## Build SW and run all tests with Verilator simulator
 sw.test.vlt: sw.vlt

hw/system/spatz_cluster/cfg/carfield.hjson

@@ -4,96 +4,96 @@
 
 // Cluster configuration for a simple system.
 {
-    cluster: {
-        boot_addr: 4096,               // 0x1000
-        cluster_base_addr: 1358954496, // 0x51000000
-        cluster_base_offset: 0,        // 0x0
-        cluster_base_hartid: 16,
-        addr_width: 48,
-        data_width: 64,
-        id_width_in: 6,
-        id_width_out: 2,
-        user_width: 10,
-        cluster_default_axi_user: 7,
-        axi_cdc_enable: true,
-        sw_rst_enable: true,
-        axi_isolate_enable: true,
-        tcdm: {
-            size: 128,
-            banks: 16,
+    "cluster": {
+        "boot_addr": 4096,               // 0x1000
+        "cluster_base_addr": 1358954496, // 0x51000000
+        "cluster_base_offset": 0,        // 0x0
+        "cluster_base_hartid": 16,
+        "addr_width": 48,
+        "data_width": 64,
+        "id_width_in": 6,
+        "id_width_out": 2,
+        "user_width": 10,
+        "cluster_default_axi_user": 7,
+        "axi_cdc_enable": true,
+        "sw_rst_enable": true,
+        "axi_isolate_enable": true,
+        "tcdm": {
+            "size": 128,
+            "banks": 16
         },
-        cluster_periph_size: 64, // kB
-        dma_data_width: 64,
-        dma_axi_req_fifo_depth: 3,
-        dma_req_fifo_depth: 3,
+        "cluster_periph_size": 64, // kB
+        "dma_data_width": 64,
+        "dma_axi_req_fifo_depth": 3,
+        "dma_req_fifo_depth": 3,
         // Spatz parameters
-        vlen: 512,
-        n_fpu: 4,
-        n_ipu: 1,
-        spatz_fpu: true,
+        "vlen": 512,
+        "n_fpu": 4,
+        "n_ipu": 1,
+        "spatz_fpu": true,
         // Timing parameters
-        timing: {
-            lat_comp_fp32: 2,
-            lat_comp_fp64: 4,
-            lat_comp_fp16: 1,
-            lat_comp_fp16_alt: 1,
-            lat_comp_fp8: 0,
-            lat_comp_fp8_alt: 0,
-            lat_noncomp: 1,
-            lat_conv: 2,
-            lat_sdotp: 4,
-            fpu_pipe_config: "BEFORE"
-            xbar_latency: "CUT_ALL_PORTS",
+        "timing": {
+            "lat_comp_fp32": 2,
+            "lat_comp_fp64": 4,
+            "lat_comp_fp16": 1,
+            "lat_comp_fp16_alt": 1,
+            "lat_comp_fp8": 0,
+            "lat_comp_fp8_alt": 0,
+            "lat_noncomp": 1,
+            "lat_conv": 2,
+            "lat_sdotp": 4,
+            "fpu_pipe_config": "BEFORE",
+            "xbar_latency": "CUT_ALL_PORTS",
 
-            register_tcdm_cuts: true,
-            register_core_req: true,
-            register_core_rsp: true,
-            register_offload_rsp: true
+            "register_tcdm_cuts": true,
+            "register_core_req": true,
+            "register_core_rsp": true,
+            "register_offload_rsp": true
         },
-        cores: [
-            { $ref: "#/dma_core_template" },
-            { $ref: "#/compute_core_template" },
+        "cores": [
+            // DMA core
+            {
+                "isa": "rv32imafd",
+                "xdma": true,
+                "xf16": true,
+                "xf8": true,
+                "xfdotp": true,
+                "num_int_outstanding_loads": 1,
+                "num_int_outstanding_mem": 4,
+                "num_spatz_outstanding_loads": 4,
+                "num_dtlb_entries": 1,
+                "num_itlb_entries": 1
+            },
+
+            // Compute core
+            {
+                "isa": "rv32imafd",
+                "xf16": true,
+                "xf8": true,
+                "xfdotp": true,
+                "xdma": false,
+                "num_int_outstanding_loads": 1,
+                "num_int_outstanding_mem": 4,
+                "num_spatz_outstanding_loads": 4,
+                "num_dtlb_entries": 1,
+                "num_itlb_entries": 1
+            }
         ],
-        icache: {
-            size: 4, // total instruction cache size in kByte
-            sets: 2, // number of ways
-            cacheline: 128 // word size in bits
+        "icache": {
+            "size": 4, // total instruction cache size in kByte
+            "sets": 2, // number of ways
+            "cacheline": 128 // word size in bits
         }
-    }
+    },
 
-    dram: {
+    "dram": {
         // 0x8000_0000
-        address: 2147483648,
+        "address": 2147483648,
         // 0x8000_0000
-        length: 2147483648
-    },
-    peripherals: {
+        "length": 2147483648
     },
 
-    // Templates.
+    "peripherals": {
 
-    compute_core_template: {
-        isa: "rv32imafd",
-        xf16: true,
-        xf8: true,
-        xfdotp: true,
-        xdma: false,
-        num_int_outstanding_loads: 1,
-        num_int_outstanding_mem: 4,
-        num_spatz_outstanding_loads: 4,
-        num_dtlb_entries: 1,
-        num_itlb_entries: 1
-    },
-    dma_core_template: {
-        isa: "rv32imafd",
-        xdma: true
-        xf16: true,
-        xf8: true,
-        xfdotp: true,
-        num_int_outstanding_loads: 1,
-        num_int_outstanding_mem: 4,
-        num_spatz_outstanding_loads: 4,
-        num_dtlb_entries: 1,
-        num_itlb_entries: 1
     }
 }