From 06227f13d0a82c0523f17a31730ece3b2a1b342d Mon Sep 17 00:00:00 2001
From: Yvan Tortorella <ytortorella@lagrev3.ee.ethz.ch>
Date: Thu, 27 Jul 2023 10:34:47 +0200
Subject: [PATCH 1/9] treewide: Add offload flow to integer cluster (PULP
 cluster)

---
 Bender.lock                                 |  8 +--
 carfield.mk                                 | 28 +++++++---
 hw/carfield_pkg.sv                          |  2 +-
 scripts/{elf_to_header.py => elf2header.py} | 61 +++++++++------------
 scripts/env-iis.sh                          |  7 +--
 sw/include/car_memory_map.h                 | 13 +++++
 sw/include/car_properties.h                 |  5 ++
 sw/include/car_util.h                       | 33 ++++++++++-
 sw/include/regs/soc_ctrl.h                  | 16 ++++--
 sw/sw.mk                                    | 39 ++++++++++---
 sw/tests/bare-metal/hostd/pulp-offload.c    | 43 +++++++++++++++
 sw/tests/bare-metal/pulpd/sw.mk             | 39 +++++++++++++
 sw/tests/bare-metal/safed/sw.mk             | 19 +++----
 sw/tests/bare-metal/secd/sw.mk              |  5 ++
 14 files changed, 239 insertions(+), 79 deletions(-)
 rename scripts/{elf_to_header.py => elf2header.py} (95%)
 create mode 100644 sw/include/car_properties.h
 create mode 100644 sw/tests/bare-metal/hostd/pulp-offload.c
 create mode 100644 sw/tests/bare-metal/pulpd/sw.mk
 create mode 100644 sw/tests/bare-metal/secd/sw.mk

diff --git a/Bender.lock b/Bender.lock
index 2c43132b..d49edb39 100644
--- a/Bender.lock
+++ b/Bender.lock
@@ -243,7 +243,7 @@ packages:
     dependencies:
     - common_cells
   fpu_interco:
-    revision: c985d54c2b078ddfbec8c2a498f453410bbdc93e
+    revision: 0769976fa51bdd820656a01161a4c46b88c59ac5
     version: null
     source:
       Git: https://github.com/pulp-platform/fpu_interco.git
@@ -296,10 +296,10 @@ packages:
     - register_interface
     - tech_cells_generic
   ibex:
-    revision: b18f7ef178ed07f5085051f96042c670a919fd5c
+    revision: 95b85ddd1c995ace9f89ee42530f9e24820c1051
     version: null
     source:
-      Git: https://github.com/pulp-platform/ibex.git
+      Git: https://github.com/lowRISC/ibex.git
     dependencies:
     - tech_cells_generic
   icache-intc:
@@ -443,7 +443,7 @@ packages:
     - common_cells
     - common_verification
   riscv:
-    revision: 6187537f9994d16bad2d721c0f5ebc5193c0f010
+    revision: 4eac53237c6d0062715d17016fe95462eb81ebc3
     version: null
     source:
       Git: git@github.com:AlSaqr-platform/riscv_nn.git
diff --git a/carfield.mk b/carfield.mk
index dd5fad44..8c4ea417 100644
--- a/carfield.mk
+++ b/carfield.mk
@@ -37,7 +37,7 @@ CHS_ROOT ?= $(shell $(BENDER) path cheshire)
 
 CHS_BOOTMODE ?= 0 # default passive bootmode
 CHS_PRELMODE ?= 1 # default serial link preload
-CHS_BINARY   ?= 
+CHS_BINARY   ?=
 CHS_IMAGE    ?=
 
 # Safety Island
@@ -52,8 +52,8 @@ SECD_BINARY   ?=
 SECD_BOOTMODE ?=
 
 # PULP cluster
-PULPCL_ROOT   ?= $(shell $(BENDER) path pulp_cluster)
-PULPCL_BINARY ?=
+PULPD_ROOT   ?= $(shell $(BENDER) path pulp_cluster)
+PULPD_BINARY ?=
 
 # Spatz cluster
 SPATZCL_ROOT    ?= $(shell $(BENDER) path spatz)
@@ -191,7 +191,7 @@ car-hw-sim:
 		 set SECD_BINARY $(SECD_BINARY); \
 		 set SAFED_BOOTMODE $(SAFED_BOOTMODE); \
 		 set SAFED_BINARY $(SAFED_BINARY); \
-		 set PULPCL_BINARY $(PULPCL_BINARY); \
+		 set PULPD_BINARY $(PULPD_BINARY); \
 		 set SPATZCL_BINARY $(SPATZCL_BINARY); \
 		 set VOPTARGS $(VOPTARGS); \
 		 set TESTBENCH $(TBENCH); \
@@ -259,10 +259,11 @@ car-sw-build: chs-sw-build car-sw-all
 
 .PHONY: car-init
 ## Shortcut to initialize carfield with all the targets described above.
-car-init: car-checkout car-hw-init car-sim-init safed-sw-init
+car-init: car-checkout car-hw-init car-sim-init safed-sw-init pulpd-sw-init
 
-# Initialize and build Safety Island SW
-.PHONY: safed-sw-init
+# Initialize and build SW for the Islands
+.PHONY: safed-sw-init pulpd-sw-init
+# Safety Island
 safed-sw-init: $(SAFED_ROOT) $(SAFED_SW_DIR)/pulp-runtime $(SAFED_SW_DIR)/pulp-freertos
 
 $(SAFED_SW_DIR)/pulp-runtime: $(SAFED_ROOT)
@@ -270,9 +271,18 @@ $(SAFED_SW_DIR)/pulp-runtime: $(SAFED_ROOT)
 $(SAFED_SW_DIR)/pulp-freertos: $(SAFED_ROOT)
 	$(MAKE) -C $(SAFED_ROOT) pulp-freertos
 
+# PULP Cluster
+pulpd-sw-init: $(PULPD_ROOT) $(PULPD_ROOT)/pulp-runtime $(PULPD_ROOT)/regression-tests
+
+$(PULPD_ROOT)/pulp-runtime: $(PULPD_ROOT)
+	$(MAKE) -C $(PULPD_ROOT) pulp-runtime
+$(PULPD_ROOT)/regression-tests: $(PULPD_ROOT)
+	$(MAKE) -C $(PULPD_ROOT) regression-tests
+
 # For independent boot of an island, we allow to compile the binary standalone.
-.PHONY: safed-sw-build
-safed-sw-build: safed-sw-init safed-sw-all
+.PHONY: safed-sw-build pulpd-sw-build
+safed-sw-build: safed-sw-all
+pulpd-sw-build: pulpd-sw-all
 
 ############
 # RTL LINT #
diff --git a/hw/carfield_pkg.sv b/hw/carfield_pkg.sv
index d4a30790..a96bc2b5 100644
--- a/hw/carfield_pkg.sv
+++ b/hw/carfield_pkg.sv
@@ -412,7 +412,7 @@ localparam int unsigned IntClusterSharedFpuDivSqrt = 0;
 localparam int unsigned IntClusterNumAxiMst = 3;
 localparam int unsigned IntClusterNumAxiSlv = 4;
 // IntClusterAxiIdInWidth is fixed from PULP Cluster
-localparam int unsigned IntClusterAxiIdInWidth = $clog2(IntClusterNumCacheBanks) + 1;
+localparam int unsigned IntClusterAxiIdInWidth = $clog2(IntClusterNumCacheBanks) + 3;
 localparam int unsigned IntClusterAxiIdOutWidth = IntClusterAxiIdInWidth     +
                                                   $clog2(IntClusterNumAxiSlv);
 localparam int unsigned IntClusterMaxUniqId = 1;
diff --git a/scripts/elf_to_header.py b/scripts/elf2header.py
similarity index 95%
rename from scripts/elf_to_header.py
rename to scripts/elf2header.py
index df3c9481..abdccd03 100644
--- a/scripts/elf_to_header.py
+++ b/scripts/elf2header.py
@@ -22,43 +22,36 @@
 import struct
 import argparse
 
-#c_header = """\
-#// Auto-generated code
-##ifdef USE_HYPER
-##include "hyperbus_test.h"
-##endif
-#
-#extern int __cluster_code_start;
-#extern int __cluster_code_end;
-#
-#"""
-#c_function = """\
-#   #ifdef CLUSTER_BIN
-#      uint32_t *p, *end, *p0;
-#      p = (uint32_t*)&__cluster_code_start;
-#      p0 = (uint32_t*)&__cluster_code_start;
-#      end = (uint32_t*)&__cluster_code_end;
-#       #ifdef USE_HYPER
-#         udma_hyper_setup();
-#         set_pagebound_hyper(2048);
-#         udma_hyper_dread((end-p)*4,p-0x80000000, 0x1C000000, 128, 0);
-#         udma_hyper_wait(0);
-#       #else
-#         uint32_t * addr;
-#         while(p<end){
-#           addr = 0x1C000000 + ((p - p0)*4);
-#           pulp_write32(addr,pulp_read32(p));
-#           p++;
-#         }
-#       #endif
-#  #else
-#"""
-
 c_header = """\
 // Auto-generated code
+#ifdef USE_HYPER
+#include "hyperbus_test.h"
+#endif
+
+extern int __cluster_code_start;
+extern int __cluster_code_end;
 
 """
 c_function = """\
+   #ifdef CLUSTER_BIN
+      uint32_t *p, *end, *p0;
+      p = (uint32_t*)&__cluster_code_start;
+      p0 = (uint32_t*)&__cluster_code_start;
+      end = (uint32_t*)&__cluster_code_end;
+       #ifdef USE_HYPER
+         udma_hyper_setup();
+         set_pagebound_hyper(2048);
+         udma_hyper_dread((end-p)*4,p-0x80000000, 0x1C000000, 128, 0);
+         udma_hyper_wait(0);
+       #else
+         uint32_t * addr;
+         while(p<end){
+           addr = 0x1C000000 + ((p - p0)*4);
+           pulp_write32(addr,pulp_read32(p));
+           p++;
+         }
+       #endif
+  #else
 """
 
 class stim(object):
@@ -149,11 +142,11 @@ def __gen_stim_header(self, filename, width):
 
     with open(filename, 'w') as file:
       file.write(c_header)
-      file.write('int load_safed_payload() {\n' )
+      file.write('int load_binary() {\n' )
       file.write(c_function)
       for key in sorted(self.mem.keys()):
         file.write('  (*(volatile unsigned int *)(uint64_t)(0x%X)) = 0x%0*X ;\n' % (int(key), width*2, self.mem.get(key)))
-      #file.write('#endif\n')
+      file.write('#endif\n')
       file.write('return 0; \n }\n')
 
 
diff --git a/scripts/env-iis.sh b/scripts/env-iis.sh
index 6aee5197..a4ffb219 100644
--- a/scripts/env-iis.sh
+++ b/scripts/env-iis.sh
@@ -5,8 +5,7 @@
 
 # set up environment variables for rtl simulation
 ROOTD=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)
-export PATH="/usr/pack/riscv-1.0-kgf/riscv64-gcc-11.2.0/bin:$PATH" # RV64 GCC toolchain
-export PATH="/usr/pack/riscv-1.0-kgf/pulp-gcc-2.6.0/bin:$PATH" # RV32 GCC toolchain
+export PATH=/usr/pack/riscv-1.0-kgf/riscv64-gcc-11.2.0/bin:$PATH # RV64 GCC toolchain
+export SAFED_RISCV=/usr/pack/riscv-1.0-kgf/pulp-gcc-2.6.0/bin/riscv32-unknown-elf
+export PULPD_RISCV=/usr/pack/pulpsdk-1.0-kgf/artifactory/pulp-sdk-release/pkg/pulp_riscv_gcc/1.0.16/bin/riscv32-unknown-elf
 export QUESTA=questa-2022.3
-[[ -d "$ROOTD/safety_island/sw/pulp-freertos" ]] && source "$ROOTD/safety_island/sw/pulp-freertos/env/safety-island.sh" #TODO add carfield target in freertos
-[[ -d "$ROOTD/safety_island/sw/pulp-runtime" ]] && source "$ROOTD/safety_island/sw/pulp-runtime/configs/carfield.sh"
diff --git a/sw/include/car_memory_map.h b/sw/include/car_memory_map.h
index 1970a2a9..5dbd358d 100644
--- a/sw/include/car_memory_map.h
+++ b/sw/include/car_memory_map.h
@@ -16,6 +16,7 @@
 #define __CAR_MEMORY_MAP_H
 
 #include "regs/safety_soc_ctrl.h"
+#include "regs/soc_ctrl.h"
 
 // Base addresses provided at link time
 extern void *__base_l2;
@@ -54,6 +55,12 @@ extern void *__base_l2;
 #define CAR_INT_CLUSTER_SPM_BASE_ADDR 0x50000000
 #define CAR_INT_CLUSTER_SPM_END_ADDR  0x50040000
 
+#define CAR_INT_CLUSTER_PERIPH_OFFS    0x00200000
+#define CAR_INT_CLUSTER_CTRL_UNIT_OFFS 0x00000000
+
+#define CAR_INT_CLUSTER_BOOT_ADDR_OFFS 0x40
+#define CAR_INT_CLUSTER_BOOT_ADDR_REG (CAR_INT_CLUSTER_SPM_BASE_ADDR + CAR_INT_CLUSTER_PERIPH_OFFS + CAR_INT_CLUSTER_CTRL_UNIT_OFFS + CAR_INT_CLUSTER_BOOT_ADDR_OFFS)
+
 // Floating Point Spatz Cluster
 #define CAR_FP_CLUSTER_SPM_BASE_ADDR 0x51000000
 #define CAR_FP_CLUSTER_SPM_END_ADDR  0x51020000
@@ -99,4 +106,10 @@ extern void *__base_l2;
 #define EINTCLEXEC 3 // Execution error integer cluster
 #define EFPCLEXEC  4 // Execution error floating point cluster
 
+// Memory-mapped registers
+#define CAR_INT_CLUSTER_FETCHEN_ADDR (CAR_SOC_CTRL_BASE_ADDR + CARFIELD_PULP_CLUSTER_FETCH_ENABLE_REG_OFFSET)
+#define CAR_INT_CLUSTER_BOOTEN_ADDR  (CAR_SOC_CTRL_BASE_ADDR + CARFIELD_PULP_CLUSTER_BOOT_ENABLE_REG_OFFSET)
+#define CAR_INT_CLUSTER_BUSY_ADDR    (CAR_SOC_CTRL_BASE_ADDR + CARFIELD_PULP_CLUSTER_BUSY_REG_OFFSET)
+#define CAR_INT_CLUSTER_EOC_ADDR     (CAR_SOC_CTRL_BASE_ADDR + CARFIELD_PULP_CLUSTER_EOC_REG_OFFSET)
+
 #endif /* __CAR_MEMORY_MAP_H */
diff --git a/sw/include/car_properties.h b/sw/include/car_properties.h
new file mode 100644
index 00000000..55e810aa
--- /dev/null
+++ b/sw/include/car_properties.h
@@ -0,0 +1,5 @@
+// Copyright 2023 ETH Zurich and University of Bologna.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+#define IntClustNumCores 12
diff --git a/sw/include/car_util.h b/sw/include/car_util.h
index 5b063285..f4b0b337 100644
--- a/sw/include/car_util.h
+++ b/sw/include/car_util.h
@@ -10,6 +10,7 @@
 
 #include "util.h"
 #include "car_memory_map.h"
+#include "car_properties.h"
 #include "regs/soc_ctrl.h"
 #include "io.h"
 
@@ -212,7 +213,7 @@ uint32_t safed_offloader_blocking () {
 	uint32_t ret = 0;
 
 	// Load binary payload
-	load_safed_payload();
+	load_binary();
 
 	// Select bootmode, write entry point, write launch signal
 	prepare_safed_boot();
@@ -228,4 +229,34 @@ uint32_t safed_offloader_blocking () {
 	return ret;
 }
 
+// PULP cluster setup and configuration
+
+void pulp_cluster_set_bootaddress(uint32_t pulp_boot_addr) {
+
+  volatile uint32_t cluster_boot_reg_addr = CAR_INT_CLUSTER_BOOT_ADDR_REG;
+
+  for (int i = 0; i < IntClustNumCores; i++) {
+    writew(pulp_boot_addr, (uint32_t*)(cluster_boot_reg_addr));
+    cluster_boot_reg_addr += 0x4;
+  }
+}
+
+void pulp_cluster_start() {
+
+  volatile uint32_t *booten_addr = (uint32_t*)(CAR_INT_CLUSTER_BOOTEN_ADDR);
+  writew(1, booten_addr);
+
+  volatile uint32_t *fetchen_addr = (uint32_t*)(CAR_INT_CLUSTER_FETCHEN_ADDR);
+	writew(1, fetchen_addr);
+}
+
+void pulp_cluster_wait_eoc() {
+
+  volatile uint32_t *pulp_eoc_addr = (uint32_t*)(CAR_INT_CLUSTER_EOC_ADDR);
+
+  while(!readw(pulp_eoc_addr))
+      ;
+
+}
+
 #endif
diff --git a/sw/include/regs/soc_ctrl.h b/sw/include/regs/soc_ctrl.h
index 033ef3a7..c1d864e9 100644
--- a/sw/include/regs/soc_ctrl.h
+++ b/sw/include/regs/soc_ctrl.h
@@ -278,25 +278,29 @@ extern "C" {
 #define CARFIELD_PULP_CLUSTER_BOOT_ENABLE_REG_OFFSET 0xe0
 #define CARFIELD_PULP_CLUSTER_BOOT_ENABLE_PULP_CLUSTER_BOOT_ENABLE_BIT 0
 
+// Spatz Cluster busy
+#define CARFIELD_SPATZ_CLUSTER_BUSY_REG_OFFSET 0xe4
+#define CARFIELD_SPATZ_CLUSTER_BUSY_SPATZ_CLUSTER_BUSY_BIT 0
+
 // PULP Cluster busy
-#define CARFIELD_PULP_CLUSTER_BUSY_REG_OFFSET 0xe4
+#define CARFIELD_PULP_CLUSTER_BUSY_REG_OFFSET 0xe8
 #define CARFIELD_PULP_CLUSTER_BUSY_PULP_CLUSTER_BUSY_BIT 0
 
 // PULP Cluster end of computation
-#define CARFIELD_PULP_CLUSTER_EOC_REG_OFFSET 0xe8
+#define CARFIELD_PULP_CLUSTER_EOC_REG_OFFSET 0xec
 #define CARFIELD_PULP_CLUSTER_EOC_PULP_CLUSTER_EOC_BIT 0
 
 // L2 RAM cfg pins -margin adjustments-
-#define CARFIELD_L2_SRAM_CONFIG0_REG_OFFSET 0xec
+#define CARFIELD_L2_SRAM_CONFIG0_REG_OFFSET 0xf0
 
 // L2 RAM cfg pins -margin adjustments-
-#define CARFIELD_L2_SRAM_CONFIG1_REG_OFFSET 0xf0
+#define CARFIELD_L2_SRAM_CONFIG1_REG_OFFSET 0xf4
 
 // L2 RAM cfg pins -margin adjustments-
-#define CARFIELD_L2_SRAM_CONFIG2_REG_OFFSET 0xf4
+#define CARFIELD_L2_SRAM_CONFIG2_REG_OFFSET 0xf8
 
 // L2 RAM cfg pins -margin adjustments-
-#define CARFIELD_L2_SRAM_CONFIG3_REG_OFFSET 0xf8
+#define CARFIELD_L2_SRAM_CONFIG3_REG_OFFSET 0xfc
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/sw/sw.mk b/sw/sw.mk
index 5c8b3465..012d5669 100644
--- a/sw/sw.mk
+++ b/sw/sw.mk
@@ -13,7 +13,7 @@
 
 CAR_LD_DIR    ?= $(CAR_SW_DIR)/link
 
-RISCV_LDFLAGS ?= $(CHS_SW_FLAGS) -nostartfiles -Wl,--gc-sections -Wl,) -L$(CHS_SW_LD_DIR) -L$(CAR_LD_DIR)
+RISCV_LDFLAGS ?= $(CHS_SW_FLAGS) -nostartfiles -Wl,--gc-sections -L$(CHS_SW_LD_DIR) -L$(CAR_LD_DIR)
 
 car-sw-all: car-sw-libs car-sw-tests
 
@@ -21,8 +21,8 @@ car-sw-all: car-sw-libs car-sw-tests
 .PHONY: car-sw-all car-sw-libs car-sw-headers car-sw-tests
 
 # Libraries
-
-CAR_SW_INCLUDES    = -I$(CAR_SW_DIR)/include -I$(CAR_SW_DIR)/tests/bare-metal/safed -I$(CHS_SW_DIR)/include $(CHS_SW_DEPS_INCS)
+CAR_PULPD_BARE ?= $(CAR_SW_DIR)/tests/bare-metal/pulpd
+CAR_SW_INCLUDES    = -I$(CAR_SW_DIR)/include -I$(CAR_SW_DIR)/tests/bare-metal/safed -I$(CAR_PULPD_BARE) -I$(CHS_SW_DIR)/include $(CHS_SW_DEPS_INCS)
 CAR_SW_LIB_SRCS_S  = $(wildcard $(CAR_SW_DIR)/lib/*.S $(CAR_SW_DIR)/lib/**/*.S)
 CAR_SW_LIB_SRCS_C  = $(wildcard $(CAR_SW_DIR)/lib/*.c $(CAR_SW_DIR)/lib/**/*.c)
 CAR_SW_LIB_SRCS_O  = $(CAR_SW_DEPS_SRCS:.c=.o) $(CAR_SW_LIB_SRCS_S:.S=.o) $(CAR_SW_LIB_SRCS_C:.c=.o)
@@ -57,9 +57,11 @@ $(foreach link,$(patsubst $(CAR_LD_DIR)/%.ld,%,$(wildcard $(CAR_LD_DIR)/*.ld)),$
 
 CAR_ELFLOAD_BLOCKING_SAFED_SRC_C := $(CAR_SW_DIR)/tests/bare-metal/hostd/safed_offloader_blocking.c
 CAR_ELFLOAD_BLOCKING_SAFED_PATH := $(basename $(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C))
+CAR_ELFLOAD_PULPD_SRC_C := $(CAR_SW_DIR)/tests/bare-metal/hostd/pulp-offload.c
+CAR_ELFLOAD_PULPD_PATH := $(basename $(CAR_ELFLOAD_PULPD_SRC_C))
 
 CAR_SW_TEST_SRCS_S	= $(wildcard $(CAR_SW_DIR)/tests/bare-metal/hostd/*.S)
-CAR_SW_TEST_SRCS_C	= $(filter-out $(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C), $(wildcard $(CAR_SW_DIR)/tests/bare-metal/hostd/*.c))
+CAR_SW_TEST_SRCS_C	= $(filter-out $(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C) $(CAR_ELFLOAD_PULPD_SRC_C), $(wildcard $(CAR_SW_DIR)/tests/bare-metal/hostd/*.c))
 
 CAR_SW_TEST_DRAM_DUMP	= $(CAR_SW_TEST_SRCS_S:.S=.car.dram.dump) $(CAR_SW_TEST_SRCS_C:.c=.car.dram.dump)
 CAR_SW_TEST_SPM_DUMP	= $(CAR_SW_TEST_SRCS_S:.S=.car.spm.dump)  $(CAR_SW_TEST_SRCS_C:.c=.car.spm.dump)
@@ -67,7 +69,10 @@ CAR_SW_TEST_L2_DUMP	= $(CAR_SW_TEST_SRCS_S:.S=.car.l2.dump)   $(CAR_SW_TEST_SRCS
 CAR_SW_TEST_SPM_ROMH	= $(CAR_SW_TEST_SRCS_S:.S=.car.rom.memh)  $(CAR_SW_TEST_SRCS_C:.c=.car.rom.memh)
 CAR_SW_TEST_SPM_GPTH	= $(CAR_SW_TEST_SRCS_S:.S=.car.gpt.memh)  $(CAR_SW_TEST_SRCS_C:.c=.car.gpt.memh)
 
-car-sw-tests: $(CAR_SW_TEST_DRAM_DUMP) $(CAR_SW_TEST_SPM_DUMP) $(CAR_SW_TEST_L2_DUMP) $(CAR_SW_TEST_SPM_ROMH) $(CAR_SW_TEST_SPM_GPTH) car-sw-offload-tests
+car-sw-tests: $(CAR_SW_TEST_DRAM_DUMP) $(CAR_SW_TEST_SPM_DUMP) $(CAR_SW_TEST_L2_DUMP) $(CAR_SW_TEST_SPM_ROMH) $(CAR_SW_TEST_SPM_GPTH) car-pulpd-sw-offload-tests car-safed-sw-offload-tests
+
+# Generate ELFs for blocking offload from cheshire. We execute from L2 or dram.
+# TODO: Make these generations nicer and using patterns. As of now, they are ugly and inefficient.
 
 #################
 # Safety Island #
@@ -75,10 +80,8 @@ car-sw-tests: $(CAR_SW_TEST_DRAM_DUMP) $(CAR_SW_TEST_SPM_DUMP) $(CAR_SW_TEST_L2_
 
 include $(CAR_SW_DIR)/tests/bare-metal/safed/sw.mk
 
-# Generate ELFs for blocking offload from cheshire. We execute from L2 or dram.
-# TODO: Make this generation nicer and using patterns. As of now, it is ugly and inefficient.
-car-sw-offload-tests: safed-sw-all
-	$(foreach header,$(HEADER_TARGETS), \
+car-safed-sw-offload-tests:
+	$(foreach header,$(SAFED_HEADER_TARGETS), \
 		cp $(header) $(CAR_SW_DIR)/tests/bare-metal/safed/payload.h; \
 		$(CHS_SW_CC) $(CAR_SW_INCLUDES) $(CHS_SW_CCFLAGS) -c $(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C) -o $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o; \
 		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CAR_LD_DIR)/l2.ld $(CHS_SW_LDFLAGS) -o $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.l2.elf  $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
@@ -87,4 +90,22 @@ car-sw-offload-tests: safed-sw-all
 		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.dram.elf > $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.dram.dump; \
 		$(RM) $(CAR_SW_DIR)/tests/bare-metal/safed/payload.h; \
 		$(RM) $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o; \
+  )
+
+###################
+# Integer Cluster #
+###################
+
+include $(CAR_SW_DIR)/tests/bare-metal/pulpd/sw.mk
+
+car-pulpd-sw-offload-tests:
+	$(foreach header,$(PULPD_HEADER_TARGETS), \
+		cp $(header) $(CAR_SW_DIR)/tests/bare-metal/pulpd/payload.h; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) $(RISCV_CCFLAGS) -c $(CAR_ELFLOAD_PULPD_SRC_C) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CAR_LD_DIR)/l2.ld $(RISCV_LDFLAGS) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.elf  $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
+		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.elf   > $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.dump; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CHS_LD_DIR)/dram.ld $(RISCV_LDFLAGS) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.elf  $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
+		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.elf > $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.dump; \
+		$(RM) $(CAR_SW_DIR)/tests/bare-metal/pulpd/payload.h; \
+		$(RM) $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o; \
 	)
diff --git a/sw/tests/bare-metal/hostd/pulp-offload.c b/sw/tests/bare-metal/hostd/pulp-offload.c
new file mode 100644
index 00000000..b9882ee4
--- /dev/null
+++ b/sw/tests/bare-metal/hostd/pulp-offload.c
@@ -0,0 +1,43 @@
+// Copyright 2023 ETH Zurich and University of Bologna.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Yvan Tortorella <yvan.tortorella@unibo.it>
+//
+// Bare-metal offload test for the PULP cluster
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include "car_memory_map.h"
+#include "car_util.h"
+#include "dif/clint.h"
+#include "dif/uart.h"
+#include "params.h"
+#include "regs/cheshire.h"
+#include "util.h"
+#include "payload.h"
+
+int main(void)
+{
+  char str[] = "Cluster boot.\r\n";
+  uint32_t rtc_freq = *reg32(&__base_regs, CHESHIRE_RTC_FREQ_REG_OFFSET);
+  uint64_t reset_freq = clint_get_core_freq(rtc_freq, 2500);
+
+  load_binary();
+
+  volatile uint32_t pulp_boot_default = 0x78008080;
+
+  pulp_cluster_set_bootaddress(pulp_boot_default);
+
+  uart_init(&__base_uart, reset_freq, 115200);
+  uart_write_str(&__base_uart, str, sizeof(str));
+  uart_write_flush(&__base_uart);
+
+  pulp_cluster_start();
+
+  pulp_cluster_wait_eoc();
+
+  return 0;
+
+}
diff --git a/sw/tests/bare-metal/pulpd/sw.mk b/sw/tests/bare-metal/pulpd/sw.mk
new file mode 100644
index 00000000..0b5fdbae
--- /dev/null
+++ b/sw/tests/bare-metal/pulpd/sw.mk
@@ -0,0 +1,39 @@
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Alessandro Ottaviano <aottaviano@iis.ee.ethz.ch>
+# Yvan Tortorella <yvan.tortorella@unibo.it>
+
+.PHONY: all clean
+
+# Make fragment for integer cluster bare-metal tests compiled with pulp-runtime.
+
+# List all the directories in the 'tests' folder
+PULPD_SW_DIR := $(PULPD_ROOT)/regression-tests/parallel_bare_tests
+PULPD_TEST_DIRS := $(wildcard $(PULPD_ROOT)/regression-tests/parallel_bare_tests/*)
+PULPD_TEST_DIRS := $(filter-out $(wildcard $(PULPD_ROOT)/regression-tests/parallel_bare_tests/*.cfg),$(PULPD_TEST_DIRS))
+
+# Generate the list of build targets based on the directories
+PULPD_BUILD_TARGETS := $(addsuffix /build,$(PULPD_TEST_DIRS))
+
+# We have a target per test. The target (1) compiles the binary and (2) generates the needed stimuli
+# file format, if any is required.
+$(PULPD_SW_DIR)/%/build: $(PULPD_ROOT) | venv
+	# Compile
+	$(MAKE) -C $(PULPD_SW_DIR)/$* all
+	@echo $(PULPD_SW_DIR)
+
+# Convert compiled binaries to header files
+PULPD_HEADER_TARGETS := $(patsubst $(PULPD_SW_DIR)/%, $(CAR_SW_DIR)/tests/bare-metal/pulpd/%.h, $(PULPD_TEST_DIRS))
+
+$(CAR_SW_DIR)/tests/bare-metal/pulpd/%.h: $(PULPD_SW_DIR)/%/build/test/test | venv
+	$(VENV)/python $(CAR_ROOT)/scripts/elf2header.py --binary $< --vectors $@
+
+# Global targets
+pulpd-sw-all: $(PULPD_BUILD_TARGETS) $(PULPD_HEADER_TARGETS)
+
+pulpd-sw-clean:
+	# Clean all the directories in 'tests'
+	$(foreach dir, $(PULPD_TEST_DIRS), $(MAKE) -C $(dir) clean;)
+	$(RM) $(CAR_SW_DIR)/tests/bare-metal/pulpd/*.h
diff --git a/sw/tests/bare-metal/safed/sw.mk b/sw/tests/bare-metal/safed/sw.mk
index e9fd7874..5e4e31a7 100644
--- a/sw/tests/bare-metal/safed/sw.mk
+++ b/sw/tests/bare-metal/safed/sw.mk
@@ -4,20 +4,17 @@
 #
 # Alessandro Ottaviano <aottaviano@iis.ee.ethz.ch>
 
-RISCV32_GCC_BINROOT ?= $(dir $(shell which riscv32-unknown-elf-gcc))
-RISCV32_CC          ?= $(RISCV32_GCC_BINROOT)/riscv32-unknown-elf-gcc
-
 .PHONY: all clean
 
 # Make fragment for safety island bare-metal tests compiled with pulp-runtime.
 
 # List all the directories in the 'tests' folder
-TEST_DIRS := $(wildcard $(SAFED_SW_DIR)/tests/*)
-TEST_DIRS := $(filter-out $(SAFED_SW_DIR)/tests/runtime_shared,$(TEST_DIRS))
-TEST_DIRS := $(filter-out $(wildcard $(SAFED_SW_DIR)/tests/freertos*),$(TEST_DIRS))
+SAFED_TEST_DIRS := $(wildcard $(SAFED_SW_DIR)/tests/*)
+SAFED_TEST_DIRS := $(filter-out $(SAFED_SW_DIR)/tests/runtime_shared,$(SAFED_TEST_DIRS))
+SAFED_TEST_DIRS := $(filter-out $(wildcard $(SAFED_SW_DIR)/tests/freertos*),$(SAFED_TEST_DIRS))
 
 # Generate the list of build targets based on the directories
-BUILD_TARGETS := $(addsuffix /build,$(TEST_DIRS))
+SAFED_BUILD_TARGETS := $(addsuffix /build,$(SAFED_TEST_DIRS))
 
 # We have a target per test. The target (1) compiles the binary and (2) generates the needed stimuli
 # file format, if any is required.
@@ -26,16 +23,16 @@ $(SAFED_SW_DIR)/tests/%/build: $(SAFED_ROOT) | venv
 	$(MAKE) -C $(SAFED_SW_DIR)/tests/$* all
 
 # Convert compiled binaries to header files
-HEADER_TARGETS := $(patsubst $(SAFED_SW_DIR)/tests/%, $(CAR_SW_DIR)/tests/bare-metal/safed/%.h, $(TEST_DIRS))
+SAFED_HEADER_TARGETS := $(patsubst $(SAFED_SW_DIR)/tests/%, $(CAR_SW_DIR)/tests/bare-metal/safed/%.h, $(SAFED_TEST_DIRS))
 
 $(CAR_SW_DIR)/tests/bare-metal/safed/%.h: $(SAFED_SW_DIR)/tests/%/build | venv
-	$(VENV)/python $(CAR_ROOT)/scripts/elf_to_header.py --binary $</$*/$* --vectors $@
+	$(VENV)/python $(CAR_ROOT)/scripts/elf2header.py --binary $</$*/$* --vectors $@
 #	xxd -i $</$*/$* > $@ )
 
 # Global targets
-safed-sw-all: $(BUILD_TARGETS) $(HEADER_TARGETS)
+safed-sw-all: $(SAFED_BUILD_TARGETS) $(SAFED_HEADER_TARGETS)
 
 safed-sw-clean:
 	# Clean all the directories in 'tests'
-	$(foreach dir, $(TEST_DIRS), $(MAKE) -C $(dir) clean;)
+	$(foreach dir, $(SAFED_TEST_DIRS), $(MAKE) -C $(dir) clean;)
 	$(RM) $(CAR_SW_DIR)/tests/bare-metal/safed/*.h
diff --git a/sw/tests/bare-metal/secd/sw.mk b/sw/tests/bare-metal/secd/sw.mk
new file mode 100644
index 00000000..aeefabb6
--- /dev/null
+++ b/sw/tests/bare-metal/secd/sw.mk
@@ -0,0 +1,5 @@
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Currently empty file, TODO

From c0db25217b0c9ae2cb7c0497edcb0ae55b986a22 Mon Sep 17 00:00:00 2001
From: Yvan Tortorella <ytortorella@lagrev3.ee.ethz.ch>
Date: Wed, 2 Aug 2023 00:49:20 +0200
Subject: [PATCH 2/9] Fixed wrong connection of register interface (and making
 it logically more sensed).

---
 hw/carfield.sv      | 28 ++++++++++++++--------------
 hw/carfield_pkg.sv  | 13 ++++++++-----
 hw/cheshire_wrap.sv | 24 ++++++++++++------------
 3 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/hw/carfield.sv b/hw/carfield.sv
index e5ad790b..bc157d43 100644
--- a/hw/carfield.sv
+++ b/hw/carfield.sv
@@ -386,8 +386,8 @@ localparam int unsigned IntClusterAxiMstRWidth  =
 // verilog_lint: waive-stop line-length
 
 // External register interface synchronous with Cheshire's clock domain
-carfield_reg_req_t [iomsb(Cfg.RegExtNumSlv-2):0] ext_reg_req;
-carfield_reg_rsp_t [iomsb(Cfg.RegExtNumSlv-2):0] ext_reg_rsp;
+carfield_reg_req_t [iomsb(NumSyncRegSlv):0] ext_reg_req;
+carfield_reg_rsp_t [iomsb(NumSyncRegSlv):0] ext_reg_rsp;
 
 localparam int unsigned LlcIdWidth = Cfg.AxiMstIdWidth   +
                                      $clog2(AxiIn.num_in)+
@@ -505,12 +505,12 @@ logic [                 LogDepth:0] axi_mst_intcluster_r_rptr ;
 carfield_reg2hw_t car_regs_reg2hw;
 carfield_hw2reg_t car_regs_hw2reg;
 
-logic     [NumAsyncRegIdx-1:0] ext_reg_async_slv_req_out;
-logic     [NumAsyncRegIdx-1:0] ext_reg_async_slv_ack_in;
-reg_req_t [NumAsyncRegIdx-1:0] ext_reg_async_slv_data_out;
-logic     [NumAsyncRegIdx-1:0] ext_reg_async_slv_req_in;
-logic     [NumAsyncRegIdx-1:0] ext_reg_async_slv_ack_out;
-reg_rsp_t [NumAsyncRegIdx-1:0] ext_reg_async_slv_data_in;
+logic     [NumAsyncRegSlv-1:0] ext_reg_async_slv_req_out;
+logic     [NumAsyncRegSlv-1:0] ext_reg_async_slv_ack_in;
+reg_req_t [NumAsyncRegSlv-1:0] ext_reg_async_slv_data_out;
+logic     [NumAsyncRegSlv-1:0] ext_reg_async_slv_req_in;
+logic     [NumAsyncRegSlv-1:0] ext_reg_async_slv_ack_out;
+reg_rsp_t [NumAsyncRegSlv-1:0] ext_reg_async_slv_data_in;
 
 // External reg interface slaves (async)
 // Currently for PLL and Padframe
@@ -1175,12 +1175,12 @@ l2_wrap #(
   .slvport_w_data_i    ( axi_slv_ext_w_data  [NumL2Ports-1:0] ),
   .slvport_w_wptr_i    ( axi_slv_ext_w_wptr  [NumL2Ports-1:0] ),
   .slvport_w_rptr_o    ( axi_slv_ext_w_rptr  [NumL2Ports-1:0] ),
-  .l2_ecc_reg_async_mst_req_i  ( ext_reg_async_slv_req_out [L2EccIdx-NumSyncRegIdx] ),
-  .l2_ecc_reg_async_mst_ack_o  ( ext_reg_async_slv_ack_in  [L2EccIdx-NumSyncRegIdx] ),
-  .l2_ecc_reg_async_mst_data_i ( ext_reg_async_slv_data_out[L2EccIdx-NumSyncRegIdx] ),
-  .l2_ecc_reg_async_mst_req_o  ( ext_reg_async_slv_req_in  [L2EccIdx-NumSyncRegIdx] ),
-  .l2_ecc_reg_async_mst_ack_i  ( ext_reg_async_slv_ack_out [L2EccIdx-NumSyncRegIdx] ),
-  .l2_ecc_reg_async_mst_data_o ( ext_reg_async_slv_data_in [L2EccIdx-NumSyncRegIdx] ),
+  .l2_ecc_reg_async_mst_req_i  ( ext_reg_async_slv_req_out [L2EccIdx-NumSyncRegSlv] ),
+  .l2_ecc_reg_async_mst_ack_o  ( ext_reg_async_slv_ack_in  [L2EccIdx-NumSyncRegSlv] ),
+  .l2_ecc_reg_async_mst_data_i ( ext_reg_async_slv_data_out[L2EccIdx-NumSyncRegSlv] ),
+  .l2_ecc_reg_async_mst_req_o  ( ext_reg_async_slv_req_in  [L2EccIdx-NumSyncRegSlv] ),
+  .l2_ecc_reg_async_mst_ack_i  ( ext_reg_async_slv_ack_out [L2EccIdx-NumSyncRegSlv] ),
+  .l2_ecc_reg_async_mst_data_o ( ext_reg_async_slv_data_in [L2EccIdx-NumSyncRegSlv] ),
   .ecc_error_o         ( l2_ecc_err                           )
 );
 
diff --git a/hw/carfield_pkg.sv b/hw/carfield_pkg.sv
index a96bc2b5..93257a2e 100644
--- a/hw/carfield_pkg.sv
+++ b/hw/carfield_pkg.sv
@@ -137,9 +137,12 @@ typedef enum int {
   PadframeIdx = 'd2, // async
   L2EccIdx    = 'd3  // async
 } cheshire_reg_out_e;
-localparam int unsigned NumSyncRegIdx = PllIdx; // This only works if PllIdx is the first async reg
-localparam int unsigned NumAsyncRegIdx = 3;
-localparam int unsigned NumTotalRegIdx = NumSyncRegIdx + NumAsyncRegIdx;
+localparam int unsigned NumSyncRegSlv = 1;
+                                      // CarRegs
+localparam int unsigned NumAsyncRegSlv =  1  + 1        + 1;
+                                       // PLL  Padframe   L2ECC
+localparam int unsigned NumTotalRegSlv = NumSyncRegSlv + NumAsyncRegSlv;
+localparam int unsigned NumTotalRegRules = NumTotalRegSlv;
 
 typedef enum doub_bt {
   CarRegsBase  = 'h0000_0000_2001_0000,
@@ -273,8 +276,8 @@ localparam cheshire_cfg_t CarfieldCfgDefault = '{
                                                 L2Port2End     ,
                                                 L2Port1End     },
   // External reg slaves (at most 8 ports and rules)
-  RegExtNumSlv      : NumTotalRegIdx,
-  RegExtNumRules    : NumTotalRegIdx,
+  RegExtNumSlv      : NumTotalRegSlv,
+  RegExtNumRules    : NumTotalRegRules,
   // For carfield, PllIdx is the first index of the async reg interfaces. Please add async reg
   // interfaces indices to the left of PllIdx, and sync reg interface indices to its right.
   RegExtRegionIdx   : '{ 0, 0, 0, 0, L2EccIdx,  PadframeIdx,  PllIdx,  CarRegsIdx  },
diff --git a/hw/cheshire_wrap.sv b/hw/cheshire_wrap.sv
index 3d9dbb10..0adea722 100644
--- a/hw/cheshire_wrap.sv
+++ b/hw/cheshire_wrap.sv
@@ -254,16 +254,16 @@ module cheshire_wrap
   output cheshire_axi_ext_slv_req_t axi_mbox_slv_req_o,
   input  cheshire_axi_ext_slv_rsp_t axi_mbox_slv_rsp_i,
   // External reg demux slaves Cheshire's clock domain (sync)
-  output cheshire_reg_ext_req_t [iomsb(Cfg.RegExtNumSlv-2):0] reg_ext_slv_req_o,
-  input  cheshire_reg_ext_rsp_t [iomsb(Cfg.RegExtNumSlv-2):0] reg_ext_slv_rsp_i,
+  output cheshire_reg_ext_req_t [iomsb(NumSyncRegSlv):0] reg_ext_slv_req_o,
+  input  cheshire_reg_ext_rsp_t [iomsb(NumSyncRegSlv):0] reg_ext_slv_rsp_i,
   // External reg demux slaves other clock domains (async)
   // Padframe and PLL
-  output logic                  [NumAsyncRegIdx-1:0] ext_reg_async_slv_req_o,
-  input  logic                  [NumAsyncRegIdx-1:0] ext_reg_async_slv_ack_i,
-  output cheshire_reg_ext_req_t [NumAsyncRegIdx-1:0] ext_reg_async_slv_data_o,
-  input  logic                  [NumAsyncRegIdx-1:0] ext_reg_async_slv_req_i,
-  output logic                  [NumAsyncRegIdx-1:0] ext_reg_async_slv_ack_o,
-  input  cheshire_reg_ext_rsp_t [NumAsyncRegIdx-1:0] ext_reg_async_slv_data_i,
+  output logic                  [NumAsyncRegSlv-1:0] ext_reg_async_slv_req_o,
+  input  logic                  [NumAsyncRegSlv-1:0] ext_reg_async_slv_ack_i,
+  output cheshire_reg_ext_req_t [NumAsyncRegSlv-1:0] ext_reg_async_slv_data_o,
+  input  logic                  [NumAsyncRegSlv-1:0] ext_reg_async_slv_req_i,
+  output logic                  [NumAsyncRegSlv-1:0] ext_reg_async_slv_ack_o,
+  input  cheshire_reg_ext_rsp_t [NumAsyncRegSlv-1:0] ext_reg_async_slv_data_i,
   // Interrupts from external devices
   input  logic [iomsb(Cfg.NumExtInIntrs):0]                                  intr_ext_i,
   output logic [iomsb(Cfg.NumExtOutIntrTgts):0][iomsb(Cfg.NumExtOutIntrs):0] intr_ext_o,
@@ -347,7 +347,7 @@ cheshire_reg_ext_req_t [iomsb(Cfg.RegExtNumSlv):0] ext_reg_req;
 cheshire_reg_ext_rsp_t [iomsb(Cfg.RegExtNumSlv):0] ext_reg_rsp;
 
 // Generate synchronous external register interface from Cheshire
-for (genvar i = 0; i < NumSyncRegIdx; i++) begin: gen_ext_reg_sync
+for (genvar i = 0; i < NumSyncRegSlv; i++) begin: gen_ext_reg_sync
   assign reg_ext_slv_req_o[i] = ext_reg_req[i];
   assign ext_reg_rsp[i]       = reg_ext_slv_rsp_i[i];
 end
@@ -728,7 +728,7 @@ axi_id_remap            #(
 
 // Async reg interface:
 // See carfield_pkg.sv for indices referring to sync and async reg interfaces.
-for (genvar i = 0; i < Cfg.RegExtNumSlv - NumSyncRegIdx; i++) begin : gen_ext_reg_async
+for (genvar i = 0; i < Cfg.RegExtNumSlv - NumAsyncRegSlv; i++) begin : gen_ext_reg_async
   reg_cdc_src #(
     .CDC_KIND ( "cdc_4phase"              ),
     .req_t     ( cheshire_reg_ext_req_t   ),
@@ -736,8 +736,8 @@ for (genvar i = 0; i < Cfg.RegExtNumSlv - NumSyncRegIdx; i++) begin : gen_ext_re
   ) i_reg_cdc_src (
       .src_clk_i    ( clk_i  ),
       .src_rst_ni   ( rst_ni ),
-      .src_req_i    ( ext_reg_req[NumSyncRegIdx + i] ),
-      .src_rsp_o    ( ext_reg_rsp[NumSyncRegIdx + i] ),
+      .src_req_i    ( ext_reg_req[NumSyncRegSlv + i] ),
+      .src_rsp_o    ( ext_reg_rsp[NumSyncRegSlv + i] ),
 
       .async_req_o  ( ext_reg_async_slv_req_o[i]  ),
       .async_ack_i  ( ext_reg_async_slv_ack_i[i]  ),

From 672059d75e5b1cf78a0f7d8436f433214c3caf7d Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 09:46:07 +0200
Subject: [PATCH 3/9] carfield.mk: Bump nonfree pointer

---
 carfield.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/carfield.mk b/carfield.mk
index 8c4ea417..3ee2da93 100644
--- a/carfield.mk
+++ b/carfield.mk
@@ -90,7 +90,7 @@ endif
 ######################
 
 CAR_NONFREE_REMOTE ?= git@iis-git.ee.ethz.ch:carfield/carfield-nonfree.git
-CAR_NONFREE_COMMIT ?= df886d3eea670dffdbe4d632a64c7b123b9fac76
+CAR_NONFREE_COMMIT ?= a51afe8cd67a47b83df79e1d0f53d5a8cf22371d
 
 ## Clone the non-free verification IP for the Carfield TB
 car-nonfree-init:

From 83069ec54b47c987e144b818057e9868634625bf Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 09:59:16 +0200
Subject: [PATCH 4/9] bender: Bump pulp cluster and redundancy cells

---
 Bender.local   |  2 +-
 Bender.lock    | 14 +++++++-------
 Bender.yml     |  2 +-
 hw/carfield.sv |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Bender.local b/Bender.local
index 967e3cc5..a8179a22 100644
--- a/Bender.local
+++ b/Bender.local
@@ -6,7 +6,7 @@ overrides:
   axi:                  { git: https://github.com/pulp-platform/axi.git                   , version: 0.39.1-beta }
   apb:                  { git: "https://github.com/pulp-platform/apb.git"                 , version: 0.2.3 }
   register_interface:   { git: "https://github.com/pulp-platform/register_interface.git"  , version: 0.4.1 }
-  redundancy_cells:     { git: "https://github.com/pulp-platform/redundancy_cells.git"    , rev: "6a011b6" } # branch: michaero/hmr-alt
+  redundancy_cells:     { git: "https://github.com/pulp-platform/redundancy_cells.git"    , rev: "482d2f5" } # branch: yt/rapidrecovery
   tech_cells_generic:   { git: "https://github.com/pulp-platform/tech_cells_generic.git"  , version: =0.2.11                                }
   riscv-dbg:            { git: "https://github.com/pulp-platform/riscv-dbg.git"           , version: =0.8.0                                 }
   idma:                 { git: "https://github.com/pulp-platform/idma.git"                , rev:     437ffa9                                }
diff --git a/Bender.lock b/Bender.lock
index d49edb39..82c751e6 100644
--- a/Bender.lock
+++ b/Bender.lock
@@ -243,13 +243,13 @@ packages:
     dependencies:
     - common_cells
   fpu_interco:
-    revision: 0769976fa51bdd820656a01161a4c46b88c59ac5
+    revision: 4aec4b68424947b0c4cf25fd7c4b907cb9ec3dfa
     version: null
     source:
       Git: https://github.com/pulp-platform/fpu_interco.git
     dependencies:
-    - cv32e40p
     - fpnew
+    - riscv
   hci:
     revision: b2e6f391aa6c10c03f45b693d80a0aaddecf169b
     version: null
@@ -296,10 +296,10 @@ packages:
     - register_interface
     - tech_cells_generic
   ibex:
-    revision: 95b85ddd1c995ace9f89ee42530f9e24820c1051
+    revision: b18f7ef178ed07f5085051f96042c670a919fd5c
     version: null
     source:
-      Git: https://github.com/lowRISC/ibex.git
+      Git: https://github.com/pulp-platform/ibex.git
     dependencies:
     - tech_cells_generic
   icache-intc:
@@ -381,7 +381,7 @@ packages:
     dependencies:
     - axi_slice
   pulp_cluster:
-    revision: ebc7d1888e2242ea8cb579aab1113e02e08367b4
+    revision: 201453dbcedc8f7235b2723363356ba5845e5120
     version: null
     source:
       Git: https://github.com/pulp-platform/pulp_cluster.git
@@ -423,7 +423,7 @@ packages:
     - hwpe-stream
     - tech_cells_generic
   redundancy_cells:
-    revision: 6a011b6f5acf3eca202ed847d6c64474a0bc7cb5
+    revision: 482d2f5ed05f25c851f4048f1bd402cc0d2b58b6
     version: null
     source:
       Git: https://github.com/pulp-platform/redundancy_cells.git
@@ -443,7 +443,7 @@ packages:
     - common_cells
     - common_verification
   riscv:
-    revision: 4eac53237c6d0062715d17016fe95462eb81ebc3
+    revision: dd086242d10f248b4051a1fc67ae2b35b989c29c
     version: null
     source:
       Git: git@github.com:AlSaqr-platform/riscv_nn.git
diff --git a/Bender.yml b/Bender.yml
index 18706716..4c89dfbf 100644
--- a/Bender.yml
+++ b/Bender.yml
@@ -16,7 +16,7 @@ dependencies:
   hyperbus:           { git: https://github.com/pulp-platform/hyperbus.git,             rev: 2adb7271438cdb96c19fbaf3e2a6bf89ffeee568 } # branch: lv/phys_in_use
   car_l2:             { git: git@iis-git.ee.ethz.ch:carfield/carfield_l2_mem.git,       rev: d6ab486b2777bf78c38b49352b5977565a272a58 } # branch: main
   safety_island:      { git: git@iis-git.ee.ethz.ch:carfield/safety-island.git,         rev: b0501345b1741fa96b781ef5d845026fec036fd2 } # branch: param_banks
-  pulp_cluster:       { git: https://github.com/pulp-platform/pulp_cluster.git,         rev: ebc7d1888e2242ea8cb579aab1113e02e08367b4 } # branch: ck/carfield-1
+  pulp_cluster:       { git: https://github.com/pulp-platform/pulp_cluster.git,         rev: 201453dbcedc8f7235b2723363356ba5845e5120 } # branch: yt/rapidrecovery
   opentitan:          { git: https://github.com/alsaqr-platform/opentitan.git,          rev: 245d92fe49dc6be32afe3bfb6a133778002d4880 } # branch: carfield
   mailbox_unit:       { git: git@github.com:pulp-platform/mailbox_unit.git,             version: 1.1.0                                }
   apb:                { git: https://github.com/pulp-platform/apb.git,                  version: 0.2.3                                }
diff --git a/hw/carfield.sv b/hw/carfield.sv
index bc157d43..1705139c 100644
--- a/hw/carfield.sv
+++ b/hw/carfield.sv
@@ -1376,7 +1376,7 @@ pulp_cluster #(
   .AXI_ID_OUT_WIDTH               ( IntClusterAxiIdOutWidth   ),
   .LOG_DEPTH                      ( LogDepth                  ),
   .BaseAddr                       ( IntClusterBaseAddr        ),
-  .CDC_SYNC_STAGES                ( SyncStages                )
+  .CdcSynchStages                 ( SyncStages                )
 ) i_integer_cluster            (
   .clk_i                       ( pulp_clk                                  ),
   .rst_ni                      ( pulp_rst_n                                ),

From 0311d63b160dfedaf7908c42e4e366dfa88c97db Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 10:16:33 +0200
Subject: [PATCH 5/9] sw/include: Improve safed polling

---
 sw/include/car_util.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/sw/include/car_util.h b/sw/include/car_util.h
index f4b0b337..2e1adf5e 100644
--- a/sw/include/car_util.h
+++ b/sw/include/car_util.h
@@ -199,9 +199,8 @@ uint32_t poll_safed_corestatus () {
 	volatile uint32_t corestatus;
 	volatile uintptr_t *corestatus_addr = (uintptr_t*)CAR_SAFETY_ISLAND_CORESTATUS_ADDR;
 	// TODO: Add a timeut to not poll indefinitely
-	while (((uint32_t)readw(corestatus_addr) & 0x80000000) == 0) {
-	    corestatus = (uint32_t) readw(corestatus_addr);
-	}
+	while (((uint32_t)readw(corestatus_addr) & 0x80000000) == 0)
+	    ;
 
 	corestatus = (uint32_t) readw(corestatus_addr);
 

From 146b6aff95ff3d87da512b1ac1a4ae2896b882e0 Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 11:32:42 +0200
Subject: [PATCH 6/9] sw/sw.mk: Simpify offload compilation flow

---
 sw/sw.mk | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/sw/sw.mk b/sw/sw.mk
index 012d5669..7ad8bdff 100644
--- a/sw/sw.mk
+++ b/sw/sw.mk
@@ -80,17 +80,22 @@ car-sw-tests: $(CAR_SW_TEST_DRAM_DUMP) $(CAR_SW_TEST_SPM_DUMP) $(CAR_SW_TEST_L2_
 
 include $(CAR_SW_DIR)/tests/bare-metal/safed/sw.mk
 
+# Template function for offload tests
+define offload_tests_template
+	$(foreach header,$(1), \
+		cp $(header) $(CAR_SW_DIR)/tests/bare-metal/$(2)/payload.h; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) $(CHS_SW_CCFLAGS) -c $(3) -o $(4).$(basename $(notdir $(header))).car.o; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CAR_LD_DIR)/l2.ld $(CHS_SW_LDFLAGS) -o $(4).$(basename $(notdir $(header))).car.l2.elf  $(4).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
+		$(CHS_SW_OBJDUMP) -d -S $(4).$(basename $(notdir $(header))).car.l2.elf > $(4).$(basename $(notdir $(header))).car.l2.dump; \
+		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CHS_LD_DIR)/dram.ld $(CHS_SW_LDFLAGS) -o $(4).$(basename $(notdir $(header))).car.dram.elf  $(4).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
+		$(CHS_SW_OBJDUMP) -d -S $(4).$(basename $(notdir $(header))).car.dram.elf > $(4).$(basename $(notdir $(header))).car.dram.dump; \
+		$(RM) $(CAR_SW_DIR)/tests/bare-metal/$(2)/payload.h; \
+		$(RM) $(4).$(basename $(notdir $(header))).car.o; \
+	)
+endef
+
 car-safed-sw-offload-tests:
-	$(foreach header,$(SAFED_HEADER_TARGETS), \
-		cp $(header) $(CAR_SW_DIR)/tests/bare-metal/safed/payload.h; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) $(CHS_SW_CCFLAGS) -c $(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C) -o $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CAR_LD_DIR)/l2.ld $(CHS_SW_LDFLAGS) -o $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.l2.elf  $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
-		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.l2.elf   > $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.l2.dump; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CHS_LD_DIR)/dram.ld $(CHS_SW_LDFLAGS) -o $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.dram.elf  $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
-		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.dram.elf > $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.dram.dump; \
-		$(RM) $(CAR_SW_DIR)/tests/bare-metal/safed/payload.h; \
-		$(RM) $(CAR_ELFLOAD_BLOCKING_SAFED_PATH).$(basename $(notdir $(header))).car.o; \
-  )
+	$(call offload_tests_template,$(SAFED_HEADER_TARGETS),safed,$(CAR_ELFLOAD_BLOCKING_SAFED_SRC_C),$(CAR_ELFLOAD_BLOCKING_SAFED_PATH))
 
 ###################
 # Integer Cluster #
@@ -99,13 +104,5 @@ car-safed-sw-offload-tests:
 include $(CAR_SW_DIR)/tests/bare-metal/pulpd/sw.mk
 
 car-pulpd-sw-offload-tests:
-	$(foreach header,$(PULPD_HEADER_TARGETS), \
-		cp $(header) $(CAR_SW_DIR)/tests/bare-metal/pulpd/payload.h; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) $(RISCV_CCFLAGS) -c $(CAR_ELFLOAD_PULPD_SRC_C) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CAR_LD_DIR)/l2.ld $(RISCV_LDFLAGS) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.elf  $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
-		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.elf   > $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.l2.dump; \
-		$(CHS_SW_CC) $(CAR_SW_INCLUDES) -T$(CHS_LD_DIR)/dram.ld $(RISCV_LDFLAGS) -o $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.elf  $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o $(CHS_SW_LIBS) $(CAR_SW_LIBS); \
-		$(CHS_SW_OBJDUMP) -d -S $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.elf > $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.dram.dump; \
-		$(RM) $(CAR_SW_DIR)/tests/bare-metal/pulpd/payload.h; \
-		$(RM) $(CAR_ELFLOAD_PULPD_PATH).$(basename $(notdir $(header))).car.o; \
-	)
+	$(call offload_tests_template,$(PULPD_HEADER_TARGETS),pulpd,$(CAR_ELFLOAD_PULPD_SRC_C),$(CAR_ELFLOAD_PULPD_PATH))
+

From 98c9e315bfa72a863761d662b5faff99ad1f1d9d Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 12:05:03 +0200
Subject: [PATCH 7/9] Update README.md

---
 README.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2e49733e..74e7c3a5 100644
--- a/README.md
+++ b/README.md
@@ -57,9 +57,20 @@ Follow these steps to launch a Carfield simulation:
 * Compile tests for Carfield. Tests resides in `sw/tests`.
 
   ```
+  // Compile Safety Island standalone software
+  source ./scripts/safed-env.sh
+  make safed-sw-build
+  // Compile Integer cluster standalone software
+  source ./scripts/pulpd-env.sh
+  make pulpd-sw-build
+  // Compile Cheshire SW
   make car-sw-build
   ```
 
+  The latter commands:
+  * Compiles safety island and pulp cluster standalone tests
+  * Compiles CVA6 standalone and offloading tests
+
 ### System bootmodes
 
 * The current supported bootmodes from Cheshire are:
@@ -78,7 +89,7 @@ Follow these steps to launch a Carfield simulation:
   of preload, if any is needed. For RTL simulation, bootmodes 0, 2 and 3 are supported. SPI SD card
   bootmode is supported on FPGA emulation.
 
-* The current supported bootmodes ffrom the Safety Island are:
+* The current supported bootmodes for the Safety Island are:
 
   | `SAFED_BOOTMODE` | Action |
   | --- | --- |

From 3ad35c396189427726aaf932dc80e86cc058943f Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 13:07:26 +0200
Subject: [PATCH 8/9] bender: Update spatz dependency

---
 Bender.lock | 2 +-
 Bender.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Bender.lock b/Bender.lock
index 82c751e6..3e02baba 100644
--- a/Bender.lock
+++ b/Bender.lock
@@ -497,7 +497,7 @@ packages:
     - common_cells
     - register_interface
   spatz:
-    revision: 1506445bea978dfa81e0d717a005e84e6fa450b5
+    revision: 56544dc2e50acd08cf2c3909267d7daf19f5be19
     version: null
     source:
       Git: git@iis-git.ee.ethz.ch:spatz/spatz.git
diff --git a/Bender.yml b/Bender.yml
index 4c89dfbf..1777da6b 100644
--- a/Bender.yml
+++ b/Bender.yml
@@ -23,7 +23,7 @@ dependencies:
   timer_unit:         { git: https://github.com/pulp-platform/timer_unit.git,           version: 1.0.2                                }
   apb_adv_timer:      { git: https://github.com/pulp-platform/apb_adv_timer.git,        version: 1.0.4                                }
   can_bus:            { git: git@github.com:AlSaqr-platform/can_bus.git,                rev: 230222cc568b49b39a3385b12edaf680657bc69d }
-  spatz:              { git: git@iis-git.ee.ethz.ch:spatz/spatz.git,                    rev: 1506445bea978dfa81e0d717a005e84e6fa450b5 } # branch: aottaviano/sync-regs
+  spatz:              { git: git@iis-git.ee.ethz.ch:spatz/spatz.git,                    rev: 56544dc2e50acd08cf2c3909267d7daf19f5be19 } # branch: aottaviano/sync-regs
   bus_err_unit:       { git: git@iis-git.ee.ethz.ch:carfield/bus_err_unit.git,          rev: 47a6436dc4b4b7f4a44f7786033b22c6d01530b2 } # branch: main
   common_cells:       { git: https://github.com/pulp-platform/common_cells.git,         version: 1.30.0                               }
 

From 6ff3527396763432582d267c5bf811fd3487a68e Mon Sep 17 00:00:00 2001
From: aottaviano <aottaviano@iis.ee.ethz.ch>
Date: Thu, 3 Aug 2023 15:03:32 +0200
Subject: [PATCH 9/9] scripts: Add scripts to source safed and pulpd SW configs

---
 scripts/pulpd-env.sh | 8 ++++++++
 scripts/safed-env.sh | 9 +++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 scripts/pulpd-env.sh
 create mode 100644 scripts/safed-env.sh

diff --git a/scripts/pulpd-env.sh b/scripts/pulpd-env.sh
new file mode 100644
index 00000000..6b870b37
--- /dev/null
+++ b/scripts/pulpd-env.sh
@@ -0,0 +1,8 @@
+# Copyright 2022 ETH Zurich and University of Bologna.
+# Solderpad Hardware License, Version 0.51, see LICENSE for details.
+# SPDX-License-Identifier: SHL-0.51
+#
+
+# set up environment variables for rtl simulation
+ROOTD=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)
+[[ -d "$ROOTD/pulp_cluster/pulp-runtime" ]] && source "$ROOTD/pulp_cluster/pulp-runtime/configs/carfield-cluster.sh"
diff --git a/scripts/safed-env.sh b/scripts/safed-env.sh
new file mode 100644
index 00000000..1ca58f32
--- /dev/null
+++ b/scripts/safed-env.sh
@@ -0,0 +1,9 @@
+# Copyright 2022 ETH Zurich and University of Bologna.
+# Solderpad Hardware License, Version 0.51, see LICENSE for details.
+# SPDX-License-Identifier: SHL-0.51
+#
+
+# set up environment variables for rtl simulation
+ROOTD=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)
+[[ -d "$ROOTD/safety_island/sw/pulp-freertos" ]] && source "$ROOTD/safety_island/sw/pulp-freertos/env/safety-island.sh" #TODO add carfield target in freertos
+[[ -d "$ROOTD/safety_island/sw/pulp-runtime" ]] && source "$ROOTD/safety_island/sw/pulp-runtime/configs/carfield.sh"