Add Chiplet AXI Interconnect (#92)

* Initial Push - Seperate Chiplet Generation

* Update OccamyGen to parse the type to testharness

* Modified Testharness for Chiplet Interconnect

* Update Bootrom

* Small Updates

Bender.yml

@@ -118,7 +118,8 @@ sources:
     files:
       - target/rtl/test/uartdpi/uartdpi.sv
       - target/sim_chip/testharness/testharness.sv
-      - target/sim/test/bootrom.sv
+      # - target/sim/test/bootrom.sv
+      - target/rtl/bootrom/bootrom.sv
       - target/rtl/src/occamy_chip.sv
 
 

Makefile

@@ -131,3 +131,4 @@ hemaia_system_vcs_preparation: # In SNAX Docker
 
 hemaia_system_vcs: # In ESAT Server
 	$(MAKE) -C ./target/sim_chip bin/occamy_chip.vcs
+# How to start the execution of the simulation: cd ./target/sim_chip/bin; ./occamy_chip.vcs -gui -R -fgp=num_threads:8

hw/occamy/occamy_chip.sv.tpl

@@ -19,6 +19,15 @@ import ${name}_pkg::*;
   input  logic        test_mode_i,
   input  chip_id_t    chip_id_i,
   input  logic [1:0]  boot_mode_i,
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+  // HeMAiA D2D AXI Interface
+  // Chiplet Requst to Router
+  output ${soc2router_bus.req_type()} soc2router_req_o,
+  input  ${soc2router_bus.rsp_type()} soc2router_rsp_i,
+  // Router Requst to Chiplet
+  input  ${router2soc_bus.req_type()} router2soc_req_i,
+  output ${router2soc_bus.rsp_type()} router2soc_rsp_o,
+% endif
   // `uart` Interface
   output logic        uart_tx_o,
   input  logic        uart_rx_i,
@@ -160,6 +169,14 @@ import ${name}_pkg::*;
     .rtc_i              (rtc_i),
     .test_mode_i        (test_mode_i),
     .chip_id_i          (chip_id_i),
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+    // Chiplet Requst to Router
+    .soc2router_req_o,
+    .soc2router_rsp_i,
+    // Router Requst to Chiplet
+    .router2soc_req_i,
+    .router2soc_rsp_o,
+% endif
     .boot_mode_i        (boot_mode_i),
     .uart_tx_o          (uart_tx_o),
     .uart_cts_ni        (uart_cts_ni),

hw/occamy/occamy_soc.sv.tpl

@@ -8,6 +8,8 @@
 // AUTOMATICALLY GENERATED by genoccamy.py; edit the script instead.
 
 <%
+  cuts_soc_to_router = occamy_cfg["cuts"]["soc_to_router"]
+  cuts_router_to_soc = occamy_cfg["cuts"]["router_to_soc"]
   cuts_narrow_to_quad = occamy_cfg["cuts"]["narrow_to_quad"]
   cuts_quad_to_narrow = occamy_cfg["cuts"]["quad_to_narrow"]
   cuts_wide_to_quad = occamy_cfg["cuts"]["wide_to_quad"]
@@ -59,6 +61,16 @@ module ${name}_soc
   output ${soc_wide_xbar.out_spm_wide.req_type()} spm_axi_wide_req_o,
   input  ${soc_wide_xbar.out_spm_wide.rsp_type()} spm_axi_wide_rsp_i,
 
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+  // HeMAiA Multi-Chip AXI Interface
+  // Chiplet Requst to Router
+  output ${soc2router_bus.req_type()} soc2router_req_o,
+  input  ${soc2router_bus.rsp_type()} soc2router_rsp_i,
+  // Router Requst to Chiplet
+  input  ${router2soc_bus.req_type()} router2soc_req_i,
+  output ${router2soc_bus.rsp_type()} router2soc_rsp_o,
+% endif
+
   // SoC control register IO
   output logic [1:0] spm_narrow_rerror_o,
   output logic [1:0] spm_wide_rerror_o,
@@ -80,6 +92,25 @@ module ${name}_soc
 
   ${module}
 
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+  ///////////////////////////////////////
+  // Connections between SoC to Router //
+  ///////////////////////////////////////
+  <% 
+    soc2router = soc_wide_xbar.out_hemaia_multichip \
+                              .change_iw(context, soc2router_bus.iw, "soc2router_iwc") \
+                              .cut(context, cuts_soc_to_router, name="soc2router_cut")
+    router2soc = router2soc_bus.copy(name="router2soc") \
+                               .declare(context)
+    router2soc.cut(context, cuts_router_to_soc, name="router2soc_cut") \
+              .change_iw(context, soc_wide_xbar.in_hemaia_multichip.iw, name="router2soc_iwc", to=soc_wide_xbar.in_hemaia_multichip)
+  %> \
+  assign soc2router_req_o = ${soc2router.req_name()};
+  assign ${soc2router.rsp_name()} = soc2router_rsp_i;
+  assign ${router2soc.req_name()} = router2soc_req_i;
+  assign router2soc_rsp_o = ${router2soc.rsp_name()};
+% endif
+
   ///////////////////////////////////
   // Connections between crossbars //
   ///////////////////////////////////
@@ -233,7 +264,7 @@ module ${name}_soc
   // SPM WIDE //
   //////////////
   <% wide_spm_mst = soc_wide_xbar.out_spm_wide \
-                    .cut(context, cuts_wide_conv_to_spm_wide)
+                    .cut(context, cuts_wide_conv_to_spm_wide, name="wide_spm_cut")
   %>\
 
   assign spm_axi_wide_req_o = ${wide_spm_mst.req_name()};

hw/occamy/occamy_top.sv.tpl

@@ -72,6 +72,16 @@ module ${name}_top
   output ${soc_wide_xbar.out_spm_wide.req_type()} spm_axi_wide_req_o,
   input  ${soc_wide_xbar.out_spm_wide.rsp_type()} spm_axi_wide_rsp_i,
 
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+  // HeMAiA Multi-Chip AXI Interface
+  // Chiplet Requst to Router
+  output ${soc2router_bus.req_type()} soc2router_req_o,
+  input  ${soc2router_bus.rsp_type()} soc2router_rsp_i,
+  // Router Requst to Chiplet
+  input  ${router2soc_bus.req_type()} router2soc_req_i,
+  output ${router2soc_bus.rsp_type()} router2soc_rsp_o,
+% endif
+
   /// Chip specific control registers
   output ${soc_axi_lite_narrow_periph_xbar.out_chip_ctrl.req_type()} chip_ctrl_req_o,
   input  ${soc_axi_lite_narrow_periph_xbar.out_chip_ctrl.rsp_type()} chip_ctrl_rsp_i,
@@ -83,7 +93,6 @@ module ${name}_top
 );
 
 <%
-
   cuts_clint_cfg = occamy_cfg["cuts"]["periph_axi_lite_narrow_clint_cfg"]
   cuts_soc_ctrl_cfg = occamy_cfg["cuts"]["periph_axi_lite_narrow_soc_ctrl_cfg"]
   cuts_chip_ctrl_cfg = occamy_cfg["cuts"]["periph_axi_lite_narrow_chip_ctrl_cfg"]
@@ -153,6 +162,14 @@ module ${name}_top
     .periph_axi_lite_narrow_rsp_i ( periph_regbus_soc2per_rsp ),
     .spm_axi_wide_req_o,
     .spm_axi_wide_rsp_i,
+% if occamy_cfg['hemaia_multichip']['single_chip'] is False: 
+    // Chiplet Requst to Router
+    .soc2router_req_o,
+    .soc2router_rsp_i,
+    // Router Requst to Chiplet
+    .router2soc_req_i,
+    .router2soc_rsp_o,
+% endif
     .spm_narrow_rerror_o (spm_narrow_rerror),
     .spm_wide_rerror_o (spm_wide_rerror),
     .mtip_i ( mtip ),

target/rtl/bootrom/src/bootrom.c

@@ -12,6 +12,8 @@
 // For values need to share between functions, use uninitialized global variable
 // + initialization function
 
+#include "chip_id.h"
+#include "sys_dma.h"
 #include "uart.h"
 #include "xmodem.h"
 
@@ -30,52 +32,133 @@ void delay_cycles(uint64_t cycle) {
 }
 
 // Boot modes.
-enum boot_mode_t { JTAG, UART, PRINTMEM, NORMAL };
+enum boot_mode_t {
+    TARGET_CHIPID,
+    UART,
+    COPY_TO_REMOTE,
+    COPY_FROM_REMOTE,
+    PRINTMEM,
+    NORMAL
+};
 
 void bootrom() {
-    enum boot_mode_t boot_mode = JTAG;
-    uint64_t start_address;
+    enum boot_mode_t boot_mode = NORMAL;
+    uint64_t local_chip_mem_start_address;
+    uint64_t remote_chip_mem_start_address;
     uint64_t memory_length;
 
-    uint32_t chip_id;
-    asm volatile("csrr %0, 0xf15" : "=r"(chip_id));
+    uint32_t chip_id = get_current_chip_id();
+    uint32_t target_chip_id = chip_id;
     uintptr_t address_prefix = ((uintptr_t)chip_id) << 40;
 
     char in_buf[8];
     init_uart(address_prefix, 50000000, 1000000);
 
     while (1) {
-        start_address = 0x80000000L | ((uint64_t)address_prefix);
+        local_chip_mem_start_address = 0x80000000L | ((uint64_t)address_prefix);
+        remote_chip_mem_start_address =
+            0x80000000L | ((uint64_t)target_chip_id << 40);
         print_str(address_prefix, "\033[2J");
         print_str(address_prefix, "\r\n\t\t Welcome to HeMAiA Bootrom");
         print_str(address_prefix, "\r\n");
         print_str(address_prefix, "\r\n\t Chip ID: 0x");
         print_u8(address_prefix, chip_id);
+        print_str(address_prefix, "\r\n\t Target Remote Chip ID: 0x");
+        print_u8(address_prefix, target_chip_id);
         print_str(address_prefix,
                   "\r\n\t Enter the number to select the mode: ");
-        print_str(address_prefix, "\r\n\t 1. Load from JTAG");
+        print_str(address_prefix, "\r\n\t 1. Change the target remote Chip ID");
         print_str(address_prefix, "\r\n\t 2. Load from UART to 0x");
-        print_u48(address_prefix, start_address);
-        print_str(address_prefix, "\r\n\t 3. Print memory from 0x");
-        print_u48(address_prefix, start_address);
-        print_str(address_prefix, "\r\n\t 4. Continue to Boot from 0x");
-        print_u48(address_prefix, start_address);
+        print_u48(address_prefix, remote_chip_mem_start_address);
+        print_str(address_prefix,
+                  "\r\n\t 3. Copy memory from local chip to remote chip");
+        print_str(address_prefix,
+                  "\r\n\t 4. Copy memory from remote chip to local chip");
+        print_str(address_prefix, "\r\n\t 5. Print memory from 0x");
+        print_u48(address_prefix, remote_chip_mem_start_address);
+        print_str(address_prefix, "\r\n\t 6. Continue to Boot from 0x");
+        print_u48(address_prefix, local_chip_mem_start_address);
         print_str(address_prefix, "\r\n");
 
         boot_mode = getchar(address_prefix) - '0' - 1;
 
+        char* cur = 0;
+
         switch (boot_mode) {
-            case JTAG:
+            case TARGET_CHIPID:
                 print_str(address_prefix,
-                          "\r\n\t Handover to debugger... \r\n\r\n");
-                __asm__ volatile(
-                    "csrr a0, mhartid;"
-                    "ebreak;");
+                          "\r\n\t Enter the target remote Chip ID: ");
+                scan_uart(address_prefix, in_buf);
+                cur = in_buf;
+                target_chip_id = 0;
+                while (*cur != '\0') {
+                    if (*cur >= '0' || *cur <= '9') {
+                        target_chip_id = (target_chip_id << 4) + *cur - '0';
+                    } else if (*cur >= 'A' || *cur <= 'F') {
+                        target_chip_id =
+                            (target_chip_id << 4) + *cur - 'A' + 10;
+                    } else if (*cur >= 'a' || *cur <= 'f') {
+                        target_chip_id =
+                            (target_chip_id << 4) + *cur - 'a' + 10;
+                    } else {
+                        print_str(address_prefix, "\r\n\t Invalid input. ");
+                        getchar(address_prefix);
+                        break;
+                    }
+                    cur++;
+                }
+                break;
+
+            case COPY_TO_REMOTE:
+                print_str(address_prefix,
+                          "\r\n\t Enter the size of the memory in byte: ");
+                scan_uart(address_prefix, in_buf);
+                cur = in_buf;
+                memory_length = 0;
+                while (*cur != '\0') {
+                    memory_length = memory_length * 10 + *cur - '0';
+                    cur++;
+                }
+                print_str(address_prefix, "\r\n\t Copying memory from 0x");
+                print_u48(address_prefix, local_chip_mem_start_address);
+                print_str(address_prefix, " to 0x");
+                print_u48(address_prefix, remote_chip_mem_start_address);
+                print_str(address_prefix, " with size ");
+                print_u48(address_prefix, memory_length);
+                print_str(address_prefix, " bytes...");
+                sys_dma_blk_memcpy(remote_chip_mem_start_address,
+                                   local_chip_mem_start_address, memory_length);
+                print_str(address_prefix, "\r\n\t Copy finished. ");
+                getchar(address_prefix);
+                break;
+
+            case COPY_FROM_REMOTE:
+                print_str(address_prefix,
+                          "\r\n\t Enter the size of the memory in byte: ");
+                scan_uart(address_prefix, in_buf);
+                cur = in_buf;
+                memory_length = 0;
+                while (*cur != '\0') {
+                    memory_length = memory_length * 10 + *cur - '0';
+                    cur++;
+                }
+                print_str(address_prefix, "\r\n\t Copying memory from 0x");
+                print_u48(address_prefix, remote_chip_mem_start_address);
+                print_str(address_prefix, " to 0x");
+                print_u48(address_prefix, local_chip_mem_start_address);
+                print_str(address_prefix, " with size ");
+                print_u48(address_prefix, memory_length);
+                print_str(address_prefix, " bytes...");
+                sys_dma_blk_memcpy(local_chip_mem_start_address,
+                                   remote_chip_mem_start_address,
+                                   memory_length);
+                print_str(address_prefix, "\r\n\t Copy finished. ");
+                getchar(address_prefix);
                 break;
 
             case UART:
                 delay_cycles(50000000);  // Delay for 1s
-                uart_xmodem(address_prefix, start_address);
+                uart_xmodem(address_prefix, remote_chip_mem_start_address);
                 print_str(address_prefix, "\r\n\t Load finished. \r\n\r\n");
                 break;
 
@@ -90,9 +173,10 @@ void bootrom() {
                     cur++;
                 }
                 print_str(address_prefix, "\r\n\t The memory from 0x");
-                print_u48(address_prefix, start_address);
+                print_u48(address_prefix, remote_chip_mem_start_address);
                 print_str(address_prefix, " is:");
-                print_mem_hex(address_prefix, (char*)start_address,
+                print_mem_hex(address_prefix,
+                              (char*)remote_chip_mem_start_address,
                               memory_length);
                 print_str(address_prefix, "\r\n\r\n\t Print finished. ");
                 getchar(address_prefix);
@@ -101,7 +185,7 @@ void bootrom() {
             case NORMAL:
                 print_str(address_prefix, "\033[2J");
                 print_str(address_prefix, "\r\n\t Booting at 0x");
-                print_u48(address_prefix, start_address);
+                print_u48(address_prefix, local_chip_mem_start_address);
                 print_str(address_prefix, "...\r\n\r\n\r\n");
                 return;
                 break;

target/rtl/bootrom/src/chip_id.h

@@ -0,0 +1,43 @@
+#pragma once
+#include <stdint.h>
+
+inline uint8_t get_current_chip_id() {
+    uint32_t chip_id;
+# if __riscv_xlen == 64
+    // 64-bit system (CVA6), get chip_id from 0xf15
+    asm volatile("csrr %0, 0xf15" : "=r"(chip_id));
+# else
+    // 32-bit system, get chip_id from 0xbc2 (base_addrh)
+    // and shift it to the right by 8 bits
+    asm volatile ("csrr %0, 0xbc2" : "=r"(chip_id));
+    chip_id = chip_id >> 8;
+# endif
+    return (uint8_t)chip_id;
+}
+
+inline uint8_t *get_current_chip_baseaddress() {
+#if __riscv_xlen == 64
+    // 64-bit system (CVA6), get chip_id from 0xf15
+    uint32_t chip_id;
+    asm volatile("csrr %0, 0xf15" : "=r"(chip_id));
+    return (uint8_t *)((uintptr_t)chip_id << 40);
+#else
+    // 32-bit system, return 0 (not supported)
+    return (uint8_t *)0;
+#endif
+}
+
+inline uint8_t *get_chip_baseaddress(uint8_t chip_id) {
+#if __riscv_xlen == 64
+    // 64-bit system, perform the shift and return the base address
+    return (uint8_t *)((uintptr_t)chip_id << 40);
+#else
+    // 32-bit system, return 0 (not supported)
+    return (uint8_t *)0;
+#endif
+}
+
+inline uint32_t get_current_chip_baseaddress_h() {
+    uint32_t chip_id = get_current_chip_id();
+    return (uint32_t)(chip_id << 8);
+}