hero: Debug omptarget

sw/hero/device/apps/libomptarget_device/link.ld

@@ -6,16 +6,17 @@ OUTPUT_ARCH( "riscv" )
 ENTRY(_start)
 
 /* Memory section should be provided in a separate, platform-specific */
-/* file. It should define at least the L1 and L3 memory blocks. */
+/* file. It should define at least the L1 and L2 memory blocks. */
 MEMORY
 {
-  L3           : ORIGIN = 0xC0000000, LENGTH = 0x800000
+  L2           : ORIGIN = 0x78000000, LENGTH = 0x800000
+  LOCAL        : ORIGIN = 0x51000000, LENGTH = 0x20000
 }
 
 SECTIONS
 {
 
-  /* Program code goes into L3 */
+  /* Program code goes into L2 */
   .text :
   {
     . = ALIGN(4);
@@ -27,48 +28,48 @@ SECTIONS
     *(.text)
     . = ALIGN(4);
     _etext = .;
-  } >L3
+  } >L2
 
-  /* By default, constant data goes into L3, right after code section */
+  /* By default, constant data goes into L2, right after code section */
   .rodata :
   {
     . = ALIGN(4);
     *(.rodata)
     *(.rodata*)
     . = ALIGN(4);
-  } >L3
+  } >L2
 
   /* HTIF section for FESVR */
-  .htif         : { } >L3
+  .htif         : { } >L2
 
   /* Thread Local Storage sections */
   .tdata    :
   {
     __tdata_start = .;
     *(.tdata .tdata.* .gnu.linkonce.td.*)
     __tdata_end = .;
-  } >L3
+  } >L2
   .tbss :
   {
     __tbss_start = .;
     *(.tbss .tbss.* .gnu.linkonce.tb.*)
     *(.tcommon)
     __tbss_end = .;
-  } >L3
+  } >L2
 
   /* Cluster Local Storage sections */
   .cdata    :
   {
     __cdata_start = .;
     *(.cdata .cdata.*)
     __cdata_end = .;
-  } >L3
+  } >LOCAL
   .cbss :
   {
     __cbss_start = .;
     *(.cbss .cbss.*)
     __cbss_end = .;
-  } >L3
+  } >LOCAL
 
   /* used by the startup to initialize data */
   _sidata = LOADADDR(.data);
@@ -80,15 +81,15 @@ SECTIONS
     __global_pointer$ = . + 0x7f0;
     *(.srodata.cst16) *(.srodata.cst8) *(.srodata.cst4) *(.srodata.cst2) *(.srodata .srodata.*)
     *(.sdata .sdata.* .gnu.linkonce.s.*)
-  } >L3
+  } >L2
 
-  /* Initialized data sections goes into L3 */
+  /* Initialized data sections goes into L2 */
   .data :
   {
     __DATA_BEGIN__ = .;
     *(.data .data.* .gnu.linkonce.d.*)
     SORT(CONSTRUCTORS)
-  } >L3
+  } >L2
   _edata = .; PROVIDE (edata = .);
 
   /* small bss section */
@@ -99,7 +100,7 @@ SECTIONS
     *(.dynsbss)
     *(.sbss .sbss.* .gnu.linkonce.sb.*)
     *(.scommon)
-  } >L3
+  } >L2
 
   /* Uninitialized data section */
   .bss            :
@@ -111,20 +112,19 @@ SECTIONS
       _end.  Align after .bss to ensure correct alignment even if the
       .bss section disappears because there are no input sections. */
    . = ALIGN(. != 0 ? 32 / 8 : 1);
-  } >L3
+  } >L2
   . = ALIGN(32 / 8);
   . = SEGMENT_START("ldata-segment", .);
   . = ALIGN(32 / 8);
   __BSS_END__ = .;
   __bss_end = .;
   _end = .; PROVIDE (end = .);
 
-  /* Uninitialized data section in L3 */
+  /* Uninitialized data section in L2 */
   .dram :
   {
     *(.dram)
     _edram = .;
-  } >L3
+  } >L2
 
-  __uart = 0x2002000;
 }

sw/hero/device/apps/libomptarget_device/src/io.h

@@ -0,0 +1,94 @@
+// Copyright 2023 ETH Zurich and University of Bologna.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Robert Balas <[email protected]>
+//
+
+/* Description: Memory mapped register I/O access
+ */
+
+#ifndef __IO_H
+#define __IO_H
+
+#include <stdint.h>
+
+
+/* generic I/O write */
+static inline void writeb(uint8_t val, uintptr_t addr)
+{
+	asm volatile("sb %0, 0(%1)"
+		     :
+		     : "r"(val), "r"((volatile uint8_t *)addr)
+		     : "memory");
+}
+
+static inline void writeh(uint16_t val, uintptr_t addr)
+{
+	asm volatile("sh %0, 0(%1)"
+		     :
+		     : "r"(val), "r"((volatile uint16_t *)addr)
+		     : "memory");
+}
+
+static inline void writew(uint32_t val, uintptr_t addr)
+{
+	asm volatile("sw %0, 0(%1)"
+		     :
+		     : "r"(val), "r"((volatile uint32_t *)addr)
+		     : "memory");
+}
+
+static inline void writed(uint64_t val, uintptr_t addr)
+{
+	asm volatile("sd %0, 0(%1)"
+		     :
+		     : "r"(val), "r"((volatile uint64_t *)addr)
+		     : "memory");
+}
+
+/* generic I/O read */
+static inline uint8_t readb(const uintptr_t addr)
+{
+	uint8_t val;
+
+	asm volatile("lb %0, 0(%1)"
+		     : "=r"(val)
+		     : "r"((const volatile uint8_t *)addr)
+		     : "memory");
+	return val;
+}
+
+static inline uint16_t readh(const uintptr_t addr)
+{
+	uint16_t val;
+
+	asm volatile("lh %0, 0(%1)"
+		     : "=r"(val)
+		     : "r"((const volatile uint16_t *)addr)
+		     : "memory");
+	return val;
+}
+
+static inline uint32_t readw(const uintptr_t addr)
+{
+	uint32_t val;
+
+	asm volatile("lw %0, 0(%1)"
+		     : "=r"(val)
+		     : "r"((const volatile uint32_t *)addr)
+		     : "memory");
+	return val;
+}
+
+static inline uint64_t readd(const uintptr_t addr)
+{
+	uint64_t val;
+
+	asm volatile("ld %0, 0(%1)"
+		     : "=r"(val)
+		     : "r"((const volatile uint64_t *)addr)
+		     : "memory");
+	return val;
+}
+#endif

sw/hero/device/apps/libomptarget_device/src/main.c

@@ -5,6 +5,35 @@
 
 #include "sw_mailbox.h"
 #include "snrt.h"
+#include "io.h"
+
+__attribute__((optimize("O0"))) void csleep(uint32_t cycles) {
+    uint32_t start = snrt_mcycle();
+    while ((snrt_mcycle() - start) < cycles) {}
+}
+
+void snrt_putchar(char c) {
+    writew(c, (uintptr_t) 0x3002000);
+    csleep(100000);
+}
+
+void snrt_puthalfbyte(uint8_t halfbyte) {
+    uint32_t ascii[16] = {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68, 69, 70};
+    snrt_putchar(ascii[halfbyte]);
+}
+
+void snrt_putbyte(uint8_t byte) {
+    snrt_puthalfbyte((byte >> 4) & 0xf);
+    snrt_puthalfbyte(byte & 0xf);
+}
+
+void snrt_putword(uint32_t word) {
+    for(int i = 3; i >= 0; i--)
+      snrt_putbyte((word >> 8 * i) & 0xff);
+    snrt_putchar(10);
+}
+
+//void _putchar(char character) {snrt_putchar(character);}
 
 //================================================================================
 // MACROS AND SETTINGS
@@ -59,7 +88,6 @@ typedef struct rab_miss_t {
 static volatile uint32_t g_printf_mutex = 0;
 
 static volatile uint32_t *soc_scratch = (uint32_t *)(0x02000014);
-struct l3_layout l3l;
 
 const uint32_t snrt_stack_size __attribute__((weak, section(".rodata"))) = 12;
 
@@ -106,6 +134,7 @@ void _snrt_hier_wakeup(void) {
  * @brief A re-entrant wrapper to printf
  *
  */
+
 void snrt_printf(const char *format, ...) {
   va_list args;
 
@@ -288,20 +317,20 @@ int main(int argc, char *argv[]) {
   unsigned core_idx = snrt_cluster_core_idx();
   unsigned core_num = snrt_cluster_core_num();
 
+
   /**
    * One core initializes the global data structures
    */
-  if (snrt_is_dm_core()) {
+  if (core_idx == 0) {
     // read memory layout from scratch2
-    memcpy(&l3l, (void *)soc_scratch[2], sizeof(struct l3_layout));
-    g_a2h_rb = (struct ring_buf *)l3l.a2h_rb;
-    g_a2h_mbox = (struct ring_buf *)l3l.a2h_mbox;
-    g_h2a_mbox = (struct ring_buf *)l3l.h2a_mbox;
+    g_a2h_rb = NULL;
+    g_h2a_mbox = (struct ring_buf *)readw(0x3000000);
+    g_a2h_mbox = (struct ring_buf *)readw(0x3000004);
   }
 
-  snrt_cluster_hw_barrier();
+  //snrt_cluster_hw_barrier();
 
-  __snrt_omp_bootstrap(core_idx);
+  //__snrt_omp_bootstrap(core_idx);
 
   //snrt_trace("omp_bootstrap complete, core_idx: %d core_num: %d\n", core_idx, core_num);
 

sw/hero/device/apps/libomptarget_device/src/sw_mailbox.c

@@ -4,6 +4,7 @@
 
 #include "sw_mailbox.h"
 #include "snrt.h" // snrt_mcycle
+#include "io.h"
 
 /***********************************************************************************
  * MACROS
@@ -25,10 +26,6 @@ volatile struct ring_buf *g_h2a_mbox;
 /***********************************************************************************
  * FUNCTIONS
  ***********************************************************************************/
-__attribute__((optimize("O0"))) void csleep(uint32_t cycles) {
-    uint32_t start = snrt_mcycle();
-    while ((snrt_mcycle() - start) < cycles) {}
-}
 
 int syscall(uint64_t which, uint64_t arg0, uint64_t arg1, uint64_t arg2,
             uint64_t arg3, uint64_t arg4) {
@@ -55,12 +52,6 @@ int syscall(uint64_t which, uint64_t arg0, uint64_t arg1, uint64_t arg2,
     return retries;
 }
 
-void snrt_putchar(char c) {
-    *(volatile uint32_t *)0x2002000 = c;
-    csleep(10000);
-    //syscall(SYS_write, 1, c, 1, 0, 0);
-}
-
 void snrt_hero_exit(int code) { syscall(SYS_exit, code, 0, 0, 0, 0); }
 
 /***********************************************************************************

sw/hero/device/runtime/src/occamy_defs.h

@@ -7,10 +7,10 @@
 #include "spatz_cluster_peripheral.h"
 
 // Hardware parameters
-#define SNRT_BASE_HARTID 1
+#define SNRT_BASE_HARTID 16
 #define SNRT_CLUSTER_CORE_NUM N_CORES_PER_CLUSTER
 #define SNRT_CLUSTER_NUM (N_QUADS * N_CLUSTERS_PER_QUAD)
-#define SNRT_CLUSTER_DM_CORE_NUM 1
+#define SNRT_CLUSTER_DM_CORE_NUM 2
 #define SNRT_TCDM_START_ADDR QUADRANT_0_CLUSTER_0_TCDM_BASE_ADDR
 #define SNRT_TCDM_SIZE                       \
     (QUADRANT_0_CLUSTER_0_PERIPH_BASE_ADDR - \

sw/hero/device/runtime/src/occamy_start.S

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 #define SNRT_INIT_INT_REGS
-#define SNRT_INIT_FP_REGS
+//#define SNRT_INIT_FP_REGS
 #define SNRT_INIT_GP
 #define SNRT_INIT_CORE_INFO
 #define SNRT_INIT_CLS

sw/hero/device/runtime/src/occamy_start.c

@@ -5,6 +5,8 @@
 #define SNRT_INIT_TLS
 #define SNRT_INIT_BSS
 #define SNRT_INIT_CLS
+#define SNRT_CRT0_CALLBACK1
+#define SNRT_CRT0_CALLBACK2
 #define SNRT_CRT0_CALLBACK3
 #define SNRT_INIT_LIBS
 #define SNRT_CRT0_PRE_BARRIER
@@ -15,8 +17,14 @@
 static inline void snrt_exit(int exit_code) {
 }
 
+static inline void snrt_crt0_callback1() {
+}
+
+static inline void snrt_crt0_callback2() {
+}
+
 static inline void snrt_crt0_callback3() {
-    _snrt_cluster_hw_barrier = cluster_hw_barrier_addr(snrt_cluster_idx());
+    //_snrt_cluster_hw_barrier = cluster_hw_barrier_addr(snrt_cluster_idx());
 }
 
 static inline void snrt_crt0_callback7() { return_to_cva6(SYNC_CLUSTERS); }

sw/hero/device/runtime/src/snrt.c

@@ -19,5 +19,4 @@
 #include "printf.c"
 #include "putchar.c"
 #include "sync.c"
-#include "sys_dma.c"
 #include "team.c"

sw/hero/device/runtime/src/snrt.h

@@ -11,7 +11,6 @@
 // Occamy specific definitions
 #include "occamy_defs.h"
 #include "occamy_memory_map.h"
-#include "sys_dma.h"
 
 // Forward declarations
 #include "alloc_decls.h"