Snitch Cluster Offloading

.gitignore

@@ -1,3 +1,5 @@
 *~
 .ninja*
 **/build/*
+
+.vscode/settings.json

.vscode/c_cpp_properties.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "cMake",
+            "configurationProvider": "ms-vscode.cmake-tools",
+            "compileCommands": "${config:cmake.buildDirectory}/compile_commands.json"
+        }
+    ],
+    "version": 4
+}

CMakeLists.txt

@@ -4,6 +4,7 @@
 #
 # Moritz Scherer <[email protected]>
 # Viviane Potocnik <[email protected]>
+# Philip Wiese <[email protected]>
 
 cmake_minimum_required(VERSION 3.13)
 
@@ -27,16 +28,83 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 project(chimera-sdk LANGUAGES C ASM)
 
+# WIESEP: It is important to set the ISA and ABI for the host and the cluster snitch
+set(ABI ilp32)
+set(ISA_CLUSTER_SNITCH rv32im)
+set(ISA_HOST rv32imc)
+
+message(STATUS "[CHIMERA-SDK] ABI                   : ${ABI}")
+message(STATUS "[CHIMERA-SDK] ISA_HOST              : ${ISA_HOST}")
+message(STATUS "[CHIMERA-SDK] ISA_CLUSTER_SNITCH    : ${ISA_CLUSTER_SNITCH}")
+if (${DISASSEMBLE_LIBRARIES})
+  message(STATUS "[CHIMERA-SDK] DISASSEMBLE_LIBRARIES : ON")
+else()
+  message(STATUS "[CHIMERA-SDK] DISASSEMBLE_LIBRARIES : OFF")
+endif()
+
 include(${CMAKE_CURRENT_LIST_DIR}/cmake/Utils.cmake)
 
-add_subdirectory(targets)
-add_subdirectory(hal)
+################################################################################
+# Host Runtime Library                                                         #
+################################################################################
+add_library(runtime_host STATIC)
+
+target_compile_options(runtime_host
+  PRIVATE
+  -O2
+)
+
+target_compile_options(runtime_host
+  PUBLIC
+  -march=${ISA_HOST}
+  -mabi=${ABI}
+)
+
+# WIESEP: Expose common link option
+target_link_options(runtime_host
+  PUBLIC
+  -march=${ISA_HOST}
+  -mabi=${ABI}
+  -nostartfiles
+  -ffreestanding
+)
 
-add_library(chimera-sdk INTERFACE)
-target_link_libraries(chimera-sdk INTERFACE hal)
-target_link_libraries(chimera-sdk INTERFACE runtime)
-target_sources(chimera-sdk INTERFACE $<TARGET_OBJECTS:runtime>)
+################################################################################
+# Snitch Cluster Runtime Library                                               #
+################################################################################
+add_library(runtime_cluster_snitch STATIC)
+
+# WIESEP: Do not export optimization flags
+target_compile_options(runtime_cluster_snitch
+  PRIVATE
+  -O2
+)
+
+target_compile_options(runtime_cluster_snitch
+  PUBLIC
+  -march=${ISA_CLUSTER_SNITCH}
+  -mabi=${ABI}
+)
+
+# WIESEP: Expose common link option
+target_link_options(runtime_cluster_snitch
+  PUBLIC
+  -march=${ISA_CLUSTER_SNITCH}
+  -mabi=${ABI}
+  -nostartfiles
+  -ffreestanding
+)
+
+################################################################################
+# Add subdirectories                                                           #
+################################################################################
+add_subdirectory(hal)
+add_subdirectory(targets)
+add_subdirectory(drivers)
 
+################################################################################
+# Testing                                                                      #
+################################################################################
 enable_testing()
 
 add_subdirectory(tests)

README.md

@@ -53,17 +53,45 @@ The resulting binaries will be stored in `build/bin`, and can be used within the
 
 To format all source files, run
 ```
-python scripts/run_clang_format.py -ir hal/
-python scripts/run_clang_format.py -ir targets/
-python scripts/run_clang_format.py -ir tests/
+python scripts/run_clang_format.py -ir hal/ targets/ tests/ drivers/
 ```
 
 Our CI uses llvm-12 for clang-format, so on IIS machines you may run
 ```
-python scripts/run_clang_format.py -ir hal/ --clang-format-executable=/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang-format
+python scripts/run_clang_format.py -ir tests/ hal/ targets/ drivers/ --clang-format-executable=/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang-format
 
-python scripts/run_clang_format.py -ir targets/ --clang-format-executable=/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang-format
-
-python scripts/run_clang_format.py -ir tests/ --clang-format-executable=/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang-format
+```
 
+## Visual Studio Code Integration
+
+To enable automatic configuration of the C/C++ extension and support for the integrated cMake build flow on the IIS workstations, add the following content to `.vscode/settings.json`:
+```json
+{
+    "cmake.configureSettings": {
+        "TOOLCHAIN_DIR": "/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin",
+        "TARGET_PLATFORM": "chimera-convolve",
+    },
+    "cmake.environment": {
+        "PATH": "/usr/pack/riscv-1.0-kgf/default/bin:${env:PATH}",
+        "LD_LIBRARY_PATH": "/usr/pack/riscv-1.0-kgf/lib64:/usr/pack/riscv-1.0-kgf/lib64",
+    }
+}
 ```
+If you are not on an IIS system, you need to adjust the paths according to your local installation.
+
+## Technical Details
+
+### Mixed ISA Compilation
+The current approach compiles all code for both the host and cluster cores into a single library. This requires precise handling to ensure compatibility between the different instruction set architectures (ISAs) and application binary interfaces (ABIs).
+This requires careful handling to avoid invalid instructions caused by mismatched ISAs between the host and cluster cores. Hence, we define four CMake variables,`ABI`, `ISA_HOST`, and `ISA_CLUSTER_SNITCH`, to specify the appropriate ISA for each core type. The ABI has to be identical to ensure correct function calls.
+Furthermore, the tests are split into `src_host` and `src_cluster` directories to clearly separate code executed on the host and cluster cores. 
+
+### cMake Build Flow
+All runtime functions executed by the host core are compiled into a dedicated `runtime_host` static library and the cluster code into `runtime_cluster_<type>` (e.g. `runtime_cluster_snitch`). Additionally, the HAL layer is compiled into the `hal_host` static libary.
+The final binary is seperated into two object libaries, one for the host and one for the cluster core. The host object library is linked with the `runtime_host` and `hal_host` libraries, while the cluster object library is linked with the `runtime_cluster_<type>` library. The final binary is then linked from the two object libraries.
+
+### Warning
+Special attention is required for functions that execute before the cluster core is fully initialized, such as the trampoline function and interrupt handlers. At this stage, critical resources like the stack, global pointer, and thread pointer are not yet configured. Consequently, the compiler must not generate code that allocates stack frames. To address this, such functions are implemented as naked functions, which prevent the compiler from adding prologues or epilogues that rely on stack operations.
+
+**It is recommended to always check the generated assembly code to ensure that the correct instructions are generated for the target core!**
+

drivers/CMakeLists.txt

@@ -0,0 +1,12 @@
+# Copyright 2024 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Philip Wiese <[email protected]>
+
+
+# WIESEP: Add all runtime drivers
+add_subdirectory(cluster)
+
+# WIESEP: Export this directory as root include directory for the drivers
+target_include_directories(runtime_host PUBLIC ${CMAKE_CURRENT_LIST_DIR})

drivers/cluster/CMakeLists.txt

@@ -0,0 +1,27 @@
+# Copyright 2024 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Philip Wiese <[email protected]>
+
+
+################################################################################
+# Host Runtime Library                                                         #
+################################################################################
+file(GLOB_RECURSE C_SOURCES
+  "offload_snitchCluster.c"
+)
+
+target_sources(runtime_host PRIVATE ${C_SOURCES})
+
+################################################################################
+# Snitch Cluster Runtime Library                                               #
+################################################################################
+file(GLOB_RECURSE C_SOURCES_SNITCH
+  "trampoline_snitchCluster.c"
+)
+
+target_sources(runtime_cluster_snitch PRIVATE ${C_SOURCES_SNITCH})
+
+
+