LLVM OpenMP runtime

.clang-tidy

@@ -0,0 +1,46 @@
+---
+Checks: >
+  bugprone-*,
+  cppcoreguidelines-*,
+  performance-*,
+  readability-*,$
+  clang-analyzer-*,
+  misc-unused-parameters,
+
+  -bugprone-easily-swappable-parameters,
+  -bugprone-reserved-identifier,
+  -cppcoreguidelines-pro-type-vararg,
+  -cppcoreguidelines-avoid-non-const-global-variables,
+
+WarningsAsErrors: ''
+HeaderFileExtensions:
+  - ''
+  - h
+  - hh
+  - hpp
+  - hxx
+ImplementationFileExtensions:
+  - c
+  - cc
+  - cpp
+  - cxx
+HeaderFilterRegex: ''
+AnalyzeTemporaryDtors: false
+FormatStyle:     file
+CheckOptions:
+  cert-dcl16-c.NewSuffixes: 'L;LL;LU;LLU'
+  google-readability-namespace-comments.ShortNamespaceLines: '10'
+  cert-err33-c.CheckedFunctions: '::aligned_alloc;::asctime_s;::at_quick_exit;::atexit;::bsearch;::bsearch_s;::btowc;::c16rtomb;::c32rtomb;::calloc;::clock;::cnd_broadcast;::cnd_init;::cnd_signal;::cnd_timedwait;::cnd_wait;::ctime_s;::fclose;::fflush;::fgetc;::fgetpos;::fgets;::fgetwc;::fopen;::fopen_s;::fprintf;::fprintf_s;::fputc;::fputs;::fputwc;::fputws;::fread;::freopen;::freopen_s;::fscanf;::fscanf_s;::fseek;::fsetpos;::ftell;::fwprintf;::fwprintf_s;::fwrite;::fwscanf;::fwscanf_s;::getc;::getchar;::getenv;::getenv_s;::gets_s;::getwc;::getwchar;::gmtime;::gmtime_s;::localtime;::localtime_s;::malloc;::mbrtoc16;::mbrtoc32;::mbsrtowcs;::mbsrtowcs_s;::mbstowcs;::mbstowcs_s;::memchr;::mktime;::mtx_init;::mtx_lock;::mtx_timedlock;::mtx_trylock;::mtx_unlock;::printf_s;::putc;::putwc;::raise;::realloc;::remove;::rename;::scanf;::scanf_s;::setlocale;::setvbuf;::signal;::snprintf;::snprintf_s;::sprintf;::sprintf_s;::sscanf;::sscanf_s;::strchr;::strerror_s;::strftime;::strpbrk;::strrchr;::strstr;::strtod;::strtof;::strtoimax;::strtok;::strtok_s;::strtol;::strtold;::strtoll;::strtoul;::strtoull;::strtoumax;::strxfrm;::swprintf;::swprintf_s;::swscanf;::swscanf_s;::thrd_create;::thrd_detach;::thrd_join;::thrd_sleep;::time;::timespec_get;::tmpfile;::tmpfile_s;::tmpnam;::tmpnam_s;::tss_create;::tss_get;::tss_set;::ungetc;::ungetwc;::vfprintf;::vfprintf_s;::vfscanf;::vfscanf_s;::vfwprintf;::vfwprintf_s;::vfwscanf;::vfwscanf_s;::vprintf_s;::vscanf;::vscanf_s;::vsnprintf;::vsnprintf_s;::vsprintf;::vsprintf_s;::vsscanf;::vsscanf_s;::vswprintf;::vswprintf_s;::vswscanf;::vswscanf_s;::vwprintf_s;::vwscanf;::vwscanf_s;::wcrtomb;::wcschr;::wcsftime;::wcspbrk;::wcsrchr;::wcsrtombs;::wcsrtombs_s;::wcsstr;::wcstod;::wcstof;::wcstoimax;::wcstok;::wcstok_s;::wcstol;::wcstold;::wcstoll;::wcstombs;::wcstombs_s;::wcstoul;::wcstoull;::wcstoumax;::wcsxfrm;::wctob;::wctrans;::wctype;::wmemchr;::wprintf_s;::wscanf;::wscanf_s;'
+  llvm-else-after-return.WarnOnUnfixable: 'false'
+  cert-str34-c.DiagnoseSignedUnsignedCharComparisons: 'false'
+  google-readability-namespace-comments.SpacesBeforeComments: '2'
+  cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: 'true'
+  google-readability-braces-around-statements.ShortStatementLines: '1'
+  google-readability-function-size.StatementThreshold: '800'
+  llvm-qualified-auto.AddConstToQualified: 'false'
+  llvm-else-after-return.WarnOnConditionVariables: 'false'
+  cert-oop54-cpp.WarnOnlyIfThisHasSuspiciousField: 'false'
+  cppcoreguidelines-avoid-do-while.IgnoreMacros: 'true'
+SystemHeaders:   false
+...
+

.clangd

@@ -0,0 +1,29 @@
+CompileFlags:
+  Add: [
+    -std=c++17,
+    -D__riscv,
+    -DNUM_CORES,
+    -DNUM_CORES_PER_TILE,
+    -DNUM_GROUPS,
+    -DSTACK_SIZE,
+    -DXQUEUE_SIZE,
+    -DSEQ_MEM_SIZE,
+    -DBANKING_FACTOR,
+    -DETL_CHECK_PUSH_POP,
+    -DETL_LOG_ERRORS,
+    -DETL_VERBOSE_ERRORS
+  ]
+
+---
+
+If:
+  PathMatch: [.*/runtime/.*/.*]
+CompileFlags:
+  Add: [-xc++, -I..]
+
+---
+
+If:
+  PathMatch: [.*/tests/.*/.*]
+CompileFlags:
+  Add: [-xc++, -I.., -I../../runtime/]

CHANGELOG.md

@@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Add `apb` dependency of version 0.2.4
 - Add support for the `FENCE` instruction
 - Add support for DRAMsys5.0 co-simulation
+- Add KMP runtime
+- Add new runtime testing infrastructure
+- Add automated benchmark running and plotting
+- Add clangd and clang-tidy configs
+- Add Banshee config
+- Add OpenMP sections and teams tests
 
 ### Changes
 - Add physical feasible TeraPool configuration with SubGroup hierarchy.
@@ -33,12 +39,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Update `common_verification` to 0.2.3
 - Update `register_interface` to 0.4.3
 - Updated Halide to version 15
+- Include SP in Snitch traces to aid in building better tracevis
+- Make the `tracevis` target depend on the `trace` target and use `-cf`
+- Rename the `omp` runtime directory to `gomp`
+- Link with `lld` when using clang
 
 ### Fixed
 - Fix type issue in `snitch_addr_demux`
 - Properly disable the debugging CSRs in ASIC implementations
 - Fix a bug in the  DMA's distributed midend
 - Fix bugs in radix2, radix4by2 parallelization and loading of data for radix4 CFFT
+- Fix building verilator on newer clang and gcc versions
+- Fix tracing targets missing some harts
+- Fix clang warnings about the .comment section
 
 ## 0.6.0 - 2023-01-09
 

Makefile

@@ -90,7 +90,7 @@ tc-llvm:
 		-DCMAKE_INSTALL_PREFIX=$(LLVM_INSTALL_DIR) \
 		-DCMAKE_CXX_COMPILER=$(CXX) \
 		-DCMAKE_C_COMPILER=$(CC) \
-		-DLLVM_ENABLE_PROJECTS="clang" \
+		-DLLVM_ENABLE_PROJECTS="clang;lld" \
 		-DLLVM_TARGETS_TO_BUILD="RISCV;host" \
 		-DLLVM_BUILD_DOCS="0" \
 		-DLLVM_ENABLE_BINDINGS="0" \
@@ -146,7 +146,7 @@ $(BENDER_INSTALL_DIR)/bender:
 verilator: $(VERILATOR_INSTALL_DIR)/bin/verilator
 $(VERILATOR_INSTALL_DIR)/bin/verilator: toolchain/verilator Makefile
 	cd $<; unset VERILATOR_ROOT; \
-	autoconf && CC=$(CLANG_CC) CXX=$(CLANG_CXX) CXXFLAGS=$(CLANG_CXXFLAGS) LDFLAGS=$(CLANG_LDFLAGS) ./configure --prefix=$(VERILATOR_INSTALL_DIR) $(VERILATOR_CI) && \
+	autoconf && CC=$(CLANG_CC) CXX=$(CLANG_CXX) CXXFLAGS="$(CLANG_CXXFLAGS) -include memory" LDFLAGS=$(CLANG_LDFLAGS) ./configure --prefix=$(VERILATOR_INSTALL_DIR) $(VERILATOR_CI) && \
 	make -j4 && make install
 
 # Update and patch hardware dependencies for MemPool

banshee-config.yaml

@@ -0,0 +1,53 @@
+# Copyright 2021 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+address:
+  scratch_reg: 0x40000000
+  wakeup_reg: 0x40000004
+  tcdm_start: 0x40000008
+  tcdm_end: 0x4000000C
+  nr_cores: 0x40000010
+  uart: 0xC0000000
+  # Not supported in MemPool
+  barrier_reg:
+    start: 0x50000000
+    offset: 0x100000
+  cluster_base_hartid: 0x50000001
+  cluster_num: 0x50000002
+  cluster_id: 0x50000003
+  cl_clint: 0x40000060
+  clint: 0xFFFF0000
+memory:
+  tcdm:
+    start: 0x0
+    size: 0x100000
+    offset: 0x100000
+    latency: 5
+  dram:
+    start: 0x80000000
+    size: 0x01000000
+    offset: 0x0
+    latency: 10
+  periphs:
+    start: 0x40000000
+    size: 0x20000
+    offset: 0x0
+    latency: 5
+    callbacks:
+      - name: zero-memory
+        size: 0x10000
+      - name: mempool-dma
+        size: 0x1C
+inst_latency:
+  mul: 3
+  mulh: 3
+  mulhsu: 3
+  mulhu: 3
+  div: 3
+  divu: 3
+  rem: 3
+  remu: 3
+ssr:
+  num_dm: 3

config/minpool-no-xpulp.mk

@@ -0,0 +1,14 @@
+# Copyright 2021 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Author: Matheus Cavalcante, ETH Zurich
+
+###############
+##  MinPool  ##
+###############
+
+include $(MEMPOOL_DIR)/config/minpool.mk
+
+# Disable Xpulpimg
+xpulpimg ?= 0

hardware/Makefile

@@ -15,6 +15,9 @@ TOOLCHAIN_DIR := $(abspath $(ROOT_DIR)/../toolchain)
 config_mk = $(abspath $(ROOT_DIR)/../config/config.mk)
 include $(config_mk)
 
+# Banshee config
+banshee_config = $(MEMPOOL_DIR)/banshee-config.yaml
+
 # build path
 buildpath       ?= build
 resultpath      ?= results
@@ -319,6 +322,13 @@ verilate: $(VERILATOR_EXE) $(buildpath) Makefile
 	# Avoid capturing the return status when running the load-throughput analysis
 	if [ $(tg) -ne 1 ]; then ./scripts/return_status.sh $(buildpath)/transcript; fi
 
+################
+# Banshee      #
+################
+
+banshee:
+	banshee --configuration $(banshee_config) --num-clusters 1 --num-cores $(num_cores) -l $(preload)
+
 #############
 # Lint      #
 #############
@@ -340,11 +350,13 @@ spyglass/tmp/files: $(bender)
 trace_env += NUM_CORES=$(num_cores)
 trace_env += SEQ_MEM_SIZE=$(seq_mem_size)
 
+.PHONY: benchmark trace log pre_trace post_trace tracevis
+
 benchmark: log simcvcs
 	# Call `make` again to get variable extension with all traces
 	result_dir=$(result_dir) $(MAKE) trace
 
-trace: pre_trace $(trace) post_trace
+trace: post_trace
 
 log:
 	mkdir -p "$(result_dir)"
@@ -361,20 +373,20 @@ log:
 pre_trace:
 	rm -rf $(tracepath)
 
-post_trace:
+post_trace: $(trace)
 	mkdir -p "$(result_dir)"
 	cp $(buildpath)/transcript "$(result_dir)/" | true
 	cp $(traceresult) "$(result_dir)"
 	cp $(trace) "$(result_dir)"
 	$(python) $(ROOT_DIR)/scripts/gen_avg.py --folder "$(result_dir)" | tee $(result_dir)/avg.txt
 
-$(buildpath)/%.trace: $(buildpath)/%.dasm
+$(buildpath)/%.trace: $(buildpath)/%.dasm pre_trace
 	mkdir -p $(tracepath)
 	$(INSTALL_DIR)/riscv-isa-sim/bin/spike-dasm < $< > $(tracepath)/$*
 	$(trace_env) $(python) $(ROOT_DIR)/scripts/gen_trace.py -p --csv $(traceresult) $(tracepath)/$* > $@
 
-tracevis:
-	$(MEMPOOL_DIR)/scripts/tracevis.py $(preload) $(buildpath)/*.trace -o $(buildpath)/tracevis.json
+tracevis: $(trace)
+	$(MEMPOOL_DIR)/scripts/tracevis.py -cf $(preload) $(buildpath)/*.trace -o $(buildpath)/tracevis.json
 
 ############################
 # Unit tests simulation    #

hardware/deps/snitch/src/snitch.sv

@@ -2415,7 +2415,7 @@ module snitch
     assign sp_new_value = gpr_wdata[i];
     always_ff @(posedge clk_i or posedge rst_i) begin
       if (!rst_i && gpr_we[i] && gpr_waddr[i] == SP && csr_stack_limit_q != 32'hFFFF_FFFF && ($signed(sp_new_value) < $signed(csr_stack_limit_q))) begin
-        $warning("[Stackoverflow: Core %0d] Set SP to 0x%08h, limit is 0x%08h", hart_id_i, sp_new_value, csr_stack_limit_q);
+        $warning("[Stackoverflow: Core %0d, PC: %0d] Set SP to 0x%08h, limit is 0x%08h", hart_id_i, inst_addr_o, sp_new_value, csr_stack_limit_q);
       end
     end
   end

hardware/scripts/gen_trace.py

@@ -28,9 +28,10 @@
 GENERAL_WARN = ('WARNING: Inconsistent final state; performance metrics may '
                 'be inaccurate. Is this trace complete?\n')
 
-TRACE_IN_REGEX = r'(\d+)\s+(\d+)\s+(0x[0-9A-Fa-fz]+)\s+([^#;]*)(\s*#;\s*(.*))?'
+TRACE_IN_REGEX = (r'(\d+)\s+(\d+)\s+(0x[0-9A-Fa-fz]+)\s+(0x[0-9A-Fa-fz]+)\s+'
+                  r'([^#;]*)(\s*#;\s*(.*))?')
 
-TRACE_OUT_FMT = '{:>8} {:>8} {:>10} {:<30}'
+TRACE_OUT_FMT = '{:>8} {:>8} {:>8} {:>10} {:<30}'
 
 
 # -------------------- Tracer configuration  --------------------
@@ -304,7 +305,7 @@ def annotate_insn(
     match = re.search(TRACE_IN_REGEX, line.strip('\n'))
     if match is None:
         raise ValueError('Not a valid trace line:\n{}'.format(line))
-    time_str, cycle_str, pc_str, insn, _, extras_str = match.groups()
+    time_str, cycle_str, pc_str, sp_str, insn, _, extras_str = match.groups()
     time_info = (int(time_str), int(cycle_str))
     show_time_info = (dupl_time_info or time_info != last_time_info)
     time_info_strs = tuple((str(elem) if show_time_info else '')
@@ -333,12 +334,13 @@ def annotate_insn(
         else:
             prev_wfi_time = 0
         return ((TRACE_OUT_FMT + ' #; {}').format(*time_info_strs,
-                                                  pc_str, insn, annot),
+                                                  pc_str, sp_str, insn, annot),
                 time_info, prev_wfi_time, retired_reg, empty)
     # Vanilla trace
     else:
-        return TRACE_OUT_FMT.format(
-            *time_info_strs, pc_str, insn), time_info, 0, retired_reg, False
+        return (TRACE_OUT_FMT.format(
+            *time_info_strs, pc_str, sp_str, insn), time_info, 0, retired_reg,
+            False)
 
 
 # -------------------- Performance metrics --------------------

hardware/src/mempool_cc.sv

@@ -406,8 +406,8 @@ module mempool_cc
           extras_fpu = $sformatf("%s}", extras_fpu);
 
           $timeformat(-9, 0, "", 10);
-          $sformat(trace_entry, "%t %8d 0x%h DASM(%h) #; %s\n",
-              $time, cycle, i_snitch.pc_q, i_snitch.inst_data_i, extras_str);
+          $sformat(trace_entry, "%t %8d 0x%h 0x%h DASM(%h) #; %s\n",
+              $time, cycle, i_snitch.pc_q, i_snitch.i_snitch_regfile.mem[2], i_snitch.inst_data_i, extras_str);
           $fwrite(f, trace_entry);
         end
 

scripts/license-checker.hjson

@@ -16,7 +16,7 @@
     'software/runtime/printf*'
     'software/runtime/encoding.h'
     'software/runtime/mempool_dma_frontend.h'
-    'software/runtime/omp/libgomp.h'
+    'software/runtime/gomp/libgomp.h'
     'software/riscv-tests/*'
     'hardware/deps/*'
     'hardware/tb/dpi/elfloader.cpp'