trace: Refactor and prepare for DMA trace generation

* bench: Refactor benchmarking utils to use JSON

* docs: Add trace and benchmarking utilities

.github/workflows/ci.yml

@@ -21,6 +21,20 @@ jobs:
       - name: Build docs
         run: make docs
 
+  #####################
+  # Python unit tests #
+  #####################
+
+  pytest:
+    name: Python unit tests
+    runs-on: ubuntu-22.04
+    container:
+      image: ghcr.io/pulp-platform/snitch_cluster:main
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run pytest
+        run: pytest
+
   ##############################################
   # Simulate SW on Snitch Cluster w/ Verilator #
   ##############################################

.gitlab-ci.yml

@@ -24,6 +24,14 @@ docs:
   script:
     - make docs
 
+#####################
+# Python unit tests #
+#####################
+
+pytest:
+  script:
+    - pytest
+
 #################################
 # Build Snitch cluster software #
 #################################

docs/rm/bench/join.md

@@ -0,0 +1 @@
+::: join

docs/rm/bench/roi.md

@@ -0,0 +1 @@
+::: roi

docs/rm/bench/visualize.md

@@ -0,0 +1 @@
+::: visualize

docs/rm/trace/annotate.md

@@ -0,0 +1 @@
+::: annotate

docs/rm/trace/events.md

@@ -0,0 +1 @@
+::: events

docs/rm/trace/gen_trace.md

@@ -1 +1,5 @@
+<<<<<<< HEAD
 ::: gen_trace
+=======
+::: gen_trace
+>>>>>>> c744477... trace: Refactor and prepare for DMA trace generation

mkdocs.yml

@@ -28,6 +28,7 @@ plugins:
           paths:
             - util/sim
             - util/trace
+            - util/bench
             - target/snitch_cluster/util
   - macros:
       on_error_fail: true
@@ -67,6 +68,12 @@ nav:
               - rm/sim/Elf.md
           - Trace Utilities:
               - gen_trace.py: rm/trace/gen_trace.md
+              - annotate.py: rm/trace/annotate.md
+              - events.py: rm/trace/events.md
+          - Benchmarking Utilities:
+              - join.py: rm/bench/join.md
+              - roi.py: rm/bench/roi.md
+              - visualize.py: rm/bench/visualize.md
           - Snitch Target Utilities:
               - run.py: rm/snitch_target_utils/run.md
               - build.py: rm/snitch_target_utils/build.md

python-requirements.txt

@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
+# Keep sorted.
 bin2coe
 dataclasses
 editorconfig-checker==2.3.51
@@ -13,18 +14,20 @@ json5
 jsonref
 jsonschema
 mako
+matplotlib
 mkdocs-material
-progressbar2
-tabulate
-yamllint
-pyyaml
-pytablewriter
-termcolor
 pandas
 prettytable
-pyelftools
+progressbar2
 psutil
+pyelftools
 pyflexfloat
+pytablewriter
+pytest
+pyyaml
+tabulate
+termcolor
+yamllint
 
 -r docs/requirements.txt
 -r sw/dnn/requirements.txt

target/common/common.mk

@@ -36,9 +36,9 @@ ADDR2LINE    ?= $(LLVM_BINROOT)/llvm-addr2line
 GENTRACE_PY      ?= $(UTIL_DIR)/trace/gen_trace.py
 ANNOTATE_PY      ?= $(UTIL_DIR)/trace/annotate.py
 EVENTS_PY        ?= $(UTIL_DIR)/trace/events.py
-PERF_CSV_PY      ?= $(UTIL_DIR)/trace/perf_csv.py
-LAYOUT_EVENTS_PY ?= $(UTIL_DIR)/trace/layout_events.py
-EVENTVIS_PY      ?= $(UTIL_DIR)/trace/eventvis.py
+JOIN_PY          ?= $(UTIL_DIR)/bench/join.py
+ROI_PY           ?= $(UTIL_DIR)/bench/roi.py
+VISUALIZE_PY     ?= $(UTIL_DIR)/bench/visualize.py
 
 # For some reason `$(VERILATOR_SEPP) which verilator` returns a
 # a two-liner with the OS on the first line, hence the tail -n1
@@ -233,8 +233,8 @@ clean-perf:
 clean-visual-trace:
 	rm -f $(VISUAL_TRACE)
 
-$(LOGS_DIR)/trace_hart_%.txt $(LOGS_DIR)/hart_%_perf.json: $(LOGS_DIR)/trace_hart_%.dasm $(GENTRACE_PY)
-	$(DASM) < $< | $(GENTRACE_PY) --permissive -d $(LOGS_DIR)/hart_$*_perf.json > $(LOGS_DIR)/trace_hart_$*.txt
+$(addprefix $(LOGS_DIR)/,trace_hart_%.txt hart_%_perf.json): $(LOGS_DIR)/trace_hart_%.dasm $(GENTRACE_PY)
+	$(DASM) < $< | $(GENTRACE_PY) --permissive -d $(LOGS_DIR)/hart_$*_perf.json -o $(LOGS_DIR)/trace_hart_$*.txt
 
 # Generate source-code interleaved traces for all harts. Reads the binary from
 # the logs/.rtlbinary file that is written at start of simulation in the vsim script

util/bench/join.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright 2024 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Author: Luca Colagrande <[email protected]>
+"""Combines performance metrics from all threads into one JSON file.
+
+This script takes the performance metrics from multiple cores or DMA
+engines, in JSON format as dumped by the [`events.py`][events] or
+[`gen_trace.py`][gen_trace] scripts, and merges them into a single
+JSON file for global inspection and further processing.
+"""
+
+import sys
+import argparse
+import re
+import json
+
+
+FILENAME_REGEX = r'([a-z]+)_([0-9a-f]+)_perf.json'
+
+
+def main():
+    # Argument parsing
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-i',
+        '--inputs',
+        metavar='<inputs>',
+        nargs='+',
+        help='Input performance metric dumps')
+    parser.add_argument(
+        '-o',
+        '--output',
+        metavar='<output>',
+        nargs='?',
+        default='perf.json',
+        help='Output JSON file')
+    args = parser.parse_args()
+
+    # Populate a list (one entry per hart) of dictionaries
+    # enumerating all the performance metrics for each hart
+    data = {}
+    for filename in sorted(args.inputs):
+
+        # Get thread ID and type (DMA or hart) from filename
+        match = re.search(FILENAME_REGEX, filename)
+        typ = match.group(1)
+        idx = int(match.group(2), base=16)
+
+        # Populate dictionary of metrics for the current hart
+        with open(filename, 'r') as f:
+            data[f'{typ}_{idx}'] = json.load(f)
+
+    # Export data
+    with open(args.output, 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

util/bench/roi.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# Copyright 2024 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Author: Luca Colagrande <[email protected]>
+"""Filters and labels execution regions for visualization.
+
+This script takes a JSON file of performance metrics, as output by
+[`join.py`][join], and generates another JSON, where the execution
+regions are filtered and labeled for visualization, according to an
+auxiliary region-of-interest (ROI) specification file (JSON format).
+The specification file can be a Mako template to parameterize
+certain parameters, such as the number of clusters in the system.
+The output JSON can be passed to the [`visualize.py`][visualize]
+script for visualization.
+
+Check out `test_data/data.json` and `test_data/spec.json` for an
+example input and specification file which can be fed as input to the
+tool respectively. The corresponding output is contained in
+`test_data/roi.json`.
+"""
+
+import argparse
+import json
+import json5
+from mako.template import Template
+import sys
+
+
+def format_roi(roi, label):
+    return {
+        "label": label,
+        "tstart": roi["tstart"],
+        "tend": roi["tend"],
+        "attrs": {key: value for key, value in roi.items() if key not in ["tstart", "tend"]}
+    }
+
+
+def get_roi(data, thread, idx):
+    thread_type, thread_idx = thread.split('_')
+    thread_idx = int(thread_idx)
+    try:
+        thread_data = data[thread]
+    except KeyError:
+        raise KeyError(f"Nonexistent thread {thread}")
+    if thread_type in ["hart", "dma"]:
+        try:
+            if thread_type == "hart":
+                return thread_data[idx]
+            elif thread_type == "dma":
+                return thread_data["transfers"][idx]
+        except IndexError:
+            raise IndexError(f"Thread {thread} does not contain region {idx}")
+    else:
+        raise ValueError(f"Unsupported thread type {thread_type}")
+
+
+def filter_and_label_rois(data, spec):
+    output = {}
+    # Iterate all threads in the rendered specification
+    for thread_spec in spec:
+        thread = thread_spec['thread']
+        output_rois = []
+        # Iterate all ROIs to keep for the current thread
+        for roi in thread_spec['roi']:
+            output_roi = format_roi(get_roi(data, thread, roi['idx']), roi['label'])
+            output_rois.append(output_roi)
+        # Add ROIs for current thread to output, if any
+        if output_rois:
+            output[thread] = output_rois
+    return output
+
+
+def load_json_inputs(input_path, spec_path, **kwargs):
+    # Read input JSON
+    with open(input_path, 'r') as f:
+        data = json5.load(f)
+    # Read and render specification template JSON
+    with open(spec_path, 'r') as f:
+        spec_template = Template(f.read())
+        rendered_spec = spec_template.render(**kwargs)
+        spec = json5.loads(rendered_spec)
+    return data, spec
+
+
+def main():
+    # Argument parsing
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'input',
+        help='Input JSON file')
+    parser.add_argument(
+        'spec',
+        help='ROI specification file (JSON format)')
+    parser.add_argument(
+        '--cfg',
+        help='Hardware configuration file used to render the specification file')
+    parser.add_argument(
+        '-o',
+        '--output',
+        nargs='?',
+        default='roi.json',
+        help='Output JSON file')
+    args = parser.parse_args()
+
+    # Load hardware configuration
+    with open(args.cfg, 'r') as f:
+        cfg = json5.load(f)
+
+    # Read and render input files
+    data, spec = load_json_inputs(args.input, args.spec, cfg=cfg)
+
+    # Process inputs and generate output JSON
+    output = filter_and_label_rois(data, spec)
+
+    # Write output to file
+    with open(args.output, 'w') as f:
+        json.dump(output, f, indent=4)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

util/bench/tests/test_data/data.json

@@ -0,0 +1,46 @@
+{
+    "hart_0": [
+        {
+            "tstart": 1759.0,
+            "tend": 6802.0,
+            "fpss_fpu_occupancy": 0.006345429307951616,
+            "total_ipc": 0.04501288915328178
+        },
+        {
+            "tstart": 6802.0,
+            "tend": 12647.0,
+            "fpss_fpu_occupancy": 0.013860369609856264,
+            "total_ipc": 0.20756331279945245
+        }
+    ],
+    "dma_9": {
+        "aggregate_bw": 11.829313543599257,
+        "transfers": [
+            {
+                "tstart": 3512,
+                "tend": 3526,
+                "bw": 1.1428571428571428
+            },
+            {
+                "tstart": 3564,
+                "tend": 3578,
+                "bw": 1.1428571428571428
+            }
+        ]
+    },
+    "dma_18": {
+        "aggregate_bw": 16.633245382585752,
+        "transfers": [
+            {
+                "tstart": 3608,
+                "tend": 3622,
+                "bw": 1.1428571428571428
+            },
+            {
+                "tstart": 3660,
+                "tend": 3674,
+                "bw": 1.1428571428571428
+            }
+        ]
+    }
+}