diff --git a/test/Ne16Weight.py b/test/Ne16Weight.py index de401ec..0151ce0 100644 --- a/test/Ne16Weight.py +++ b/test/Ne16Weight.py @@ -100,7 +100,9 @@ def decode( return weight @staticmethod - def source_generate(wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter) -> None: + def source_generate( + wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter + ) -> None: assert wmem == "tcdm", f"Invalid wmem source provided: {wmem}" section = "PI_L1" diff --git a/test/NeurekaV2TestConf.py b/test/NeurekaV2TestConf.py index 6094610..72542a7 100644 --- a/test/NeurekaV2TestConf.py +++ b/test/NeurekaV2TestConf.py @@ -103,7 +103,8 @@ def check_valid_out_type_with_norm_quant(self) -> NeurekaV2TestConf: @field_validator("wmem") @classmethod def check_valid_wmem(cls, v: WmemLiteral) -> WmemLiteral: - _supported_wmem = ["tcdm", "sram", "mram"] + breakpoint() + _supported_wmem = ["sram", "mram"] assert ( v in _supported_wmem ), f"Unsupported wmem {v}. Supported {_supported_wmem}." diff --git a/test/NeurekaV2Weight.py b/test/NeurekaV2Weight.py index 5b2c5d2..6455f47 100644 --- a/test/NeurekaV2Weight.py +++ b/test/NeurekaV2Weight.py @@ -132,7 +132,9 @@ def decode( return weight @staticmethod - def source_generate(wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter) -> None: + def source_generate( + wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter + ) -> None: if wmem == "sram": section = '__attribute__((section(".weightmem_sram")))' elif wmem == "mram": diff --git a/test/NeurekaWeight.py b/test/NeurekaWeight.py index 1f7db8f..9f9a9cd 100644 --- a/test/NeurekaWeight.py +++ b/test/NeurekaWeight.py @@ -158,7 +158,9 @@ def decode( return weight @staticmethod - def source_generate(wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter) -> None: + def source_generate( + wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter + ) -> None: if wmem == "sram": section = '__attribute__((section(".weightmem_sram")))' elif wmem == "mram": diff --git a/test/NnxTestClasses.py b/test/NnxTestClasses.py index ef482b6..ac83c3c 100644 --- a/test/NnxTestClasses.py +++ b/test/NnxTestClasses.py @@ -21,12 +21,12 @@ import os from abc import ABC, abstractmethod from enum import Enum -from typing import Callable, Literal, Optional, Set, Tuple, Type, Union +from typing import Literal, Optional, Set, Tuple, Type, Union import numpy as np import numpy.typing as npt import torch -from pydantic import BaseModel, PositiveInt, field_validator, model_validator +from pydantic import BaseModel, PositiveInt, model_validator from HeaderWriter import HeaderWriter from NeuralEngineFunctionalModel import NeuralEngineFunctionalModel @@ -226,11 +226,15 @@ def incr_generator(): x += 1 if x > _type.max: x = 0 - return torch.from_numpy( - np.fromiter(incr_generator(), count=np.prod(shape), dtype=np.int64) - ).reshape( - (shape[0], shape[2], shape[3], shape[1]) - ).permute((0, 3, 1, 2)).type(torch.int64) + + return ( + torch.from_numpy( + np.fromiter(incr_generator(), count=np.prod(shape), dtype=np.int64) + ) + .reshape((shape[0], shape[2], shape[3], shape[1])) + .permute((0, 3, 1, 2)) + .type(torch.int64) + ) class DataGenerationMethod(Enum): RANDOM = 0 @@ -238,7 +242,9 @@ class DataGenerationMethod(Enum): INCREMENTED = 2 @staticmethod - def _generate_data(_type: IntegerType, shape: Tuple, method: NnxTestGenerator.DataGenerationMethod): + def _generate_data( + _type: IntegerType, shape: Tuple, method: NnxTestGenerator.DataGenerationMethod + ): if method == NnxTestGenerator.DataGenerationMethod.RANDOM: return NnxTestGenerator._generate_random(_type, shape) elif method == NnxTestGenerator.DataGenerationMethod.ONES: @@ -287,13 +293,15 @@ def from_conf( if scale is None: assert conf.scale_type is not None scale = NnxTestGenerator._generate_data( - conf.scale_type, shape=scale_shape, + conf.scale_type, + shape=scale_shape, method=data_generation_method, ) if conf.has_bias and bias is None: assert conf.bias_type is not None bias = NnxTestGenerator._generate_data( - conf.bias_type, shape=bias_shape, + conf.bias_type, + shape=bias_shape, method=data_generation_method, ).type(torch.int32) if global_shift is None: @@ -342,9 +350,7 @@ class NnxWeight(ABC): @staticmethod @abstractmethod def encode( - weight: npt.NDArray[np.uint8], - bits: int, - depthwise: bool = False + weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = False ) -> npt.NDArray[np.uint8]: """Unroll weight into expected memory format @@ -368,9 +374,7 @@ def decode( @staticmethod @abstractmethod def source_generate( - wmem: WmemLiteral, - init: npt.NDArray[np.uint8], - header_writer: HeaderWriter + wmem: WmemLiteral, init: npt.NDArray[np.uint8], header_writer: HeaderWriter ) -> None: """Function implementing generation of weight's sources""" ... @@ -429,7 +433,9 @@ def generate(self, test_name: str, test: NnxTest): test.conf.depthwise, ) - self.nnxWeightCls.source_generate(test.conf.wmem, weight_init, self.header_writer) + self.nnxWeightCls.source_generate( + test.conf.wmem, weight_init, self.header_writer + ) # Render scale if test.scale is not None: diff --git a/test/conftest.py b/test/conftest.py index bb68f24..07030e4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,6 +17,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import subprocess from typing import Union import pydantic @@ -28,7 +29,7 @@ from NeurekaV2TestConf import NeurekaV2TestConf from NeurekaV2Weight import NeurekaV2Weight from NeurekaWeight import NeurekaWeight -from NnxTestClasses import NnxTest, NnxTestGenerator, NnxWeight +from NnxTestClasses import NnxTest, NnxTestGenerator _SUPPORTED_ACCELERATORS = ["ne16", "neureka", "neureka_v2"] @@ -70,6 +71,13 @@ def pytest_addoption(parser): default=120, help="Execution timeout in seconds. Default: 120s", ) + parser.addoption( + "--build-flow", + dest="build_flow", + choices=["make", "cmake"], + default="make", + help="Choose the build flow. Default: make", + ) def _find_test_dirs(path: Union[str, os.PathLike]): @@ -82,6 +90,7 @@ def pytest_generate_tests(metafunc): regenerate = metafunc.config.getoption("regenerate") timeout = metafunc.config.getoption("timeout") nnxName = metafunc.config.getoption("accelerator") + build_flow = metafunc.config.getoption("build_flow") if nnxName == "ne16": nnxWeightCls = Ne16Weight @@ -115,7 +124,7 @@ def pytest_generate_tests(metafunc): nnxTestAndNames.append((test, test_dir)) except pydantic.ValidationError as e: for error in e.errors(): - if error['type'] == 'missing': + if error["type"] == "missing": raise e nnxTestAndNames.append( @@ -127,7 +136,16 @@ def pytest_generate_tests(metafunc): ) ) + if build_flow == "cmake": + os.makedirs("app/build/gvsoc_workdir", exist_ok=True) + assert "GVSOC" in os.environ, "The GVSOC environment variable is not set." + subprocess.run( + f"cmake -Sapp -Bapp/build -GNinja -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain_llvm.cmake -DACCELERATOR={nnxName}".split(), + check=True, + ) + metafunc.parametrize("nnxTestAndName", nnxTestAndNames) metafunc.parametrize("timeout", [timeout]) metafunc.parametrize("nnxName", [nnxName]) metafunc.parametrize("nnxWeightCls", [nnxWeightCls]) + metafunc.parametrize("build_flow", [build_flow]) diff --git a/test/requirements.txt b/test/requirements.txt index eee0644..f0c2d71 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -3,3 +3,4 @@ pydantic pytest pytorch==1.11.0 toml +ninja diff --git a/test/test.py b/test/test.py index 9ba8270..57f8435 100644 --- a/test/test.py +++ b/test/test.py @@ -21,7 +21,7 @@ import re import subprocess from pathlib import Path -from typing import Dict, Optional, Tuple, Type, Union +from typing import Dict, Literal, Optional, Tuple, Type, Union from NnxTestClasses import NnxTest, NnxTestConf, NnxTestHeaderGenerator, NnxWeight @@ -93,12 +93,11 @@ def execute_command( return status, msg, stdout, stderr -def assert_message( - msg: str, test_name: str, cmd: str, stdout: str, stderr: Optional[str] = None -): +def assert_message(msg: str, test_name: str, stdout: str, stderr: Optional[str] = None): retval = ( f"Test {test_name} failed: {msg}\n" - f"Command: {cmd}\n" + HORIZONTAL_LINE + f"\nCaptured stdout:\n{stdout}\n" + + HORIZONTAL_LINE + + f"\nCaptured stdout:\n{stdout}\n" ) if stderr is not None: @@ -107,35 +106,58 @@ def assert_message( return retval +def build(nnxName: str, flow: Literal["make", "cmake"]) -> None: + env = os.environ + + if flow == "make": + cmd = "make -C app all platform=gvsoc" + env["ACCELERATOR"] = nnxName + elif flow == "cmake": + cmd = "cmake --build app/build" + + subprocess.run(cmd.split(), check=True, capture_output=True, text=True, env=env) + + +def run(nnxName: str, flow: Literal["make", "cmake"]) -> str: + env = os.environ + + if flow == "make": + cmd = "make -C app run platform=gvsoc" + env["ACCELERATOR"] = nnxName + elif flow == "cmake": + bin = os.path.abspath("app/build/test-pulp-nnx") + gvsoc = os.environ["GVSOC"] + cmd = f"{gvsoc} --binary {bin} --work-dir app/build/gvsoc_workdir --target siracusa image flash run" + + proc = subprocess.run( + cmd.split(), check=True, capture_output=True, text=True, env=env + ) + + return proc.stdout + + def test( nnxTestAndName: Tuple[NnxTest, str], timeout: int, nnxName: str, nnxWeightCls: Type[NnxWeight], + build_flow: Literal["cmake", "make"], ): nnxTest, nnxTestName = nnxTestAndName - NnxTestHeaderGenerator(nnxWeightCls).generate( - nnxTestName, nnxTest - ) - - Path("app/src/nnx_layer.c").touch() - cmd = f"make -C app all run platform=gvsoc" - passed, msg, stdout, stderr = execute_command( - cmd=cmd, timeout=timeout, envflags={"ACCELERATOR": nnxName} - ) + NnxTestHeaderGenerator(nnxWeightCls).generate(nnxTestName, nnxTest) - assert passed, assert_message(msg, nnxTestName, cmd, stdout, stderr) + build(nnxName, build_flow) + stdout = run(nnxName, build_flow) match_success = re.search(r"> Success! No errors found.", stdout) match_fail = re.search(r"> Failure! Found (\d*)/(\d*) errors.", stdout) assert match_success or match_fail, assert_message( - "No regexes matched.", nnxTestName, cmd, stdout + "No regexes matched.", nnxTestName, stdout ) assert not match_fail, assert_message( f"Errors found: {match_fail.group(1)}/{match_fail.group(2)}", nnxTestName, - cmd, stdout, ) diff --git a/test/testgen.py b/test/testgen.py index deb7c11..70fb8be 100644 --- a/test/testgen.py +++ b/test/testgen.py @@ -19,13 +19,10 @@ import argparse import json import os -from typing import Callable, Optional, Set, Type, Union +from typing import Optional, Set, Type, Union -import numpy as np -import numpy.typing as npt import toml -from HeaderWriter import HeaderWriter from Ne16TestConf import Ne16TestConf from Ne16Weight import Ne16Weight from NeurekaTestConf import NeurekaTestConf @@ -38,7 +35,6 @@ NnxTestGenerator, NnxTestHeaderGenerator, NnxWeight, - WmemLiteral, )