Skip to content

Commit

Permalink
Skip invalid tests
Browse files Browse the repository at this point in the history
  • Loading branch information
lukamac committed Jan 18, 2024
1 parent ccc835b commit df0eb6b
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 24 deletions.
39 changes: 24 additions & 15 deletions test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import os
from typing import Union

import pydantic
from Ne16 import Ne16
from Ne16TestConf import Ne16TestConf
from Neureka import Neureka
Expand Down Expand Up @@ -78,19 +80,6 @@ def pytest_generate_tests(metafunc):
timeout = metafunc.config.getoption("timeout")
nnxName = metafunc.config.getoption("accelerator")

if recursive:
tests_dirs = test_dirs
test_dirs = []
for tests_dir in tests_dirs:
test_dirs.extend(_find_test_dirs(tests_dir))

# (Re)Generate test data
for test_dir in test_dirs:
test = NnxTest.load(Ne16TestConf, test_dir)
if not test.is_valid() or regenerate:
test = NnxTestGenerator.from_conf(test.conf, Ne16.ACCUMULATOR_TYPE)
test.save_data(test_dir)

if nnxName == "ne16":
nnxCls = Ne16
nnxTestConfCls = Ne16TestConf
Expand All @@ -102,8 +91,28 @@ def pytest_generate_tests(metafunc):
False
), f"Given accelerator {nnxName} not supported. Supported accelerators: {_SUPPORTED_ACCELERATORS}"

metafunc.parametrize("path", test_dirs)
if recursive:
tests_dirs = test_dirs
test_dirs = []
for tests_dir in tests_dirs:
test_dirs.extend(_find_test_dirs(tests_dir))

# Load valid tests
valid_paths = []
valid_tests = []
for test_dir in test_dirs:
try:
test = NnxTest.load(nnxTestConfCls, test_dir)
# (Re)generate data
if not test.is_valid() or regenerate:
test = NnxTestGenerator.from_conf(test.conf, nnxCls.ACCUMULATOR_TYPE)
test.save_data(test_dir)
valid_tests.append(test)
valid_paths.append(test_dir)
except pydantic.ValidationError as e:
_ = e

metafunc.parametrize("nnxTestAndName", zip(valid_tests, valid_paths))
metafunc.parametrize("timeout", [timeout])
metafunc.parametrize("nnxName", [nnxName])
metafunc.parametrize("nnxCls", [nnxCls])
metafunc.parametrize("nnxTestConfCls", [nnxTestConfCls])
15 changes: 6 additions & 9 deletions test/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,35 +109,32 @@ def assert_message(


def test(
path: str,
nnxTestAndName: Tuple[NnxTest, str],
timeout: int,
nnxName: str,
nnxCls: Union[Type[Ne16], Type[Neureka]],
nnxTestConfCls: Type[NnxTestConf],
):
test_name = path
test = NnxTest.load(nnxTestConfCls, path)

NnxTestHeaderGenerator(nnxCls.weight_unroll).generate(test_name, test)
nnxTest, nnxTestName = nnxTestAndName
NnxTestHeaderGenerator(nnxCls.weight_unroll).generate(nnxTestName, nnxTest)

Path("app/src/nnx_layer.c").touch()
cmd = f"make -C app all run platform=gvsoc"
passed, msg, stdout, stderr = execute_command(
cmd=cmd, timeout=timeout, envflags={"ACCELERATOR": nnxName}
)

assert passed, assert_message(msg, test_name, cmd, stdout, stderr)
assert passed, assert_message(msg, nnxTestName, cmd, stdout, stderr)

match_success = re.search(r"> Success! No errors found.", stdout)
match_fail = re.search(r"> Failure! Found (\d*)/(\d*) errors.", stdout)

assert match_success or match_fail, assert_message(
"No regexes matched.", test_name, cmd, stdout
"No regexes matched.", nnxTestName, cmd, stdout
)

assert not match_fail, assert_message(
f"Errors found: {match_fail.group(1)}/{match_fail.group(2)}",
test_name,
nnxTestName,
cmd,
stdout,
)

0 comments on commit df0eb6b

Please sign in to comment.