Better test parametrization

test/NnxMapping.py

@@ -0,0 +1,37 @@
+from typing import List, Literal, get_args
+
+from Ne16TestConf import Ne16TestConf
+from Ne16Weight import Ne16Weight
+from NeurekaTestConf import NeurekaTestConf
+from NeurekaV2TestConf import NeurekaV2TestConf
+from NeurekaV2Weight import NeurekaV2Weight
+from NeurekaWeight import NeurekaWeight
+from NnxTestClasses import NnxTestConf, NnxWeight
+
+NnxName = Literal["ne16", "neureka", "neureka_v2"]
+
+
+def valid_nnx_names() -> List[str]:
+    return get_args(NnxName)
+
+
+def is_valid_nnx_name(name: str) -> bool:
+    return name in valid_nnx_names()
+
+
+def NnxWeightClsFromName(name: NnxName) -> NnxWeight:
+    if name == "ne16":
+        return Ne16Weight
+    elif name == "neureka":
+        return NeurekaWeight
+    elif name == "neureka_v2":
+        return NeurekaV2Weight
+
+
+def NnxTestConfClsFromName(name: NnxName) -> NnxTestConf:
+    if name == "ne16":
+        return Ne16TestConf
+    elif name == "neureka":
+        return NeurekaTestConf
+    elif name == "neureka_v2":
+        return NeurekaV2TestConf

test/conftest.py

@@ -23,15 +23,14 @@
 import pydantic
 import pytest
 
-from Ne16TestConf import Ne16TestConf
-from Ne16Weight import Ne16Weight
-from NeurekaTestConf import NeurekaTestConf
-from NeurekaV2TestConf import NeurekaV2TestConf
-from NeurekaV2Weight import NeurekaV2Weight
-from NeurekaWeight import NeurekaWeight
+from NnxMapping import (
+    NnxName,
+    NnxTestConfClsFromName,
+    is_valid_nnx_name,
+    valid_nnx_names,
+)
 from NnxTestClasses import NnxTest, NnxTestGenerator
-
-_SUPPORTED_ACCELERATORS = ["ne16", "neureka", "neureka_v2"]
+from TestClasses import implies
 
 
 def pytest_addoption(parser):
@@ -55,7 +54,7 @@ def pytest_addoption(parser):
     parser.addoption(
         "-A",
         "--accelerator",
-        choices=_SUPPORTED_ACCELERATORS,
+        choices=valid_nnx_names(),
         default="ne16",
         help="Choose an accelerator to test. Default: ne16",
     )
@@ -92,19 +91,11 @@ def pytest_generate_tests(metafunc):
     nnxName = metafunc.config.getoption("accelerator")
     build_flow = metafunc.config.getoption("build_flow")
 
-    if nnxName == "ne16":
-        nnxWeightCls = Ne16Weight
-        nnxTestConfCls = Ne16TestConf
-    elif nnxName == "neureka":
-        nnxWeightCls = NeurekaWeight
-        nnxTestConfCls = NeurekaTestConf
-    elif nnxName == "neureka_v2":
-        nnxWeightCls = NeurekaV2Weight
-        nnxTestConfCls = NeurekaV2TestConf
-    else:
-        assert (
-            False
-        ), f"Given accelerator {nnxName} not supported. Supported accelerators: {_SUPPORTED_ACCELERATORS}"
+    assert is_valid_nnx_name(nnxName), \
+        f"Given accelerator {nnxName} not supported. Supported accelerators: {valid_nnx_names()}"
+
+    assert implies(build_flow == "cmake", nnxName in ["neureka", "neureka_v2"]), \
+        "The cmake build flow has been tested only with neureka and neureka_v2 accelerators"
 
     if recursive:
         tests_dirs = test_dirs
@@ -113,39 +104,41 @@ def pytest_generate_tests(metafunc):
             test_dirs.extend(_find_test_dirs(tests_dir))
 
     # Load valid tests
-    nnxTestAndNames = []
+    nnxTestNames = []
+    nnxTestConfCls = NnxTestConfClsFromName(nnxName)
     for test_dir in test_dirs:
         try:
             test = NnxTest.load(nnxTestConfCls, test_dir)
             # (Re)generate data
             if not test.is_valid() or regenerate:
                 test = NnxTestGenerator.from_conf(test.conf)
                 test.save_data(test_dir)
-            nnxTestAndNames.append((test, test_dir))
+            nnxTestNames.append(test_dir)
         except pydantic.ValidationError as e:
             for error in e.errors():
                 if error["type"] == "missing":
                     raise e
 
-            nnxTestAndNames.append(
+            nnxTestNames.append(
                 pytest.param(
-                    (None, test_dir),
+                    test_dir,
                     marks=pytest.mark.skipif(
                         True, reason=f"Invalid test {test_dir}: {e.errors}"
                     ),
                 )
             )
 
     if build_flow == "cmake":
-        os.makedirs("app/build/gvsoc_workdir", exist_ok=True)
+        build_dir = os.path.abspath(f'app/build_{nnxName}')
+        gvsoc_workdir = os.path.join(build_dir, "gvsoc_workdir")
+        os.makedirs(gvsoc_workdir, exist_ok=True)
         assert "GVSOC" in os.environ, "The GVSOC environment variable is not set."
         subprocess.run(
-            f"cmake -Sapp -Bapp/build -GNinja -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain_llvm.cmake -DACCELERATOR={nnxName}".split(),
+            f"cmake -Sapp -B{build_dir} -GNinja -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain_llvm.cmake -DACCELERATOR={nnxName}".split(),
             check=True,
         )
 
-    metafunc.parametrize("nnxTestAndName", nnxTestAndNames)
-    metafunc.parametrize("timeout", [timeout])
     metafunc.parametrize("nnxName", [nnxName])
-    metafunc.parametrize("nnxWeightCls", [nnxWeightCls])
     metafunc.parametrize("build_flow", [build_flow])
+    metafunc.parametrize("nnxTestName", nnxTestNames)
+    metafunc.parametrize("timeout", [timeout])

test/test.py

@@ -23,6 +23,7 @@
 from pathlib import Path
 from typing import Dict, Literal, Optional, Tuple, Type, Union
 
+from NnxMapping import NnxName, NnxTestConfClsFromName, NnxWeightClsFromName
 from NnxTestClasses import NnxTest, NnxTestConf, NnxTestHeaderGenerator, NnxWeight
 
 HORIZONTAL_LINE = "\n" + "-" * 100 + "\n"
@@ -125,9 +126,11 @@ def run(nnxName: str, flow: Literal["make", "cmake"]) -> str:
         cmd = "make -C app run platform=gvsoc"
         env["ACCELERATOR"] = nnxName
     elif flow == "cmake":
-        bin = os.path.abspath("app/build/test-pulp-nnx")
-        gvsoc = os.environ["GVSOC"]
-        cmd = f"{gvsoc} --binary {bin} --work-dir app/build/gvsoc_workdir --target siracusa image flash run"
+        build_dir = os.path.abspath(f'app/build_{nnxName}')
+        bin = os.path.join(build_dir, "test-pulp-nnx")
+        gvsoc = env["GVSOC"]
+        gvsoc_workdir = os.path.join(build_dir, "gvsoc_workdir")
+        cmd = f"{gvsoc} --binary {bin} --work-dir {gvsoc_workdir} --target siracusa image flash run"
 
     proc = subprocess.run(
         cmd.split(), check=True, capture_output=True, text=True, env=env
@@ -137,13 +140,16 @@ def run(nnxName: str, flow: Literal["make", "cmake"]) -> str:
 
 
 def test(
-    nnxTestAndName: Tuple[NnxTest, str],
-    timeout: int,
-    nnxName: str,
-    nnxWeightCls: Type[NnxWeight],
+    nnxName: NnxName,
     build_flow: Literal["cmake", "make"],
+    nnxTestName: Tuple[NnxTest, str],
+    timeout: int,
 ):
-    nnxTest, nnxTestName = nnxTestAndName
+    nnxTestConfCls = NnxTestConfClsFromName(nnxName)
+    # conftest.py makes sure the test is valid and generated
+    nnxTest = NnxTest.load(nnxTestConfCls, nnxTestName)
+
+    nnxWeightCls = NnxWeightClsFromName(nnxName)
     NnxTestHeaderGenerator(nnxWeightCls).generate(nnxTestName, nnxTest)
 
     build(nnxName, build_flow)