Patch 1

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "python.pythonPath": ".tox/local/bin/python",
+    "python.defaultInterpreterPath": "${workspaceFolder}/.tox/local",
     "python.linting.flake8Enabled": true,
     "python.linting.pylintEnabled": false,
     "python.linting.mypyEnabled": true,
@@ -11,7 +11,6 @@
     ],
     "python.testing.cwd": "${workspaceFolder}",
     "python.testing.unittestEnabled": false,
-    "python.testing.nosetestsEnabled": false,
     "python.testing.pytestEnabled": true,
     "python.formatting.provider": "black",
     "python.formatting.blackArgs": [
@@ -41,8 +40,8 @@
     "C_Cpp.configurationWarnings": "Disabled",
     "files.associations": {
         "*.py.template": "python",
-        "*.cc": "c++",
-        "*.hpp": "c++",
+        "*.cc": "cpp",
+        "*.hpp": "cpp",
         "__bit_reference": "cpp",
         "__config": "cpp",
         "__debug": "cpp",
@@ -137,7 +136,11 @@
         "any": "cpp",
         "span": "cpp",
         "variant": "cpp",
-        "__bits": "cpp"
+        "__bits": "cpp",
+        "filesystem": "cpp",
+        "__memory": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp"
     },
     "python.linting.mypyArgs": [
         "--config-file=${workspaceFolder}/tox.ini"

CONTRIBUTING.rst

@@ -82,6 +82,7 @@ To run the language verification build you'll need to use a different docker con
 
     docker pull uavcan/c_cpp:ubuntu-20.04
     docker run --rm -it -v $PWD:/repo uavcan/c_cpp:ubuntu-20.04
+    cd /repo
     ./.github/verify.py -l c
     ./.github/verify.py -l cpp
 

conf.py

@@ -77,7 +77,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "monokai"

conftest.py

@@ -14,6 +14,7 @@
 import tempfile
 import textwrap
 import typing
+import urllib
 from doctest import ELLIPSIS
 
 import pydsdl
@@ -187,7 +188,7 @@ def out_dir(self) -> pathlib.Path:
         The directory to place test output under for this test case.
         """
         if self._out_dir is None:
-            self._out_dir = self.create_new_temp_dir(self.test_name)
+            self._out_dir = self.create_new_temp_dir(urllib.parse.quote_plus(self.test_name))
         return self._out_dir
 
     @property

docs/index.rst

@@ -4,6 +4,7 @@
    :hidden:
 
    api/library
+   languages
    templates
    CLI (nnvg) <cli>
    dev

docs/languages.rst

@@ -0,0 +1,7 @@
+################################################
+Software Language Generation Guide
+################################################
+
+.. note ::
+    This is a placeholder for documentation this project owes you, the user, for how to integrate nnvg with build
+    systems and how to tune and optimize source code generation for each supported language.

setup.cfg

@@ -7,7 +7,7 @@ description = Generate code from DSDL using Jinja2 templates.
 long_description = file: README.rst
 long_description_content_type = text/x-rst
 license = MIT
-license_file = LICENSE.rst
+license_files = LICENSE.rst
 keywords = uavcan, dsdl, can, can-bus, codegen, cyphal, opencyphal
 classifiers =
     Development Status :: 3 - Alpha

src/nunavut/_utilities.py

@@ -93,6 +93,18 @@ def test_truth(cls, ynd_value: "YesNoDefault", default_value: bool) -> bool:
     DEFAULT = 2
 
 
+@enum.unique
+class ResourceType(enum.Enum):
+    """
+    Common Nunavut classifications for Python package resources.
+    """
+
+    ANY = 0
+    CONFIGURATION = 1
+    SERIALIZATION_SUPPORT = 2
+    TYPE_SUPPORT = 3
+
+
 def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pathlib.Path, None, None]:
     """
     >>> from nunavut._utilities import iter_package_resources
@@ -104,5 +116,22 @@ def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pat
 
     """
     for resource in importlib_resources.files(pkg_name).iterdir():
-        if any(suffix == resource.suffix for suffix in suffix_filters):  # type: ignore
-            yield cast(pathlib.Path, resource)
+        if resource.is_file() and isinstance(resource, pathlib.Path):
+            # Not sure why this works but it's seemed to so far. importlib_resources.as_file(resource)
+            # may be more correct but this can create temporary files which would disappear after the iterator
+            # had copied their paths. If you are reading this because this method isn't working for some packaging
+            # scheme then we may need to use importlib_resources.as_file(resource) to create a runtime cache of
+            # temporary objects that live for a given nunavut session. This, of course, wouldn't help across sessions
+            # which is a common use case when integrating Nunavut with build systems. So...here be dragons.
+            file_resource = cast(pathlib.Path, resource)
+            if any(suffix == file_resource.suffix for suffix in suffix_filters):
+                yield file_resource
+
+
+def empty_list_support_files() -> Generator[pathlib.Path, None, None]:
+    """
+    Helper for implementing the list_support_files method in language support packages. This provides an empty
+    iterator with the correct type annotations.
+    """
+    # works in Python 3.3 and newer. Thanks https://stackoverflow.com/a/13243870
+    yield from ()

src/nunavut/cli/__init__.py

@@ -511,7 +511,7 @@ def extension_type(raw_arg: str) -> str:
 def _extra_includes_from_env(env_var_name: str) -> typing.List[str]:
     try:
         extra_includes_from_env = os.environ[env_var_name].split(os.pathsep)
-        logging.info("Additional include directories from {}: %s", env_var_name, str(extra_includes_from_env))
+        logging.info("Additional include directories from {}: {}".format(env_var_name, str(extra_includes_from_env)))
         return extra_includes_from_env
     except KeyError:
         return []

src/nunavut/jinja/__init__.py

@@ -19,7 +19,7 @@
 import nunavut.lang
 import nunavut.postprocessors
 import pydsdl
-from nunavut._utilities import YesNoDefault
+from nunavut._utilities import ResourceType, YesNoDefault
 from yaml import Dumper as YamlDumper
 from yaml import dump as yaml_dump
 
@@ -779,9 +779,9 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any):
         target_language = self.language_context.get_target_language()
 
         self._sub_folders = None  # type: typing.Optional[pathlib.Path]
-        self._support_enabled = False  # If not enabled then we remove any support files found
+        self._serialization_support_enabled = False
         if target_language is not None:
-            self._support_enabled = not target_language.omit_serialization_support
+            self._serialization_support_enabled = not target_language.omit_serialization_support
 
             #  Create the sub-folder to copy-to based on the support namespace.
             self._sub_folders = pathlib.Path("")
@@ -794,11 +794,11 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any):
     # +-----------------------------------------------------------------------+
     def get_templates(self) -> typing.Iterable[pathlib.Path]:
         files = []
-        target_language = self.language_context.get_target_language()
-
-        if target_language is not None:
-            for resource in target_language.support_files:
+        if self._serialization_support_enabled:
+            for resource in self._get_templates_by_support_type(ResourceType.SERIALIZATION_SUPPORT):
                 files.append(resource)
+        for resource in self._get_templates_by_support_type(ResourceType.TYPE_SUPPORT):
+            files.append(resource)
         return files
 
     def generate_all(self, is_dryrun: bool = False, allow_overwrite: bool = True) -> typing.Iterable[pathlib.Path]:
@@ -812,6 +812,14 @@ def generate_all(self, is_dryrun: bool = False, allow_overwrite: bool = True) ->
     # +-----------------------------------------------------------------------+
     # | Private
     # +-----------------------------------------------------------------------+
+    def _get_templates_by_support_type(self, resource_type: ResourceType) -> typing.Iterable[pathlib.Path]:
+        files = []
+        target_language = self.language_context.get_target_language()
+
+        if target_language is not None:
+            for resource in target_language.get_support_files(resource_type):
+                files.append(resource)
+        return files
 
     def _generate_all(
         self, target_language: nunavut.lang.Language, sub_folders: pathlib.Path, is_dryrun: bool, allow_overwrite: bool
@@ -833,27 +841,14 @@ def _generate_all(
         for resource in self.get_templates():
             target = (target_path / resource.name).with_suffix(target_language.extension)
             logger.info("Generating support file: %s", target)
-            if not self._support_enabled:
-                self._remove_header(target, is_dryrun, allow_overwrite)
-            elif resource.suffix == TEMPLATE_SUFFIX:
+            if resource.suffix == TEMPLATE_SUFFIX:
                 self._generate_header(resource, target, is_dryrun, allow_overwrite)
                 generated.append(target)
             else:
                 self._copy_header(resource, target, is_dryrun, allow_overwrite, line_pps, file_pps)
                 generated.append(target)
         return generated
 
-    def _remove_header(self, target: pathlib.Path, is_dryrun: bool, allow_overwrite: bool) -> None:
-        if not is_dryrun:
-            if not allow_overwrite and target.exists():
-                raise PermissionError("{} exists. Refusing to remove.".format(str(target)))
-            try:
-                target.unlink()
-            except FileNotFoundError:
-                # missing_ok was added in python 3.8 so this try/except statement will
-                # go away someday when python 3.7 support is dropped.
-                pass
-
     def _generate_header(
         self, template_path: pathlib.Path, output_path: pathlib.Path, is_dryrun: bool, allow_overwrite: bool
     ) -> pathlib.Path:

src/nunavut/lang/__init__.py

@@ -18,8 +18,8 @@
 
 import pydsdl
 
+from .._utilities import ResourceType, YesNoDefault, iter_package_resources, empty_list_support_files
 from ..dependencies import Dependencies, DependencyBuilder
-from .._utilities import YesNoDefault, iter_package_resources
 from ._config import LanguageConfig, VersionReader
 
 logger = logging.getLogger(__name__)
@@ -467,10 +467,13 @@ def omit_serialization_support(self) -> bool:
         """
         return self._omit_serialization_support
 
-    @property
-    def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
+    def get_support_files(
+        self, resource_type: ResourceType = ResourceType.ANY
+    ) -> typing.Generator[pathlib.Path, None, None]:
         """
-        Iterates over non-templated supporting files embedded within the Nunavut distribution.
+        Iterates over supporting files embedded within the Nunavut distribution.
+
+        :param resource_type: The type of support resources to enumerate.
 
         .. invisible-code-block: python
 
@@ -483,7 +486,7 @@ def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
 
             my_lang = _GenericLanguage(mock_module, mock_config, True)
             my_lang._section = "nunavut.lang.not_a_language_really_not_a_language"
-            for support_file in my_lang.support_files:
+            for support_file in my_lang.get_support_files():
                 # if the module doesn't exist it shouldn't have any support files.
                 assert False
 
@@ -495,15 +498,10 @@ def support_files(self) -> typing.Generator[pathlib.Path, None, None]:
             # to allow the copy generator access to the packaged support files.
             list_support_files = getattr(
                 module, "list_support_files"
-            )  # type: typing.Callable[[], typing.Generator[pathlib.Path, None, None]]
-            return list_support_files()
+            )  # type: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]]
+            return list_support_files(resource_type)
         else:
-            # No serialization support for this language
-            def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
-                # This makes both MyPy and sonarqube happy.
-                return typing.cast(typing.Generator[pathlib.Path, None, None], iter(()))
-
-            return list_support_files()
+            return empty_list_support_files()
 
     def get_option(
         self, option_key: str, default_value: typing.Union[typing.Mapping[str, typing.Any], str, None] = None

src/nunavut/lang/_common.py

@@ -14,6 +14,7 @@
 import typing
 
 import pydsdl
+from nunavut._utilities import ResourceType
 
 from . import Language
 
@@ -30,13 +31,13 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t
             self.make_path(dt, self._language, output_extension).as_posix() for dt in dep_types.composite_types
         ]
 
+        namespace_path = pathlib.Path("")
+        for namespace_part in self._language.support_namespace:
+            namespace_path = namespace_path / pathlib.Path(namespace_part)
         if not self._language.omit_serialization_support:
-            namespace_path = pathlib.Path("")
-            for namespace_part in self._language.support_namespace:
-                namespace_path = namespace_path / pathlib.Path(namespace_part)
             path_list += [
                 (namespace_path / pathlib.Path(p.name).with_suffix(output_extension)).as_posix()
-                for p in self._language.support_files
+                for p in self._language.get_support_files(ResourceType.SERIALIZATION_SUPPORT)
             ]
 
         prefer_system_includes = self._language.get_config_value_as_bool("prefer_system_includes", False)
@@ -46,7 +47,7 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t
             path_list_with_punctuation = ['"{}"'.format(p) for p in path_list]
 
         if sort:
-            return sorted(path_list_with_punctuation) + self._language.get_includes(dep_types)
+            return sorted(path_list_with_punctuation + self._language.get_includes(dep_types))
         else:
             return path_list_with_punctuation + self._language.get_includes(dep_types)
 

src/nunavut/lang/c/support/__init__.py

@@ -8,15 +8,17 @@
 """
 import pathlib
 import typing
-from nunavut._utilities import iter_package_resources
+
+from nunavut._utilities import ResourceType, empty_list_support_files, iter_package_resources
 
 __version__ = "1.0.0"
 """Version of the c support headers."""
 
 
-def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
+def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]:
     """
     Get a list of C support headers embedded in this package.
+    :param resource_type: A type of support file to list.
 
     .. invisible-code-block: python
 
@@ -37,4 +39,8 @@ def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
 
     :return: A list of C support header resources.
     """
-    return iter_package_resources(__name__, ".h", ".j2")
+    # The c support only has serialization support resources
+    if resource_type not in (ResourceType.ANY, ResourceType.SERIALIZATION_SUPPORT):
+        return empty_list_support_files()
+    else:
+        return iter_package_resources(__name__, ".h", ".j2")