markdown/rst: Support __map_ and nested parameters (#164)

* Support nested parameters and maps for RST

* Add read_only info to RST template

* Use map parameters for AutoDocumentation

* Add test for markdown generation

* Use jinja indentation instead of regex

* Add dynamic_parameters also to DefaultConfig

* Proper fromat of default yaml config

* Proper format of the default yaml file

* Fix imported function names

* Add a jinja2 filter to create valid multiline c++ string literals

* Handle empty strings for descriptions

generate_parameter_library_py/generate_parameter_library_py/generate_markdown.py

@@ -31,13 +31,16 @@
 
 import argparse
 import os
+import re
 import sys
 from jinja2 import Template
 from typeguard import typechecked
+import yaml
 
 from generate_parameter_library_py.parse_yaml import (
     GenerateCode,
     DeclareParameter,
+    DeclareRuntimeParameter,
     ValidationFunction,
 )
 
@@ -95,7 +98,44 @@ def __str__(self):
         constraints = '\n'.join(str(val) for val in self.param_validations)
 
         data = {
-            'name': self.declare_parameters.code_gen_variable.name,
+            'name': self.declare_parameters.parameter_name,
+            'read_only': self.declare_parameters.parameter_read_only,
+            'type': self.declare_parameters.code_gen_variable.defined_type,
+            'default_value': self.declare_parameters.code_gen_variable.lang_str_value,
+            'constraints': constraints,
+            # remove leading whitespace from description, this is necessary for correct indentation of multi-line descriptions
+            'description': re.sub(
+                r'(?m)^(?!$)\s*',
+                '',
+                str(self.declare_parameters.parameter_description),
+                flags=re.MULTILINE,
+            ),
+        }
+
+        j2_template = Template(GenerateCode.templates['parameter_detail'])
+        code = j2_template.render(data, trim_blocks=True)
+        return code
+
+
+class RuntimeParameterDetailMarkdown:
+    @typechecked
+    def __init__(self, declare_parameters: DeclareRuntimeParameter):
+        self.declare_parameters = declare_parameters
+        self.param_validations = [
+            ParameterValidationMarkdown(val)
+            for val in declare_parameters.parameter_validations
+        ]
+
+    def __str__(self):
+        constraints = '\n'.join(str(val) for val in self.param_validations)
+        data = {
+            # replace __map_key with <key>
+            'name': re.sub(
+                r'__map_(\w+)',
+                lambda match: '<' + match.group(1) + '>',
+                self.declare_parameters.parameter_name,
+            ),
+            'read_only': self.declare_parameters.parameter_read_only,
             'type': self.declare_parameters.code_gen_variable.defined_type,
             'default_value': self.declare_parameters.code_gen_variable.lang_str_value,
             'constraints': constraints,
@@ -119,14 +159,65 @@ def __init__(self, gen_param_struct: GenerateCode):
     def __str__(self):
         j2_template = Template(GenerateCode.templates['default_config'])
 
-        tmp = '\n'.join(
-            param.parameter_name + ': ' + str(param.code_gen_variable.lang_str_value)
-            for param in self.gen_param_struct.declare_parameters
+        tmp = (
+            '\n'.join(
+                param.parameter_name
+                + ': '
+                + str(param.code_gen_variable.lang_str_value)
+                for param in self.gen_param_struct.declare_parameters
+            )
+            + '\n'
+            + '\n'.join(
+                # replace __map_key with <key>
+                re.sub(
+                    r'__map_(\w+)',
+                    lambda match: '<' + match.group(1) + '>',
+                    param.parameter_name,
+                )
+                + ': '
+                + str(param.code_gen_variable.lang_str_value)
+                for param in self.gen_param_struct.declare_dynamic_parameters
+            )
         )
 
+        # Split the string into lines and group them by the first part
+        def nest_dict(d, keys, value):
+            # Check if the value is a string
+            if isinstance(value, str):
+                # Remove double quotes from the string
+                value = value.replace('"', '')
+                # Try to convert the value to a boolean, number, or leave it as a string
+                if value.lower() == 'true':
+                    value = True
+                elif value.lower() == 'false':
+                    value = False
+                else:
+                    try:
+                        value = float(value)
+                    except ValueError:
+                        pass
+
+            if len(keys) == 1:
+                d[keys[0]] = value
+            else:
+                key = keys.pop(0)
+                if key not in d:
+                    d[key] = {}
+                nest_dict(d[key], keys, value)
+
+        # Split the string into lines and create a dictionary
+        d = {}
+        for line in tmp.strip().split('\n'):
+            name, value = line.split(':', 1)
+            keys = name.split('.')
+            nest_dict(d, keys, value.strip())
+
+        # Convert the dictionary to a string
+        result = yaml.dump(d, default_flow_style=False)
+
         data = {
             'namespace': self.gen_param_struct.namespace,
-            'default_param_values': tmp,
+            'default_param_values': result,
         }
         code = j2_template.render(data, trim_blocks=True)
 
@@ -142,6 +233,10 @@ def __init__(self, gen_param_struct: GenerateCode):
             ParameterDetailMarkdown(param)
             for param in self.gen_param_struct.declare_parameters
         ]
+        self.runtime_param_details = [
+            RuntimeParameterDetailMarkdown(param)
+            for param in self.gen_param_struct.declare_dynamic_parameters
+        ]
 
     def __str__(self):
         words = self.gen_param_struct.namespace.split('_')
@@ -150,7 +245,9 @@ def __str__(self):
         data = {
             'title': title,
             'default_config': str(self.default_config),
-            'parameter_details': '\n'.join(str(val) for val in self.param_details),
+            'parameter_details': '\n'.join(str(val) for val in self.param_details)
+            + '\n'
+            + '\n'.join(str(val) for val in self.runtime_param_details),
         }
 
         j2_template = Template(GenerateCode.templates['documentation'])

generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_parameter

@@ -1,7 +1,7 @@
 if (!parameters_interface_->has_parameter(prefix_ + "{{parameter_name}}")) {
 {%- filter indent(width=4) %}
 rcl_interfaces::msg::ParameterDescriptor descriptor;
-descriptor.description = "{{parameter_description}}";
+descriptor.description = {{parameter_description | valid_string_cpp}};
 descriptor.read_only = {{parameter_read_only}};
 {%- for validation in parameter_validations if ("bounds" in validation.function_name or "lt" in validation.function_name or "gt" in validation.function_name) %}
 {%- if "DOUBLE" in parameter_type %}

generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_runtime_parameter

@@ -5,7 +5,7 @@ auto param_name = fmt::format("{}{}.{}.{}", prefix_, "{{struct_name}}", value, "
 if (!parameters_interface_->has_parameter(param_name)) {
 {%- filter indent(width=4) %}
 rcl_interfaces::msg::ParameterDescriptor descriptor;
-descriptor.description = "{{parameter_description}}";
+descriptor.description = {{parameter_description | valid_string_cpp}};
 descriptor.read_only = {{parameter_read_only}};
 {%- for validation in parameter_validations if ("bounds" in validation.function_name or "lt" in validation.function_name or "gt" in validation.function_name) %}
 {%- if "DOUBLE" in parameter_type %}

generate_parameter_library_py/generate_parameter_library_py/jinja_templates/markdown/parameter_detail

@@ -4,10 +4,10 @@
 {% endif %}
 * Type: `{{type}}`
 {%- if default_value|length %}
-* Default Value: {{default_value}}
-{% endif %}
-{%- if constraints|length %}
+* Default Value: {{default_value}}{% endif %}{% if read_only %}
+* Read only: {{read_only}}{% endif %}{%- if constraints|length %}
 
 *Constraints:*
 {{constraints}}
+{% else %}
 {% endif %}

generate_parameter_library_py/generate_parameter_library_py/jinja_templates/rst/parameter_detail

@@ -1,16 +1,19 @@
-{{name}} ({{type}}){% if description|length %}
-  {{description}}
+{{name}} ({{type}}){%- filter indent(width=2) %}{% if description|length %}
+{{description}}
+{% endif %}
+{%- if read_only %}
+Read only: {{read_only}}
 {% endif %}
 {%- if default_value|length %}
-  Default:  {{default_value}}
+Default:  {{default_value}}
 {% endif %}
 {%- if constraints|length %}
-
-  Constraints:
+Constraints:
 
 {%- filter indent(width=2) %}
 
 {{constraints}}
 {% endfilter -%}
 
 {% endif %}
+{% endfilter -%}

generate_parameter_library_py/generate_parameter_library_py/parse_yaml.py

@@ -29,7 +29,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from jinja2 import Template
+from jinja2 import Template, Environment
 from typeguard import typechecked
 from typing import Any, List, Optional
 from yaml.parser import ParserError
@@ -39,6 +39,7 @@
 
 from generate_parameter_library_py.cpp_convertions import CPPConverstions
 from generate_parameter_library_py.python_convertions import PythonConvertions
+from generate_parameter_library_py.string_filters_cpp import valid_string_cpp
 
 
 # YAMLSyntaxError standardizes compiler error messages
@@ -499,6 +500,7 @@ def __str__(self):
         bool_to_str = self.code_gen_variable.conversation.bool_to_str
 
         parameter_validations = self.parameter_validations
+
         data = {
             'parameter_name': self.parameter_name,
             'parameter_value': self.parameter_value,
@@ -507,7 +509,11 @@ def __str__(self):
             'parameter_read_only': bool_to_str(self.parameter_read_only),
             'parameter_validations': parameter_validations,
         }
-        j2_template = Template(GenerateCode.templates['declare_parameter'])
+
+        # Create a Jinja2 environment to register the custom filter
+        env = Environment()
+        env.filters['valid_string_cpp'] = valid_string_cpp
+        j2_template = env.from_string(GenerateCode.templates['declare_parameter'])
         code = j2_template.render(data, trim_blocks=True)
         return code
 
@@ -566,7 +572,12 @@ def __str__(self):
             'parameter_validations': self.parameter_validations,
         }
 
-        j2_template = Template(GenerateCode.templates['declare_runtime_parameter'])
+        # Create a Jinja2 environment to register the custom filter
+        env = Environment()
+        env.filters['valid_string_cpp'] = valid_string_cpp
+        j2_template = env.from_string(
+            GenerateCode.templates['declare_runtime_parameter']
+        )
         code = j2_template.render(data, trim_blocks=True)
         return code
 
@@ -689,6 +700,18 @@ class GenerateCode:
     templates = None
 
     def __init__(self, language: str):
+        if language == 'cpp':
+            self.comments = '// auto-generated DO NOT EDIT'
+        elif language == 'rst':
+            self.comments = '.. auto-generated DO NOT EDIT'
+        elif language == 'markdown':
+            self.comments = '<!--- auto-generated DO NOT EDIT -->'
+        elif language == 'python' or language == 'markdown':
+            self.comments = '# auto-generated DO NOT EDIT'
+        else:
+            raise compile_error(
+                'Invalid language, only cpp, markdown, rst, and python are currently supported.'
+            )
         GenerateCode.templates = get_all_templates(language)
         self.language = language
         self.namespace = ''
@@ -702,16 +725,6 @@ def __init__(self, language: str):
         self.remove_dynamic_parameter = []
         self.declare_parameter_sets = []
         self.set_stack_params = []
-        if language == 'cpp':
-            self.comments = '// auto-generated DO NOT EDIT'
-        elif language == 'rst':
-            self.comments = '.. auto-generated DO NOT EDIT'
-        elif language == 'python' or language == 'markdown':
-            self.comments = '# auto-generated DO NOT EDIT'
-        else:
-            raise compile_error(
-                'Invalid language, only c++ and python are currently supported.'
-            )
         self.user_validation_file = ''
 
     def parse(self, yaml_file, validate_header):

generate_parameter_library_py/generate_parameter_library_py/string_filters_cpp.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+def valid_string_cpp(description):
+    """
+    Filter a string to make it a valid C++ string literal.
+
+    Args:
+      description (str): The input string to be filtered.
+
+    Returns:
+      str: The filtered string that is a valid C++ string.
+    """
+    if description:
+        filtered_description = (
+            description.replace('\\', '\\\\').replace('"', '\\"').replace('`', '')
+        )
+        # create a quote delimited string for every line
+        filtered_description = '\n'.join(
+            f'"{line}"' for line in filtered_description.splitlines()
+        )
+        return filtered_description
+    else:
+        return '""'

generate_parameter_library_py/generate_parameter_library_py/test/YAML_parse_error_test.py

@@ -19,8 +19,9 @@
 import os
 
 from ament_index_python.packages import get_package_share_path
-from generate_parameter_library_py.generate_cpp_header import run as run_python
-from generate_parameter_library_py.generate_python_module import run as run_cpp
+from generate_parameter_library_py.generate_cpp_header import run as run_cpp
+from generate_parameter_library_py.generate_python_module import run as run_python
+from generate_parameter_library_py.generate_markdown import run as run_md
 from generate_parameter_library_py.parse_yaml import YAMLSyntaxError
 from generate_parameter_library_py.generate_cpp_header import parse_args
 
@@ -29,16 +30,42 @@ def set_up(yaml_test_file):
     full_file_path = os.path.join(
         get_package_share_path('generate_parameter_library_py'), 'test', yaml_test_file
     )
-    testargs = [sys.argv[0], '/tmp/admittance_controller.h', full_file_path]
+    testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.h', full_file_path]
 
     with patch.object(sys, 'argv', testargs):
         args = parse_args()
         output_file = args.output_cpp_header_file
         yaml_file = args.input_yaml_file
         validate_header = args.validate_header
         run_cpp(output_file, yaml_file, validate_header)
+
+    testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.py', full_file_path]
+
+    with patch.object(sys, 'argv', testargs):
+        args = parse_args()
+        output_file = args.output_cpp_header_file
+        yaml_file = args.input_yaml_file
+        validate_header = args.validate_header
         run_python(output_file, yaml_file, validate_header)
 
+    testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.md', full_file_path]
+
+    with patch.object(sys, 'argv', testargs):
+        args = parse_args()
+        output_file = args.output_cpp_header_file
+        yaml_file = args.input_yaml_file
+        validate_header = args.validate_header
+        run_md(yaml_file, output_file, 'markdown')
+
+    testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.rst', full_file_path]
+
+    with patch.object(sys, 'argv', testargs):
+        args = parse_args()
+        output_file = args.output_cpp_header_file
+        yaml_file = args.input_yaml_file
+        validate_header = args.validate_header
+        run_md(yaml_file, output_file, 'rst')
+
 
 # class TestViewValidCodeGen(unittest.TestCase):
 @pytest.mark.parametrize(