From 360be2807a09b8e93c18e5768f450ba6c8366247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20Fr=C3=B6hlich?= Date: Wed, 3 Jan 2024 16:50:00 +0100 Subject: [PATCH] markdown/rst: Support __map_ and nested parameters (#164) * Support nested parameters and maps for RST * Add read_only info to RST template * Use map parameters for AutoDocumentation * Add test for markdown generation * Use jinja indentation instead of regex * Add dynamic_parameters also to DefaultConfig * Proper fromat of default yaml config * Proper format of the default yaml file * Fix imported function names * Add a jinja2 filter to create valid multiline c++ string literals * Handle empty strings for descriptions --- .../generate_markdown.py | 109 +++++++++++++++++- .../jinja_templates/cpp/declare_parameter | 2 +- .../cpp/declare_runtime_parameter | 2 +- .../jinja_templates/markdown/parameter_detail | 6 +- .../jinja_templates/rst/parameter_detail | 13 ++- .../parse_yaml.py | 39 ++++--- .../string_filters_cpp.py | 22 ++++ .../test/YAML_parse_error_test.py | 33 +++++- 8 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 generate_parameter_library_py/generate_parameter_library_py/string_filters_cpp.py diff --git a/generate_parameter_library_py/generate_parameter_library_py/generate_markdown.py b/generate_parameter_library_py/generate_parameter_library_py/generate_markdown.py index 34dd1f2..fcfef4b 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/generate_markdown.py +++ b/generate_parameter_library_py/generate_parameter_library_py/generate_markdown.py @@ -31,13 +31,16 @@ import argparse import os +import re import sys from jinja2 import Template from typeguard import typechecked +import yaml from generate_parameter_library_py.parse_yaml import ( GenerateCode, DeclareParameter, + DeclareRuntimeParameter, ValidationFunction, ) @@ -95,7 +98,44 @@ def __str__(self): constraints = '\n'.join(str(val) for val in self.param_validations) data = { - 'name': self.declare_parameters.code_gen_variable.name, + 'name': self.declare_parameters.parameter_name, + 'read_only': self.declare_parameters.parameter_read_only, + 'type': self.declare_parameters.code_gen_variable.defined_type, + 'default_value': self.declare_parameters.code_gen_variable.lang_str_value, + 'constraints': constraints, + # remove leading whitespace from description, this is necessary for correct indentation of multi-line descriptions + 'description': re.sub( + r'(?m)^(?!$)\s*', + '', + str(self.declare_parameters.parameter_description), + flags=re.MULTILINE, + ), + } + + j2_template = Template(GenerateCode.templates['parameter_detail']) + code = j2_template.render(data, trim_blocks=True) + return code + + +class RuntimeParameterDetailMarkdown: + @typechecked + def __init__(self, declare_parameters: DeclareRuntimeParameter): + self.declare_parameters = declare_parameters + self.param_validations = [ + ParameterValidationMarkdown(val) + for val in declare_parameters.parameter_validations + ] + + def __str__(self): + constraints = '\n'.join(str(val) for val in self.param_validations) + data = { + # replace __map_key with + 'name': re.sub( + r'__map_(\w+)', + lambda match: '<' + match.group(1) + '>', + self.declare_parameters.parameter_name, + ), + 'read_only': self.declare_parameters.parameter_read_only, 'type': self.declare_parameters.code_gen_variable.defined_type, 'default_value': self.declare_parameters.code_gen_variable.lang_str_value, 'constraints': constraints, @@ -119,14 +159,65 @@ def __init__(self, gen_param_struct: GenerateCode): def __str__(self): j2_template = Template(GenerateCode.templates['default_config']) - tmp = '\n'.join( - param.parameter_name + ': ' + str(param.code_gen_variable.lang_str_value) - for param in self.gen_param_struct.declare_parameters + tmp = ( + '\n'.join( + param.parameter_name + + ': ' + + str(param.code_gen_variable.lang_str_value) + for param in self.gen_param_struct.declare_parameters + ) + + '\n' + + '\n'.join( + # replace __map_key with + re.sub( + r'__map_(\w+)', + lambda match: '<' + match.group(1) + '>', + param.parameter_name, + ) + + ': ' + + str(param.code_gen_variable.lang_str_value) + for param in self.gen_param_struct.declare_dynamic_parameters + ) ) + # Split the string into lines and group them by the first part + def nest_dict(d, keys, value): + # Check if the value is a string + if isinstance(value, str): + # Remove double quotes from the string + value = value.replace('"', '') + # Try to convert the value to a boolean, number, or leave it as a string + if value.lower() == 'true': + value = True + elif value.lower() == 'false': + value = False + else: + try: + value = float(value) + except ValueError: + pass + + if len(keys) == 1: + d[keys[0]] = value + else: + key = keys.pop(0) + if key not in d: + d[key] = {} + nest_dict(d[key], keys, value) + + # Split the string into lines and create a dictionary + d = {} + for line in tmp.strip().split('\n'): + name, value = line.split(':', 1) + keys = name.split('.') + nest_dict(d, keys, value.strip()) + + # Convert the dictionary to a string + result = yaml.dump(d, default_flow_style=False) + data = { 'namespace': self.gen_param_struct.namespace, - 'default_param_values': tmp, + 'default_param_values': result, } code = j2_template.render(data, trim_blocks=True) @@ -142,6 +233,10 @@ def __init__(self, gen_param_struct: GenerateCode): ParameterDetailMarkdown(param) for param in self.gen_param_struct.declare_parameters ] + self.runtime_param_details = [ + RuntimeParameterDetailMarkdown(param) + for param in self.gen_param_struct.declare_dynamic_parameters + ] def __str__(self): words = self.gen_param_struct.namespace.split('_') @@ -150,7 +245,9 @@ def __str__(self): data = { 'title': title, 'default_config': str(self.default_config), - 'parameter_details': '\n'.join(str(val) for val in self.param_details), + 'parameter_details': '\n'.join(str(val) for val in self.param_details) + + '\n' + + '\n'.join(str(val) for val in self.runtime_param_details), } j2_template = Template(GenerateCode.templates['documentation']) diff --git a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_parameter b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_parameter index 9add858..45dcb61 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_parameter +++ b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_parameter @@ -1,7 +1,7 @@ if (!parameters_interface_->has_parameter(prefix_ + "{{parameter_name}}")) { {%- filter indent(width=4) %} rcl_interfaces::msg::ParameterDescriptor descriptor; -descriptor.description = "{{parameter_description}}"; +descriptor.description = {{parameter_description | valid_string_cpp}}; descriptor.read_only = {{parameter_read_only}}; {%- for validation in parameter_validations if ("bounds" in validation.function_name or "lt" in validation.function_name or "gt" in validation.function_name) %} {%- if "DOUBLE" in parameter_type %} diff --git a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_runtime_parameter b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_runtime_parameter index 9d9395d..b891691 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_runtime_parameter +++ b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/cpp/declare_runtime_parameter @@ -5,7 +5,7 @@ auto param_name = fmt::format("{}{}.{}.{}", prefix_, "{{struct_name}}", value, " if (!parameters_interface_->has_parameter(param_name)) { {%- filter indent(width=4) %} rcl_interfaces::msg::ParameterDescriptor descriptor; -descriptor.description = "{{parameter_description}}"; +descriptor.description = {{parameter_description | valid_string_cpp}}; descriptor.read_only = {{parameter_read_only}}; {%- for validation in parameter_validations if ("bounds" in validation.function_name or "lt" in validation.function_name or "gt" in validation.function_name) %} {%- if "DOUBLE" in parameter_type %} diff --git a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/markdown/parameter_detail b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/markdown/parameter_detail index a53ab28..1f4a544 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/markdown/parameter_detail +++ b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/markdown/parameter_detail @@ -4,10 +4,10 @@ {% endif %} * Type: `{{type}}` {%- if default_value|length %} -* Default Value: {{default_value}} -{% endif %} -{%- if constraints|length %} +* Default Value: {{default_value}}{% endif %}{% if read_only %} +* Read only: {{read_only}}{% endif %}{%- if constraints|length %} *Constraints:* {{constraints}} +{% else %} {% endif %} diff --git a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/rst/parameter_detail b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/rst/parameter_detail index 6475874..cc87455 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/rst/parameter_detail +++ b/generate_parameter_library_py/generate_parameter_library_py/jinja_templates/rst/parameter_detail @@ -1,12 +1,14 @@ -{{name}} ({{type}}){% if description|length %} - {{description}} +{{name}} ({{type}}){%- filter indent(width=2) %}{% if description|length %} +{{description}} +{% endif %} +{%- if read_only %} +Read only: {{read_only}} {% endif %} {%- if default_value|length %} - Default: {{default_value}} +Default: {{default_value}} {% endif %} {%- if constraints|length %} - - Constraints: +Constraints: {%- filter indent(width=2) %} @@ -14,3 +16,4 @@ {% endfilter -%} {% endif %} +{% endfilter -%} diff --git a/generate_parameter_library_py/generate_parameter_library_py/parse_yaml.py b/generate_parameter_library_py/generate_parameter_library_py/parse_yaml.py index c32c3a3..9728e72 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/parse_yaml.py +++ b/generate_parameter_library_py/generate_parameter_library_py/parse_yaml.py @@ -29,7 +29,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from jinja2 import Template +from jinja2 import Template, Environment from typeguard import typechecked from typing import Any, List, Optional from yaml.parser import ParserError @@ -39,6 +39,7 @@ from generate_parameter_library_py.cpp_convertions import CPPConverstions from generate_parameter_library_py.python_convertions import PythonConvertions +from generate_parameter_library_py.string_filters_cpp import valid_string_cpp # YAMLSyntaxError standardizes compiler error messages @@ -499,6 +500,7 @@ def __str__(self): bool_to_str = self.code_gen_variable.conversation.bool_to_str parameter_validations = self.parameter_validations + data = { 'parameter_name': self.parameter_name, 'parameter_value': self.parameter_value, @@ -507,7 +509,11 @@ def __str__(self): 'parameter_read_only': bool_to_str(self.parameter_read_only), 'parameter_validations': parameter_validations, } - j2_template = Template(GenerateCode.templates['declare_parameter']) + + # Create a Jinja2 environment to register the custom filter + env = Environment() + env.filters['valid_string_cpp'] = valid_string_cpp + j2_template = env.from_string(GenerateCode.templates['declare_parameter']) code = j2_template.render(data, trim_blocks=True) return code @@ -566,7 +572,12 @@ def __str__(self): 'parameter_validations': self.parameter_validations, } - j2_template = Template(GenerateCode.templates['declare_runtime_parameter']) + # Create a Jinja2 environment to register the custom filter + env = Environment() + env.filters['valid_string_cpp'] = valid_string_cpp + j2_template = env.from_string( + GenerateCode.templates['declare_runtime_parameter'] + ) code = j2_template.render(data, trim_blocks=True) return code @@ -689,6 +700,18 @@ class GenerateCode: templates = None def __init__(self, language: str): + if language == 'cpp': + self.comments = '// auto-generated DO NOT EDIT' + elif language == 'rst': + self.comments = '.. auto-generated DO NOT EDIT' + elif language == 'markdown': + self.comments = '' + elif language == 'python' or language == 'markdown': + self.comments = '# auto-generated DO NOT EDIT' + else: + raise compile_error( + 'Invalid language, only cpp, markdown, rst, and python are currently supported.' + ) GenerateCode.templates = get_all_templates(language) self.language = language self.namespace = '' @@ -702,16 +725,6 @@ def __init__(self, language: str): self.remove_dynamic_parameter = [] self.declare_parameter_sets = [] self.set_stack_params = [] - if language == 'cpp': - self.comments = '// auto-generated DO NOT EDIT' - elif language == 'rst': - self.comments = '.. auto-generated DO NOT EDIT' - elif language == 'python' or language == 'markdown': - self.comments = '# auto-generated DO NOT EDIT' - else: - raise compile_error( - 'Invalid language, only c++ and python are currently supported.' - ) self.user_validation_file = '' def parse(self, yaml_file, validate_header): diff --git a/generate_parameter_library_py/generate_parameter_library_py/string_filters_cpp.py b/generate_parameter_library_py/generate_parameter_library_py/string_filters_cpp.py new file mode 100644 index 0000000..3af310b --- /dev/null +++ b/generate_parameter_library_py/generate_parameter_library_py/string_filters_cpp.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +def valid_string_cpp(description): + """ + Filter a string to make it a valid C++ string literal. + + Args: + description (str): The input string to be filtered. + + Returns: + str: The filtered string that is a valid C++ string. + """ + if description: + filtered_description = ( + description.replace('\\', '\\\\').replace('"', '\\"').replace('`', '') + ) + # create a quote delimited string for every line + filtered_description = '\n'.join( + f'"{line}"' for line in filtered_description.splitlines() + ) + return filtered_description + else: + return '""' diff --git a/generate_parameter_library_py/generate_parameter_library_py/test/YAML_parse_error_test.py b/generate_parameter_library_py/generate_parameter_library_py/test/YAML_parse_error_test.py index b85af04..7afb837 100644 --- a/generate_parameter_library_py/generate_parameter_library_py/test/YAML_parse_error_test.py +++ b/generate_parameter_library_py/generate_parameter_library_py/test/YAML_parse_error_test.py @@ -19,8 +19,9 @@ import os from ament_index_python.packages import get_package_share_path -from generate_parameter_library_py.generate_cpp_header import run as run_python -from generate_parameter_library_py.generate_python_module import run as run_cpp +from generate_parameter_library_py.generate_cpp_header import run as run_cpp +from generate_parameter_library_py.generate_python_module import run as run_python +from generate_parameter_library_py.generate_markdown import run as run_md from generate_parameter_library_py.parse_yaml import YAMLSyntaxError from generate_parameter_library_py.generate_cpp_header import parse_args @@ -29,7 +30,7 @@ def set_up(yaml_test_file): full_file_path = os.path.join( get_package_share_path('generate_parameter_library_py'), 'test', yaml_test_file ) - testargs = [sys.argv[0], '/tmp/admittance_controller.h', full_file_path] + testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.h', full_file_path] with patch.object(sys, 'argv', testargs): args = parse_args() @@ -37,8 +38,34 @@ def set_up(yaml_test_file): yaml_file = args.input_yaml_file validate_header = args.validate_header run_cpp(output_file, yaml_file, validate_header) + + testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.py', full_file_path] + + with patch.object(sys, 'argv', testargs): + args = parse_args() + output_file = args.output_cpp_header_file + yaml_file = args.input_yaml_file + validate_header = args.validate_header run_python(output_file, yaml_file, validate_header) + testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.md', full_file_path] + + with patch.object(sys, 'argv', testargs): + args = parse_args() + output_file = args.output_cpp_header_file + yaml_file = args.input_yaml_file + validate_header = args.validate_header + run_md(yaml_file, output_file, 'markdown') + + testargs = [sys.argv[0], '/tmp/' + yaml_test_file + '.rst', full_file_path] + + with patch.object(sys, 'argv', testargs): + args = parse_args() + output_file = args.output_cpp_header_file + yaml_file = args.input_yaml_file + validate_header = args.validate_header + run_md(yaml_file, output_file, 'rst') + # class TestViewValidCodeGen(unittest.TestCase): @pytest.mark.parametrize(