Add --yaml-extend option to allow modifying rosdoc2.yaml

rosdoc2/verbs/build/impl.py

@@ -98,6 +98,11 @@ def prepare_arguments(parser):
         action='store_true',
         help='enable more output to debug problems'
     )
+    parser.add_argument(
+        '--yaml-extend',
+        '-y',
+        help='Extend rosdoc2.yaml'
+    )
     return parser
 
 

rosdoc2/verbs/build/inspect_package_for_settings.py

@@ -12,12 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 
+import yaml
+
 from .build_context import BuildContext
+from .builders import create_builder_by_name
 from .create_format_map_from_package import create_format_map_from_package
 from .parse_rosdoc2_yaml import parse_rosdoc2_yaml
 
+logger = logging.getLogger('rosdoc2')
+
 DEFAULT_ROSDOC_CONFIG_FILE = """\
 ## Default configuration, generated by rosdoc2.
 
@@ -139,5 +145,66 @@ def inspect_package_for_settings(package, tool_options):
     for depends in package['buildtool_depends']:
         if str(depends) == 'ament_cmake_python':
             build_context.ament_cmake_python = True
-
-    return parse_rosdoc2_yaml(rosdoc_config_file, build_context)
+    configs = list(yaml.load_all(rosdoc_config_file, Loader=yaml.SafeLoader))
+
+    (settings_dict, builders_list) = parse_rosdoc2_yaml(configs, build_context)
+
+    # Extend the configs if requested. See test/ex_test.yaml as example of format.
+    yaml_extend = tool_options.yaml_extend
+    if yaml_extend:
+        if not os.path.isfile(yaml_extend):
+            raise ValueError(
+                f"yaml_extend path '{yaml_extend}' is not a file")
+        with open(yaml_extend, 'r') as f:
+            yaml_extend_text = f.read()
+        extended_settings = yaml.load(yaml_extend_text, Loader=yaml.SafeLoader)
+        for item in extended_settings:
+            ex_name = next(iter(item))
+            options = item[ex_name]['options'] if 'options' in item[ex_name] else {}
+            logger.info(
+                f'Searching rosdoc2.yaml extension {ex_name} for {package.name} options {options}')
+            if 'packages' in item[ex_name] and package.name in item[ex_name]['packages']:
+                extended_object = item[ex_name]['packages'][package.name]
+                if 'settings' in extended_object:
+                    for key, value in extended_object['settings'].items():
+                        settings_dict[key] = value
+                        # Don't override an existing value if 'only_if_missing' is true
+                        if 'only_if_missing' in options and options['only_if_missing']:
+                            if key in settings_dict:
+                                logger.warning(f'yaml extension wants to set {key} '
+                                               'but it is already set. Using existing value.')
+                                continue
+                        logger.info(f'Overriding rosdoc2.yaml setting  <{key}> with <{value}>')
+                if 'builders' in extended_object:
+                    for ex_builder in extended_object['builders']:
+                        ex_builder_name = next(iter(ex_builder))
+                        # find this object in the builders list
+                        for user_builder in builders_list:
+                            user_builder_name = next(iter(user_builder))
+                            if user_builder_name == ex_builder_name:
+                                for builder_k, builder_v in ex_builder[ex_builder_name].items():
+                                    # Don't override an existing value if 'only_if_missing' is true
+                                    if 'only_if_missing' in options and options['only_if_missing']:
+                                        if builder_k in user_builder[user_builder_name]:
+                                            logger.warning('yaml extension wants to set '
+                                                           f'{builder_k} but it is already set. '
+                                                           'Using existing value.')
+                                            continue
+                                    logger.info(f'Overriding rosdoc2 builder <{ex_builder_name}> '
+                                                f'property <{builder_k}> with <{builder_v}>')
+                                    user_builder[user_builder_name][builder_k] = builder_v
+
+    # if None, python_source is set to either './<package.name>' or 'src/<package.name>'
+    build_context.python_source = settings_dict.get('python_source', None)
+    build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False)
+    build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False)
+    build_context.build_type = settings_dict.get('override_build_type', build_context.build_type)
+
+    builders = []
+    for builder in builders_list:
+        builder_name = next(iter(builder))
+        builders.append(create_builder_by_name(builder_name,
+                                               builder_dict=builder[builder_name],
+                                               build_context=build_context))
+
+    return (settings_dict, builders)

rosdoc2/verbs/build/parse_rosdoc2_yaml.py

@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
 
-from .builders import create_builder_by_name
-
-
-def parse_rosdoc2_yaml(yaml_string, build_context):
+def parse_rosdoc2_yaml(configs, build_context):
     """
     Parse a rosdoc2.yaml configuration string, returning it as a tuple of settings and builders.
 
     :return: a tuple with the first item being the tool settings as a dictionary,
         and the second item being a list of Builder objects.
     """
-    configs = list(yaml.load_all(yaml_string, Loader=yaml.SafeLoader))
     file_name = build_context.configuration_file_path
     if len(configs) != 2:
         raise ValueError(
@@ -57,12 +52,6 @@ def parse_rosdoc2_yaml(yaml_string, build_context):
             f'expected a dict{{output_dir: build_settings, ...}}, '
             f"got a '{type(settings_dict)}' instead")
 
-    # if None, python_source is set to either './<package.name>' or 'src/<package.name>'
-    build_context.python_source = settings_dict.get('python_source', None)
-    build_context.always_run_doxygen = settings_dict.get('always_run_doxygen', False)
-    build_context.always_run_sphinx_apidoc = settings_dict.get('always_run_sphinx_apidoc', False)
-    build_context.build_type = settings_dict.get('override_build_type', build_context.build_type)
-
     if 'builders' not in config:
         raise ValueError(
             f"Error parsing file '{file_name}', in the second section, "
@@ -74,15 +63,10 @@ def parse_rosdoc2_yaml(yaml_string, build_context):
             'expected a list of builders, '
             f"got a '{type(builders_list)}' instead")
 
-    builders = []
     for builder in builders_list:
         if len(builder) != 1:
             raise ValueError(
                 f"Error parsing file '{file_name}', in the second section, each builder "
                 'must have exactly one key (which is the type of builder to use)')
-        builder_name = next(iter(builder))
-        builders.append(create_builder_by_name(builder_name,
-                                               builder_dict=builder[builder_name],
-                                               build_context=build_context))
 
-    return (settings_dict, builders)
+    return (settings_dict, builders_list)

test/ex_test.yaml

@@ -0,0 +1,19 @@
+---
+- docs_support:
+    options: {'only_if_missing': true}
+    packages:
+      invalid_python_source:
+        builders:
+          - sphinx:
+              user_doc_dir: funny_docs
+      # Test of the 'only_if_missing' option.
+      default_yaml:
+        builders:
+          - sphinx:
+              user_doc_dir: i_do_not_exist
+- python_location:
+    options: {}
+    packages:
+      src_alt_python:
+        settings:
+          python_source: launch

test/packages/invalid_python_source/funny_docs/funny_doc.rst

@@ -0,0 +1,4 @@
+This is in a funny place
+========================
+
+blah, blah

test/test_builder.py

@@ -35,19 +35,22 @@ def module_dir(tmp_path_factory):
     return tmp_path_factory.getbasetemp()
 
 
-def do_build_package(package_path, work_path) -> None:
+def do_build_package(package_path, work_path, with_extension=False) -> None:
     build_dir = work_path / 'build'
     output_dir = work_path / 'output'
     cr_dir = work_path / 'cross_references'
 
     # Create a top level parser
     parser = prepare_arguments(argparse.ArgumentParser())
-    options = parser.parse_args([
+    args = [
         '-p', str(package_path),
         '-c', str(cr_dir),
         '-o', str(output_dir),
         '-d', str(build_dir),
-    ])
+    ]
+    if with_extension:
+        args.extend(['-y', str(pathlib.Path('test') / 'ex_test.yaml')])
+    options = parser.parse_args(args)
     logger.info(f'*** Building package(s) at {package_path} with options {options}')
 
     # run rosdoc2 on the package
@@ -93,15 +96,16 @@ def test_full_package(module_dir):
 def test_default_yaml(module_dir):
     """Test a package with C++, python, and docs using specified default rosdoc2.yaml configs."""
     PKG_NAME = 'default_yaml'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     do_test_full_package(module_dir, pkg_name=PKG_NAME)
 
 
 def test_only_python(module_dir):
     """Test a pure python package."""
     PKG_NAME = 'only_python'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    # Use with_extension=True to show that nothing changes if the package is not there.
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     includes = [
         PKG_NAME,
@@ -151,10 +155,13 @@ def test_false_python(module_dir):
 
 def test_invalid_python_source(module_dir):
     PKG_NAME = 'invalid_python_source'
-    do_build_package(DATAPATH / PKG_NAME, module_dir)
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
 
     excludes = ['python api']
-    includes = ['This packages incorrectly specifies python source']
+    includes = [
+        'This packages incorrectly specifies python source',
+        'this is in a funny place',  # Documentation found using extended yaml
+    ]
 
     do_test_package(PKG_NAME, module_dir,
                     includes=includes,
@@ -250,3 +257,14 @@ def test_empty_doc_dir(module_dir):
                     links_exist=links_exist)
 
     do_test_package(PKG_NAME, module_dir)
+
+
+def test_src_alt_python(module_dir):
+    PKG_NAME = 'src_alt_python'
+    do_build_package(DATAPATH / PKG_NAME, module_dir, with_extension=True)
+
+    includes = ['python api']  # We found the python source with the extended yaml
+    links_exist = ['dummy.html']  # We found python source with extended yaml
+    do_test_package(PKG_NAME, module_dir,
+                    includes=includes,
+                    links_exist=links_exist)