From 5a372775d8572e27d408b62737294f4feac18a79 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:58:46 +0100 Subject: [PATCH] fixed basics --- cylc/sphinx_ext/cylc_lang/__init__.py | 46 +- cylc/sphinx_ext/cylc_lang/autodocumenters.py | 512 +++++++++---------- etc/flow/global.cylc | 15 +- etc/{ => workflow}/flow.cylc | 19 +- 4 files changed, 289 insertions(+), 303 deletions(-) rename etc/{ => workflow}/flow.cylc (62%) diff --git a/cylc/sphinx_ext/cylc_lang/__init__.py b/cylc/sphinx_ext/cylc_lang/__init__.py index 6469726..918c827 100644 --- a/cylc/sphinx_ext/cylc_lang/__init__.py +++ b/cylc/sphinx_ext/cylc_lang/__init__.py @@ -19,7 +19,8 @@ from cylc.sphinx_ext.cylc_lang.autodocumenters import ( CylcAutoDirective, CylcAutoTypeDirective, - CylcMetadataDirective + CylcWorkflowDirective, + CylcGlobalDirective, ) from cylc.sphinx_ext.cylc_lang.domains import ( ParsecDomain, @@ -203,48 +204,32 @@ ''' rawdoc2 = """ -.. rst:directive:: cylc-metadata +.. rst:directive:: .. auto-global-cylc:: source - Get a Cylc Configuration and render metadata fields. + Get a Cylc Global Configuration and render metadata fields. - .. rst:directive:option:: source - :type: string + If the optional source argument is give, + set ``CYLC_SITE_CONF_PATH`` to this value. - If set, renders the metadata of a workflow, otherwise the global - config. + .. note:: - .. rst:directive:option:: global - :type: string - - Set CYLC_SITE_CONF_PATH to this value. - - .. note:: - - If you have a user config this will still override the site - config! - - ---- - - Workflow Config - ^^^^^^^^^^^^^^^ + If you have a user config this will still override the site + config! .. rst-example:: - .. cylc-metadata:: - :source: {workflow_path} + .. auto-cylc-global:: {workflow_path} - ---- +.. rst:directive:: .. auto-cylc-workflow:: source - Global Config - ^^^^^^^^^^^^^ + Get a Cylc Workflow Configuration from source and document the settings. .. rst-example:: - .. cylc-metadata:: - :global: {workflow_path} - + .. auto-cylc-workflow:: {workflow_path}/workflow """ + workflow_path = Path(__file__).parent.parent.parent.parent / 'etc' __doc__ = ( rawdoc1 @@ -271,6 +256,7 @@ def setup(app): app.add_domain(ParsecDomain) app.add_directive('auto-cylc-conf', CylcAutoDirective) app.add_directive('auto-cylc-type', CylcAutoTypeDirective) - app.add_directive('cylc-metadata', CylcMetadataDirective) + app.add_directive('auto-cylc-workflow', CylcWorkflowDirective) + app.add_directive('auto-cylc-global', CylcGlobalDirective) app.add_directive('cylc-scope', CylcScopeDirective) return {'version': __version__, 'parallel_read_safe': True} diff --git a/cylc/sphinx_ext/cylc_lang/autodocumenters.py b/cylc/sphinx_ext/cylc_lang/autodocumenters.py index 8ed4c90..4f0ef35 100644 --- a/cylc/sphinx_ext/cylc_lang/autodocumenters.py +++ b/cylc/sphinx_ext/cylc_lang/autodocumenters.py @@ -1,3 +1,22 @@ +# ----------------------------------------------------------------------------- +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# ----------------------------------------------------------------------------- + +from copy import copy from importlib import import_module import json import os @@ -7,11 +26,12 @@ dedent, indent ) +from typing import Any, Dict, List, Optional from docutils.parsers.rst import Directive from docutils.statemachine import StringList, ViewList -from sphinx import addnodes +from sphinx import addnodes, project from sphinx.util.docutils import SphinxDirective from cylc.flow.parsec.config import ConfigNode @@ -19,6 +39,7 @@ INDENT = ' ' +CYLC_CONF = 'cylc:conf' class DependencyError(Exception): @@ -88,6 +109,7 @@ def directive( ['.. my_directive:: a b c', ''] """ + arguments = arguments or [] ret = [ f'.. {directive}::{" " if arguments else ""}{" ".join(arguments)}' ] @@ -104,6 +126,8 @@ def directive( ]) ret.append('') if content: + if isinstance(content, List): + content = '\n'.join(content) ret.extend( indent( # remove indentation and head,tail blanklines @@ -369,31 +393,14 @@ def run(self): return [node] -class CylcMetadataDirective(SphinxDirective): - """Represent a Cylc Config. +class CylcGlobalDirective(SphinxDirective): + """Represent a Cylc Global Config. """ - optional_arguments = 3 + optional_arguments = 1 def run(self): - # Parse input options: - for key, value in zip( - [i.strip(':') for i in self.arguments[::2]], - list(self.arguments[1::2]) - ): - self.options.update({key: value}) - - # Get global or workflow metadara - if 'source' in self.options: - config = self.load_workflow_cfg(self.options['source']) - metadata = self.get_workflow_metadata( - config, self.options['source']) - ret = self.workflow_to_node(metadata, config) - else: - src = self.options['global'] - config = self.load_global_cfg(self.options['global']) - metadata = self.get_global_metadata(config) - ret = self.global_to_node(metadata, src) - + src = self.arguments[0] if self.arguments else None + ret = self.config_to_node(load_cfg(src), src) node = addnodes.desc_content() self.state.nested_parse( StringList(ret), @@ -403,278 +410,257 @@ def run(self): return [node] @staticmethod - def load_global_cfg(conf_path=None): - """Get Global Configuration metadata: - - Args: - Path: Global conf path. + def config_to_node(config: [Dict, Any], src: str) -> List[str]: + """Take a global config and create a node for display. - """ - # Load Global Config: - if conf_path: - env = os.environ - sub = run( - ['cylc', 'config', '--json'], - capture_output=True, - env=env.update({'CYLC_SITE_CONF_PATH': conf_path}) - ) - else: - sub = run(['cylc', 'config', '--json'], capture_output=True) + * Displays `platform groups` and then `platforms`. + * For each group and platform: + * Adds a title, either the platform name, or the ``[meta]title`` + field. + * Creates a key item list containing: + * The name of the item as :regex: if the ``[meta]title`` set. + * Job Runner. + * Hosts/Platforms to be selected from. + * ``[meta]URL``. - CylcMetadataDirective.check_subproc_output(sub) - - return json.loads(sub.stdout) - - @staticmethod - def get_global_metadata(config): - """ - Additional Processing: - * Get lists of hosts/platforms and job runner from the config. - * If no title is provided, use the platform/group regex as the title. - * If title != regex then insert text saying which regex - needs matching to select this platform. Returns: - A dictionary in the form: - 'platforms': {'platform regex': {..metadata..}}, - 'platform groups': {'platform regex': {..metadata..}} + A list of lines for inclusion in the document. """ - metadata = {} - for section, select_from in zip( - ['platforms', 'platform groups'], - ['hosts', 'platforms'] - ): - metadata[section] = config.get(section) - if not metadata[section]: - continue - for key in config.get(section).keys(): - # Grab a list of hosts or platforms that this - # platform or group will select from: - select_from = ( - config.get(section).get(key).get('hosts') - or config.get(section).get(key).get('platforms')) - select_from = select_from or [key] - metadata[section][key]['select_from'] = select_from - - # Grab the job runner this platform uses: - if section == 'platforms': - metadata[section][key]['job_runner'] = config.get( - section).get(key).get('job runner', 'background') - return metadata + # Basic info about platforms. + ret = [] - @staticmethod - def global_to_node(meta, src): - """Convert the global metadata into rst format. - """ - ret = [ - f'.. cylc:conf:: Global Config: {src}', '', - f'{INDENT}.. note::', '', - f'{INDENT * 2}Platforms and platform groups are listed' - ' in the order in which Cylc will check for matches to the' - ' ``[runtime][NAMESPACE]platform`` setting.', ''] - - # For platforms in platform groupts then for hosts in platforms: - for settings, selects in zip( + note = directive( + 'note', + content=( + 'Platforms and platform groups are listed' + ' in the order in which Cylc will check for matches to the' + ' ``[runtime][NAMESPACE]platform`` setting.' + ) + ) + + for section_name, selectable in zip( ['platform groups', 'platforms'], ['platforms', 'hosts'] ): - if meta.get(settings, {}): - ret += [f'{INDENT}.. cylc:conf:: {settings}', ''] - for regex, info in reversed(meta[settings].items()): - platform_meta = info.get('meta', {}) - title = platform_meta.get('title', '') - if not title: - title = regex - ret += [ - f'{INDENT * 3}.. cylc:conf:: {title}', '', - f'{INDENT * 4}.. csv-table:: Key Details', ''] - - # Add the definition regex if an explicit title - # has been given to the platform. - if title != regex: - ret.append(f'{INDENT * 5}Regex, ``{regex}``.') - - if info.get('job_runner'): - ret.append( - f'{INDENT * 5}job runner,' - f' ``{info.get("job_runner")}``') - - # Lists platforms in group or hosts in platform. - selectables = '- ' + '\n- '.join( - f'``{i}``' for i in info["select_from"]) - ret += [f'{INDENT * 5}{selects},"{selectables}"', ''] - - ret.append('') - url = platform_meta.get('url', '') - if url: - ret += [ - f'{INDENT * 4}.. seealso::', - '', - f'{INDENT * 5}{url}', - '' - ] - - # Add description. - desc = platform_meta.get("description", "") - ret += [ - f'{INDENT * 4}' - f'{desc}', - ''] - - # Add any other metadata as a definition list: - ret += CylcMetadataDirective.meta_to_def_list( - metadata=platform_meta.items(), - lowest_indent=4 + section = config.get(section_name, {}) + if not section: + continue + + content = [] + for regex, conf in section.items(): + # Build info about a given platform or platform group. + section_content = [] + + meta = conf.get('meta', {}) + + # Title - Use regex if [meta]title not supplied; + # but include regex field if title is supplied: + title = meta.get('title', '') + if title: + section_content.append(f':regex: ``{regex}``') + else: + title = regex + + # Job Runner + section_content.append( + f":job runner: {conf.get('job runner', 'background')}") + + # List of hosts or platforms: + section_content.append( + f':{selectable}: ' + ', '.join( + f'``{s}``' for s in + conf.get(selectable, [regex]) ) + ) + + # Get [meta]URL - if it exists put it in a seealso directive: + url = meta.get('URL', '') + if url: + section_content.append(f':URL: {url}') + + # Custom keys: + section_content += custom_items(meta) + + # Key list needs a closing space: + section_content.append('') + + # Add description tag. + description = meta.get('description', '') + if description: + section_content.append(description) + + content += directive( + CYLC_CONF, [title], content=section_content) - # Handy for debugging: - # [print(f'{i + 1:03}|{j}') for i, j in enumerate(ret)] + ret += directive( + CYLC_CONF, [section_name], content=content) + + ret = directive( + CYLC_CONF, [src], content=note + ret) + + # Prettified Debug to help with finding errors: + if project.logger.getEffectiveLevel() > 9: + [print(f'{i + 1:03}|{line}') for i, line in enumerate(ret)] return ret - @staticmethod - def load_workflow_cfg(conf_path): - """Get Workflow Configuration metadata: - Args: - conf_path: workflow conf path. - """ - # Load Global Config: - sub = run( - ['cylc', 'config', '--json', conf_path], - capture_output=True, +class CylcWorkflowDirective(SphinxDirective): + """Represent a Cylc Workflow Config. + """ + required_arguments = 1 + + def run(self): + ret = self.config_to_node( + load_cfg(self.arguments[0]), + self.arguments[0] ) - CylcMetadataDirective.check_subproc_output(sub) - return json.loads(sub.stdout) + node = addnodes.desc_content() + self.state.nested_parse( + StringList(ret), + self.content_offset, + node + ) + return [node] @staticmethod - def get_workflow_metadata(config, conf_path): - """Get workflow metadata. + def config_to_node(config, src): + """Document Workflow Additional processing: * If no title field is provided use either the workflow folder or the task/family name. - * Don't return the root family if there is no metadata. - - Returns: - 'workflow': {.. top level metadata ..}, - 'runtime': {'namespace': '.. task or family metadata ..'} """ - # Extract Data - meta = {} - - # Copy metadata to the two top level sections: - meta['workflow'] = config.get('meta') - meta['runtime'] = { - k: v.get('meta', {}) - for k, v in config.get('runtime', {}).items()} - - # Title is parent directory if otherwise unset: - if not meta.get('workflow', {}).get('title', ''): - meta['workflow']['title'] = Path(conf_path).name - - # Title of namespace is parent if otherwise unset: - poproot = False - for namespace, info in meta['runtime'].items(): - # don't display root unless it's actually had some - # metadata added, but save a flag rather than modifying - # the iterable being looped over: - if ( - namespace == 'root' - and not any(meta['runtime'].get('root').values()) - ): - poproot = True - - # If metadata doesn't have a title set title to the namespace name: - if not info.get('title', ''): - meta['runtime'][namespace]['title'] = namespace - - if poproot: - meta['runtime'].pop('root') - - return meta + workflow_content = [] + + # Handle workflow level metadata: + workflow_meta = config.get('meta', {}) + # Title or path + workflow_name = workflow_meta.get('title', '') + if not workflow_name: + workflow_name = src + + # URL if available + url = workflow_meta.get('URL', '') + if url: + workflow_content += directive('seealso', [url]) + + # Custom keys: + workflow_content += custom_items(workflow_meta) + + # Description: + workflow_content += ['', workflow_meta.get('description', ''), ''] + + # Add details of the runtime section: + for task_name, taskdef in config.get('runtime', {}).items(): + task_content = [] + task_meta = taskdef.get('meta', {}) + + # Does task have a title? + title = task_meta.get('title', '') + if title: + title = f'{title} ({task_name})' + else: + title = task_name - @staticmethod - def workflow_to_node(meta, src): - """Convert workflow metadata to list of strings for use as a node. + # Task URL + url = task_meta.get('URL', '') + if url: + task_content.append(f':URL: {url}') - """ - ret = [] + # Custom keys: + task_content += custom_items(task_meta) - # Handle the workflow config metadata: - workflow_title = meta.get('workflow', {}).get('title', src) - workflow_desc = meta.get('workflow', {}).get( - 'description', '').split('\n') + desc = task_meta.get('description', '') + if desc: + task_content += ['', desc, ''] - ret += [f'.. cylc:conf:: {workflow_title}', ''] - ret += [f' {t.strip()}' for t in workflow_desc] + [''] + workflow_content += directive( + CYLC_CONF, [title], content=task_content) - # Handle the runtime config metadata: - ret += [f'{INDENT}Runtime', f'{INDENT}=======', ''] + ret = directive(CYLC_CONF, [workflow_name], content=workflow_content) - for taskmeta in meta['runtime'].values(): - indent = INDENT - title = taskmeta["title"] - ret += [f'{indent}.. cylc:conf:: {title}', ''] - indent += INDENT + # Pretty debug statement: + if project.logger.getEffectiveLevel() > 9: + [print(f'{i + 1:03}|{line}') for i, line in enumerate(ret)] - url = taskmeta.get('url', '') - if url: - ret += [ - f'{indent}.. seealso::', - '', - f'{indent + INDENT}{url}', - '' - ] - - desc = taskmeta.get('description', '').replace('\n', ' ') - ret += [f'{indent}{desc}', ''] - - ret += CylcMetadataDirective.meta_to_def_list( - metadata=taskmeta.items(), - lowest_indent=1 - ) + return ret - # Handy for debugging: - # [print(f'{i + 1:03}|{j}') for i, j in enumerate(ret)] - return ret +def load_cfg(conf_path: Optional[str] = None) -> Dict[str, Any]: + """Get Workflow Configuration metadata: - @staticmethod - def check_subproc_output(sub): - """Check subprocess outputs - catch failure. - """ - if sub.returncode: - # Very specifically handle the case where the correct - # version of Cylc isn't installed: - if 'no such option: --json' in sub.stderr.decode(): - msg = ( - 'Requires cylc config --json, not available' - ' for this version of Cylc') - raise DependencyError(msg) - # Handle any and all other errors in the subprocess: - else: - msg = 'Cylc config metadata failed with: \n' - msg += '\n'.join( - i.strip("\n") for i in sub.stderr.decode().split('\n')) - raise Exception(msg) + Args: + conf_path: global or workflow conf path. - @staticmethod - def meta_to_def_list(metadata, lowest_indent): - result = [] - other_meta_items = False - for key, value in metadata: - if key in ['title', 'description', 'url']: - continue - if other_meta_items is False: - result += [ - f'{INDENT * lowest_indent}Other metadata', - f'{INDENT * lowest_indent}^^^^^^^^^^^^^^', ''] - other_meta_items = True - value = value.replace('\n', ' ') - result += [ - f'{INDENT * (lowest_indent + 1)}{key}:', - f'{INDENT * (lowest_indent + 2)}{value}', ''] - - return result + Raises: + DependencyError: If a version of Cylc without the + ``cylc config --json`` facility is installed. + """ + env = None + if conf_path is None: + cmd = ['cylc', 'config', '--json'] + elif (Path(conf_path) / 'flow.cylc').exists(): + # Load workflow Config: + cmd = ['cylc', 'config', '--json', conf_path] + elif (Path(conf_path) / 'flow/global.cylc').exists(): + # Load Global Config: + if conf_path: + env = copy(os.environ) + env = env.update({'CYLC_SITE_CONF_PATH': conf_path}) + cmd = ['cylc', 'config', '--json'] + else: + raise FileNotFoundError( + f'No Cylc config file found at {conf_path}') + + sub = run( + cmd, + capture_output=True, + env=env or os.environ + ) + + # Catches failure caused by a version of Cylc without + # the ``cylc config --json`` option. + if sub.returncode: + # cylc config --json not available: + if 'no such option: --json' in sub.stderr.decode(): + msg = ( + 'Requires cylc config --json, not available' + ' for this version of Cylc') + raise DependencyError(msg) + # all other errors in the subprocess: + else: + msg = 'Cylc config metadata failed with: \n' + msg += '\n'.join( + i.strip("\n") for i in sub.stderr.decode().split('\n')) + raise Exception(msg) + + return json.loads(sub.stdout) + + +def custom_items( + data: Dict[str, Any], + not_these: Optional[List[str]] = None, + these: Optional[List[str]] = None +) -> List[str]: + """Given a dict return a keylist. + + Args: + data: The input dictionary. + not_these: Keys to ignore. + these: Keys to include. + """ + ret = [] + if these: + for key in these: + value = data.get('key', '') + value = value.replace("\n", "\n ") + ret.append(f':{key}:\n {value}') + else: + for key, val in data.items(): + if key not in (not_these or ['title', 'description', 'URL']): + value = val.replace("\n", "\n ") + ret.append(f':{key}:\n {value}') + return ret diff --git a/etc/flow/global.cylc b/etc/flow/global.cylc index 8389d9a..cd28297 100644 --- a/etc/flow/global.cylc +++ b/etc/flow/global.cylc @@ -3,14 +3,23 @@ [[[meta]]] title = "Kings/Charing/Bounds Cross" description = """ - If platform name is a regex you might want - an explicit title + * Demonstrate that you can insert RST here. + + .. warning:: + + If platform name is a regex you might want + an explicit title. + + And another thing + ^^^^^^^^^^^^^^^^^ """ - url = https://www.mysupercomputer.ac.uk + URL = https://www.mysupercomputer.ac.uk [[mornington_crescent]] [[[meta]]] description = """ + ###### I win! + ###### """ location = Otago contractor = Takahē Industries diff --git a/etc/flow.cylc b/etc/workflow/flow.cylc similarity index 62% rename from etc/flow.cylc rename to etc/workflow/flow.cylc index ba69819..37ff72c 100644 --- a/etc/flow.cylc +++ b/etc/workflow/flow.cylc @@ -4,7 +4,7 @@ This flow.cylc file is placed here to allow the testing of the metadata config extension. """ - url = 'https://www.myproject.com' + URL = 'https://www.myproject.com' custom key yan = "Perhaps it's relevent?" custom key tan = "Perhaps it's also relevent?" custom key tethera = "Or perhaps not?" @@ -19,17 +19,22 @@ [[[meta]]] title = 'task title' description = """ - All about my task - I will document my working - All about my task - for a P12M + P1D + .. admonition:: To the tune of "All around my hat" + + All about my task + + I will document my working + + All about my task + + for a P12M + P1D """ - url = 'https://www.myproject.com/tasks/foo' + URL = 'https://www.myproject.com/tasks/foo' [[bar]] [[[meta]]] description = """ Lorem Ipsum blah blah blah """ - url = 'https://www.myproject.com/tasks/bar' + URL = 'https://www.myproject.com/tasks/bar' and another thing = Morse