Merge remote-tracking branch 'upstream/master' into feature.sim_mode_…

…at_runtime

* upstream/master:
  tui: avoid deprecated import (cylc#5858)
  Exclude broken pytest-asyncio version
  remove lint (cylc#5856)
  Async pipe decorator: preserve wrapped func signature
  Maintain order for CYLC_PYTHONPATH (cylc#5853)
  Update tests/integration/tui/conftest.py
  tests/i: fix flaky tui/test_updater tests (cylc#5849)
  tests/i: tui - attempt to stabilise tests
  Bump pypa/gh-action-pypi-publish from 1.8.10 to 1.8.11 (cylc#5851)
  tests/i: fix flaky tui/test_updater tests
  Improve changelog entries
  Fix bad TOML example in `cylc lint` docstring (cylc#5843)
  lint - S011 no longer warns against 0{{a}} (cylc#5841)
  `cylc lint`: add rule to catch `rose date`
  tui: show view
  tui: log view
  tui: tidy dangling interfaces
  tui: fix an obscure freezing issue
  tui: update screenshots
  Tui 1.0

.github/workflows/2_auto_publish_release.yml

@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.10
+      uses: pypa/gh-action-pypi-publish@v1.8.11
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

changes.d/5731.feat.md

@@ -0,0 +1 @@
+Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows.

changes.d/5772.feat.md

@@ -1 +1 @@
-Add a check for indentation being 4N spaces.
+`cylc lint`: added a check for indentation being 4N spaces.

changes.d/5838.feat.md

@@ -0,0 +1 @@
+`cylc lint`: added rule to check for `rose date` usage (should be replaced with `isodatetime`).

changes.d/5841.fix.md

@@ -0,0 +1 @@
+`cylc lint`: improved handling of S011 to not warn if the `#` is `#$` (e.g. shell base arithmetic).

cylc/flow/async_util.py

@@ -17,6 +17,7 @@
 
 import asyncio
 from functools import partial, wraps
+from inspect import signature
 import os
 from pathlib import Path
 from typing import List, Union
@@ -262,10 +263,22 @@ def __str__(self):
     def __repr__(self):
         return _AsyncPipe(self.func).__repr__()
 
+    @property
+    def __name__(self):
+        return self.func.__name__
+
     @property
     def __doc__(self):
         return self.func.__doc__
 
+    @property
+    def __signature__(self):
+        return signature(self.func)
+
+    @property
+    def __annotations__(self):
+        return self.func.__annotations__
+
 
 def pipe(func=None, preproc=None):
     """An asynchronous pipe implementation in pure Python.

cylc/flow/cfgspec/workflow.py

@@ -1794,7 +1794,7 @@ def upg(cfg, descr):
         ['cylc', 'simulation', 'disable suite event handlers'])
     u.obsolete('8.0.0', ['cylc', 'simulation'], is_section=True)
     u.obsolete('8.0.0', ['visualization'], is_section=True)
-    u.obsolete('8.0.0', ['scheduling', 'spawn to max active cycle points']),
+    u.obsolete('8.0.0', ['scheduling', 'spawn to max active cycle points'])
     u.deprecate(
         '8.0.0',
         ['cylc', 'task event mail interval'],

cylc/flow/data_store_mgr.py

@@ -1587,7 +1587,7 @@ def insert_job(self, name, cycle_point, status, job_conf):
             name=tproxy.name,
             cycle_point=tproxy.cycle_point,
             execution_time_limit=job_conf.get('execution_time_limit'),
-            platform=job_conf.get('platform')['name'],
+            platform=job_conf['platform']['name'],
             job_runner_name=job_conf.get('job_runner_name'),
         )
         # Not all fields are populated with some submit-failures,

cylc/flow/option_parsers.py

@@ -48,6 +48,7 @@
 )
 
 WORKFLOW_ID_ARG_DOC = ('WORKFLOW', 'Workflow ID')
+OPT_WORKFLOW_ID_ARG_DOC = ('[WORKFLOW]', 'Workflow ID')
 WORKFLOW_ID_MULTI_ARG_DOC = ('WORKFLOW ...', 'Workflow ID(s)')
 WORKFLOW_ID_OR_PATH_ARG_DOC = ('WORKFLOW | PATH', 'Workflow ID or path')
 ID_MULTI_ARG_DOC = ('ID ...', 'Workflow/Cycle/Family/Task ID(s)')

cylc/flow/scripts/cylc.py

@@ -31,9 +31,12 @@ def pythonpath_manip():
         https://github.com/cylc/cylc-flow/issues/5124
     """
     if 'CYLC_PYTHONPATH' in os.environ:
-        for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep):
-            abspath = os.path.abspath(item)
-            sys.path.insert(0, abspath)
+        paths = [
+            os.path.abspath(item)
+            for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep)
+        ]
+        paths.extend(sys.path)
+        sys.path = paths
     if 'PYTHONPATH' in os.environ:
         for item in os.environ['PYTHONPATH'].split(os.pathsep):
             abspath = os.path.abspath(item)

cylc/flow/scripts/graph.py

@@ -414,9 +414,21 @@ async def graph_diff(
     graph_a: List[str] = []
     graph_b: List[str] = []
     graph_reference(
-        opts, workflow_a, start, stop, flow_file, write=graph_a.append),
+        opts,
+        workflow_a,
+        start,
+        stop,
+        flow_file,
+        write=graph_a.append,
+    )
     graph_reference(
-        opts, workflow_b, start, stop, flow_file_b, write=graph_b.append),
+        opts,
+        workflow_b,
+        start,
+        stop,
+        flow_file_b,
+        write=graph_b.append,
+    )
 
     # compare graphs
     diff_lines = list(

cylc/flow/scripts/lint.py

@@ -40,7 +40,7 @@
 TOMLDOC = """
 pyproject.toml configuration:{}
    [cylc-lint]                     # any of {}
-       ignore = ['S001', 'S002]    # List of rules to ignore
+       ignore = ['S001', 'S002']    # List of rules to ignore
        exclude = ['etc/foo.cylc']  # List of files to ignore
        rulesets = ['style', '728'] # Sets default rulesets to check
        max-line-length = 130       # Max line length for linting
@@ -437,7 +437,7 @@ def check_indentation(line: str) -> bool:
         'evaluate commented lines': True,
         FUNCTION: functools.partial(
             check_if_jinja2,
-            function=re.compile(r'(?<!{)#.*?{[{%]').findall
+            function=re.compile(r'(?<!{)#[^$].*?{[{%]').findall
         )
     },
     'S012': {
@@ -569,7 +569,21 @@ def check_indentation(line: str) -> bool:
             'job-script-vars/index.html'
         ),
         FUNCTION: check_for_obsolete_environment_variables,
-    }
+    },
+    'U014': {
+        'short': 'Use "isodatetime [ref]" instead of "rose date [-c]"',
+        'rst': (
+            'For datetime operations in task scripts:\n\n'
+            ' * Use ``isodatetime`` instead of ``rose date``\n'
+            ' * Use ``isodatetime ref`` instead of ``rose date -c`` for '
+            'the current cycle point\n'
+        ),
+        'url': (
+            'https://cylc.github.io/cylc-doc/stable/html/7-to-8/'
+            'cheat-sheet.html#datetime-operations'
+        ),
+        FUNCTION: re.compile(r'rose +date').findall,
+    },
 }
 RULESETS = ['728', 'style', 'all']
 EXTRA_TOML_VALIDATION = {

cylc/flow/scripts/tui.py

@@ -15,34 +15,35 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""cylc tui WORKFLOW
+"""cylc tui [WORKFLOW]
 
 View and control running workflows in the terminal.
 
 (Tui = Terminal User Interface)
 
-WARNING: Tui is experimental and may break with large flows.
-An upcoming change to the way Tui receives data from the scheduler will make it
-much more efficient in the future.
+Tui allows you to monitor and interact with workflows in a manner similar
+to the GUI.
+
+Press "h" whilst running Tui to bring up the help screen, use the arrow
+keys to navigage.
+
 """
-# TODO: remove this warning once Tui is delta-driven
-# https://github.com/cylc/cylc-flow/issues/3527
 
+from getpass import getuser
 from textwrap import indent
-from typing import TYPE_CHECKING
-from urwid import html_fragment
+from typing import TYPE_CHECKING, Optional
 
+from cylc.flow.id import Tokens
 from cylc.flow.id_cli import parse_id
 from cylc.flow.option_parsers import (
-    WORKFLOW_ID_ARG_DOC,
+    OPT_WORKFLOW_ID_ARG_DOC,
     CylcOptionParser as COP,
 )
 from cylc.flow.terminal import cli_function
 from cylc.flow.tui import TUI
+from cylc.flow.tui.util import suppress_logging
 from cylc.flow.tui.app import (
     TuiApp,
-    TREE_EXPAND_DEPTH
-    # ^ a nasty solution
 )
 
 if TYPE_CHECKING:
@@ -55,57 +56,25 @@
 def get_option_parser() -> COP:
     parser = COP(
         __doc__,
-        argdoc=[WORKFLOW_ID_ARG_DOC],
+        argdoc=[OPT_WORKFLOW_ID_ARG_DOC],
         # auto_add=False,  NOTE: at present auto_add can not be turned off
         color=False
     )
 
-    parser.add_option(
-        '--display',
-        help=(
-            'Specify the display technology to use.'
-            ' "raw" for interactive in-terminal display.'
-            ' "html" for non-interactive html output.'
-        ),
-        action='store',
-        choices=['raw', 'html'],
-        default='raw',
-    )
-    parser.add_option(
-        '--v-term-size',
-        help=(
-            'The virtual terminal size for non-interactive'
-            '--display options.'
-        ),
-        action='store',
-        default='80,24'
-    )
-
     return parser
 
 
 @cli_function(get_option_parser)
-def main(_, options: 'Values', workflow_id: str) -> None:
-    workflow_id, *_ = parse_id(
-        workflow_id,
-        constraint='workflows',
-    )
-    screen = None
-    if options.display == 'html':
-        TREE_EXPAND_DEPTH[0] = -1  # expand tree fully
-        screen = html_fragment.HtmlGenerator()
-        screen.set_terminal_properties(256)
-        screen.register_palette(TuiApp.palette)
-        html_fragment.screenshot_init(
-            [tuple(map(int, options.v_term_size.split(',')))],
-            []
+def main(_, options: 'Values', workflow_id: Optional[str] = None) -> None:
+    # get workflow ID if specified
+    if workflow_id:
+        workflow_id, *_ = parse_id(
+            workflow_id,
+            constraint='workflows',
         )
+        tokens = Tokens(workflow_id)
+        workflow_id = tokens.duplicate(user=getuser()).id
 
-    try:
-        TuiApp(workflow_id, screen=screen).main()
-
-        if options.display == 'html':
-            for fragment in html_fragment.screenshot_collect():
-                print(fragment)
-    except KeyboardInterrupt:
+    # start Tui
+    with suppress_logging(), TuiApp().main(workflow_id):
         pass

cylc/flow/tui/__init__.py

@@ -106,12 +106,26 @@
 
 
 class Bindings:
+    """Represets key bindings for the Tui app."""
 
     def __init__(self):
         self.bindings = []
         self.groups = {}
 
     def bind(self, keys, group, desc, callback):
+        """Register a key binding.
+
+        Args:
+            keys:
+                The keys to bind.
+            group:
+                The group to which this binding should belong.
+            desc:
+                Description for this binding, used to generate help.
+            callback:
+                The thing to call when this binding is pressed.
+
+        """
         if group not in self.groups:
             raise ValueError(f'Group {group} not registered.')
         binding = {
@@ -124,6 +138,15 @@ def bind(self, keys, group, desc, callback):
         self.groups[group]['bindings'].append(binding)
 
     def add_group(self, group, desc):
+        """Add a new binding group.
+
+        Args:
+            group:
+                The name of the group.
+            desc:
+                A description of the group, used to generate help.
+
+        """
         self.groups[group] = {
             'name': group,
             'desc': desc,
@@ -134,6 +157,12 @@ def __iter__(self):
         return iter(self.bindings)
 
     def list_groups(self):
+        """List groups and the bindings in them.
+
+        Yields:
+            (group_name, [binding, ...])
+
+        """
         for name, group in self.groups.items():
             yield (
                 group,
@@ -143,6 +172,3 @@ def list_groups(self):
                     if binding['group'] == name
                 ]
             )
-
-
-BINDINGS = Bindings()