From 2c9c93878e74184984394ad3e3550baf479737e1 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Wed, 2 Aug 2023 18:02:11 +1000 Subject: [PATCH] added support for pydra usage for python cmd with algorithm --- lib/mrtrix3/algorithm.py | 83 ++++++++++++++++++++++-------------- lib/mrtrix3/app.py | 25 ++++++++--- pydra/pydra-auto-gen.py | 90 +++++++++++++++++++++++----------------- 3 files changed, 123 insertions(+), 75 deletions(-) diff --git a/lib/mrtrix3/algorithm.py b/lib/mrtrix3/algorithm.py index 88386e2acc..724e20f10e 100644 --- a/lib/mrtrix3/algorithm.py +++ b/lib/mrtrix3/algorithm.py @@ -19,51 +19,72 @@ import importlib, inspect, os, pkgutil, sys - +from pathlib import Path # Helper function for finding where the files representing different script algorithms will be stored # These will be in a sub-directory relative to this library file def _algorithms_path(): - from mrtrix3 import path #pylint: disable=import-outside-toplevel - return os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1])), os.pardir, 'lib', 'mrtrix3', path.script_subdir_name())) - + from mrtrix3 import path # pylint: disable=import-outside-toplevel + return str(Path(__file__).parent / "dwi2mask") # DEBUGGING, PLEASE REMOVE + return os.path.realpath( + os.path.join( + os.path.dirname( + os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1]) + ), + os.pardir, + "lib", + "mrtrix3", + path.script_subdir_name(), + ) + ) # This function needs to be safe to run in order to populate the help page; that is, no app initialisation has been run -def get_list(): #pylint: disable=unused-variable - from mrtrix3 import app #pylint: disable=import-outside-toplevel - algorithm_list = [ ] - for filename in os.listdir(_algorithms_path()): - filename = filename.split('.') - if len(filename) == 2 and filename[1] == 'py' and filename[0] != '__init__': - algorithm_list.append(filename[0]) - algorithm_list = sorted(algorithm_list) - app.debug('Found algorithms: ' + str(algorithm_list)) - return algorithm_list +def get_list(): # pylint: disable=unused-variable + from mrtrix3 import app # pylint: disable=import-outside-toplevel + algorithm_list = [] + for filename in os.listdir(_algorithms_path()): + filename = filename.split(".") + if len(filename) == 2 and filename[1] == "py" and filename[0] != "__init__": + algorithm_list.append(filename[0]) + algorithm_list = sorted(algorithm_list) + app.debug("Found algorithms: " + str(algorithm_list)) + return algorithm_list # Note: This function essentially duplicates the current state of app.cmdline in order for command-line # options common to all algorithms of a particular script to be applicable once any particular sub-parser # is invoked. Therefore this function must be called _after_ all such options are set up. -def usage(cmdline): #pylint: disable=unused-variable - from mrtrix3 import app, path #pylint: disable=import-outside-toplevel - sys.path.insert(0, os.path.realpath(os.path.join(_algorithms_path(), os.pardir))) - initlist = [ ] - # Don't let Python 3 try to read incompatible .pyc files generated by Python 2 for no-longer-existent .py files - pylist = get_list() - base_parser = app.Parser(description='Base parser for construction of subparsers', parents=[cmdline]) - subparsers = cmdline.add_subparsers(title='Algorithm choices', help='Select the algorithm to be used to complete the script operation; additional details and options become available once an algorithm is nominated. Options are: ' + ', '.join(get_list()), dest='algorithm') - for dummy_importer, package_name, dummy_ispkg in pkgutil.iter_modules( [ _algorithms_path() ] ): - if package_name in pylist: - module = importlib.import_module(path.script_subdir_name() + '.' + package_name) - module.usage(base_parser, subparsers) - initlist.extend(package_name) - app.debug('Initialised algorithms: ' + str(initlist)) +def usage(cmdline): # pylint: disable=unused-variable + from mrtrix3 import app, path # pylint: disable=import-outside-toplevel + + sys.path.insert(0, os.path.realpath(os.path.join(_algorithms_path(), os.pardir))) + initlist = [] + # Don't let Python 3 try to read incompatible .pyc files generated by Python 2 for no-longer-existent .py files + pylist = get_list() + base_parser = app.Parser( + description="Base parser for construction of subparsers", parents=[cmdline] + ) + subparsers = cmdline.add_subparsers( + title="Algorithm choices", + help="Select the algorithm to be used to complete the script operation; additional details and options become available once an algorithm is nominated. Options are: " + + ", ".join(get_list()), + dest="algorithm", + ) + for dummy_importer, package_name, dummy_ispkg in pkgutil.iter_modules( + [_algorithms_path()] + ): + if package_name in pylist: + module = importlib.import_module("dwi2mask." + package_name) + # module = importlib.import_module(path.script_subdir_name() + "." + package_name) + module.usage(base_parser, subparsers) + initlist.extend(package_name) + app.debug("Initialised algorithms: " + str(initlist)) +def get_module(name): # pylint: disable=unused-variable + from mrtrix3 import path # pylint: disable=import-outside-toplevel -def get_module(name): #pylint: disable=unused-variable - from mrtrix3 import path #pylint: disable=import-outside-toplevel - return sys.modules[path.script_subdir_name() + '.' + name] + return sys.modules[path.script_subdir_name() + "." + name] diff --git a/lib/mrtrix3/app.py b/lib/mrtrix3/app.py index 433259259c..de87ab5354 100644 --- a/lib/mrtrix3/app.py +++ b/lib/mrtrix3/app.py @@ -1091,6 +1091,19 @@ def print_group_options(group): def print_usage_pydra(self): + if self._subparsers: + + if len(sys.argv) == 3: + for alg in self._subparsers._group_actions[0].choices: + if alg == sys.argv[-2]: + self._subparsers._group_actions[0].choices[alg].print_usage_pydra() + return + self.error('Invalid subparser nominated: ' + sys.argv[-2]) + assert len(sys.argv) == 2 + sys.stdout.write(",".join(self._subparsers._group_actions[0].choices)) + sys.stdout.flush() + return + def get_arg_metadata(arg): metadata = { "help_string": arg.help, @@ -1172,6 +1185,8 @@ def parse_type(type_): ) outputs_str = re.sub(r"'#([a-zA-Z0-9_\[\]]+)#'", r"\1", str(outputs)) + task_name = self.prog.replace(" ", "_") + text = ( "import typing\n" "from pathlib import Path # noqa: F401\n" @@ -1184,10 +1199,10 @@ def parse_type(type_): ) text += f"input_fields = {inputs_str}\n" - text += f"{self.prog}_input_spec = specs.SpecInfo(name='Input', fields=input_fields, bases=(specs.ShellSpec,))\n\n" + text += f"{task_name}_input_spec = specs.SpecInfo(name='Input', fields=input_fields, bases=(specs.ShellSpec,))\n\n" text += f"output_fields = {outputs_str}\n" - text += f"{self.prog}_output_spec = specs.SpecInfo(name='Output', fields=output_fields, bases=(specs.ShellOutSpec,))\n\n" - text += f"class {self.prog}(ShellCommandTask):\n" + text += f"{task_name}_output_spec = specs.SpecInfo(name='Output', fields=output_fields, bases=(specs.ShellOutSpec,))\n\n" + text += f"class {task_name}(ShellCommandTask):\n" indent = " " text += indent + "\"\"\"\n" text += indent + (self.description if self.description else "") @@ -1204,8 +1219,8 @@ def parse_type(type_): text += indent + '**Author:** ' + self._author + '\n\n' text += indent + '**Copyright:** ' + self._copyright + '\n\n' text += indent + "\"\"\"\n" - text += f" input_spec = {self.prog}_input_spec\n" - text += f" output_spec = {self.prog}_output_spec\n" + text += f" input_spec = {task_name}_input_spec\n" + text += f" output_spec = {task_name}_output_spec\n" text += f" executable='{self.prog}'\n\n" if HAVE_BLACK: diff --git a/pydra/pydra-auto-gen.py b/pydra/pydra-auto-gen.py index 16c27dfbd5..74ed496ddf 100644 --- a/pydra/pydra-auto-gen.py +++ b/pydra/pydra-auto-gen.py @@ -1,7 +1,9 @@ import os from pathlib import Path import subprocess as sp +import typing as ty import logging +import re import click import black.report import black.parsing @@ -45,46 +47,56 @@ def auto_gen_mrtrix3_pydra(cmd_dir: Path, output_dir: Path, log_errors: bool): for cmd_name in sorted(os.listdir(cmd_dir)): if cmd_name.startswith("_") or "." in cmd_name or cmd_name in IGNORE: continue - try: - code_str = sp.check_output( - [str(cmd_dir / cmd_name), "__print_usage_pydra__"] - ).decode("utf-8") - except sp.CalledProcessError: - if log_errors: - logger.error("Could not generate interface for '%s'", cmd_name) - continue - else: - raise - - # Since Python identifiers can't start with numbers we need to rename 5tt* - # with fivetissuetype* - if cmd_name.startswith("5tt"): - old_name = cmd_name - cmd_name = old_name.replace("5tt", "fivetissuetype") - code_str = code_str.replace( - f"class {old_name}", f"class {cmd_name}" - ) - code_str = code_str.replace( - f"{old_name}_input_spec", f"{cmd_name}_input_spec" - ) - code_str = code_str.replace( - f"{old_name}_output_spec", f"{cmd_name}_input_spec" - ) - try: - code_str = black.format_file_contents( - code_str, fast=False, mode=black.FileMode() + cmd = [str(cmd_dir / cmd_name)] + auto_gen_cmd(cmd, cmd_name, output_dir, log_errors) + + +def auto_gen_cmd(cmd: ty.List[str], cmd_name: str, output_dir: Path, log_errors: bool): + + try: + code_str = sp.check_output(cmd + ["__print_usage_pydra__"]).decode("utf-8") + except sp.CalledProcessError: + if log_errors: + logger.error("Could not generate interface for '%s'", cmd_name) + return + else: + raise + + if re.match(r"(\w+,)+\w+", code_str): + for algorithm in code_str.split(","): + auto_gen_cmd( + cmd + [algorithm], f"{cmd_name}_{algorithm}", output_dir, log_errors ) - except black.report.NothingChanged: - pass - except black.parsing.InvalidInput: - if log_errors: - logger.error("Could not parse generated interface for '%s'", cmd_name) - else: - raise - output_dir.mkdir(exist_ok=True) - with open(output_dir / (cmd_name + ".py"), "w") as f: - f.write(code_str) - logger.info("%s", cmd_name) + + # Since Python identifiers can't start with numbers we need to rename 5tt* + # with fivetissuetype* + if cmd_name.startswith("5tt"): + old_name = cmd_name + cmd_name = old_name.replace("5tt", "fivetissuetype") + code_str = code_str.replace( + f"class {old_name}", f"class {cmd_name}" + ) + code_str = code_str.replace( + f"{old_name}_input_spec", f"{cmd_name}_input_spec" + ) + code_str = code_str.replace( + f"{old_name}_output_spec", f"{cmd_name}_input_spec" + ) + try: + code_str = black.format_file_contents( + code_str, fast=False, mode=black.FileMode() + ) + except black.report.NothingChanged: + pass + except black.parsing.InvalidInput: + if log_errors: + logger.error("Could not parse generated interface for '%s'", cmd_name) + else: + raise + output_dir.mkdir(exist_ok=True) + with open(output_dir / (cmd_name + ".py"), "w") as f: + f.write(code_str) + logger.info("%s", cmd_name) if __name__ == "__main__":