diff --git a/mlcube/mlcube/__main__.py b/mlcube/mlcube/__main__.py index 5f08b0a..e6fa6a3 100644 --- a/mlcube/mlcube/__main__.py +++ b/mlcube/mlcube/__main__.py @@ -6,20 +6,24 @@ import typing as t import click - import coloredlogs +from omegaconf import OmegaConf +from mlcube.cli import ( + MLCubeCommand, + MultiValueOption, + Options, + UsageExamples, + parse_cli_args, +) +from mlcube.errors import ExecutionError, IllegalParameterValueError, MLCubeError from mlcube.parser import CliParser -from mlcube.cli import (MLCubeCommand, MultiValueOption, Options, parse_cli_args, UsageExamples) -from mlcube.errors import (ExecutionError, IllegalParameterValueError, MLCubeError) from mlcube.shell import Shell from mlcube.system_settings import SystemSettings -from omegaconf import OmegaConf - logger = logging.getLogger(__name__) -_TERMINAL_WIDTH = shutil.get_terminal_size()[0] # Since Python version 3.3 +_TERMINAL_WIDTH = shutil.get_terminal_size()[0] # Since Python version 3.3 """Width of a user terminal. MLCube overrides default (80) character width to make usage examples look better.""" @@ -120,7 +124,8 @@ def parser_process(value: str, state: click.parser.ParsingState): help="CPU options defined during MLCube container execution.", ) -@click.group(name='mlcube', add_help_option=False) + +@click.group(name="mlcube", add_help_option=False) @Options.loglevel @Options.help def cli(log_level: t.Optional[str]): @@ -142,8 +147,15 @@ def cli(log_level: t.Optional[str]): @cli.command( - name='show_config', cls=MLCubeCommand, add_help_option=False, epilog=UsageExamples.show_config, - context_settings={'ignore_unknown_options': True, 'allow_extra_args': True, 'max_content_width': _TERMINAL_WIDTH} + name="show_config", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.show_config, + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + "max_content_width": _TERMINAL_WIDTH, + }, ) @Options.mlcube @Options.platform @@ -152,8 +164,14 @@ def cli(log_level: t.Optional[str]): @Options.parameter @Options.help @click.pass_context -def show_config(ctx: click.core.Context, mlcube: t.Optional[str], platform: str, workspace: str, - resolve: bool, p: t.Tuple[str]) -> None: +def show_config( + ctx: click.core.Context, + mlcube: t.Optional[str], + platform: str, + workspace: str, + resolve: bool, + p: t.Tuple[str], +) -> None: """Show effective MLCube configuration. Effective MLCube configuration is the one used by one of MLCube runners to run this MLCube. This configuration is @@ -172,16 +190,23 @@ def show_config(ctx: click.core.Context, mlcube: t.Optional[str], platform: str, if mlcube is None: mlcube = os.getcwd() _, mlcube_config = parse_cli_args( - unparsed_args=ctx.args + ['-P' + param for param in p], + unparsed_args=ctx.args + ["-P" + param for param in p], parsed_args={"mlcube": mlcube, "platform": platform, "workspace": workspace}, - resolve=resolve + resolve=resolve, ) print(OmegaConf.to_yaml(mlcube_config)) @cli.command( - name='configure', cls=MLCubeCommand, add_help_option=False, epilog=UsageExamples.configure, - context_settings={'ignore_unknown_options': True, 'allow_extra_args': True, 'max_content_width': _TERMINAL_WIDTH} + name="configure", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.configure, + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + "max_content_width": _TERMINAL_WIDTH, + }, ) @Options.mlcube @Options.platform @@ -203,15 +228,21 @@ def configure(mlcube: t.Optional[str], platform: str, p: t.Tuple[str]) -> None: p: Additional MLCube configuration parameters (these parameters are those parameters that normally start with `-P` prefix). Here, due to original implementation, we need to `unparse` by adding `-P` prefix. """ - logger.debug("mlcube::configure, mlcube=%s, platform=%s, p=%s", mlcube, platform, str(p)) + logger.debug( + "mlcube::configure, mlcube=%s, platform=%s, p=%s", mlcube, platform, str(p) + ) if mlcube is None: mlcube = os.getcwd() - logger.info("Configuring MLCube (`%s`) for `%s` platform.", os.path.abspath(mlcube), platform) + logger.info( + "Configuring MLCube (`%s`) for `%s` platform.", + os.path.abspath(mlcube), + platform, + ) try: runner_cls, mlcube_config = parse_cli_args( - unparsed_args=['-P' + param for param in p], + unparsed_args=["-P" + param for param in p], parsed_args={"mlcube": mlcube, "platform": platform}, - resolve=True + resolve=True, ) runner = runner_cls(mlcube_config, task=None) runner.configure() @@ -229,8 +260,15 @@ def configure(mlcube: t.Optional[str], platform: str, p: t.Tuple[str]) -> None: @cli.command( - name='run', cls=MLCubeCommand, add_help_option=False, epilog=UsageExamples.run, - context_settings={'ignore_unknown_options': True, 'allow_extra_args': True, 'max_content_width': _TERMINAL_WIDTH} + name="run", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.run, + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + "max_content_width": _TERMINAL_WIDTH, + }, ) @mlcube_option @platform_option @@ -272,15 +310,30 @@ def run( """ logger.info( "run input_arg mlcube=%s, platform=%s, task=%s, workspace=%s, network=%s, security=%s, gpus=%s, memory=%s, " - "cpu=%s", mlcube, platform, task, workspace, network, security, gpus, memory, cpu + "cpu=%s", + mlcube, + platform, + task, + workspace, + network, + security, + gpus, + memory, + cpu, ) runner_cls, mlcube_config = parse_cli_args( unparsed_args=ctx.args, parsed_args={ - "mlcube": mlcube, "platform": platform, "workspace": workspace, - "network": network, "security": security, "gpus": gpus, "memory": memory, "cpu": cpu + "mlcube": mlcube, + "platform": platform, + "workspace": workspace, + "network": network, + "security": security, + "gpus": gpus, + "memory": memory, + "cpu": cpu, }, - resolve=True + resolve=True, ) mlcube_tasks: t.List[str] = list( (mlcube_config.get("tasks", None) or {}).keys() @@ -332,8 +385,11 @@ def run( @cli.command( - name='describe', cls=MLCubeCommand, add_help_option=False, epilog=UsageExamples.describe, - context_settings={'max_content_width': _TERMINAL_WIDTH} + name="describe", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.describe, + context_settings={"max_content_width": _TERMINAL_WIDTH}, ) @Options.mlcube @Options.help @@ -347,9 +403,7 @@ def describe(mlcube: t.Optional[str]) -> None: if mlcube is None: mlcube = os.getcwd() _, mlcube_config = parse_cli_args( - unparsed_args=[], - parsed_args={"mlcube": mlcube}, - resolve=True + unparsed_args=[], parsed_args={"mlcube": mlcube}, resolve=True ) print("MLCube") print(f" path = {mlcube_config.runtime.root}") @@ -385,59 +439,108 @@ def describe(mlcube: t.Optional[str]) -> None: @cli.command( - name='config', cls=MLCubeCommand, add_help_option=False, epilog=UsageExamples.config, - context_settings={'ignore_unknown_options': True, 'allow_extra_args': True, 'max_content_width': _TERMINAL_WIDTH} + name="config", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.config, + context_settings={ + "ignore_unknown_options": True, + "allow_extra_args": True, + "max_content_width": _TERMINAL_WIDTH, + }, ) @click.option( - '--list', 'list_all', is_flag=True, - help="Print out the content of system settings file." + "--list", + "list_all", + is_flag=True, + help="Print out the content of system settings file.", ) @click.option( - '--get', required=False, type=str, default=None, - help="Return value of the key (use OmegaConf notation, e.g. `mlcube config --get runners.docker`)." + "--get", + required=False, + type=str, + default=None, + help="Return value of the key (use OmegaConf notation, e.g. `mlcube config --get runners.docker`).", ) @click.option( - '--create_platform', '--create-platform', required=False, cls=MultiValueOption, type=tuple, default=None, + "--create_platform", + "--create-platform", + required=False, + cls=MultiValueOption, + type=tuple, + default=None, help="Create a new platform instance for this runner. Default runner parameters are used to initialize this new " - "platform." + "platform.", ) @click.option( - '--remove_platform', '--remove-platform', required=False, type=str, default=None, + "--remove_platform", + "--remove-platform", + required=False, + type=str, + default=None, help="Remove this platform. If this is one of the default platforms (e.g., `docker`), it will be recreated (with " - "default values) next time `mlcube` runs." + "default values) next time `mlcube` runs.", ) @click.option( - '--rename_platform', '--rename-platform', required=False, cls=MultiValueOption, type=tuple, default=None, + "--rename_platform", + "--rename-platform", + required=False, + cls=MultiValueOption, + type=tuple, + default=None, help="Rename existing platform. If default platform is to be renamed (e.g., `docker`), it will be recreated " - "(with default values) next time `mlcube` runs." + "(with default values) next time `mlcube` runs.", ) @click.option( - '--copy_platform', '--copy-platform', required=False, cls=MultiValueOption, type=tuple, default=None, + "--copy_platform", + "--copy-platform", + required=False, + cls=MultiValueOption, + type=tuple, + default=None, help="Copy existing platform. This can be useful for creating new platforms off existing platforms, for instance," - "creating a new SSH runner configuration that runs MLCubes on a new remote server." + "creating a new SSH runner configuration that runs MLCubes on a new remote server.", ) @click.option( - '--rename_runner', '--rename-runner', required=False, cls=MultiValueOption, type=tuple, default=None, + "--rename_runner", + "--rename-runner", + required=False, + cls=MultiValueOption, + type=tuple, + default=None, help="Rename existing MLCube runner. If platforms exist that reference this runner, users must explicitly provide " - "`--update-platforms` flag to confirm they want to update platforms' description too." + "`--update-platforms` flag to confirm they want to update platforms' description too.", ) @click.option( - '--remove_runner', '--remove-runner', required=False, type=str, default=None, + "--remove_runner", + "--remove-runner", + required=False, + type=str, + default=None, help="Remove existing runner. If platforms exist that reference this runner, users must explicitly provide " - "`--remove-platforms` flag to confirm they want to remove platforms too." + "`--remove-platforms` flag to confirm they want to remove platforms too.", ) @Options.help @click.pass_context -def config(ctx: click.core.Context, - list_all: bool, # mlcube config --list - get: t.Optional[str], # mlcube config --get KEY - create_platform: t.Optional[t.Tuple], # mlcube config --create-platform RUNNER PLATFORM - remove_platform: t.Optional[str], # mlcube config --remove-platform NAME - rename_platform: t.Optional[t.Tuple], # mlcube config --rename-platform OLD_NAME NEW_NAME - copy_platform: t.Optional[t.Tuple], # mlcube config --copy-platform EXISTING_PLATFORM NEW_PLATFORM - rename_runner: t.Optional[t.Tuple], # mlcube config --rename-runner OLD_NAME NEW_NAME - remove_runner: t.Optional[str] # mlcube config --remove-runner NAME - ) -> None: +def config( + ctx: click.core.Context, + list_all: bool, # mlcube config --list + get: t.Optional[str], # mlcube config --get KEY + create_platform: t.Optional[ + t.Tuple + ], # mlcube config --create-platform RUNNER PLATFORM + remove_platform: t.Optional[str], # mlcube config --remove-platform NAME + rename_platform: t.Optional[ + t.Tuple + ], # mlcube config --rename-platform OLD_NAME NEW_NAME + copy_platform: t.Optional[ + t.Tuple + ], # mlcube config --copy-platform EXISTING_PLATFORM NEW_PLATFORM + rename_runner: t.Optional[ + t.Tuple + ], # mlcube config --rename-runner OLD_NAME NEW_NAME + remove_runner: t.Optional[str], # mlcube config --remove-runner NAME +) -> None: """Work with MLCube [system settings](https://mlcommons.github.io/mlcube/getting-started/system-settings/) similar to `git config`. @@ -479,10 +582,14 @@ def _check_tuple( settings.copy_platform(rename_platform, delete_source=False) elif rename_runner: _check_tuple(rename_runner, "rename_runner", 2, "OLD_NAME NEW_NAME") - update_platforms: bool = "--update-platforms" in ctx.args or "--update_platforms" in ctx.args + update_platforms: bool = ( + "--update-platforms" in ctx.args or "--update_platforms" in ctx.args + ) settings.rename_runner(rename_runner, update_platforms=update_platforms) elif remove_runner: - remove_platforms: bool = "--remove-platforms" in ctx.args or "--remove_platforms" in ctx.args + remove_platforms: bool = ( + "--remove-platforms" in ctx.args or "--remove_platforms" in ctx.args + ) settings.remove_runner(remove_runner, remove_platforms=remove_platforms) except MLCubeError as e: logger.error( @@ -491,8 +598,11 @@ def _check_tuple( @cli.command( - name='create', add_help_option=False, cls=MLCubeCommand, epilog=UsageExamples.create, - context_settings={'max_content_width': _TERMINAL_WIDTH} + name="create", + add_help_option=False, + cls=MLCubeCommand, + epilog=UsageExamples.create, + context_settings={"max_content_width": _TERMINAL_WIDTH}, ) @Options.help def create() -> None: @@ -515,5 +625,57 @@ def create() -> None: print(f"\tMore details: {mlcube_cookiecutter_url}") +@cli.command( + name="inspect", + cls=MLCubeCommand, + add_help_option=False, + epilog=UsageExamples.inspect, + context_settings={"max_content_width": _TERMINAL_WIDTH}, +) +@Options.mlcube +@Options.platform +@click.option( + "--force", + is_flag=True, + help="Force inspecting the MLCube object. For instance, if MLCube has not been pulled or built it, then pull " + "or build it.", +) +@click.option( + "--format", + "format_", + metavar="FORMAT", + required=False, + type=click.Choice(["json", "yaml"]), + default="json", + help="Format for reporting results.", +) +@Options.help +def inspect( + mlcube: t.Optional[str], platform: str, force: bool = False, format_: str = "json" +) -> None: + """Return low-level information on MLCube objects.""" + runner_cls, mlcube_config = parse_cli_args( + parsed_args={"mlcube": mlcube, "platform": platform}, + unparsed_args=[], + resolve=True, + ) + try: + runner = runner_cls(mlcube_config, task=None) + info: t.Dict = runner.inspect(force=force) + logger.debug("inspect info=%s", info) + if format_ == "json": + import json + + print(json.dumps(info)) + else: + import yaml + + yaml.dump(info, sys.stdout) + except MLCubeError as err: + logger.exception(err) + print(str(err)) + exit(1) + + if __name__ == "__main__": cli() diff --git a/mlcube/mlcube/cli.py b/mlcube/mlcube/cli.py index 190c340..3f4d4db 100644 --- a/mlcube/mlcube/cli.py +++ b/mlcube/mlcube/cli.py @@ -5,28 +5,35 @@ from xml.etree.ElementTree import Element import click + try: from click.core import DEPRECATED_HELP_NOTICE except ImportError: DEPRECATED_HELP_NOTICE = "(Deprecated)" from markdown import Markdown - -__all__ = ["parse_cli_args", "markdown2text", "MultiValueOption", "Options", "MLCubeCommand", "UsageExamples", - "parse_cli_args"] - from omegaconf import DictConfig from mlcube.config import MLCubeConfig -from mlcube.parser import MLCubeDirectory, CliParser +from mlcube.parser import CliParser, MLCubeDirectory from mlcube.platform import Platform from mlcube.runner import Runner from mlcube.system_settings import SystemSettings from mlcube.validate import Validate +__all__ = [ + "parse_cli_args", + "markdown2text", + "MultiValueOption", + "Options", + "MLCubeCommand", + "UsageExamples", + "parse_cli_args", +] + def parse_cli_args( - unparsed_args: t.List[str], parsed_args: t.Dict, resolve: bool + unparsed_args: t.List[str], parsed_args: t.Dict, resolve: bool ) -> t.Tuple[t.Optional[t.Type[Runner]], DictConfig]: """Parse command line arguments. @@ -46,11 +53,15 @@ def parse_cli_args( mlcube_inst: MLCubeDirectory = CliParser.parse_mlcube_arg(parsed_args["mlcube"]) Validate.validate_type(mlcube_inst, MLCubeDirectory) - mlcube_cli_args, task_cli_args = CliParser.parse_extra_arg(unparsed_args, parsed_args) + mlcube_cli_args, task_cli_args = CliParser.parse_extra_arg( + unparsed_args, parsed_args + ) if parsed_args.get("platform", None) is not None: system_settings = SystemSettings() - runner_config: t.Optional[DictConfig] = system_settings.get_platform(parsed_args["platform"]) + runner_config: t.Optional[DictConfig] = system_settings.get_platform( + parsed_args["platform"] + ) runner_cls: t.Optional[t.Type[Runner]] = Platform.get_runner( system_settings.runners.get(runner_config.runner, None) ) @@ -80,8 +91,9 @@ def markdown2text(text: str) -> str: Returns: Plain text. """ - _markdown: t.Optional[Markdown] = getattr(markdown2text, '_markdown', None) + _markdown: t.Optional[Markdown] = getattr(markdown2text, "_markdown", None) if _markdown is None: + def unmark_element(element: Element, stream: t.Optional[StringIO] = None): if stream is None: stream = StringIO() @@ -93,11 +105,11 @@ def unmark_element(element: Element, stream: t.Optional[StringIO] = None): stream.write(element.tail) return stream.getvalue() - Markdown.output_formats['plain'] = unmark_element - _markdown = Markdown(output_format='plain') + Markdown.output_formats["plain"] = unmark_element + _markdown = Markdown(output_format="plain") _markdown.stripTopLevelTags = False - setattr(markdown2text, '_markdown', _markdown) + setattr(markdown2text, "_markdown", _markdown) try: text = _markdown.convert(text) @@ -113,13 +125,13 @@ class MultiValueOption(click.Option): `--rename-key OLD_VALUE NEW_VALUE`. This will assign a tuple `(OLD_VALUE, NEW_VALUE)` to a function's`rename_key` parameter. """ + def __init__(self, *args, **kwargs) -> None: super(MultiValueOption, self).__init__(*args, **kwargs) self._previous_parser_process: t.Optional[t.Callable] = None self._eat_all_parser: t.Optional[click.parser.Option] = None def add_to_parser(self, parser: click.parser.OptionParser, ctx: click.core.Context): - def parser_process(value: str, state: click.parser.ParsingState): values: t.List[str] = [value] prefixes: t.Tuple[str] = tuple(self._eat_all_parser.prefixes) @@ -131,8 +143,9 @@ def parser_process(value: str, state: click.parser.ParsingState): super(MultiValueOption, self).add_to_parser(parser, ctx) for opt_name in self.opts: - our_parser: t.Optional[click.parser.Option] = \ - parser._long_opt.get(opt_name) or parser._short_opt.get(opt_name) + our_parser: t.Optional[click.parser.Option] = parser._long_opt.get( + opt_name + ) or parser._short_opt.get(opt_name) if our_parser: self._eat_all_parser = our_parser self._previous_parser_process = our_parser.process @@ -150,15 +163,19 @@ class HelpEpilog(object): examples: List of tuples. Each tuple contains two elements. First element is the example title, and the second element is the list of commands for this example. """ + class Example: """Context manager for helping with formatting epilogues when users invoke help on a command line.""" - def __init__(self, formatter: click.formatting.HelpFormatter, title: str) -> None: + + def __init__( + self, formatter: click.formatting.HelpFormatter, title: str + ) -> None: self.formatter = formatter self.title = title def __enter__(self) -> None: self.formatter.indent() - self.formatter.write_heading('- ' + self.title) + self.formatter.write_heading("- " + self.title) self.formatter.write_paragraph() self.formatter.current_indent += 2 * self.formatter.indent_increment @@ -169,14 +186,16 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None: def __init__(self, examples: t.List[t.Tuple[str, t.List[str]]]) -> None: self.examples = examples - def format_epilog(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None: + def format_epilog( + self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter + ) -> None: if not self.examples: return - formatter.write_heading('\nEXAMPLES') + formatter.write_heading("\nEXAMPLES") for title, commands in self.examples: with HelpEpilog.Example(formatter, title): for cmd in commands: - formatter.write_text('$ ' + cmd) + formatter.write_text("$ " + cmd) class MLCubeCommand(click.Command): @@ -192,6 +211,7 @@ def my_cmd() -> None: ... ``` """ + def format_help_text(self, ctx, formatter): """Writes the help text to the formatter if it exists.""" if self.help: @@ -206,7 +226,9 @@ def format_help_text(self, ctx, formatter): with formatter.indentation(): formatter.write_text(DEPRECATED_HELP_NOTICE) - def format_options(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None: + def format_options( + self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter + ) -> None: """Writes all the options into the formatter if they exist. This implementation removes Markdown format from the options' help messages should they exist. Any errors @@ -222,7 +244,9 @@ def format_options(self, ctx: click.core.Context, formatter: click.formatting.He with formatter.section("Options"): formatter.write_dl(opts) - def format_epilog(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None: + def format_epilog( + self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter + ) -> None: """Format epilog if its type `mlcube.EpilogWithExamples`, else fallback to default implementation.""" if self.epilog: try: @@ -263,167 +287,205 @@ def runner_url(runner: str) -> str: class Options: """Options for various MLCube commands""" - help = click.help_option( - '--help', '-h', - help="Show help message and exit." - ) + help = click.help_option("--help", "-h", help="Show help message and exit.") loglevel = click.option( - '--log-level', '--log_level', required=False, default='warning', - type=click.Choice(['critical', 'error', 'warning', 'info', 'debug']), + "--log-level", + "--log_level", + required=False, + default="warning", + type=click.Choice(["critical", "error", "warning", "info", "debug"]), help="Logging level is a lower-case string value for Python's logging library (see " - "[Logging Levels]({log_level}) for more details). Only messages with this logging level or higher are " - "logged.".format( - log_level="https://docs.python.org/3/library/logging.html#logging-levels" - ) + "[Logging Levels]({log_level}) for more details). Only messages with this logging level or higher are " + "logged.".format( + log_level="https://docs.python.org/3/library/logging.html#logging-levels" + ), ) mlcube = click.option( - '--mlcube', required=False, type=str, default=None, metavar='PATH', + "--mlcube", + required=False, + type=str, + default=None, + metavar="PATH", help="Path to an MLCube project. It can be a [directory path]({mlcube_root_dir}), or a path to an MLCube " - "[configuration file]({mlcube_config}). When it is a directory path, MLCube runtime assumes this " - "directory is the MLCube root directory containing `mlcube.yaml` file. When it is a file path, this file " - "is assumed to be the MLCube configuration file (`mlcube.yaml`), and a parent directory of this file is " - "considered to be the MLCube root directory. Default value is current directory.".format( - mlcube_root_dir=OnlineDocs.concept_url("mlcube-root-directory"), - mlcube_config=OnlineDocs.concept_url("mlcube-configuration") - ) + "[configuration file]({mlcube_config}). When it is a directory path, MLCube runtime assumes this " + "directory is the MLCube root directory containing `mlcube.yaml` file. When it is a file path, this file " + "is assumed to be the MLCube configuration file (`mlcube.yaml`), and a parent directory of this file is " + "considered to be the MLCube root directory. Default value is current directory.".format( + mlcube_root_dir=OnlineDocs.concept_url("mlcube-root-directory"), + mlcube_config=OnlineDocs.concept_url("mlcube-configuration"), + ), ) platform = click.option( - '--platform', required=False, type=str, default='docker', metavar='NAME', + "--platform", + required=False, + type=str, + default="docker", + metavar="NAME", help="[Platform]({platform}) name to run MLCube on (a platform is a configured instance of an MLCube runner). " - "Multiple platforms are supported, including `docker` ([Docker and Podman]({docker})), `singularity` " - "([Singularity]({singularity})). Other runners are in experimental stage: `gcp` " - "([Google Cloud Platform]({gcp})), `k8s` ([Kubernetes]({k8s})), `kubeflow` " - "([KubeFlow]({kubeflow})), ssh ([SSH runner]({ssh})). Default is `docker`. Platforms are defined and " - "configured in MLCube [system settings file]({sys_settings}).".format( - platform=OnlineDocs.concept_url("platform"), - docker=OnlineDocs.runner_url("docker-runner"), - singularity=OnlineDocs.runner_url("singularity-runner"), - gcp=OnlineDocs.runner_url("gcp-runner"), - k8s=OnlineDocs.runner_url("kubernetes"), - kubeflow=OnlineDocs.runner_url("kubeflow"), - ssh=OnlineDocs.runner_url("ssh-runner"), - sys_settings=OnlineDocs.url("getting-started/system-settings/") - ) + "Multiple platforms are supported, including `docker` ([Docker and Podman]({docker})), `singularity` " + "([Singularity]({singularity})). Other runners are in experimental stage: `gcp` " + "([Google Cloud Platform]({gcp})), `k8s` ([Kubernetes]({k8s})), `kubeflow` " + "([KubeFlow]({kubeflow})), ssh ([SSH runner]({ssh})). Default is `docker`. Platforms are defined and " + "configured in MLCube [system settings file]({sys_settings}).".format( + platform=OnlineDocs.concept_url("platform"), + docker=OnlineDocs.runner_url("docker-runner"), + singularity=OnlineDocs.runner_url("singularity-runner"), + gcp=OnlineDocs.runner_url("gcp-runner"), + k8s=OnlineDocs.runner_url("kubernetes"), + kubeflow=OnlineDocs.runner_url("kubeflow"), + ssh=OnlineDocs.runner_url("ssh-runner"), + sys_settings=OnlineDocs.url("getting-started/system-settings/"), + ), ) task = click.option( - '--task', required=False, type=str, default=None, + "--task", + required=False, + type=str, + default=None, help="MLCube [task]({task}) name(s) to run, default is `main`. This parameter can take a list of values, in " - "which case task names are separated with comma (,).".format( - task=OnlineDocs.concept_url('task') - ) + "which case task names are separated with comma (,).".format( + task=OnlineDocs.concept_url("task") + ), ) workspace = click.option( - '--workspace', required=False, type=str, default=None, metavar='PATH', + "--workspace", + required=False, + type=str, + default=None, + metavar="PATH", help="Location of a [workspace]({workspace}) to store input and output artifacts of MLCube [tasks]({task}). " - "If not specified (None), `${{MLCUBE_ROOT}}/workspace/` is used.".format( - workspace=OnlineDocs.concept_url("workspace"), - task=OnlineDocs.concept_url("task") - ) + "If not specified (None), `${{MLCUBE_ROOT}}/workspace/` is used.".format( + workspace=OnlineDocs.concept_url("workspace"), + task=OnlineDocs.concept_url("task"), + ), ) parameter = click.option( - '-P', '-p', required=False, type=str, default=None, metavar='PARAMS', multiple=True, + "-P", + "-p", + required=False, + type=str, + default=None, + metavar="PARAMS", + multiple=True, help="MLCube [configuration parameter]({config_param}) is a key-value pair. Must start with `-P` or '-p'. The " - "dot (.) is used to refer to nested parameters, for instance, `-Pdocker.build_strategy=always`. These " - "parameters have the highest priority and override any other parameters in " - "[system settings]({sys_settings}) and [MLCube configuration]({config}). ".format( - config_param=OnlineDocs.concept_url("mlcube-configuration-parameter"), - sys_settings=OnlineDocs.concept_url("system-settings"), - config=OnlineDocs.concept_url("mlcube-configuration") - ) + "dot (.) is used to refer to nested parameters, for instance, `-Pdocker.build_strategy=always`. These " + "parameters have the highest priority and override any other parameters in " + "[system settings]({sys_settings}) and [MLCube configuration]({config}). ".format( + config_param=OnlineDocs.concept_url("mlcube-configuration-parameter"), + sys_settings=OnlineDocs.concept_url("system-settings"), + config=OnlineDocs.concept_url("mlcube-configuration"), + ), ) resolve = click.option( - '--resolve', is_flag=True, + "--resolve", + is_flag=True, help="Resolve [MLCube parameters]({config_param}). The `mlcube` uses [OmegaConf]({omega_conf}) library to " - "manage its configuration, including [configuration files]({config}), [system settings]({sys_settings}) " - "files and configuration parameters provided by users on command lines. OmegaConf supports variable " - "interpolation (when one variables depend on other variables, e.g., `{{'docker.image': " - "'mlcommons/{{name}}:${{version}}'}}`). When this flag is set to true, the `mlcube` computes actual " - "values of all variables.".format( - config_param=OnlineDocs.concept_url("mlcube-configuration-parameter"), - omega_conf="https://omegaconf.readthedocs.io/", - config=OnlineDocs.concept_url("mlcube-configuration"), - sys_settings=OnlineDocs.concept_url("system-settings") - ) + "manage its configuration, including [configuration files]({config}), [system settings]({sys_settings}) " + "files and configuration parameters provided by users on command lines. OmegaConf supports variable " + "interpolation (when one variables depend on other variables, e.g., `{{'docker.image': " + "'mlcommons/{{name}}:${{version}}'}}`). When this flag is set to true, the `mlcube` computes actual " + "values of all variables.".format( + config_param=OnlineDocs.concept_url("mlcube-configuration-parameter"), + omega_conf="https://omegaconf.readthedocs.io/", + config=OnlineDocs.concept_url("mlcube-configuration"), + sys_settings=OnlineDocs.concept_url("system-settings"), + ), ) def _mnist(steps: t.List[str]) -> t.List[str]: return [ - 'git clone https://github.com/mlcommons/mlcube_examples', - 'cd ./mlcube_examples', - ] + steps + "git clone https://github.com/mlcommons/mlcube_examples", + "cd ./mlcube_examples", + ] + steps class UsageExamples: - show_config = HelpEpilog([ - ( - 'Show effective MLCube configuration', - _mnist(['mlcube show_config --mlcube=mnist']) - ), - ( - 'Show effective MLCube configuration overriding parameters on a command line', - _mnist(['mlcube show_config --mlcube=mnist -Pdocker.build_strategy=auto']) - ) - ]) + show_config = HelpEpilog( + [ + ( + "Show effective MLCube configuration", + _mnist(["mlcube show_config --mlcube=mnist"]), + ), + ( + "Show effective MLCube configuration overriding parameters on a command line", + _mnist( + ["mlcube show_config --mlcube=mnist -Pdocker.build_strategy=auto"] + ), + ), + ] + ) """Usage examples for `mlcube show_config` command.""" - configure = HelpEpilog([ - ( - 'Configure MNIST MLCube project', - _mnist(['mlcube configure --mlcube=mnist --platform=docker']) - ) - ]) + configure = HelpEpilog( + [ + ( + "Configure MNIST MLCube project", + _mnist(["mlcube configure --mlcube=mnist --platform=docker"]), + ) + ] + ) """Usage examples for `mlcube configure` command.""" - run = HelpEpilog([ - ( - 'Run MNIST MLCube project', - _mnist(['mlcube run --mlcube=mnist --platform=docker --task=download,train']) - ) - ]) + run = HelpEpilog( + [ + ( + "Run MNIST MLCube project", + _mnist( + [ + "mlcube run --mlcube=mnist --platform=docker --task=download,train" + ] + ), + ) + ] + ) """Usage examples for `mlcube run` command.""" - describe = HelpEpilog([ - ( - 'Run MNIST MLCube project', - _mnist(['mlcube describe --mlcube=mnist']) - ) - ]) + describe = HelpEpilog( + [("Run MNIST MLCube project", _mnist(["mlcube describe --mlcube=mnist"]))] + ) """Usage examples for `mlcube describe` command.""" - create = HelpEpilog([ - ( - 'Create a new empty MLCube project', - ['mlcube create'] - ) - ]) + create = HelpEpilog([("Create a new empty MLCube project", ["mlcube create"])]) """Usage examples for `mlcube create` command.""" - config = HelpEpilog([ - ( - 'Print the content of MLCube system settings file', - ['mlcube config --list'] - ), - ( - 'Get default environmental variables for mlcube run command with docker platform', - ['mlcube config --get platforms.docker.env_args'] - ), - ( - 'Create, rename and remove a custom docker platform by copying existing configuration', - [ - 'mlcube config --create-platform docker docker_v01', - 'mlcube config --get platforms.docker_v01', - 'mlcube config --rename-platform docker_v01 docker_v02', - 'mlcube config --get platforms.docker_v02', - 'mlcube config --remove-platform docker_v02' - ] - ) - ]) + config = HelpEpilog( + [ + ( + "Print the content of MLCube system settings file", + ["mlcube config --list"], + ), + ( + "Get default environmental variables for mlcube run command with docker platform", + ["mlcube config --get platforms.docker.env_args"], + ), + ( + "Create, rename and remove a custom docker platform by copying existing configuration", + [ + "mlcube config --create-platform docker docker_v01", + "mlcube config --get platforms.docker_v01", + "mlcube config --rename-platform docker_v01 docker_v02", + "mlcube config --get platforms.docker_v02", + "mlcube config --remove-platform docker_v02", + ], + ), + ] + ) """Usage examples for `mlcube config` command.""" + + inspect = HelpEpilog( + [ + ( + "Return low-level information on MLCube objects", + _mnist(["mlcube inspect --mlcube=mnist --platform=docker"]), + ) + ] + ) + """Usage examples for `mlcube inspect` command.""" diff --git a/mlcube/mlcube/runner.py b/mlcube/mlcube/runner.py index f0bc4ab..4a0f1d9 100644 --- a/mlcube/mlcube/runner.py +++ b/mlcube/mlcube/runner.py @@ -6,12 +6,11 @@ import logging import typing as t -from mlcube.errors import ConfigurationError +from omegaconf import DictConfig, OmegaConf -from omegaconf import (DictConfig, OmegaConf) +from mlcube.errors import ConfigurationError, MLCubeError - -__all__ = ['RunnerConfig', 'Runner'] +__all__ = ["RunnerConfig", "Runner"] logger = logging.getLogger(__name__) @@ -54,7 +53,7 @@ def validate(mlcube: DictConfig) -> None: ... -RunnerConfigType = t.TypeVar('RunnerConfigType', bound='RunnerConfig') +RunnerConfigType = t.TypeVar("RunnerConfigType", bound="RunnerConfig") class Runner(object): @@ -62,7 +61,9 @@ class Runner(object): CONFIG: RunnerConfigType = RunnerConfig - def __init__(self, mlcube: t.Union[DictConfig, t.Dict], task: t.Optional[str]) -> None: + def __init__( + self, mlcube: t.Union[DictConfig, t.Dict], task: t.Optional[str] + ) -> None: """Initialize the base runner. Args: @@ -72,12 +73,18 @@ def __init__(self, mlcube: t.Union[DictConfig, t.Dict], task: t.Optional[str]) - if isinstance(mlcube, dict): mlcube: DictConfig = OmegaConf.create(mlcube) if not isinstance(mlcube, DictConfig): - raise ConfigurationError(f"Invalid mlcube type ('{type(DictConfig)}'). Expecting 'DictConfig'.") + raise ConfigurationError( + f"Invalid mlcube type ('{type(DictConfig)}'). Expecting 'DictConfig'." + ) self.mlcube = mlcube self.task = task - logger.debug("%s.__init__ configuration: %s", self.__class__.__name__, str(self.mlcube.runner)) + logger.debug( + "%s.__init__ configuration: %s", + self.__class__.__name__, + str(self.mlcube.runner), + ) def configure(self) -> None: """Configure this MLCube.""" @@ -86,3 +93,14 @@ def configure(self) -> None: def run(self) -> None: """Run one MLCube task.""" ... + + def inspect(self, force: bool = False) -> t.Dict: + """Return low-level information about MLCube objects. + + Args: + force: If true, and MLCube does not exist (e.g., has not been pulled or built yet), then pull/build it. + """ + raise MLCubeError( + f"The `inspect` command has not been implemented yet (runner={self.mlcube['runner']['runner']}, " + f"cls={self.__class__.__name__})." + )