From ad03377e6606af161f62d5558e3c65ae9df22901 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Sun, 14 Jan 2024 19:54:36 +0100
Subject: [PATCH 001/225] Add `pydantic-argparse` dependency

---
 requirements/requirements-all.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index a811c95dd..3b8fa208c 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -11,3 +11,4 @@ pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2
 yacman>=0.9.2
+pydantic-argparse==0.8.0

From 1b08809c59511f19995555de5507383d0435bf87 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 10 Jan 2024 16:17:51 +0100
Subject: [PATCH 002/225] Add pydantic model scaffold for top-level parser

---
 looper/cli_pydantic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 looper/cli_pydantic.py

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
new file mode 100644
index 000000000..6ddf7b3e6
--- /dev/null
+++ b/looper/cli_pydantic.py
@@ -0,0 +1,13 @@
+import pydantic
+
+class TopLevelParser(pydantic.BaseModel):
+    """
+    Top level parser that takes
+    - commands (run, runp, check...)
+    - arguments that are required no matter the subcommand
+    """
+    # commands
+    ...
+
+    # top-level arguments
+    ...

From bbf3c75db62062511fa6b3c32ea85eb55beb136a Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 11 Jan 2024 10:42:14 +0100
Subject: [PATCH 003/225] Add entrypoint

---
 looper/cli_pydantic.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 6ddf7b3e6..8928c5e18 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,4 +1,5 @@
 import pydantic
+import pydantic_argparse
 
 class TopLevelParser(pydantic.BaseModel):
     """
@@ -11,3 +12,14 @@ class TopLevelParser(pydantic.BaseModel):
 
     # top-level arguments
     ...
+
+
+if __name__ == "__main__":
+    parser = pydantic_argparse.ArgumentParser(
+        model=TopLevelParser,
+        prog="looper",
+        description="pydantic-argparse demo",
+        add_help=True,
+    )
+    args = parser.parse_typed_args()
+    print(args)

From d2ac1845e8c7eac79adf83ea283124af4f6e78ef Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 11 Jan 2024 10:47:01 +0100
Subject: [PATCH 004/225] Add scaffold for dynamic command model generation

---
 looper/cli_pydantic.py | 65 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 8928c5e18..8de65de4f 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,6 +1,69 @@
+from dataclasses import dataclass
+import enum
+from typing import Optional, TypeAlias
+
 import pydantic
 import pydantic_argparse
 
+from const import MESSAGE_BY_SUBCOMMAND
+
+AllowedArgumentType: TypeAlias = str | int | bool | list
+
+
+class Command(enum.Enum):
+    """
+    Lists all supported commands
+    """
+    RUN = enum.auto()
+
+@dataclass
+class Argument:
+    """
+    CLI argument / flag definition
+    """
+    # argument name, e.g. "ignore-args"
+    name: str
+    # argument type, e.g. `bool`
+    type: type[AllowedArgumentType]
+    # argument description (will be the CLI help text)
+    description: str
+    # default value for argument (needs to be an instance of `type`)
+    # TODO: how can we constrain the type of this to be an instance of
+    # the value of the `type` field?
+    default: AllowedArgumentType
+    # set of commands this argument is used by
+    used_by: set[Command]
+
+arguments = [
+    Argument(
+        "ignore-flags",
+        bool,
+        "Ignore run status flags",
+        False,
+        {Command.RUN}
+    ),
+    Argument(
+        "time-delay",
+        int,
+        "Time delay in seconds between job submissions (min: 0, max: 30)",
+        0,
+        {Command.RUN}
+    )
+]
+
+def create_model_from_arguments(name: str, command: Command, arguments: list[Argument]) -> type[pydantic.BaseModel]:
+    """
+    Creates a `pydantic` model for a command from a list of arguments
+    """
+    return pydantic.create_model(
+    name, **{
+        arg.name: (arg.type, pydantic.Field(description=arg.description, default=arg.default))
+        for arg in arguments if command in arg.used_by
+    }
+)
+
+RunParser = create_model_from_arguments("RunParser", Command.RUN, arguments)
+
 class TopLevelParser(pydantic.BaseModel):
     """
     Top level parser that takes
@@ -8,7 +71,7 @@ class TopLevelParser(pydantic.BaseModel):
     - arguments that are required no matter the subcommand
     """
     # commands
-    ...
+    run: Optional[RunParser] = pydantic.Field(description=MESSAGE_BY_SUBCOMMAND["run"])
 
     # top-level arguments
     ...

From 44fa244e6b7e0aa2cb6f3d1c0b49ee03ac711485 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 11 Jan 2024 11:53:00 +0100
Subject: [PATCH 005/225] Validate type of `default` field

---
 looper/cli_pydantic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 8de65de4f..c030d534f 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -9,7 +9,6 @@
 
 AllowedArgumentType: TypeAlias = str | int | bool | list
 
-
 class Command(enum.Enum):
     """
     Lists all supported commands
@@ -34,6 +33,13 @@ class Argument:
     # set of commands this argument is used by
     used_by: set[Command]
 
+    def __post_init__(self):
+        if not isinstance(self.default, self.type):
+            raise TypeError(
+                "Value for `default` needs to be of the type given in "
+                f"the `type` field ({self.type})"
+                )
+
 arguments = [
     Argument(
         "ignore-flags",

From 7b9273a9b1aeadd8aecbca2a3e0fb48150ad039d Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 11 Jan 2024 15:53:42 +0100
Subject: [PATCH 006/225] Allow `None` as value next to allowed argument types

---
 looper/cli_pydantic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index c030d534f..0dbfae330 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -29,12 +29,12 @@ class Argument:
     # default value for argument (needs to be an instance of `type`)
     # TODO: how can we constrain the type of this to be an instance of
     # the value of the `type` field?
-    default: AllowedArgumentType
+    default: None | AllowedArgumentType
     # set of commands this argument is used by
     used_by: set[Command]
 
     def __post_init__(self):
-        if not isinstance(self.default, self.type):
+        if self.default is not None and not isinstance(self.default, self.type):
             raise TypeError(
                 "Value for `default` needs to be of the type given in "
                 f"the `type` field ({self.type})"

From 2333870e7348e84f40e48e3e91f7d2bf8d9dc403 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 11 Jan 2024 15:53:59 +0100
Subject: [PATCH 007/225] [WIP] Add missing arguments for hello_looper to run

---
 looper/cli_pydantic.py | 73 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 0dbfae330..acffc2653 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 import enum
+import os
 from typing import Optional, TypeAlias
 
 import pydantic
@@ -54,6 +55,55 @@ def __post_init__(self):
         "Time delay in seconds between job submissions (min: 0, max: 30)",
         0,
         {Command.RUN}
+    ),
+    Argument(
+        "dry-run",
+        bool,
+        "Don't actually submit jobs",
+        False,
+        {Command.RUN}
+    ),
+    Argument(
+        "command-extra",
+        str,
+        "String to append to every command",
+        "",
+        {Command.RUN}
+    ),
+    Argument(
+        "command-extra-override",
+        str,
+        "Same as command-extra, but overrides values in PEP",
+        "",
+        {Command.RUN}
+    ),
+    Argument(
+        "lump",
+        int,
+        "Total input file size (GB) to batch into one job",
+        None,
+        {Command.RUN}
+    ),
+    Argument(
+        "lumpn",
+        int,
+        "Number of commands to batch into one job",
+        None,
+        {Command.RUN}
+    ),
+    Argument(
+        "limit",
+        int,
+        "Limit to n samples",
+        None,
+        {Command.RUN}
+    ),
+    Argument(
+        "skip",
+        int,
+        "Skip samples by numerical index",
+        None,
+        {Command.RUN}
     )
 ]
 
@@ -80,7 +130,28 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParser] = pydantic.Field(description=MESSAGE_BY_SUBCOMMAND["run"])
 
     # top-level arguments
-    ...
+    config_file: Optional[str] = pydantic.Field(
+        description="Project configuration file"
+    )
+    pep_config: Optional[str] = pydantic.Field(description="PEP configuration file")
+    output_dir: Optional[str] = pydantic.Field(description="Output directory")
+    sample_pipeline_interfaces: Optional[str] = pydantic.Field(
+        description="Sample pipeline interfaces definition"
+    )
+    project_pipeline_interfaces: Optional[str] = pydantic.Field(
+        description="Project pipeline interfaces definition"
+    )
+    amend: Optional[bool] = pydantic.Field(description="List of amendments to activate")
+    sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
+    exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
+    divvy: Optional[str] = pydantic.Field(
+        description=(
+            "Path to divvy configuration file. Default=$DIVCFG env "
+            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set")
+        )
+    )
+
+
 
 
 if __name__ == "__main__":

From 2ecef808add0c4765b7a9a2266ae4680a0693f10 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Sun, 14 Jan 2024 19:37:59 +0100
Subject: [PATCH 008/225] Add `cli_pydantic.py` script to setuptools scripts

---
 looper/cli_pydantic.py | 5 ++++-
 setup.py               | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index acffc2653..7e9fff5f1 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -154,7 +154,7 @@ class TopLevelParser(pydantic.BaseModel):
 
 
 
-if __name__ == "__main__":
+def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
@@ -163,3 +163,6 @@ class TopLevelParser(pydantic.BaseModel):
     )
     args = parser.parse_typed_args()
     print(args)
+
+if __name__ == "__main__":
+    main()
diff --git a/setup.py b/setup.py
index a1150555b..e6d940868 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,7 @@ def get_static(name, condition=None):
         "console_scripts": [
             "looper = looper.__main__:main",
             "divvy = looper.__main__:divvy_main",
+            "looper-pydantic-argparse = looper.cli_pydantic:main"
         ],
     },
     scripts=scripts,

From bed82923e3fb596a5927049b242ecafac77e3418 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Sun, 14 Jan 2024 19:57:27 +0100
Subject: [PATCH 009/225] Rework design

- make arguments inherit from `pydantic.fields.FieldInfo` and augment
them with a `name` attribute
- make a `Command` class that generates a `pydantic` model with the
arguments the command takes
- make commands take lists of specific arguments they take instead of
having arguments know by which commands they are supported
---
 looper/cli_pydantic.py | 223 ++++++++++++++++++++---------------------
 1 file changed, 107 insertions(+), 116 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7e9fff5f1..42affb831 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,124 +1,139 @@
 from dataclasses import dataclass
 import enum
-import os
-from typing import Optional, TypeAlias
+from typing import Any, Optional
 
 import pydantic
 import pydantic_argparse
 
-from const import MESSAGE_BY_SUBCOMMAND
+from .const import MESSAGE_BY_SUBCOMMAND
 
-AllowedArgumentType: TypeAlias = str | int | bool | list
 
-class Command(enum.Enum):
+class Argument(pydantic.fields.FieldInfo):
     """
-    Lists all supported commands
+    CLI argument / flag definition
+
+    Naively, one would think one could just subclass `pydantic.Field`,
+    but actually `pydantic.Field` is a function, and not a class.
+    `pydantic.Field()` returns a validated `FieldInfo` instance,
+    so we instead sublcass `FieldInfo` directly and validate it in the
+    constructor.
+
+    :param str name: argument name, e.g. "ignore-args"
+    :param str description: argument description, which will appear as the
+        help text for this argument
+    :param Any default: a tuple of the form (type, default_value). If the
+        default value is `...` (Ellipsis), then the argument is required.
     """
-    RUN = enum.auto()
+
+    def __init__(self, name: str, default: Any, description: str) -> None:
+        self._name = name
+        super().__init__(default=default, description=description)
+        self._validate()
+
+    @property
+    def name(self):
+        """
+        Argument name as used in the CLI, e.g. "ignore-args"
+        """
+        return self._name
 
 @dataclass
-class Argument:
+class Command:
     """
-    CLI argument / flag definition
+    Representation of a command
+
+    :param str name: command name
+    :param str description: command description
+    :param list[Argument] arguments: list of arguments supported by this command
     """
-    # argument name, e.g. "ignore-args"
     name: str
-    # argument type, e.g. `bool`
-    type: type[AllowedArgumentType]
-    # argument description (will be the CLI help text)
     description: str
-    # default value for argument (needs to be an instance of `type`)
-    # TODO: how can we constrain the type of this to be an instance of
-    # the value of the `type` field?
-    default: None | AllowedArgumentType
-    # set of commands this argument is used by
-    used_by: set[Command]
-
-    def __post_init__(self):
-        if self.default is not None and not isinstance(self.default, self.type):
-            raise TypeError(
-                "Value for `default` needs to be of the type given in "
-                f"the `type` field ({self.type})"
-                )
-
-arguments = [
-    Argument(
+    arguments: list[Argument]
+
+    def create_model(self) -> type[pydantic.BaseModel]:
+        """
+        Creates a `pydantic` model for this command
+        """
+        arguments = dict()
+        for arg in self.arguments:
+            # These gymnastics are necessary because of
+            # https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
+            arg_type, arg_default_value = arg.default
+            arguments[arg.name] = (
+                arg_type,
+                pydantic.Field(arg_default_value, description=arg.description)
+            )
+        return pydantic.create_model(
+            self.name, **arguments
+        )
+
+class ArgumentEnum(enum.Enum):
+    """
+    Lists all available arguments
+
+    TODO: not sure whether an enum is the ideal data structure for that
+    """
+    IGNORE_FLAGS = Argument(
         "ignore-flags",
-        bool,
+        (bool, False),
         "Ignore run status flags",
-        False,
-        {Command.RUN}
-    ),
-    Argument(
+    )
+    TIME_DELAY = Argument(
         "time-delay",
-        int,
+        (int, 0),
         "Time delay in seconds between job submissions (min: 0, max: 30)",
-        0,
-        {Command.RUN}
-    ),
-    Argument(
+    )
+    DRY_RUN = Argument(
         "dry-run",
-        bool,
-        "Don't actually submit jobs",
-        False,
-        {Command.RUN}
-    ),
-    Argument(
+        (bool, False),
+        "Don't actually submit jobs"
+    )
+    COMMAND_EXTRA = Argument(
         "command-extra",
-        str,
-        "String to append to every command",
-        "",
-        {Command.RUN}
-    ),
-    Argument(
+        (str, ""),
+        "String to append to every command"
+    )
+    COMMAND_EXTRA_OVERRIDE = Argument(
         "command-extra-override",
-        str,
-        "Same as command-extra, but overrides values in PEP",
-        "",
-        {Command.RUN}
-    ),
-    Argument(
+        (str, ""),
+        "Same as command-extra, but overrides values in PEP"
+    )
+    LUMP = Argument(
         "lump",
-        int,
-        "Total input file size (GB) to batch into one job",
-        None,
-        {Command.RUN}
-    ),
-    Argument(
+        (int, None),
+        "Total input file size (GB) to batch into one job"
+    )
+    LUMPN = Argument(
         "lumpn",
-        int,
-        "Number of commands to batch into one job",
-        None,
-        {Command.RUN}
-    ),
-    Argument(
+        (int, None),
+        "Number of commands to batch into one job"
+    )
+    LIMIT = Argument(
         "limit",
-        int,
-        "Limit to n samples",
-        None,
-        {Command.RUN}
-    ),
-    Argument(
+        (int, None),
+        "Limit to n samples"
+    )
+    SKIP = Argument(
         "skip",
-        int,
-        "Skip samples by numerical index",
-        None,
-        {Command.RUN}
+        (int, None),
+        "Skip samples by numerical index"
     )
-]
 
-def create_model_from_arguments(name: str, command: Command, arguments: list[Argument]) -> type[pydantic.BaseModel]:
-    """
-    Creates a `pydantic` model for a command from a list of arguments
-    """
-    return pydantic.create_model(
-    name, **{
-        arg.name: (arg.type, pydantic.Field(description=arg.description, default=arg.default))
-        for arg in arguments if command in arg.used_by
-    }
+RunParser = Command(
+    "run", MESSAGE_BY_SUBCOMMAND["run"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value
+    ]
 )
-
-RunParser = create_model_from_arguments("RunParser", Command.RUN, arguments)
+RunParserModel = RunParser.create_model()
 
 class TopLevelParser(pydantic.BaseModel):
     """
@@ -127,31 +142,7 @@ class TopLevelParser(pydantic.BaseModel):
     - arguments that are required no matter the subcommand
     """
     # commands
-    run: Optional[RunParser] = pydantic.Field(description=MESSAGE_BY_SUBCOMMAND["run"])
-
-    # top-level arguments
-    config_file: Optional[str] = pydantic.Field(
-        description="Project configuration file"
-    )
-    pep_config: Optional[str] = pydantic.Field(description="PEP configuration file")
-    output_dir: Optional[str] = pydantic.Field(description="Output directory")
-    sample_pipeline_interfaces: Optional[str] = pydantic.Field(
-        description="Sample pipeline interfaces definition"
-    )
-    project_pipeline_interfaces: Optional[str] = pydantic.Field(
-        description="Project pipeline interfaces definition"
-    )
-    amend: Optional[bool] = pydantic.Field(description="List of amendments to activate")
-    sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
-    exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
-    divvy: Optional[str] = pydantic.Field(
-        description=(
-            "Path to divvy configuration file. Default=$DIVCFG env "
-            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set")
-        )
-    )
-
-
+    run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
 
 def main() -> None:

From dfb07cf3a6a1de26cf23cda66299374fb033d327 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Mon, 15 Jan 2024 11:33:55 +0100
Subject: [PATCH 010/225] Fix typo

Co-authored-by: Zhihan Zhang <32028117+zz1874@users.noreply.github.com>
---
 looper/cli_pydantic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 42affb831..083e1e273 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -15,7 +15,7 @@ class Argument(pydantic.fields.FieldInfo):
     Naively, one would think one could just subclass `pydantic.Field`,
     but actually `pydantic.Field` is a function, and not a class.
     `pydantic.Field()` returns a validated `FieldInfo` instance,
-    so we instead sublcass `FieldInfo` directly and validate it in the
+    so we instead subclass `FieldInfo` directly and validate it in the
     constructor.
 
     :param str name: argument name, e.g. "ignore-args"

From bc5c78fd307543727b7eaa21a657bd7b33e26fd4 Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Tue, 16 Jan 2024 12:55:36 +0800
Subject: [PATCH 011/225] Use **kwargs in Argument and ArgumentEnum

---
 looper/cli_pydantic.py | 83 +++++++++++++++++++++---------------------
 1 file changed, 41 insertions(+), 42 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 083e1e273..a2108db6b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,6 +1,6 @@
-from dataclasses import dataclass
 import enum
-from typing import Any, Optional
+from dataclasses import dataclass
+from typing import Optional
 
 import pydantic
 import pydantic_argparse
@@ -19,15 +19,13 @@ class Argument(pydantic.fields.FieldInfo):
     constructor.
 
     :param str name: argument name, e.g. "ignore-args"
-    :param str description: argument description, which will appear as the
-        help text for this argument
-    :param Any default: a tuple of the form (type, default_value). If the
-        default value is `...` (Ellipsis), then the argument is required.
+    :param dict kwargs: additional keyword arguments supported by
+        `FieldInfo`, such as description, default value, etc.
     """
 
-    def __init__(self, name: str, default: Any, description: str) -> None:
+    def __init__(self, name, **kwargs) -> None:
         self._name = name
-        super().__init__(default=default, description=description)
+        super().__init__(**kwargs)
         self._validate()
 
     @property
@@ -37,6 +35,7 @@ def name(self):
         """
         return self._name
 
+
 @dataclass
 class Command:
     """
@@ -46,6 +45,7 @@ class Command:
     :param str description: command description
     :param list[Argument] arguments: list of arguments supported by this command
     """
+
     name: str
     description: str
     arguments: list[Argument]
@@ -61,11 +61,10 @@ def create_model(self) -> type[pydantic.BaseModel]:
             arg_type, arg_default_value = arg.default
             arguments[arg.name] = (
                 arg_type,
-                pydantic.Field(arg_default_value, description=arg.description)
+                pydantic.Field(arg_default_value, description=arg.description),
             )
-        return pydantic.create_model(
-            self.name, **arguments
-        )
+        return pydantic.create_model(self.name, **arguments)
+
 
 class ArgumentEnum(enum.Enum):
     """
@@ -73,54 +72,51 @@ class ArgumentEnum(enum.Enum):
 
     TODO: not sure whether an enum is the ideal data structure for that
     """
+
     IGNORE_FLAGS = Argument(
-        "ignore-flags",
-        (bool, False),
-        "Ignore run status flags",
+        name="ignore-flags",
+        default=(bool, False),
+        description="Ignore run status flags",
     )
     TIME_DELAY = Argument(
-        "time-delay",
-        (int, 0),
-        "Time delay in seconds between job submissions (min: 0, max: 30)",
+        name="time-delay",
+        default=(int, 0),
+        description="Time delay in seconds between job submissions (min: 0, max: 30)",
     )
     DRY_RUN = Argument(
-        "dry-run",
-        (bool, False),
-        "Don't actually submit jobs"
+        name="dry-run", default=(bool, False), description="Don't actually submit jobs"
     )
     COMMAND_EXTRA = Argument(
-        "command-extra",
-        (str, ""),
-        "String to append to every command"
+        name="command-extra",
+        default=(str, ""),
+        description="String to append to every command",
     )
     COMMAND_EXTRA_OVERRIDE = Argument(
-        "command-extra-override",
-        (str, ""),
-        "Same as command-extra, but overrides values in PEP"
+        name="command-extra-override",
+        default=(str, ""),
+        description="Same as command-extra, but overrides values in PEP",
     )
     LUMP = Argument(
-        "lump",
-        (int, None),
-        "Total input file size (GB) to batch into one job"
+        name="lump",
+        default=(int, None),
+        description="Total input file size (GB) to batch into one job",
     )
     LUMPN = Argument(
-        "lumpn",
-        (int, None),
-        "Number of commands to batch into one job"
+        name="lumpn",
+        default=(int, None),
+        description="Number of commands to batch into one job",
     )
     LIMIT = Argument(
-        "limit",
-        (int, None),
-        "Limit to n samples"
+        name="limit", default=(int, None), description="Limit to n samples"
     )
     SKIP = Argument(
-        "skip",
-        (int, None),
-        "Skip samples by numerical index"
+        name="skip", default=(int, None), description="Skip samples by numerical index"
     )
 
+
 RunParser = Command(
-    "run", MESSAGE_BY_SUBCOMMAND["run"],
+    "run",
+    MESSAGE_BY_SUBCOMMAND["run"],
     [
         ArgumentEnum.IGNORE_FLAGS.value,
         ArgumentEnum.TIME_DELAY.value,
@@ -130,17 +126,19 @@ class ArgumentEnum(enum.Enum):
         ArgumentEnum.LUMP.value,
         ArgumentEnum.LUMPN.value,
         ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value
-    ]
+        ArgumentEnum.SKIP.value,
+    ],
 )
 RunParserModel = RunParser.create_model()
 
+
 class TopLevelParser(pydantic.BaseModel):
     """
     Top level parser that takes
     - commands (run, runp, check...)
     - arguments that are required no matter the subcommand
     """
+
     # commands
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
@@ -155,5 +153,6 @@ def main() -> None:
     args = parser.parse_typed_args()
     print(args)
 
+
 if __name__ == "__main__":
     main()

From c4a20d9038a06d53f09e6d1e660db7a29422eae2 Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <32028117+zz1874@users.noreply.github.com>
Date: Tue, 16 Jan 2024 15:39:11 +0800
Subject: [PATCH 012/225] Update looper/cli_pydantic.py

Co-authored-by: Nathan Sheffield <nsheff@users.noreply.github.com>
---
 looper/cli_pydantic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index a2108db6b..6b3c6a48e 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -69,6 +69,8 @@ def create_model(self) -> type[pydantic.BaseModel]:
 class ArgumentEnum(enum.Enum):
     """
     Lists all available arguments
+    
+    Having a single "repository" of arguments allows us to re-use them easily across different commands.
 
     TODO: not sure whether an enum is the ideal data structure for that
     """

From c45deb9b0022815dcb27e4839f58f7b2bfb229d1 Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Tue, 16 Jan 2024 15:43:22 +0800
Subject: [PATCH 013/225] Polish docstring of class Argument

---
 looper/cli_pydantic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 6b3c6a48e..7c2342531 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -12,6 +12,10 @@ class Argument(pydantic.fields.FieldInfo):
     """
     CLI argument / flag definition
 
+    This class is designed to define CLI arguments or flags. It leverages
+    Pydantic for data validation and serves as a source of truth for multiple
+    interfaces, including a CLI.
+
     Naively, one would think one could just subclass `pydantic.Field`,
     but actually `pydantic.Field` is a function, and not a class.
     `pydantic.Field()` returns a validated `FieldInfo` instance,

From 82f84d884a11ba30aec8ca77258c8e83881c5c4f Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 10:13:38 +0100
Subject: [PATCH 014/225] Explicitly list `default` and `description` as args
 for Argument

Co-authored-by: Zhihan Zhang <zhihan.zhang@tweag.io>
---
 looper/cli_pydantic.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7c2342531..943cf23f1 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,6 +1,6 @@
 import enum
 from dataclasses import dataclass
-from typing import Optional
+from typing import Any, Optional
 
 import pydantic
 import pydantic_argparse
@@ -23,13 +23,17 @@ class Argument(pydantic.fields.FieldInfo):
     constructor.
 
     :param str name: argument name, e.g. "ignore-args"
+    :param Any default: a tuple of the form (type, default_value). If the
+        default value is `...` (Ellipsis), then the argument is required.
+    :param str description: argument description, which will appear as the
+        help text for this argument
     :param dict kwargs: additional keyword arguments supported by
-        `FieldInfo`, such as description, default value, etc.
+        `FieldInfo`. These are passed along as they are.
     """
 
-    def __init__(self, name, **kwargs) -> None:
+    def __init__(self, name: str, default: Any, description: str, **kwargs) -> None:
         self._name = name
-        super().__init__(**kwargs)
+        super().__init__(default=default, description=description, **kwargs)
         self._validate()
 
     @property
@@ -73,7 +77,7 @@ def create_model(self) -> type[pydantic.BaseModel]:
 class ArgumentEnum(enum.Enum):
     """
     Lists all available arguments
-    
+
     Having a single "repository" of arguments allows us to re-use them easily across different commands.
 
     TODO: not sure whether an enum is the ideal data structure for that

From 1e6620f0119468e2fbffd95144a37aea99623be8 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 10:45:59 +0100
Subject: [PATCH 015/225] Refactor into separate module

Co-authored-by: Zhihan Zhang <zhihan.zhang@tweag.io>
---
 looper/cli_pydantic.py             | 152 +----------------------------
 looper/command_models/README.md    |   4 +
 looper/command_models/__init__.py  |   6 ++
 looper/command_models/arguments.py |  90 +++++++++++++++++
 looper/command_models/commands.py  |  65 ++++++++++++
 5 files changed, 166 insertions(+), 151 deletions(-)
 create mode 100644 looper/command_models/README.md
 create mode 100644 looper/command_models/__init__.py
 create mode 100644 looper/command_models/arguments.py
 create mode 100644 looper/command_models/commands.py

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 943cf23f1..7fd6162b2 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,156 +1,6 @@
-import enum
-from dataclasses import dataclass
-from typing import Any, Optional
-
-import pydantic
 import pydantic_argparse
 
-from .const import MESSAGE_BY_SUBCOMMAND
-
-
-class Argument(pydantic.fields.FieldInfo):
-    """
-    CLI argument / flag definition
-
-    This class is designed to define CLI arguments or flags. It leverages
-    Pydantic for data validation and serves as a source of truth for multiple
-    interfaces, including a CLI.
-
-    Naively, one would think one could just subclass `pydantic.Field`,
-    but actually `pydantic.Field` is a function, and not a class.
-    `pydantic.Field()` returns a validated `FieldInfo` instance,
-    so we instead subclass `FieldInfo` directly and validate it in the
-    constructor.
-
-    :param str name: argument name, e.g. "ignore-args"
-    :param Any default: a tuple of the form (type, default_value). If the
-        default value is `...` (Ellipsis), then the argument is required.
-    :param str description: argument description, which will appear as the
-        help text for this argument
-    :param dict kwargs: additional keyword arguments supported by
-        `FieldInfo`. These are passed along as they are.
-    """
-
-    def __init__(self, name: str, default: Any, description: str, **kwargs) -> None:
-        self._name = name
-        super().__init__(default=default, description=description, **kwargs)
-        self._validate()
-
-    @property
-    def name(self):
-        """
-        Argument name as used in the CLI, e.g. "ignore-args"
-        """
-        return self._name
-
-
-@dataclass
-class Command:
-    """
-    Representation of a command
-
-    :param str name: command name
-    :param str description: command description
-    :param list[Argument] arguments: list of arguments supported by this command
-    """
-
-    name: str
-    description: str
-    arguments: list[Argument]
-
-    def create_model(self) -> type[pydantic.BaseModel]:
-        """
-        Creates a `pydantic` model for this command
-        """
-        arguments = dict()
-        for arg in self.arguments:
-            # These gymnastics are necessary because of
-            # https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
-            arg_type, arg_default_value = arg.default
-            arguments[arg.name] = (
-                arg_type,
-                pydantic.Field(arg_default_value, description=arg.description),
-            )
-        return pydantic.create_model(self.name, **arguments)
-
-
-class ArgumentEnum(enum.Enum):
-    """
-    Lists all available arguments
-
-    Having a single "repository" of arguments allows us to re-use them easily across different commands.
-
-    TODO: not sure whether an enum is the ideal data structure for that
-    """
-
-    IGNORE_FLAGS = Argument(
-        name="ignore-flags",
-        default=(bool, False),
-        description="Ignore run status flags",
-    )
-    TIME_DELAY = Argument(
-        name="time-delay",
-        default=(int, 0),
-        description="Time delay in seconds between job submissions (min: 0, max: 30)",
-    )
-    DRY_RUN = Argument(
-        name="dry-run", default=(bool, False), description="Don't actually submit jobs"
-    )
-    COMMAND_EXTRA = Argument(
-        name="command-extra",
-        default=(str, ""),
-        description="String to append to every command",
-    )
-    COMMAND_EXTRA_OVERRIDE = Argument(
-        name="command-extra-override",
-        default=(str, ""),
-        description="Same as command-extra, but overrides values in PEP",
-    )
-    LUMP = Argument(
-        name="lump",
-        default=(int, None),
-        description="Total input file size (GB) to batch into one job",
-    )
-    LUMPN = Argument(
-        name="lumpn",
-        default=(int, None),
-        description="Number of commands to batch into one job",
-    )
-    LIMIT = Argument(
-        name="limit", default=(int, None), description="Limit to n samples"
-    )
-    SKIP = Argument(
-        name="skip", default=(int, None), description="Skip samples by numerical index"
-    )
-
-
-RunParser = Command(
-    "run",
-    MESSAGE_BY_SUBCOMMAND["run"],
-    [
-        ArgumentEnum.IGNORE_FLAGS.value,
-        ArgumentEnum.TIME_DELAY.value,
-        ArgumentEnum.DRY_RUN.value,
-        ArgumentEnum.COMMAND_EXTRA.value,
-        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
-        ArgumentEnum.LUMP.value,
-        ArgumentEnum.LUMPN.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-    ],
-)
-RunParserModel = RunParser.create_model()
-
-
-class TopLevelParser(pydantic.BaseModel):
-    """
-    Top level parser that takes
-    - commands (run, runp, check...)
-    - arguments that are required no matter the subcommand
-    """
-
-    # commands
-    run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
+from .command_models.commands import TopLevelParser
 
 
 def main() -> None:
diff --git a/looper/command_models/README.md b/looper/command_models/README.md
new file mode 100644
index 000000000..dea00d8bd
--- /dev/null
+++ b/looper/command_models/README.md
@@ -0,0 +1,4 @@
+# `pydantic`-based definitions of `looper` commands and their arguments
+
+With the goal of writing an HTTP API that is in sync with the `looper` CLI, this module defines `looper` commands as `pydantic` models and arguments as fields in there.
+These can then be used by the [`pydantic-argparse`](https://pydantic-argparse.supimdos.com/) library to create a type-validated CLI (see `../cli_pydantic.py`), and by the future HTTP API for validating `POST`ed JSON data. Eventually, the `pydantic-argparse`-based CLI will replace the existing `argparse`-based CLI defined in `../cli_looper.py`.
diff --git a/looper/command_models/__init__.py b/looper/command_models/__init__.py
new file mode 100644
index 000000000..92f3a9e69
--- /dev/null
+++ b/looper/command_models/__init__.py
@@ -0,0 +1,6 @@
+"""
+This module holds `pydantic` models that describe commands and their arguments.
+
+These can be used either with the `pydantic-argparse` library to build a CLI or
+by an HTTP API.
+"""
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
new file mode 100644
index 000000000..f29a2bd49
--- /dev/null
+++ b/looper/command_models/arguments.py
@@ -0,0 +1,90 @@
+import enum
+from typing import Any
+
+import pydantic
+
+
+class Argument(pydantic.fields.FieldInfo):
+    """
+    CLI argument / flag definition
+
+    This class is designed to define CLI arguments or flags. It leverages
+    Pydantic for data validation and serves as a source of truth for multiple
+    interfaces, including a CLI.
+
+    Naively, one would think one could just subclass `pydantic.Field`,
+    but actually `pydantic.Field` is a function, and not a class.
+    `pydantic.Field()` returns a validated `FieldInfo` instance,
+    so we instead subclass `FieldInfo` directly and validate it in the
+    constructor.
+
+    :param str name: argument name, e.g. "ignore-args"
+    :param Any default: a tuple of the form (type, default_value). If the
+        default value is `...` (Ellipsis), then the argument is required.
+    :param str description: argument description, which will appear as the
+        help text for this argument
+    :param dict kwargs: additional keyword arguments supported by
+        `FieldInfo`. These are passed along as they are.
+    """
+
+    def __init__(self, name: str, default: Any, description: str, **kwargs) -> None:
+        self._name = name
+        super().__init__(default=default, description=description, **kwargs)
+        self._validate()
+
+    @property
+    def name(self):
+        """
+        Argument name as used in the CLI, e.g. "ignore-args"
+        """
+        return self._name
+
+
+class ArgumentEnum(enum.Enum):
+    """
+    Lists all available arguments
+
+    Having a single "repository" of arguments allows us to re-use them easily across different commands.
+
+    TODO: not sure whether an enum is the ideal data structure for that
+    """
+
+    IGNORE_FLAGS = Argument(
+        name="ignore-flags",
+        default=(bool, False),
+        description="Ignore run status flags",
+    )
+    TIME_DELAY = Argument(
+        name="time-delay",
+        default=(int, 0),
+        description="Time delay in seconds between job submissions (min: 0, max: 30)",
+    )
+    DRY_RUN = Argument(
+        name="dry-run", default=(bool, False), description="Don't actually submit jobs"
+    )
+    COMMAND_EXTRA = Argument(
+        name="command-extra",
+        default=(str, ""),
+        description="String to append to every command",
+    )
+    COMMAND_EXTRA_OVERRIDE = Argument(
+        name="command-extra-override",
+        default=(str, ""),
+        description="Same as command-extra, but overrides values in PEP",
+    )
+    LUMP = Argument(
+        name="lump",
+        default=(int, None),
+        description="Total input file size (GB) to batch into one job",
+    )
+    LUMPN = Argument(
+        name="lumpn",
+        default=(int, None),
+        description="Number of commands to batch into one job",
+    )
+    LIMIT = Argument(
+        name="limit", default=(int, None), description="Limit to n samples"
+    )
+    SKIP = Argument(
+        name="skip", default=(int, None), description="Skip samples by numerical index"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
new file mode 100644
index 000000000..39701de5e
--- /dev/null
+++ b/looper/command_models/commands.py
@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+from typing import Optional
+
+import pydantic
+
+from .arguments import Argument, ArgumentEnum
+from ..const import MESSAGE_BY_SUBCOMMAND
+
+@dataclass
+class Command:
+    """
+    Representation of a command
+
+    :param str name: command name
+    :param str description: command description
+    :param list[Argument] arguments: list of arguments supported by this command
+    """
+
+    name: str
+    description: str
+    arguments: list[Argument]
+
+    def create_model(self) -> type[pydantic.BaseModel]:
+        """
+        Creates a `pydantic` model for this command
+        """
+        arguments = dict()
+        for arg in self.arguments:
+            # These gymnastics are necessary because of
+            # https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
+            arg_type, arg_default_value = arg.default
+            arguments[arg.name] = (
+                arg_type,
+                pydantic.Field(arg_default_value, description=arg.description),
+            )
+        return pydantic.create_model(self.name, **arguments)
+
+
+RunParser = Command(
+    "run",
+    MESSAGE_BY_SUBCOMMAND["run"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+    ],
+)
+RunParserModel = RunParser.create_model()
+
+
+class TopLevelParser(pydantic.BaseModel):
+    """
+    Top level parser that takes
+    - commands (run, runp, check...)
+    - arguments that are required no matter the subcommand
+    """
+
+    # commands
+    run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)

From 71593e494f7bd07a72e0c3617cc2a1d0031c3f01 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 11:49:49 +0100
Subject: [PATCH 016/225] Add `CONFIG_FILE` argument

Co-authored-by: Zhihan Zhang <zhihan.zhang@tweag.io>
---
 looper/cli_pydantic.py             | 116 +++++++++++++++++++++++++++++
 looper/command_models/arguments.py |   5 ++
 2 files changed, 121 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7fd6162b2..07c0e8552 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,7 +1,36 @@
+import os
+import sys
+
+import logmuse
 import pydantic_argparse
+import yaml
+from eido import inspect_project
+from pephubclient import PEPHubClient
+from ubiquerg import VersionInHelpParser
 
 from .command_models.commands import TopLevelParser
 
+from divvy import select_divvy_config
+
+from . import __version__
+from .cli_looper import _proc_resources_spec
+from .const import *
+from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
+from .exceptions import *
+from .looper import *
+from .parser_types import *
+from .project import Project, ProjectContext
+from .utils import (
+    dotfile_path,
+    enrich_args_via_cfg,
+    init_generic_pipeline,
+    initiate_looper_config,
+    is_registry_path,
+    read_looper_config_file,
+    read_looper_dotfile,
+    read_yaml_file,
+)
+
 
 def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
@@ -13,6 +42,93 @@ def main() -> None:
     args = parser.parse_typed_args()
     print(args)
 
+    # here comes adapted `cli_looper.py` code
+    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+    try:
+        looper_config_dict = read_looper_dotfile()
+
+        for looper_config_key, looper_config_item in looper_config_dict.items():
+            print(looper_config_key, looper_config_item)
+            setattr(args, looper_config_key, looper_config_item)
+
+    except OSError:
+        parser.print_help(sys.stderr)
+        raise ValueError(
+            f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
+        )
+
+    args = enrich_args_via_cfg(args, parser, False)
+    divcfg = (
+        select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
+    )
+    # Ignore flags if user is selecting or excluding on flags:
+    if args.sel_flag or args.exc_flag:
+        args.ignore_flags = True
+
+    # Initialize project
+    if is_registry_path(args.config_file):
+        if vars(args)[SAMPLE_PL_ARG]:
+            p = Project(
+                amendments=args.amend,
+                divcfg_path=divcfg,
+                runp=args.command == "runp",
+                project_dict=PEPHubClient()._load_raw_pep(
+                    registry_path=args.config_file
+                ),
+                **{
+                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                },
+            )
+        else:
+            raise MisconfigurationException(
+                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
+            )
+    else:
+        try:
+            p = Project(
+                cfg=args.config_file,
+                amendments=args.amend,
+                divcfg_path=divcfg,
+                runp=False,
+                **{
+                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                },
+            )
+        except yaml.parser.ParserError as e:
+            _LOGGER.error(f"Project config parse failed -- {e}")
+            sys.exit(1)
+
+    selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
+    if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
+        _LOGGER.info(
+            "Failed to activate '{}' computing package. "
+            "Using the default one".format(selected_compute_pkg)
+        )
+
+    with ProjectContext(
+        prj=p,
+        selector_attribute="toggle",
+        selector_include=None,
+        selector_exclude=None,
+        selector_flag=None,
+        exclusion_flag=None,
+    ) as prj:
+        command = "run"
+        if command == "run":
+            run = Runner(prj)
+            try:
+                compute_kwargs = _proc_resources_spec(args)
+                return run(args, rerun=False, **compute_kwargs)
+            except SampleFailedException:
+                sys.exit(1)
+            except IOError:
+                _LOGGER.error(
+                    "{} pipeline_interfaces: '{}'".format(
+                        prj.__class__.__name__, prj.pipeline_interface_sources
+                    )
+                )
+                raise
+
 
 if __name__ == "__main__":
     main()
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index f29a2bd49..1938ff1fa 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -88,3 +88,8 @@ class ArgumentEnum(enum.Enum):
     SKIP = Argument(
         name="skip", default=(int, None), description="Skip samples by numerical index"
     )
+    CONFIG_FILE = Argument(
+        name="config-file",
+        default=(str, None),
+        description="Project configuration file",
+    )

From b990e62c36a402c00248dc9e7d6ebf2669b212ec Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 11:50:40 +0100
Subject: [PATCH 017/225] Add `pydantic` fields

---
 looper/command_models/commands.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 39701de5e..c986bca5d 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -63,3 +63,19 @@ class TopLevelParser(pydantic.BaseModel):
 
     # commands
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
+
+    # arguments
+    config_file: Optional[str] = pydantic.Field(
+        description="Project configuration file"
+    )
+    pep_config: Optional[str] = pydantic.Field(description="PEP configuration file")
+    output_dir: Optional[str] = pydantic.Field(description="Output directory")
+    sample_pipeline_interfaces: Optional[str] = pydantic.Field(
+        description="Sample pipeline interfaces definition"
+    )
+    project_pipeline_interfaces: Optional[str] = pydantic.Field(
+        description="Project pipeline interfaces definition"
+    )
+    amend: Optional[bool] = pydantic.Field(description="Amend stuff?")
+    sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
+    exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")

From 74e89dbd04766382f012f40a580df5f0aeb2ad98 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 13:55:42 +0100
Subject: [PATCH 018/225] Add a `with_reduced_default` method to `Argument`

This method returns the same data, but with the `default`
tuple (<type>, <value>) replaced by only <value>. This allows
`Argument`s to be used when directly defining `pydantic` models
instead of using `create_model()`.
---
 looper/command_models/arguments.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 1938ff1fa..1a5b71c79 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -1,3 +1,4 @@
+from copy import copy
 import enum
 from typing import Any
 
@@ -39,6 +40,26 @@ def name(self):
         """
         return self._name
 
+    def with_reduced_default(self) -> pydantic.fields.FieldInfo:
+        """
+        Convert to a `FieldInfo` instance with reduced default value
+
+        Returns a copy of an instance, but with the `default` attribute
+        replaced by only the default value, without the type information.
+        This is required when using an instance in a direct `pydantic`
+        model definition, instead of creating a model dynamically using
+        `pydantic.create_model`.
+
+        TODO: this is due to this issue:
+        https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
+        and it's a bit tedious.
+
+        """
+        c = copy(self)
+        _, default_value = self.default
+        c.default = default_value
+        return c
+
 
 class ArgumentEnum(enum.Enum):
     """

From a498889bec4a44fcb0d6e524857452db4bcb4942 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 13:57:35 +0100
Subject: [PATCH 019/225] Add `SETTINGS` argument

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 1 +
 2 files changed, 6 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 1a5b71c79..7ee88a983 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -114,3 +114,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Project configuration file",
     )
+    SETTINGS = Argument(
+        name="settings",
+        default=(str, ""),
+        description="Path to a YAML settings file with compute settings"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index c986bca5d..cd0d8cb2f 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -79,3 +79,4 @@ class TopLevelParser(pydantic.BaseModel):
     amend: Optional[bool] = pydantic.Field(description="Amend stuff?")
     sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
+    settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()

From b20ab5719e51ebac3f3938dc8cccffe541a3a435 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:13:17 +0100
Subject: [PATCH 020/225] Converted manual `pep_config` option into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 7ee88a983..4f33249e6 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -119,3 +119,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, ""),
         description="Path to a YAML settings file with compute settings"
     )
+    PEP_CONFIG = Argument(
+        name="pep_config",
+        default=(str, None),
+        description="PEP configuration file",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index cd0d8cb2f..45b3b5e29 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -68,7 +68,6 @@ class TopLevelParser(pydantic.BaseModel):
     config_file: Optional[str] = pydantic.Field(
         description="Project configuration file"
     )
-    pep_config: Optional[str] = pydantic.Field(description="PEP configuration file")
     output_dir: Optional[str] = pydantic.Field(description="Output directory")
     sample_pipeline_interfaces: Optional[str] = pydantic.Field(
         description="Sample pipeline interfaces definition"
@@ -80,3 +79,4 @@ class TopLevelParser(pydantic.BaseModel):
     sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
+    pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()

From 5753a2a90c1e1f4b7e483471fab0a11eb2af9bb4 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:19:37 +0100
Subject: [PATCH 021/225] Converted manual `output_dir` option into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 4f33249e6..8047fa345 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -124,3 +124,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="PEP configuration file",
     )
+    OUTPUT_DIR = Argument(
+        name="output_dir",
+        default=(str, None),
+        description="Output directory",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 45b3b5e29..f2174bfcc 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -68,7 +68,6 @@ class TopLevelParser(pydantic.BaseModel):
     config_file: Optional[str] = pydantic.Field(
         description="Project configuration file"
     )
-    output_dir: Optional[str] = pydantic.Field(description="Output directory")
     sample_pipeline_interfaces: Optional[str] = pydantic.Field(
         description="Sample pipeline interfaces definition"
     )
@@ -80,3 +79,4 @@ class TopLevelParser(pydantic.BaseModel):
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
+    output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()

From da94064d2e140dd858f83cda99f251283eb9b105 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:26:27 +0100
Subject: [PATCH 022/225] Converted manual `config_file` option into `Argument`

---
 looper/command_models/commands.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index f2174bfcc..a29fb46be 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,9 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    config_file: Optional[str] = pydantic.Field(
-        description="Project configuration file"
-    )
     sample_pipeline_interfaces: Optional[str] = pydantic.Field(
         description="Sample pipeline interfaces definition"
     )
@@ -80,3 +77,4 @@ class TopLevelParser(pydantic.BaseModel):
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
+    config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()

From 6a6c28dd48c616dddd28c5eb2a509c7e1fdaaaba Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:31:58 +0100
Subject: [PATCH 023/225] Converted manual `sample_pipeline_interfaces` option
 into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 4 +---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 8047fa345..044927d4c 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -129,3 +129,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Output directory",
     )
+    SAMPLE_PIPELINE_INTERFACES = Argument(
+        name="sample_pipeline_interfaces",
+        default=(list, ...),
+        description="Paths to looper sample config files"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index a29fb46be..38d5c665b 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,9 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    sample_pipeline_interfaces: Optional[str] = pydantic.Field(
-        description="Sample pipeline interfaces definition"
-    )
     project_pipeline_interfaces: Optional[str] = pydantic.Field(
         description="Project pipeline interfaces definition"
     )
@@ -78,3 +75,4 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
+    sample_pipeline_interfaces: list[str] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()

From 6dcf3dd6dad98f9dc5ae7ea88ffdecdd815514cb Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:34:06 +0100
Subject: [PATCH 024/225] Converted manual `project_pipeline_interfaces` option
 into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 4 +---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 044927d4c..6d7f63c76 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -134,3 +134,8 @@ class ArgumentEnum(enum.Enum):
         default=(list, ...),
         description="Paths to looper sample config files"
     )
+    PROJECT_PIPELINE_INTERFACES = Argument(
+        name="project_pipeline_interfaces",
+        default=(list, ...),
+        description="Paths to looper project config files"
+        )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 38d5c665b..ec0d2550d 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,9 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    project_pipeline_interfaces: Optional[str] = pydantic.Field(
-        description="Project pipeline interfaces definition"
-    )
     amend: Optional[bool] = pydantic.Field(description="Amend stuff?")
     sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
@@ -76,3 +73,4 @@ class TopLevelParser(pydantic.BaseModel):
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
     sample_pipeline_interfaces: list[str] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
+    project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()

From 88b1fcfe48ed3dd6dd92eb8d70442681afa2470b Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:36:56 +0100
Subject: [PATCH 025/225] Converted manual `amend` option into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 6d7f63c76..8a570ce71 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -139,3 +139,8 @@ class ArgumentEnum(enum.Enum):
         default=(list, ...),
         description="Paths to looper project config files"
         )
+    AMEND = Argument(
+        name="amend",
+        default=(list, ...),
+        description="List of amendments to activate"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index ec0d2550d..9196a1c7a 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,7 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    amend: Optional[bool] = pydantic.Field(description="Amend stuff?")
     sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
@@ -74,3 +73,4 @@ class TopLevelParser(pydantic.BaseModel):
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
     sample_pipeline_interfaces: list[str] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()

From 6e8aa34de9c053d5a7d50e8a31b437849dacacbe Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:39:44 +0100
Subject: [PATCH 026/225] Converted manual `sel_flag` option into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 8a570ce71..d8c850309 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -144,3 +144,8 @@ class ArgumentEnum(enum.Enum):
         default=(list, ...),
         description="List of amendments to activate"
     )
+    SEL_FLAG = Argument(
+        name="sel_flag",
+        default=(str, ""),
+        description="Sample selection flag"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 9196a1c7a..8ee42d34b 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,7 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    sel_flag: Optional[bool] = pydantic.Field(description="Selection flag")
     exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
@@ -74,3 +73,4 @@ class TopLevelParser(pydantic.BaseModel):
     sample_pipeline_interfaces: list[str] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()
+    sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()

From 1339d42237c90afa25abcb268e79f14642048d57 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 14:41:00 +0100
Subject: [PATCH 027/225] Converted manual `exc_flag` option into `Argument`

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index d8c850309..5061015e9 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -149,3 +149,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, ""),
         description="Sample selection flag"
     )
+    EXC_FLAG = Argument(
+        name="exc_flag",
+        default=(str, ""),
+        description="Sample exclusion flag"
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 8ee42d34b..1095e3fc8 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -65,7 +65,6 @@ class TopLevelParser(pydantic.BaseModel):
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
 
     # arguments
-    exc_flag: Optional[bool] = pydantic.Field(description="Exclusion flag")
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
@@ -74,3 +73,4 @@ class TopLevelParser(pydantic.BaseModel):
     project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
+    exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From 2aa3a03c5f65caf181006ccbed8093931c947cc6 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:06:54 +0100
Subject: [PATCH 028/225] Make `sample_pipeline_interfaces` optional

---
 looper/command_models/arguments.py | 2 +-
 looper/command_models/commands.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 5061015e9..4af3ed196 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -131,7 +131,7 @@ class ArgumentEnum(enum.Enum):
     )
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
-        default=(list, ...),
+        default=(list, []),
         description="Paths to looper sample config files"
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 1095e3fc8..55c0efc32 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -69,8 +69,8 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    sample_pipeline_interfaces: list[str] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()
+    sample_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From c9974c401f7b45ceec82454d4868b86c16ef77e3 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:07:07 +0100
Subject: [PATCH 029/225] Make `project_pipeline_interfaces` optional

---
 looper/command_models/arguments.py | 2 +-
 looper/command_models/commands.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 4af3ed196..8783cb58a 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -136,7 +136,7 @@ class ArgumentEnum(enum.Enum):
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
         name="project_pipeline_interfaces",
-        default=(list, ...),
+        default=(list, []),
         description="Paths to looper project config files"
         )
     AMEND = Argument(
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 55c0efc32..e0357afbb 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -69,8 +69,8 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    project_pipeline_interfaces: list[str] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()
     sample_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
+    project_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From bb10f1b1871aa861eddac9c5bbcfc774d710e73f Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:07:18 +0100
Subject: [PATCH 030/225] Make `amend` optional

---
 looper/command_models/arguments.py | 2 +-
 looper/command_models/commands.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 8783cb58a..2b06df166 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -141,7 +141,7 @@ class ArgumentEnum(enum.Enum):
         )
     AMEND = Argument(
         name="amend",
-        default=(list, ...),
+        default=(list, []),
         description="List of amendments to activate"
     )
     SEL_FLAG = Argument(
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index e0357afbb..ec66ac0ab 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -69,8 +69,8 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    amend: list[str] = ArgumentEnum.AMEND.value.with_reduced_default()
     sample_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     project_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    amend: Optional[list[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From 701afab41b9ddcfca6ea01a003eff59f83e81c84 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:34:46 +0100
Subject: [PATCH 031/225] Replace dashes by underscores in argument names

---
 looper/command_models/arguments.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 2b06df166..9c4f9c007 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -71,25 +71,25 @@ class ArgumentEnum(enum.Enum):
     """
 
     IGNORE_FLAGS = Argument(
-        name="ignore-flags",
+        name="ignore_flags",
         default=(bool, False),
         description="Ignore run status flags",
     )
     TIME_DELAY = Argument(
-        name="time-delay",
+        name="time_delay",
         default=(int, 0),
         description="Time delay in seconds between job submissions (min: 0, max: 30)",
     )
     DRY_RUN = Argument(
-        name="dry-run", default=(bool, False), description="Don't actually submit jobs"
+        name="dry_run", default=(bool, False), description="Don't actually submit jobs"
     )
     COMMAND_EXTRA = Argument(
-        name="command-extra",
+        name="command_extra",
         default=(str, ""),
         description="String to append to every command",
     )
     COMMAND_EXTRA_OVERRIDE = Argument(
-        name="command-extra-override",
+        name="command_extra_override",
         default=(str, ""),
         description="Same as command-extra, but overrides values in PEP",
     )
@@ -110,7 +110,7 @@ class ArgumentEnum(enum.Enum):
         name="skip", default=(int, None), description="Skip samples by numerical index"
     )
     CONFIG_FILE = Argument(
-        name="config-file",
+        name="config_file",
         default=(str, None),
         description="Project configuration file",
     )

From 3fc08b63c16c9e7437a45aee2588c85c164c5ce4 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:35:02 +0100
Subject: [PATCH 032/225] Fix argument accessions in `Runner`

---
 looper/looper.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 32e97a0d8..e8259215c 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -400,12 +400,12 @@ def __call__(self, args, rerun=False, **compute_kwargs):
                 pipeline_interface=piface,
                 prj=self.prj,
                 compute_variables=comp_vars,
-                delay=args.time_delay,
-                extra_args=args.command_extra,
-                extra_args_override=args.command_extra_override,
-                ignore_flags=args.ignore_flags,
-                max_cmds=args.lumpn,
-                max_size=args.lump,
+                delay=args.run.time_delay,
+                extra_args=args.run.command_extra,
+                extra_args_override=args.run.command_extra_override,
+                ignore_flags=args.run.ignore_flags,
+                max_cmds=args.run.lumpn,
+                max_size=args.run.lump,
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
@@ -486,7 +486,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
         )
         _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
         self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
-        if args.dry_run:
+        if args.run.dry_run:
             job_sub_total_if_real = job_sub_total
             job_sub_total = 0
             _LOGGER.info(

From 3fb1aefc41a08b32ea893423fe45d9f4b30763b2 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:52:48 +0100
Subject: [PATCH 033/225] Fix `lump` argument type

---
 looper/command_models/arguments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 9c4f9c007..5367f5cc3 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -95,7 +95,7 @@ class ArgumentEnum(enum.Enum):
     )
     LUMP = Argument(
         name="lump",
-        default=(int, None),
+        default=(float, None),
         description="Total input file size (GB) to batch into one job",
     )
     LUMPN = Argument(

From 6438a356991ad176c672fa383096da0f9ba9257e Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Tue, 16 Jan 2024 15:55:00 +0100
Subject: [PATCH 034/225] Print separator line for better visibility

---
 looper/cli_pydantic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 07c0e8552..9ed84be45 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -41,6 +41,7 @@ def main() -> None:
     )
     args = parser.parse_typed_args()
     print(args)
+    print("#########################################")
 
     # here comes adapted `cli_looper.py` code
     looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)

From e786ea2e60345624ac6589982f02e0797c4a94ae Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 10:55:59 +0100
Subject: [PATCH 035/225] Formatting changes

---
 looper/cli_pydantic.py             |  3 +--
 looper/command_models/arguments.py | 22 ++++++++--------------
 looper/command_models/commands.py  | 11 ++++++++---
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 9ed84be45..7e593034e 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -8,12 +8,11 @@
 from pephubclient import PEPHubClient
 from ubiquerg import VersionInHelpParser
 
-from .command_models.commands import TopLevelParser
-
 from divvy import select_divvy_config
 
 from . import __version__
 from .cli_looper import _proc_resources_spec
+from .command_models.commands import TopLevelParser
 from .const import *
 from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
 from .exceptions import *
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 5367f5cc3..53389d7b6 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -1,5 +1,5 @@
-from copy import copy
 import enum
+from copy import copy
 from typing import Any
 
 import pydantic
@@ -117,7 +117,7 @@ class ArgumentEnum(enum.Enum):
     SETTINGS = Argument(
         name="settings",
         default=(str, ""),
-        description="Path to a YAML settings file with compute settings"
+        description="Path to a YAML settings file with compute settings",
     )
     PEP_CONFIG = Argument(
         name="pep_config",
@@ -132,25 +132,19 @@ class ArgumentEnum(enum.Enum):
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
         default=(list, []),
-        description="Paths to looper sample config files"
+        description="Paths to looper sample config files",
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
         name="project_pipeline_interfaces",
         default=(list, []),
-        description="Paths to looper project config files"
-        )
+        description="Paths to looper project config files",
+    )
     AMEND = Argument(
-        name="amend",
-        default=(list, []),
-        description="List of amendments to activate"
+        name="amend", default=(list, []), description="List of amendments to activate"
     )
     SEL_FLAG = Argument(
-        name="sel_flag",
-        default=(str, ""),
-        description="Sample selection flag"
+        name="sel_flag", default=(str, ""), description="Sample selection flag"
     )
     EXC_FLAG = Argument(
-        name="exc_flag",
-        default=(str, ""),
-        description="Sample exclusion flag"
+        name="exc_flag", default=(str, ""), description="Sample exclusion flag"
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index ec66ac0ab..c1621f9ff 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -3,8 +3,9 @@
 
 import pydantic
 
-from .arguments import Argument, ArgumentEnum
 from ..const import MESSAGE_BY_SUBCOMMAND
+from .arguments import Argument, ArgumentEnum
+
 
 @dataclass
 class Command:
@@ -69,8 +70,12 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    sample_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
-    project_pipeline_interfaces: Optional[list[str]] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    sample_pipeline_interfaces: Optional[
+        list[str]
+    ] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
+    project_pipeline_interfaces: Optional[
+        list[str]
+    ] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     amend: Optional[list[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From 5644b7fa14036c8f4cd0b39f6f0b8cf8c62bb717 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 10:56:52 +0100
Subject: [PATCH 036/225] Add LOOPER_CONFIG as an argument

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 3 +++
 2 files changed, 8 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 53389d7b6..b1ee53525 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -114,6 +114,11 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Project configuration file",
     )
+    LOOPER_CONFIG = Argument(
+        name="looper_config",
+        default=(str, None),
+        description="Looper configuration file (YAML)",
+    )
     SETTINGS = Argument(
         name="settings",
         default=(str, ""),
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index c1621f9ff..8886802be 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -70,6 +70,9 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
+    looper_config: Optional[
+        str
+    ] = ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
     sample_pipeline_interfaces: Optional[
         list[str]
     ] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()

From a0bf3ffcfdafc85cd727f73c387320a5e206f365 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 10:59:58 +0100
Subject: [PATCH 037/225] `run` special treatment: `enrich_args_via_cfg`

---
 looper/utils.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/looper/utils.py b/looper/utils.py
index 3796cbc6f..bac98dcbf 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -273,19 +273,32 @@ def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
     else:
         cli_args, _ = aux_parser.parse_known_args()
 
-    for dest in vars(parser_args):
-        if dest not in POSITIONAL or not hasattr(result, dest):
-            if dest in cli_args:
-                x = getattr(cli_args, dest)
-                r = convert_value(x) if isinstance(x, str) else x
-            elif cfg_args_all is not None and dest in cfg_args_all:
-                if isinstance(cfg_args_all[dest], list):
-                    r = [convert_value(i) for i in cfg_args_all[dest]]
+
+    def set_single_arg(argname, default_source_namespace, result_namespace):
+        if argname not in POSITIONAL or not hasattr(result, argname):
+            if argname in cli_args:
+                cli_provided_value = getattr(cli_args, argname)
+                r = convert_value(cli_provided_value) if isinstance(cli_provided_value, str) else cli_provided_value
+            elif cfg_args_all is not None and argname in cfg_args_all:
+                if isinstance(cfg_args_all[argname], list):
+                    r = [convert_value(i) for i in cfg_args_all[argname]]
                 else:
-                    r = convert_value(cfg_args_all[dest])
+                    r = convert_value(cfg_args_all[argname])
             else:
-                r = getattr(parser_args, dest)
-            setattr(result, dest, r)
+                r = getattr(default_source_namespace, argname)
+            setattr(result_namespace, argname, r)
+
+    for top_level_argname in vars(parser_args):
+        if top_level_argname not in ["run"]:
+            # this argument is a top-level argument
+            set_single_arg(top_level_argname, parser_args, result)
+        else:
+            # this argument actually is a subcommand
+            enriched_command_namespace = argparse.Namespace()
+            command_namespace = getattr(parser_args, top_level_argname)
+            for argname in vars(command_namespace):
+                set_single_arg(argname, command_namespace, enriched_command_namespace)
+            setattr(result, top_level_argname, enriched_command_namespace)
     return result
 
 

From 0845d2fa446ce812e4254e37e1daf850abccef5d Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 11:59:56 +0100
Subject: [PATCH 038/225] Add `DIVVY` argument

---
 looper/command_models/arguments.py | 6 ++++++
 looper/command_models/commands.py  | 1 +
 2 files changed, 7 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index b1ee53525..d8cd5f9c1 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -1,5 +1,6 @@
 import enum
 from copy import copy
+import os
 from typing import Any
 
 import pydantic
@@ -153,3 +154,8 @@ class ArgumentEnum(enum.Enum):
     EXC_FLAG = Argument(
         name="exc_flag", default=(str, ""), description="Sample exclusion flag"
     )
+    DIVVY = Argument(
+        name="divvy", default=(str, os.getenv("DIVCFG", None)), description=(
+            "Path to divvy configuration file. Default=$DIVCFG env "
+            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set"))
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 8886802be..32f9abdc7 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -50,6 +50,7 @@ def create_model(self) -> type[pydantic.BaseModel]:
         ArgumentEnum.LUMPN.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.DIVVY.value
     ],
 )
 RunParserModel = RunParser.create_model()

From 38366139dcc628c4acbf2c14734ade73888e06e8 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 12:05:39 +0100
Subject: [PATCH 039/225] Fix `divvy` argument accession

---
 looper/cli_pydantic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7e593034e..b03b3885a 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -59,7 +59,7 @@ def main() -> None:
 
     args = enrich_args_via_cfg(args, parser, False)
     divcfg = (
-        select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
+        select_divvy_config(filepath=args.run.divvy) if hasattr(args.run, "divvy") else None
     )
     # Ignore flags if user is selecting or excluding on flags:
     if args.sel_flag or args.exc_flag:

From ea95529b3c6a057ec869ddf5b71ca9622693a8bc Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 19:03:06 +0100
Subject: [PATCH 040/225] Clean up imports

---
 looper/cli_pydantic.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index b03b3885a..ef1e6411b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,12 +1,9 @@
 import os
 import sys
 
-import logmuse
 import pydantic_argparse
 import yaml
-from eido import inspect_project
 from pephubclient import PEPHubClient
-from ubiquerg import VersionInHelpParser
 
 from divvy import select_divvy_config
 
@@ -22,12 +19,8 @@
 from .utils import (
     dotfile_path,
     enrich_args_via_cfg,
-    init_generic_pipeline,
-    initiate_looper_config,
     is_registry_path,
-    read_looper_config_file,
     read_looper_dotfile,
-    read_yaml_file,
 )
 
 

From b3debfb1b07da27ba9721fa4213384c6064fd76f Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 19:06:17 +0100
Subject: [PATCH 041/225] Add docstring to `cli_pydantic.py`

It explicitly points out that this is only a test script.
---
 looper/cli_pydantic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index ef1e6411b..1ea55e33d 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -1,3 +1,17 @@
+"""
+CLI script using `pydantic-argparse` for parsing of arguments
+
+Arguments / commands are defined in `command_models/` and are given, eventually, as
+`pydantic` models, allowing for type-checking and validation of arguments.
+
+Note: this is only a test script so far, and coexists next to the current CLI
+(`cli_looper.py`), which uses `argparse` directly. The goal is to eventually
+replace the current CLI with a CLI based on above-mentioned `pydantic` models,
+but whether this will happen with `pydantic-argparse` or another, possibly self-
+written library is not yet clear.
+It is well possible that this script will be removed again.
+"""
+
 import os
 import sys
 

From f2f366935e672ffcd774daf803afd149c5599942 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 19:13:12 +0100
Subject: [PATCH 042/225] Add docstrings

---
 looper/command_models/arguments.py | 4 ++++
 looper/command_models/commands.py  | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index d8cd5f9c1..ca3c2824e 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -1,3 +1,7 @@
+"""
+Argument definitions via a thin wrapper around `pydantic.fields.FieldInfo`
+"""
+
 import enum
 from copy import copy
 import os
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 32f9abdc7..9cee62050 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -1,3 +1,7 @@
+"""
+`pydantic` models for `looper` commands and a wrapper class.
+"""
+
 from dataclasses import dataclass
 from typing import Optional
 

From 7a6495e977a1438d7c9c46dc75f82e17e5df2ee1 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 17 Jan 2024 19:13:25 +0100
Subject: [PATCH 043/225] Refactor to easily support future commands

---
 looper/command_models/commands.py | 1 +
 looper/utils.py                   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 9cee62050..a71fd1b43 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -59,6 +59,7 @@ def create_model(self) -> type[pydantic.BaseModel]:
 )
 RunParserModel = RunParser.create_model()
 
+SUPPORTED_COMMANDS = [RunParser]
 
 class TopLevelParser(pydantic.BaseModel):
     """
diff --git a/looper/utils.py b/looper/utils.py
index bac98dcbf..3b59f3ebd 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -19,8 +19,8 @@
 from pydantic.error_wrappers import ValidationError
 
 from .const import *
+from .command_models.commands import SUPPORTED_COMMANDS
 from .exceptions import MisconfigurationException, RegistryPathException
-
 _LOGGER = getLogger(__name__)
 
 
@@ -289,7 +289,7 @@ def set_single_arg(argname, default_source_namespace, result_namespace):
             setattr(result_namespace, argname, r)
 
     for top_level_argname in vars(parser_args):
-        if top_level_argname not in ["run"]:
+        if top_level_argname not in [cmd.name for cmd in SUPPORTED_COMMANDS]:
             # this argument is a top-level argument
             set_single_arg(top_level_argname, parser_args, result)
         else:

From c6319a48483b152756f0024bd1666c820cc0a7f0 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 18 Jan 2024 18:10:18 +0100
Subject: [PATCH 044/225] Run formatter

---
 looper/cli_pydantic.py             | 4 +++-
 looper/command_models/arguments.py | 7 +++++--
 looper/command_models/commands.py  | 3 ++-
 looper/utils.py                    | 8 ++++++--
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 1ea55e33d..0ba540080 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -66,7 +66,9 @@ def main() -> None:
 
     args = enrich_args_via_cfg(args, parser, False)
     divcfg = (
-        select_divvy_config(filepath=args.run.divvy) if hasattr(args.run, "divvy") else None
+        select_divvy_config(filepath=args.run.divvy)
+        if hasattr(args.run, "divvy")
+        else None
     )
     # Ignore flags if user is selecting or excluding on flags:
     if args.sel_flag or args.exc_flag:
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index ca3c2824e..11e7912ef 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -159,7 +159,10 @@ class ArgumentEnum(enum.Enum):
         name="exc_flag", default=(str, ""), description="Sample exclusion flag"
     )
     DIVVY = Argument(
-        name="divvy", default=(str, os.getenv("DIVCFG", None)), description=(
+        name="divvy",
+        default=(str, os.getenv("DIVCFG", None)),
+        description=(
             "Path to divvy configuration file. Default=$DIVCFG env "
-            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set"))
+            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set")
+        ),
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index a71fd1b43..3cb040aa2 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -54,13 +54,14 @@ def create_model(self) -> type[pydantic.BaseModel]:
         ArgumentEnum.LUMPN.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
-        ArgumentEnum.DIVVY.value
+        ArgumentEnum.DIVVY.value,
     ],
 )
 RunParserModel = RunParser.create_model()
 
 SUPPORTED_COMMANDS = [RunParser]
 
+
 class TopLevelParser(pydantic.BaseModel):
     """
     Top level parser that takes
diff --git a/looper/utils.py b/looper/utils.py
index 3b59f3ebd..531ea01a6 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -21,6 +21,7 @@
 from .const import *
 from .command_models.commands import SUPPORTED_COMMANDS
 from .exceptions import MisconfigurationException, RegistryPathException
+
 _LOGGER = getLogger(__name__)
 
 
@@ -273,12 +274,15 @@ def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
     else:
         cli_args, _ = aux_parser.parse_known_args()
 
-
     def set_single_arg(argname, default_source_namespace, result_namespace):
         if argname not in POSITIONAL or not hasattr(result, argname):
             if argname in cli_args:
                 cli_provided_value = getattr(cli_args, argname)
-                r = convert_value(cli_provided_value) if isinstance(cli_provided_value, str) else cli_provided_value
+                r = (
+                    convert_value(cli_provided_value)
+                    if isinstance(cli_provided_value, str)
+                    else cli_provided_value
+                )
             elif cfg_args_all is not None and argname in cfg_args_all:
                 if isinstance(cfg_args_all[argname], list):
                     r = [convert_value(i) for i in cfg_args_all[argname]]

From 8fc9590e533498728656cb4a691b5f4790f6d900 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 19 Jan 2024 09:50:52 +0100
Subject: [PATCH 045/225] Infer currently used subcommand instead of hardcoding
 it

---
 looper/cli_pydantic.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 0ba540080..42b01a55b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -23,7 +23,7 @@
 
 from . import __version__
 from .cli_looper import _proc_resources_spec
-from .command_models.commands import TopLevelParser
+from .command_models.commands import SUPPORTED_COMMANDS, TopLevelParser
 from .const import *
 from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
 from .exceptions import *
@@ -49,6 +49,16 @@ def main() -> None:
     print(args)
     print("#########################################")
 
+    # Find out which subcommand was used
+    supported_command_names = [cmd.name for cmd in SUPPORTED_COMMANDS]
+    subcommand_valued_args = [
+        (arg, value)
+        for arg, value in vars(args).items()
+        if arg and arg in supported_command_names
+    ]
+    # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
+    [(subcommand_name, subcommand_args)] = subcommand_valued_args
+
     # here comes adapted `cli_looper.py` code
     looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
     try:
@@ -66,8 +76,8 @@ def main() -> None:
 
     args = enrich_args_via_cfg(args, parser, False)
     divcfg = (
-        select_divvy_config(filepath=args.run.divvy)
-        if hasattr(args.run, "divvy")
+        select_divvy_config(filepath=subcommand_args.divvy)
+        if hasattr(subcommand_args, "divvy")
         else None
     )
     # Ignore flags if user is selecting or excluding on flags:
@@ -122,8 +132,7 @@ def main() -> None:
         selector_flag=None,
         exclusion_flag=None,
     ) as prj:
-        command = "run"
-        if command == "run":
+        if subcommand_name == "run":
             run = Runner(prj)
             try:
                 compute_kwargs = _proc_resources_spec(args)

From c2e7ca38eb942d9defbf6b5cba272914e1d3b322 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 19 Jan 2024 09:52:16 +0100
Subject: [PATCH 046/225] Format `setup.py`

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e6d940868..c05314fe6 100644
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,7 @@ def get_static(name, condition=None):
         "console_scripts": [
             "looper = looper.__main__:main",
             "divvy = looper.__main__:divvy_main",
-            "looper-pydantic-argparse = looper.cli_pydantic:main"
+            "looper-pydantic-argparse = looper.cli_pydantic:main",
         ],
     },
     scripts=scripts,

From 4b715c32259ddc68fad1ab68ee35b3f39920a030 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 19 Jan 2024 10:09:22 +0100
Subject: [PATCH 047/225] Loosen `pydantic-argparse` version constraint

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 3b8fa208c..71c0df877 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -11,4 +11,4 @@ pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2
 yacman>=0.9.2
-pydantic-argparse==0.8.0
+pydantic-argparse>=0.8.0

From b2a1e1c5b5c8d80c59c5259ae1bc124b3343234c Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 19 Jan 2024 10:12:42 +0100
Subject: [PATCH 048/225] Apply suggestions from code review

Co-authored-by: Vince <vince.reuter@gmail.com>
---
 looper/command_models/__init__.py  | 6 +++---
 looper/command_models/arguments.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/looper/command_models/__init__.py b/looper/command_models/__init__.py
index 92f3a9e69..4258506b0 100644
--- a/looper/command_models/__init__.py
+++ b/looper/command_models/__init__.py
@@ -1,6 +1,6 @@
 """
-This module holds `pydantic` models that describe commands and their arguments.
+This package holds `pydantic` models that describe commands and their arguments.
 
-These can be used either with the `pydantic-argparse` library to build a CLI or
-by an HTTP API.
+These can be used either by an HTTP API or with the `pydantic-argparse` 
+library to build a CLI.
 """
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 11e7912ef..658ca051c 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -163,6 +163,6 @@ class ArgumentEnum(enum.Enum):
         default=(str, os.getenv("DIVCFG", None)),
         description=(
             "Path to divvy configuration file. Default=$DIVCFG env "
-            "variable. Currently: {}".format(os.getenv("DIVCFG", None) or "not set")
+            "variable. Currently: {}".format(os.getenv("DIVCFG") or "not set")
         ),
     )

From 2eb2e77b4221bfd75548d4653213a51c4d058ccf Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Fri, 26 Jan 2024 12:44:44 +0800
Subject: [PATCH 049/225] Fix typing to support Python 3.8

---
 looper/cli_pydantic.py             |  5 +++++
 looper/command_models/arguments.py |  8 ++++----
 looper/command_models/commands.py  | 12 ++++++------
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 42b01a55b..a93965df2 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -12,6 +12,11 @@
 It is well possible that this script will be removed again.
 """
 
+# Note: The following import is used for forward annotations (Python 3.8)
+# to prevent potential 'TypeError' related to the use of the '|' operator
+# with types.
+from __future__ import annotations
+
 import os
 import sys
 
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 658ca051c..09a813747 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -5,7 +5,7 @@
 import enum
 from copy import copy
 import os
-from typing import Any
+from typing import Any, List
 
 import pydantic
 
@@ -141,16 +141,16 @@ class ArgumentEnum(enum.Enum):
     )
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
-        default=(list, []),
+        default=(List, []),
         description="Paths to looper sample config files",
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
         name="project_pipeline_interfaces",
-        default=(list, []),
+        default=(List, []),
         description="Paths to looper project config files",
     )
     AMEND = Argument(
-        name="amend", default=(list, []), description="List of amendments to activate"
+        name="amend", default=(List, []), description="List of amendments to activate"
     )
     SEL_FLAG = Argument(
         name="sel_flag", default=(str, ""), description="Sample selection flag"
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 3cb040aa2..be2330d18 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -3,7 +3,7 @@
 """
 
 from dataclasses import dataclass
-from typing import Optional
+from typing import List, Optional, Type
 
 import pydantic
 
@@ -23,9 +23,9 @@ class Command:
 
     name: str
     description: str
-    arguments: list[Argument]
+    arguments: List[Argument]
 
-    def create_model(self) -> type[pydantic.BaseModel]:
+    def create_model(self) -> Type[pydantic.BaseModel]:
         """
         Creates a `pydantic` model for this command
         """
@@ -81,11 +81,11 @@ class TopLevelParser(pydantic.BaseModel):
         str
     ] = ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
     sample_pipeline_interfaces: Optional[
-        list[str]
+        List[str]
     ] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
     project_pipeline_interfaces: Optional[
-        list[str]
+        List[str]
     ] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
-    amend: Optional[list[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
+    amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()

From 8b39b7622692314f91021a9d624e4cf15e6c83c3 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Mon, 29 Jan 2024 10:29:26 +0100
Subject: [PATCH 050/225] Add logging setup

Co-authored-by: Zhihan Zhang <zhihan.zhang@tweag.io>
---
 looper/cli_pydantic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index a93965df2..336ba12d5 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -20,6 +20,7 @@
 import os
 import sys
 
+import logmuse
 import pydantic_argparse
 import yaml
 from pephubclient import PEPHubClient
@@ -77,6 +78,13 @@ def main() -> None:
         parser.print_help(sys.stderr)
         raise ValueError(
             f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
+    global _LOGGER
+
+    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
+    args_command = [
+        attr for attr in [cmd.name for cmd in SUPPORTED_COMMANDS] if hasattr(args, attr)
+    ]
+    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, args_command))
         )
 
     args = enrich_args_via_cfg(args, parser, False)

From 97d0e57a42c05a6db3c6a88a3abeb309a70db45a Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Mon, 29 Jan 2024 10:31:19 +0100
Subject: [PATCH 051/225] Make CLI script accept `looper-config` argument

Co-authored-by: Zhihan Zhang <zhihan.zhang@tweag.io>
---
 looper/cli_pydantic.py | 78 ++++++++++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 29 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 336ba12d5..945ddd919 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -24,6 +24,7 @@
 import pydantic_argparse
 import yaml
 from pephubclient import PEPHubClient
+from pydantic_argparse.argparse.parser import ArgumentParser
 
 from divvy import select_divvy_config
 
@@ -40,20 +41,16 @@
     dotfile_path,
     enrich_args_via_cfg,
     is_registry_path,
+    read_looper_config_file,
     read_looper_dotfile,
 )
 
 
-def main() -> None:
-    parser = pydantic_argparse.ArgumentParser(
-        model=TopLevelParser,
-        prog="looper",
-        description="pydantic-argparse demo",
-        add_help=True,
-    )
-    args = parser.parse_typed_args()
-    print(args)
-    print("#########################################")
+def run_looper(args: TopLevelParser, parser: ArgumentParser):
+    # here comes adapted `cli_looper.py` code
+    global _LOGGER
+
+    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
 
     # Find out which subcommand was used
     supported_command_names = [cmd.name for cmd in SUPPORTED_COMMANDS]
@@ -65,29 +62,39 @@ def main() -> None:
     # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
     [(subcommand_name, subcommand_args)] = subcommand_valued_args
 
-    # here comes adapted `cli_looper.py` code
-    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
-    try:
-        looper_config_dict = read_looper_dotfile()
-
-        for looper_config_key, looper_config_item in looper_config_dict.items():
-            print(looper_config_key, looper_config_item)
-            setattr(args, looper_config_key, looper_config_item)
-
-    except OSError:
-        parser.print_help(sys.stderr)
-        raise ValueError(
-            f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
-    global _LOGGER
+    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
-    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
-    args_command = [
-        attr for attr in [cmd.name for cmd in SUPPORTED_COMMANDS] if hasattr(args, attr)
-    ]
-    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, args_command))
+    if args.config_file is None:
+        looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+        try:
+            if args.looper_config:
+                looper_config_dict = read_looper_config_file(args.looper_config)
+            else:
+                looper_config_dict = read_looper_dotfile()
+                _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+            for looper_config_key, looper_config_item in looper_config_dict.items():
+                setattr(args, looper_config_key, looper_config_item)
+
+        except OSError:
+            parser.print_help(sys.stderr)
+            _LOGGER.warning(
+                f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
+            )
+            sys.exit(1)
+    else:
+        _LOGGER.warning(
+            "This PEP configures looper through the project config. This approach is deprecated and will "
+            "be removed in future versions. Please use a looper config file. For more information see "
+            "looper.databio.org/en/latest/looper-config"
         )
 
     args = enrich_args_via_cfg(args, parser, False)
+
+    # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
+    if vars(args)[PROJECT_PL_ARG]:
+        args.pipeline_interfaces = vars(args)[PROJECT_PL_ARG]
+
     divcfg = (
         select_divvy_config(filepath=subcommand_args.divvy)
         if hasattr(subcommand_args, "divvy")
@@ -161,5 +168,18 @@ def main() -> None:
                 raise
 
 
+def main() -> None:
+    parser = pydantic_argparse.ArgumentParser(
+        model=TopLevelParser,
+        prog="looper",
+        description="pydantic-argparse demo",
+        add_help=True,
+    )
+    args = parser.parse_typed_args()
+    print(args)
+    print("#########################################")
+    run_looper(args, parser)
+
+
 if __name__ == "__main__":
     main()

From ea11a4f46d21be759bf8a95e384ebd8dd397126c Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Mon, 22 Jan 2024 17:16:27 +0800
Subject: [PATCH 052/225] Add arguments for logging

These arguments are compatible with logmuse generated parser
---
 looper/command_models/arguments.py | 18 ++++++++++++++++++
 looper/command_models/commands.py  |  5 +++++
 2 files changed, 23 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 09a813747..2b639e622 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -166,3 +166,21 @@ class ArgumentEnum(enum.Enum):
             "variable. Currently: {}".format(os.getenv("DIVCFG") or "not set")
         ),
     )
+
+    # Arguments for logger compatible with logmuse
+    SILENT = Argument(
+        name="silent", default=(bool, False), description="Whether to silence logging"
+    )
+    VERBOSITY = Argument(
+        name="verbosity",
+        default=(int, None),
+        description="Alternate mode of expression for logging level that better "
+        "accords with intuition about how to convey this.",
+    )
+    LOGDEV = Argument(
+        name="logdev",
+        default=(bool, False),
+        description="Whether to log in development mode; possibly among other "
+        "behavioral changes to logs handling, use a more information-rich "
+        "message format template.",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index be2330d18..5b437997a 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -89,3 +89,8 @@ class TopLevelParser(pydantic.BaseModel):
     amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()
+
+    # arguments for logging
+    silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
+    verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
+    logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()

From f2f4cc8b84e019bf69aaeb5d73828ca16d39b4bb Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 26 Jan 2024 13:40:26 +0100
Subject: [PATCH 053/225] Add `SKIP_FILE_CHECKS`, `PACKAGE` and `COMPUTE`
 arguments

---
 looper/command_models/arguments.py | 23 +++++++++++++++++++++--
 looper/command_models/commands.py  |  5 ++++-
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 2b639e622..b23c3a6c0 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -3,8 +3,8 @@
 """
 
 import enum
-from copy import copy
 import os
+from copy import copy
 from typing import Any, List
 
 import pydantic
@@ -158,6 +158,21 @@ class ArgumentEnum(enum.Enum):
     EXC_FLAG = Argument(
         name="exc_flag", default=(str, ""), description="Sample exclusion flag"
     )
+    SKIP_FILE_CHECKS = Argument(
+        name="skip_file_checks",
+        default=(bool, False),
+        description="Do not perform input file checks"
+    )
+    PACKAGE = Argument(
+        name="package",
+        default=(str, None),
+        description="Name of computing resource package to use"
+        )
+    COMPUTE = Argument(
+        name="compute",
+        default=(List, []),
+        description="List of key-value pairs (k1=v1)"
+    )
     DIVVY = Argument(
         name="divvy",
         default=(str, os.getenv("DIVCFG", None)),
@@ -166,7 +181,6 @@ class ArgumentEnum(enum.Enum):
             "variable. Currently: {}".format(os.getenv("DIVCFG") or "not set")
         ),
     )
-
     # Arguments for logger compatible with logmuse
     SILENT = Argument(
         name="silent", default=(bool, False), description="Whether to silence logging"
@@ -184,3 +198,8 @@ class ArgumentEnum(enum.Enum):
         "behavioral changes to logs handling, use a more information-rich "
         "message format template.",
     )
+    PIPESTAT = Argument(
+        name="pipestat",
+        default=(str, None),
+        description="Path to pipestat files.",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 5b437997a..8ceab93a7 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -55,6 +55,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
         ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value
     ],
 )
 RunParserModel = RunParser.create_model()
@@ -89,8 +92,8 @@ class TopLevelParser(pydantic.BaseModel):
     amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()
-
     # arguments for logging
     silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
     verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
     logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
+    pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()

From 5963357655db66c5548a4a714cd11ea26f1899c9 Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Wed, 31 Jan 2024 16:52:13 +0800
Subject: [PATCH 054/225] `run` special treatment: move arguments to a
 second-level namespace

Otherwise `looper run` will have, for example, issues with the
`time-delay` argument.
---
 looper/cli_looper.py | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index 82cb7997f..412c43687 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -10,6 +10,7 @@
 from ubiquerg import VersionInHelpParser
 
 from . import __version__
+from .command_models.commands import RunParser
 from .const import *
 from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
 from .exceptions import *
@@ -573,6 +574,48 @@ def _proc_resources_spec(args):
     return settings_data
 
 
+def make_hierarchical_if_needed(args):
+    """
+    Make an `argparse` namespace hierarchic for commands that support it
+
+    In the course of introducing `pydantic` models as ground truths, some logic pertaining
+    to the `run` command in `looper/looper.py` was changed in order to do accurately reflect
+    which arguments are top-level arguments and which arguments are specific to the `run`
+    command.
+
+    If the command in the given arguments is 'run', this function creates a sub-namespace
+    named 'run' and moves selected arguments specified in RUN_ARGS to the 'run' namespace.
+    The selected arguments are also removed from the original namespace.
+    This thus morphes the original namespace into the hierarchy the `run` command logic
+    expects downstream.
+
+    :param argparse.Namespace: The argparse namespace containing program arguments.
+    :return argparse.Namespace: The modified, partially hierarchical argparse namespace.
+    """
+
+    def add_command_hierarchy(command_args):
+        # make a new namespace that will be the resulting second-level namespace
+        command_namespace = argparse.Namespace()
+        for arg in vars(args):
+            if arg in command_args:
+                setattr(command_namespace, arg, getattr(args, arg))
+
+        # remove arguments that have been moved into the second-level namespace
+        # from the top-level namespace
+        for arg in command_args:
+            if hasattr(args, arg):
+                delattr(args, arg)
+
+        setattr(args, args.command, command_namespace)
+
+    if args.command == "run":
+        # we only want to only move arguments to the `run` second-level namespace
+        # that are in fact specific to the `run` subcommand
+        run_args = [argument.name for argument in RunParser.arguments]
+        add_command_hierarchy(run_args)
+
+    return args
+
 def main(test_args=None):
     """Primary workflow"""
     global _LOGGER
@@ -585,6 +628,7 @@ def main(test_args=None):
     else:
         args, remaining_args = parser.parse_known_args()
 
+    args = make_hierarchical_if_needed(args)
     cli_use_errors = validate_post_parse(args)
     if cli_use_errors:
         parser.print_help(sys.stderr)

From dfc4664e9eb9f950498045e01b46d2c6675c2fb1 Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Wed, 31 Jan 2024 17:59:35 +0800
Subject: [PATCH 055/225] `run` special treatment: adapt `divvy` argument
 retrieval

Co-authored-by: Simeon Carstens <simeon.carstens@tweag.io>
---
 looper/cli_looper.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index 412c43687..00a9c826e 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -696,9 +696,16 @@ def main(test_args=None):
             )
         )
 
-    divcfg = (
-        select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
-    )
+    if args.command == "run":
+        divcfg = (
+            select_divvy_config(filepath=args.run.divvy)
+            if hasattr(args.run, "divvy")
+            else None
+        )
+    else:
+        divcfg = (
+            select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
+        )
 
     # Ignore flags if user is selecting or excluding on flags:
     if args.sel_flag or args.exc_flag:

From 712f08d24065d943a0dadda416bdfb7c6969adc6 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 1 Feb 2024 14:43:24 +0100
Subject: [PATCH 056/225] Move `limit` and `skip` arguments from `run` to
 top-level model

---
 looper/command_models/commands.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 8ceab93a7..3b197d46f 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -52,8 +52,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
         ArgumentEnum.LUMP.value,
         ArgumentEnum.LUMPN.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
         ArgumentEnum.DIVVY.value,
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
@@ -97,3 +95,5 @@ class TopLevelParser(pydantic.BaseModel):
     verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
     logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
     pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()
+    limit: Optional[int] = ArgumentEnum.LIMIT.value.with_reduced_default()
+    skip: Optional[int] = ArgumentEnum.SKIP.value.with_reduced_default()

From 363cb2018937a231e53ac973d0c5d7fa5110d5f8 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 1 Feb 2024 14:43:43 +0100
Subject: [PATCH 057/225] Add `settings` argument to `run` model

---
 looper/command_models/commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 3b197d46f..72a6af217 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -55,7 +55,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.DIVVY.value,
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
-        ArgumentEnum.PACKAGE.value
+        ArgumentEnum.PACKAGE.value,
+        ArgumentEnum.SETTINGS.value
     ],
 )
 RunParserModel = RunParser.create_model()

From fcb70e520b9b0b1ec25869af27b97626228db1ab Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 31 Jan 2024 17:37:58 +0100
Subject: [PATCH 058/225] `run` special treatment: `_proc_resources_spec`

---
 looper/cli_looper.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index 00a9c826e..bf6df9d82 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -546,13 +546,18 @@ def _proc_resources_spec(args):
     :raise ValueError: if interpretation of the given specification as encoding
         of key-value pairs fails
     """
-    spec = getattr(args, "compute", None)
+    if (hasattr(args, "run") and args.run) or args.command in ("run",):
+        spec = getattr(args.run, "compute", None)
+        settings = args.run.settings
+    else:
+        spec = getattr(args, "compute", None)
+        settings = args.settings
     try:
-        settings_data = read_yaml_file(args.settings) or {}
+        settings_data = read_yaml_file(settings) or {}
     except yaml.YAMLError:
         _LOGGER.warning(
             "Settings file ({}) does not follow YAML format,"
-            " disregarding".format(args.settings)
+            " disregarding".format(settings)
         )
         settings_data = {}
     if not spec:

From 0182e43172c71ac83afb654619601ef7991ef1ae Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 31 Jan 2024 17:39:17 +0100
Subject: [PATCH 059/225] `run` special treatment: `validate_post_parse`

---
 looper/cli_looper.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index bf6df9d82..c2661d2c4 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -523,7 +523,10 @@ def validate_post_parse(args: argparse.Namespace) -> List[str]:
                 SAMPLE_INCLUSION_OPTNAME,
             ],
         )
-        if getattr(args, attr, None)
+        # Depending on the subcommand used, the above options might either be in
+        # the top-level namespace or in the subcommand namespace (the latter due
+        # to a `modify_args_namespace()`)
+        if getattr(args, attr, None)# or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
     ]
     if len(used_exclusives) > 1:
         problems.append(
@@ -689,7 +692,6 @@ def main(test_args=None):
             )
 
     args = enrich_args_via_cfg(args, aux_parser, test_args)
-
     # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
     if vars(args)[PROJECT_PL_ARG]:
         args.pipeline_interfaces = vars(args)[PROJECT_PL_ARG]

From e3bc2157b15932a8560593583fc3e7479a89d518 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Wed, 31 Jan 2024 23:17:09 +0100
Subject: [PATCH 060/225] `run` special treatment: project CLI attributes

---
 looper/cli_looper.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index c2661d2c4..792ec0057 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -738,14 +738,19 @@ def main(test_args=None):
             )
     else:
         try:
+            project_args = {
+                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+            }
+            if args.command == "run":
+                project_args.update(**{
+                    attr: getattr(args.run, attr) for attr in CLI_PROJ_ATTRS if attr in args.run
+                })
             p = Project(
                 cfg=args.config_file,
                 amendments=args.amend,
                 divcfg_path=divcfg,
                 runp=args.command == "runp",
-                **{
-                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
-                },
+                **project_args,
             )
         except yaml.parser.ParserError as e:
             _LOGGER.error(f"Project config parse failed -- {e}")

From c21d8ed52defd1a29a238832c16b82ab1d9b592b Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 1 Feb 2024 15:35:29 +0100
Subject: [PATCH 061/225] Remove debug print statements

---
 looper/cli_pydantic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 945ddd919..831788383 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -176,8 +176,6 @@ def main() -> None:
         add_help=True,
     )
     args = parser.parse_typed_args()
-    print(args)
-    print("#########################################")
     run_looper(args, parser)
 
 

From 2229334adc5b3d46ed6715d255347ce24b1e03de Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 1 Feb 2024 16:01:07 +0100
Subject: [PATCH 062/225] Run formatter

---
 looper/cli_looper.py               | 17 ++++++++++++-----
 looper/command_models/arguments.py |  8 ++++----
 looper/command_models/commands.py  |  2 +-
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index 792ec0057..06d3b4e1c 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -526,7 +526,9 @@ def validate_post_parse(args: argparse.Namespace) -> List[str]:
         # Depending on the subcommand used, the above options might either be in
         # the top-level namespace or in the subcommand namespace (the latter due
         # to a `modify_args_namespace()`)
-        if getattr(args, attr, None)# or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
+        if getattr(
+            args, attr, None
+        )  # or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
     ]
     if len(used_exclusives) > 1:
         problems.append(
@@ -624,6 +626,7 @@ def add_command_hierarchy(command_args):
 
     return args
 
+
 def main(test_args=None):
     """Primary workflow"""
     global _LOGGER
@@ -739,12 +742,16 @@ def main(test_args=None):
     else:
         try:
             project_args = {
-                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
             }
             if args.command == "run":
-                project_args.update(**{
-                    attr: getattr(args.run, attr) for attr in CLI_PROJ_ATTRS if attr in args.run
-                })
+                project_args.update(
+                    **{
+                        attr: getattr(args.run, attr)
+                        for attr in CLI_PROJ_ATTRS
+                        if attr in args.run
+                    }
+                )
             p = Project(
                 cfg=args.config_file,
                 amendments=args.amend,
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index b23c3a6c0..8ceea244a 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -161,17 +161,17 @@ class ArgumentEnum(enum.Enum):
     SKIP_FILE_CHECKS = Argument(
         name="skip_file_checks",
         default=(bool, False),
-        description="Do not perform input file checks"
+        description="Do not perform input file checks",
     )
     PACKAGE = Argument(
         name="package",
         default=(str, None),
-        description="Name of computing resource package to use"
-        )
+        description="Name of computing resource package to use",
+    )
     COMPUTE = Argument(
         name="compute",
         default=(List, []),
-        description="List of key-value pairs (k1=v1)"
+        description="List of key-value pairs (k1=v1)",
     )
     DIVVY = Argument(
         name="divvy",
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 72a6af217..605864563 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -56,7 +56,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
-        ArgumentEnum.SETTINGS.value
+        ArgumentEnum.SETTINGS.value,
     ],
 )
 RunParserModel = RunParser.create_model()

From b0d5b4a142449d94ef73ba315c60a8a7c076ad9d Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Thu, 1 Feb 2024 16:17:56 +0100
Subject: [PATCH 063/225] Appease `black` version in CI

---
 looper/command_models/commands.py | 18 +++++++++---------
 tests/smoketests/test_run.py      | 18 +++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 605864563..e451ae10e 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -79,15 +79,15 @@ class TopLevelParser(pydantic.BaseModel):
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
     output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
     config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    looper_config: Optional[
-        str
-    ] = ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
-    sample_pipeline_interfaces: Optional[
-        List[str]
-    ] = ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
-    project_pipeline_interfaces: Optional[
-        List[str]
-    ] = ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    looper_config: Optional[str] = (
+        ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
+    )
+    sample_pipeline_interfaces: Optional[List[str]] = (
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
+    )
+    project_pipeline_interfaces: Optional[List[str]] = (
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    )
     amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index c646103fc..f34adf926 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -123,9 +123,9 @@ def test_looper_single_pipeline(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
 
         x = test_args_expansion(tp, "run")
         try:
@@ -140,9 +140,9 @@ def test_looper_var_templates(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
         x = test_args_expansion(tp, "run")
         try:
             # Test that {looper.piface_dir} is correctly rendered to a path which will show up in the final .sub file
@@ -211,9 +211,9 @@ def test_looper_pipeline_invalid(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
         piface_path = os.path.join(os.path.dirname(tp), pifaces[1])
         with mod_yaml_data(piface_path) as piface_data:
             del piface_data["pipeline_name"]

From 1601fddc1750ff90ebf1261dac5f8bed0a5f496a Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Fri, 26 Jan 2024 16:20:20 +0800
Subject: [PATCH 064/225] Add necessary arguments: sel_attr, sel_incl, sel_excl

---
 looper/cli_pydantic.py             | 14 +++++++-------
 looper/command_models/arguments.py | 15 +++++++++++++++
 looper/command_models/commands.py  |  3 +++
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 831788383..d895b7ac3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -110,7 +110,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             p = Project(
                 amendments=args.amend,
                 divcfg_path=divcfg,
-                runp=args.command == "runp",
+                runp=subcommand_name == "runp",
                 project_dict=PEPHubClient()._load_raw_pep(
                     registry_path=args.config_file
                 ),
@@ -128,7 +128,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
                 cfg=args.config_file,
                 amendments=args.amend,
                 divcfg_path=divcfg,
-                runp=False,
+                runp=subcommand_name == "runp",
                 **{
                     attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
                 },
@@ -146,11 +146,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
 
     with ProjectContext(
         prj=p,
-        selector_attribute="toggle",
-        selector_include=None,
-        selector_exclude=None,
-        selector_flag=None,
-        exclusion_flag=None,
+        selector_attribute=args.sel_attr,
+        selector_include=args.sel_incl,
+        selector_exclude=args.sel_excl,
+        selector_flag=args.sel_flag,
+        exclusion_flag=args.exc_flag,
     ) as prj:
         if subcommand_name == "run":
             run = Runner(prj)
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 8ceea244a..66516852e 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -152,6 +152,21 @@ class ArgumentEnum(enum.Enum):
     AMEND = Argument(
         name="amend", default=(List, []), description="List of amendments to activate"
     )
+    SEL_ATTR = Argument(
+        name="sel_attr",
+        default=(str, "toggle"),
+        description="Attribute for sample exclusion OR inclusion",
+    )
+    SEL_INCL = Argument(
+        name="sel_incl",
+        default=(str, ""),
+        description="Include only samples with these values",
+    )
+    SEL_EXCL = Argument(
+        name="sel_excl",
+        default=(str, ""),
+        description="Exclude samples with these values",
+    )
     SEL_FLAG = Argument(
         name="sel_flag", default=(str, ""), description="Sample selection flag"
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index e451ae10e..f9c494b3f 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -89,6 +89,9 @@ class TopLevelParser(pydantic.BaseModel):
         ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
     )
     amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
+    sel_attr: Optional[str] = ArgumentEnum.SEL_ATTR.value.with_reduced_default()
+    sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
+    sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()
     # arguments for logging

From 380de82c6ab7a5cd40dcb3c71a3c523e5b4111b8 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Thu, 1 Feb 2024 16:13:31 -0500
Subject: [PATCH 065/225] clean up publish action

---
 .github/workflows/python-publish.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index cf8fa182b..365a8b1b1 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -12,12 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     name: upload release to PyPI
     permissions:
-      # IMPORTANT: this permission is mandatory for trusted publishing
-      id-token: write
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -25,9 +24,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install setuptools wheel twine
       - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           python setup.py sdist bdist_wheel
       - name: Publish package distributions to PyPI

From 6d70a6948ae1ebcc0b791574a6a6ae127b51a78b Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Thu, 1 Feb 2024 18:35:02 +0800
Subject: [PATCH 066/225] Add Developer documentation for adding new models

---
 looper/command_models/DEVELOPER.md | 64 ++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 looper/command_models/DEVELOPER.md

diff --git a/looper/command_models/DEVELOPER.md b/looper/command_models/DEVELOPER.md
new file mode 100644
index 000000000..514f41297
--- /dev/null
+++ b/looper/command_models/DEVELOPER.md
@@ -0,0 +1,64 @@
+# Developer Documentation
+
+## Adding New Models
+
+To add a new model (command) to the project, follow these steps:
+
+1. Add new arguments in `looper/command_models/arguments.py` if necessary.
+
+- Add a new entry for the `ArgumentEnum` class.
+- For example:
+
+```python
+# arguments.py
+
+class ArgumentEnum(enum.Enum):
+    ...
+
+    NEW_ARGUMENT = Argument(
+        name="new_argument",
+        default=(new_argument_type, "default_value"),
+        description="Description of the new argument",
+    )
+
+```
+
+2. Create a new command in the existing command creation logic in `looper/command_models/commands.py`.
+
+- Create a new `Command` instance.
+- Create a `pydantic` model for this new command.
+- Add the new `Command` instance to `SUPPORTED_COMMANDS`.
+- For example:
+
+```python
+NewCommandParser = Command(
+    "new_command",
+    MESSAGE_BY_SUBCOMMAND["new_command"],
+    [
+        ...
+        ArgumentEnum.NEW_ARGUMENT.value,
+        # Add more arguments as needed for the new command
+    ],
+)
+NewCommandParserModel = NewCommandParser.create_model()
+
+SUPPORTED_COMMANDS = [..., NewCommandParser]
+```
+
+3. Update the new argument(s) and command in `TopLevelParser` from `looper/command_models/commands.py`.
+
+- Add a new field for the new command.
+- Add a new field for the new argument(s).
+- For example:
+
+```python
+class TopLevelParser(pydantic.BaseModel):
+
+    # commands
+    ...
+    new_command: Optional[NewCommandParserModel] = pydantic.Field(description=NewCommandParser.description)
+
+    # arguments
+    ...
+    new_argument: Optional[new_argument_type] = ArgumentEnum.NEW_ARGUMENT.value.with_reduced_default()
+```

From bb30baec35e1af6138ed025aae6028098bc447ef Mon Sep 17 00:00:00 2001
From: Zhihan Zhang <zhihan.zhang@tweag.io>
Date: Fri, 2 Feb 2024 13:35:27 +0800
Subject: [PATCH 067/225] Add section: special treatment for the `run` command

---
 looper/command_models/DEVELOPER.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/looper/command_models/DEVELOPER.md b/looper/command_models/DEVELOPER.md
index 514f41297..13db6c24d 100644
--- a/looper/command_models/DEVELOPER.md
+++ b/looper/command_models/DEVELOPER.md
@@ -62,3 +62,18 @@ class TopLevelParser(pydantic.BaseModel):
     ...
     new_argument: Optional[new_argument_type] = ArgumentEnum.NEW_ARGUMENT.value.with_reduced_default()
 ```
+
+## Special Treatment for the `run` Command
+
+The `run` command in our project requires special treatment to accommodate hierarchical namespaces
+and properly handle its unique characteristics. Several functions have been adapted to ensure the
+correct behavior of the run command, and similar adaptations may be necessary for other commands.
+
+For developers looking to understand the details of the special treatment given to the `run`
+command and its associated changes, commits with titles starting with **"`run` special treatment:"**
+provide valuable insights. These commits encapsulate modifications made to functions such as
+`make_hierarchical_if_needed`, adjustments in `divvy` argument retrieval, adaptations to
+functions like `_proc_resources_spec`, `validate_post_parse`, and also project CLI
+attributes. If you are considering adding new commands to the project, it is recommended to follow
+these commits for guidance on adapting functions and ensuring consistent behavior across different
+commands.

From d74d6087e75da94d3374dd64d7e722bcdbb46fe4 Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 2 Feb 2024 10:21:25 +0100
Subject: [PATCH 068/225] Detail locations of `run` command special treatments
 in DEVELOPER.md

I realized that referring to commits / commit message titles is not a
good idea (although it was originally mine), because commits might get
squashed. So this replaces the mention of commits with the relevant precise
function names / places in the code.
---
 looper/command_models/DEVELOPER.md | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/looper/command_models/DEVELOPER.md b/looper/command_models/DEVELOPER.md
index 13db6c24d..68ef080ff 100644
--- a/looper/command_models/DEVELOPER.md
+++ b/looper/command_models/DEVELOPER.md
@@ -63,17 +63,23 @@ class TopLevelParser(pydantic.BaseModel):
     new_argument: Optional[new_argument_type] = ArgumentEnum.NEW_ARGUMENT.value.with_reduced_default()
 ```
 
-## Special Treatment for the `run` Command
+## Special treatment for the `run` command
 
 The `run` command in our project requires special treatment to accommodate hierarchical namespaces
 and properly handle its unique characteristics. Several functions have been adapted to ensure the
 correct behavior of the run command, and similar adaptations may be necessary for other commands.
 
 For developers looking to understand the details of the special treatment given to the `run`
-command and its associated changes, commits with titles starting with **"`run` special treatment:"**
-provide valuable insights. These commits encapsulate modifications made to functions such as
-`make_hierarchical_if_needed`, adjustments in `divvy` argument retrieval, adaptations to
-functions like `_proc_resources_spec`, `validate_post_parse`, and also project CLI
-attributes. If you are considering adding new commands to the project, it is recommended to follow
-these commits for guidance on adapting functions and ensuring consistent behavior across different
-commands.
+command and its associated changes, we recommend to inspect the following functions / part of the
+code:
+- `looper/cli_looper.py`:
+  - `make_hierarchical_if_needed()`
+  - assignment of the `divcfg` variable
+  - assignment of the `project_args` variable
+  - `_proc_resources_spec()`
+  - `validate_post_parse()`
+- `looper/utils.py`:
+  - `enrich_args_via_cfg()`
+
+If you are adding new commands to the project / migrate existing commands to a `pydantic` model-based definition, adapt these parts of the codes with equivalent behavior for your new command.
+Likewise, adapt argument accessions in the corresponding executor in `looper/looper.py` to take into account the hierarchical organization of the command's arguments.

From 3e9127ae66cb8a8f442eb6faf10ff37605c1202f Mon Sep 17 00:00:00 2001
From: Simeon Carstens <simeon.carstens@tweag.io>
Date: Fri, 2 Feb 2024 10:26:18 +0100
Subject: [PATCH 069/225] Fix some capitalizations

---
 looper/command_models/DEVELOPER.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/DEVELOPER.md b/looper/command_models/DEVELOPER.md
index 68ef080ff..d71f7bf65 100644
--- a/looper/command_models/DEVELOPER.md
+++ b/looper/command_models/DEVELOPER.md
@@ -1,6 +1,6 @@
-# Developer Documentation
+# Developer documentation
 
-## Adding New Models
+## Adding new command models
 
 To add a new model (command) to the project, follow these steps:
 

From 0a49dd2e2095ff84a309919b56d2abcf9e6c5959 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:29:40 -0500
Subject: [PATCH 070/225] finish removing attmap from divvy

---
 looper/divvy.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 9107907f9..0a2af1c0f 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -6,11 +6,15 @@
 import sys
 import shutil
 import yaml
-from yaml import SafeLoader
-from shutil import copytree
 
+
+from shutil import copytree
+from yacman import YAMLConfigManager as YAMLConfigManager
+from yacman import FILEPATH_KEY, load_yaml, select_config
+from yaml import SafeLoader
 from ubiquerg import is_writable, VersionInHelpParser
-import yacman
+
+
 
 from .const import (
     COMPUTE_SETTINGS_VARNAME,
@@ -28,7 +32,7 @@
 # This is the divvy.py submodule from divvy
 
 
-class ComputingConfiguration(yacman.YAMLConfigManager):
+class ComputingConfiguration(YAMLConfigManager):
     """
     Represents computing configuration objects.
 
@@ -73,7 +77,7 @@ def __init__(self, entries=None, filepath=None):
 
     def write(self, filename=None):
         super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
-        filename = filename or getattr(self, yacman.FILEPATH_KEY)
+        filename = filename or getattr(self, FILEPATH_KEY)
         filedir = os.path.dirname(filename)
         # For this object, we *also* have to write the template files
         for pkg_name, pkg in self["compute_packages"].items():
@@ -151,7 +155,7 @@ def activate_package(self, package_name):
             # Augment compute, creating it if needed.
             if self.compute is None:
                 _LOGGER.debug("Creating Project compute")
-                self.compute = yacman.YAMLConfigManager()
+                self.compute = YAMLConfigManager()
                 _LOGGER.debug(
                     "Adding entries for package_name '{}'".format(package_name)
                 )
@@ -200,11 +204,11 @@ def clean_start(self, package_name):
         self.reset_active_settings()
         return self.activate_package(package_name)
 
-    def get_active_package(self):
+    def get_active_package(self) -> YAMLConfigManager:
         """
         Returns settings for the currently active compute package
 
-        :return yacman.YacAttMap: data defining the active compute package
+        :return YAMLConfigManager: data defining the active compute package
         """
         return self.compute
 
@@ -222,7 +226,7 @@ def reset_active_settings(self):
 
         :return bool: success flag
         """
-        self.compute = yacman.YacAttMap()
+        self.compute = YAMLConfigManager()
         return True
 
     def update_packages(self, config_file):
@@ -235,11 +239,11 @@ def update_packages(self, config_file):
 
         :param str config_file: path to file with new divvy configuration data
         """
-        entries = yacman.load_yaml(config_file)
+        entries = load_yaml(config_file)
         self.update(entries)
         return True
 
-    def get_adapters(self):
+    def get_adapters(self) -> YAMLConfigManager:
         """
         Get current adapters, if defined.
 
@@ -248,9 +252,9 @@ def get_adapters(self):
         package-specific set of adapters, if any defined in 'adapters' section
         under currently active compute package.
 
-        :return yacman.YAMLConfigManager: current adapters mapping
+        :return YAMLConfigManager: current adapters mapping
         """
-        adapters = yacman.YAMLConfigManager()
+        adapters = YAMLConfigManager()
         if "adapters" in self and self["adapters"] is not None:
             adapters.update(self["adapters"])
         if "compute" in self and "adapters" in self.compute:
@@ -376,7 +380,7 @@ def select_divvy_config(filepath):
     :param str | NoneType filepath: direct file path specification
     :return str: path to the config file to read
     """
-    divcfg = yacman.select_config(
+    divcfg = select_config(
         config_filepath=filepath,
         config_env_vars=COMPUTE_SETTINGS_VARNAME,
         default_config_filepath=DEFAULT_CONFIG_FILEPATH,

From 98ec9f593b303b95e41eb4aa5ba3e4d87e2937ac Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:31:36 -0500
Subject: [PATCH 071/225] lint

---
 looper/divvy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 0a2af1c0f..2ed776ac3 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -15,7 +15,6 @@
 from ubiquerg import is_writable, VersionInHelpParser
 
 
-
 from .const import (
     COMPUTE_SETTINGS_VARNAME,
     DEFAULT_COMPUTE_RESOURCES_NAME,

From 5ee9a73db61499facc58118048fc541ce10dc5f8 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:32:59 -0500
Subject: [PATCH 072/225] lint unrelated file with new black

---
 tests/smoketests/test_run.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 5d166fe38..059c2721a 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -123,9 +123,9 @@ def test_looper_single_pipeline(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
 
         x = test_args_expansion(tp, "run")
         try:
@@ -140,9 +140,9 @@ def test_looper_var_templates(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
         x = test_args_expansion(tp, "run")
         try:
             # Test that {looper.piface_dir} is correctly rendered to a path which will show up in the final .sub file
@@ -211,9 +211,9 @@ def test_looper_pipeline_invalid(self, prep_temp_pep):
             pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
                 PIPELINE_INTERFACES_KEY
             ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ] = pifaces[1]
+            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
+                pifaces[1]
+            )
         piface_path = os.path.join(os.path.dirname(tp), pifaces[1])
         with mod_yaml_data(piface_path) as piface_data:
             del piface_data["pipeline_name"]

From 4b1b2225b0662a0e3de9cd48f10a45fa28252233 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:36:35 -0500
Subject: [PATCH 073/225] use FutureYAMLConfigManager

---
 looper/conductor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 807d34f3e..5301772a8 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -21,7 +21,7 @@
 from pipestat import PipestatError
 from ubiquerg import expandpath, is_command_callable
 from yaml import dump
-from yacman import YAMLConfigManager
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
 
 from .const import *
 from .exceptions import JobSubmissionException, SampleFailedException

From f479b800a286f0392151a76d90fb215a6ba29f21 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:47:20 -0500
Subject: [PATCH 074/225] require yacman with FutureYAMLConfigManager

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 0f3963a5e..3851734f4 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -10,4 +10,4 @@ pipestat>=0.8.0
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2
-yacman>=0.9.2
+yacman==0.9.3

From 0c41a9a87cc980e64df41e8a0b0fb499e3055ae8 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 7 Feb 2024 14:44:40 -0500
Subject: [PATCH 075/225] add newest pipestat pre-release for yacman 0.9.3
 compatibility

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 3851734f4..3e81e07bf 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.1.2
 peppy>=0.40.0
-pipestat>=0.8.0
+pipestat>=0.8.2a1
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2

From 0d9194f10a2033158bba5b4ab7c9b69bf74d5113 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 8 Feb 2024 11:23:56 -0500
Subject: [PATCH 076/225] Computing Configuration inherits from
 FutureYamlConfigManager 0.9.3

---
 looper/divvy.py              | 26 ++++++++++++++++----------
 tests/divvytests/conftest.py |  8 +++++++-
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 2ed776ac3..4c4127cfc 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -9,7 +9,7 @@
 
 
 from shutil import copytree
-from yacman import YAMLConfigManager as YAMLConfigManager
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
 from yacman import FILEPATH_KEY, load_yaml, select_config
 from yaml import SafeLoader
 from ubiquerg import is_writable, VersionInHelpParser
@@ -47,25 +47,31 @@ class ComputingConfiguration(YAMLConfigManager):
         `DIVCFG` file)
     """
 
-    def __init__(self, entries=None, filepath=None):
+    def __init__(
+        self,
+        entries=None,
+        filepath=None,
+        schema_source=None,
+        validate_on_write=False,
+    ):
+        super(ComputingConfiguration, self).__init__(
+            validate_on_write=validate_on_write
+        )
+
         if not entries and not filepath:
             # Handle the case of an empty one, when we'll use the default
             filepath = select_divvy_config(None)
 
-        super(ComputingConfiguration, self).__init__(
-            entries=entries,
-            filepath=filepath,
-            schema_source=DEFAULT_CONFIG_SCHEMA,
-            validate_on_write=True,
+        temp_ym = YAMLConfigManager.from_yaml_file(
+            filepath=filepath, schema_source=DEFAULT_CONFIG_SCHEMA
         )
-
+        self.data.update(temp_ym.data)
+        self.filepath = filepath
         if not "compute_packages" in self:
             raise Exception(
                 "Your divvy config file is not in divvy config format "
                 "(it lacks a compute_packages section): '{}'".format(filepath)
             )
-            # We require that compute_packages be present, even if empty
-            self["compute_packages"] = {}
 
         # Initialize default compute settings.
         _LOGGER.debug("Establishing project compute settings")
diff --git a/tests/divvytests/conftest.py b/tests/divvytests/conftest.py
index c194a82af..eca9c8413 100644
--- a/tests/divvytests/conftest.py
+++ b/tests/divvytests/conftest.py
@@ -3,11 +3,17 @@
 import looper.divvy as divvy
 import pytest
 
+from looper.divvy import select_divvy_config, DEFAULT_CONFIG_SCHEMA
+
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(THIS_DIR, "data/divcfg-master")
 FILES = glob.glob(DATA_DIR + "/*.yaml")
-DCC_ATTRIBUTES = divvy.ComputingConfiguration().keys()
+DCC_ATTRIBUTES = divvy.ComputingConfiguration(
+    filepath=select_divvy_config(None),
+    schema_source=DEFAULT_CONFIG_SCHEMA,
+    validate_on_write=True,
+).keys()
 
 
 @pytest.fixture

From 23ea9abc7ddf26c57410d77d540db716e859109b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 8 Feb 2024 11:35:20 -0500
Subject: [PATCH 077/225] add schema info via temp_ym

---
 looper/divvy.py              | 7 +++++--
 tests/divvytests/conftest.py | 6 +-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 4c4127cfc..a128da1f7 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -52,8 +52,9 @@ def __init__(
         entries=None,
         filepath=None,
         schema_source=None,
-        validate_on_write=False,
+        validate_on_write=True,
     ):
+
         super(ComputingConfiguration, self).__init__(
             validate_on_write=validate_on_write
         )
@@ -66,8 +67,10 @@ def __init__(
             filepath=filepath, schema_source=DEFAULT_CONFIG_SCHEMA
         )
         self.data.update(temp_ym.data)
+        setattr(self, "schema", temp_ym.schema)
+        self.schema_source = temp_ym.schema_source
         self.filepath = filepath
-        if not "compute_packages" in self:
+        if "compute_packages" not in self:
             raise Exception(
                 "Your divvy config file is not in divvy config format "
                 "(it lacks a compute_packages section): '{}'".format(filepath)
diff --git a/tests/divvytests/conftest.py b/tests/divvytests/conftest.py
index eca9c8413..8a6ecff51 100644
--- a/tests/divvytests/conftest.py
+++ b/tests/divvytests/conftest.py
@@ -9,11 +9,7 @@
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(THIS_DIR, "data/divcfg-master")
 FILES = glob.glob(DATA_DIR + "/*.yaml")
-DCC_ATTRIBUTES = divvy.ComputingConfiguration(
-    filepath=select_divvy_config(None),
-    schema_source=DEFAULT_CONFIG_SCHEMA,
-    validate_on_write=True,
-).keys()
+DCC_ATTRIBUTES = divvy.ComputingConfiguration().keys()
 
 
 @pytest.fixture

From 286893c04143b767f8f5de682cbdda4ed7d52db6 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 8 Feb 2024 11:47:35 -0500
Subject: [PATCH 078/225] control statements for filepath vs entries

---
 looper/divvy.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index a128da1f7..d32b19688 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -63,13 +63,24 @@ def __init__(
             # Handle the case of an empty one, when we'll use the default
             filepath = select_divvy_config(None)
 
-        temp_ym = YAMLConfigManager.from_yaml_file(
-            filepath=filepath, schema_source=DEFAULT_CONFIG_SCHEMA
-        )
+        if filepath:
+            temp_ym = YAMLConfigManager.from_yaml_file(
+                filepath=filepath, schema_source=DEFAULT_CONFIG_SCHEMA
+            )
+            self.filepath = filepath
+        elif entries:
+            temp_ym = YAMLConfigManager.from_obj(
+                entries=entries, schema_source=DEFAULT_CONFIG_SCHEMA
+            )
+        else:
+            raise Exception(
+                "Must provide either filepath or entries when creating ComputingConfiguration."
+            )
+
         self.data.update(temp_ym.data)
         setattr(self, "schema", temp_ym.schema)
         self.schema_source = temp_ym.schema_source
-        self.filepath = filepath
+
         if "compute_packages" not in self:
             raise Exception(
                 "Your divvy config file is not in divvy config format "
@@ -81,7 +92,9 @@ def __init__(
         self.compute = None
         self.setdefault("adapters", None)
         self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
-        self.config_file = self.filepath
+        self.config_file = (
+            self.filepath
+        )  # TODO this seems problematic if using entries for creation. There is no filepath.
 
     def write(self, filename=None):
         super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)

From f14af3b3018c6ae02415285ade1237230ee63fe4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 8 Feb 2024 12:14:14 -0500
Subject: [PATCH 079/225] implement write_lock in write func

---
 looper/divvy.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index d32b19688..3d49168a1 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -10,7 +10,7 @@
 
 from shutil import copytree
 from yacman import FutureYAMLConfigManager as YAMLConfigManager
-from yacman import FILEPATH_KEY, load_yaml, select_config
+from yacman import write_lock, FILEPATH_KEY, load_yaml, select_config
 from yaml import SafeLoader
 from ubiquerg import is_writable, VersionInHelpParser
 
@@ -97,7 +97,9 @@ def __init__(
         )  # TODO this seems problematic if using entries for creation. There is no filepath.
 
     def write(self, filename=None):
-        super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
+        with write_lock(self) as locked_ym:
+            locked_ym.rebase()
+            locked_ym.write()
         filename = filename or getattr(self, FILEPATH_KEY)
         filedir = os.path.dirname(filename)
         # For this object, we *also* have to write the template files

From 4fdecf3cd15debf15c99fea824329450f94be66e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 12 Feb 2024 10:27:25 -0500
Subject: [PATCH 080/225] implement working FutureYamlConfigManager using class
 methods, rename testing funcs

---
 looper/divvy.py                               | 45 ++++---------------
 looper/project.py                             |  2 +-
 tests/divvytests/conftest.py                  | 10 +++--
 tests/divvytests/divvy_tests/test_divvy.py    | 12 ++---
 .../regression/test_write_script.py           |  5 ++-
 tests/divvytests/test_divvy_simple.py         |  7 ++-
 6 files changed, 30 insertions(+), 51 deletions(-)

diff --git a/looper/divvy.py b/looper/divvy.py
index 3d49168a1..bd880f948 100644
--- a/looper/divvy.py
+++ b/looper/divvy.py
@@ -50,51 +50,22 @@ class ComputingConfiguration(YAMLConfigManager):
     def __init__(
         self,
         entries=None,
-        filepath=None,
+        wait_max=None,
+        strict_ro_locks=False,
         schema_source=None,
-        validate_on_write=True,
+        validate_on_write=False,
     ):
-
-        super(ComputingConfiguration, self).__init__(
-            validate_on_write=validate_on_write
+        super().__init__(
+            entries, wait_max, strict_ro_locks, schema_source, validate_on_write
         )
 
-        if not entries and not filepath:
-            # Handle the case of an empty one, when we'll use the default
-            filepath = select_divvy_config(None)
-
-        if filepath:
-            temp_ym = YAMLConfigManager.from_yaml_file(
-                filepath=filepath, schema_source=DEFAULT_CONFIG_SCHEMA
-            )
-            self.filepath = filepath
-        elif entries:
-            temp_ym = YAMLConfigManager.from_obj(
-                entries=entries, schema_source=DEFAULT_CONFIG_SCHEMA
-            )
-        else:
-            raise Exception(
-                "Must provide either filepath or entries when creating ComputingConfiguration."
-            )
-
-        self.data.update(temp_ym.data)
-        setattr(self, "schema", temp_ym.schema)
-        self.schema_source = temp_ym.schema_source
-
         if "compute_packages" not in self:
-            raise Exception(
-                "Your divvy config file is not in divvy config format "
-                "(it lacks a compute_packages section): '{}'".format(filepath)
-            )
-
+            self["compute_packages"] = {}
         # Initialize default compute settings.
         _LOGGER.debug("Establishing project compute settings")
         self.compute = None
         self.setdefault("adapters", None)
         self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
-        self.config_file = (
-            self.filepath
-        )  # TODO this seems problematic if using entries for creation. There is no filepath.
 
     def write(self, filename=None):
         with write_lock(self) as locked_ym:
@@ -183,7 +154,7 @@ def activate_package(self, package_name):
                     "Adding entries for package_name '{}'".format(package_name)
                 )
 
-            self.compute.update(self["compute_packages"][package_name])
+            self.compute.update_from_obj(self["compute_packages"][package_name])
 
             # Ensure submission template is absolute. This *used to be* handled
             # at update (so the paths were stored as absolutes in the packages),
@@ -192,7 +163,7 @@ def activate_package(self, package_name):
             if not os.path.isabs(self.compute["submission_template"]):
                 try:
                     self.compute["submission_template"] = os.path.join(
-                        os.path.dirname(self.filepath),
+                        os.path.dirname(self.default_config_file),
                         self.compute["submission_template"],
                     )
                 except AttributeError as e:
diff --git a/looper/project.py b/looper/project.py
index 6607db6e2..c67f1ce08 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -144,7 +144,7 @@ def __init__(self, cfg=None, amendments=None, divcfg_path=None, **kwargs):
         self.dcc = (
             None
             if divcfg_path is None
-            else ComputingConfiguration(filepath=divcfg_path)
+            else ComputingConfiguration.from_yaml_file(filepath=divcfg_path)
         )
         if DRY_RUN_KEY in self and not self[DRY_RUN_KEY]:
             _LOGGER.debug("Ensuring project directories exist")
diff --git a/tests/divvytests/conftest.py b/tests/divvytests/conftest.py
index 8a6ecff51..7d1111f41 100644
--- a/tests/divvytests/conftest.py
+++ b/tests/divvytests/conftest.py
@@ -9,19 +9,23 @@
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(THIS_DIR, "data/divcfg-master")
 FILES = glob.glob(DATA_DIR + "/*.yaml")
-DCC_ATTRIBUTES = divvy.ComputingConfiguration().keys()
+
+
+dcc_filepath = select_divvy_config(None)
+DCC = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
+DCC_ATTRIBUTES = DCC.keys()
 
 
 @pytest.fixture
 def empty_dcc():
     """Provide the empty/default ComputingConfiguration object"""
-    return divvy.ComputingConfiguration()
+    return divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
 
 
 @pytest.fixture(params=FILES)
 def dcc(request):
     """Provide ComputingConfiguration objects for all files in divcfg repository"""
-    return divvy.ComputingConfiguration(filepath=request.param)
+    return divvy.ComputingConfiguration.from_yaml_file(filepath=request.param)
 
 
 @pytest.fixture
diff --git a/tests/divvytests/divvy_tests/test_divvy.py b/tests/divvytests/divvy_tests/test_divvy.py
index aa8fa85ee..da9ce7c83 100644
--- a/tests/divvytests/divvy_tests/test_divvy.py
+++ b/tests/divvytests/divvy_tests/test_divvy.py
@@ -6,7 +6,7 @@
 from tests.divvytests.conftest import DCC_ATTRIBUTES, FILES, mock_env_missing
 
 
-class DefaultDCCTests:
+class TestDefaultDCC:
     """Tests the default divvy.ComputingConfiguration object creation"""
 
     def test_no_args(self, empty_dcc):
@@ -22,7 +22,7 @@ def test_no_env_var(self, mock_env_missing, empty_dcc):
         empty_dcc
 
 
-class DCCTests:
+class TestDCC:
     """Tests the divvy.ComputingConfiguration object creation"""
 
     def test_object_creation(self, dcc):
@@ -35,7 +35,7 @@ def test_attrs_produced(self, att, dcc):
         dcc[att]
 
 
-class ActivatingTests:
+class TestActivating:
     """Test for the activate_package method"""
 
     def test_activating_default_package(self, dcc):
@@ -56,7 +56,7 @@ def test_not_activating_faulty_package(self, dcc, package):
         assert not dcc.activate_package(package)
 
 
-class GettingActivePackageTests:
+class TestGettingActivePackage:
     """Test for the get_active_package method"""
 
     def test_settings_nonempty(self, dcc):
@@ -65,7 +65,7 @@ def test_settings_nonempty(self, dcc):
         assert settings != YacAttMap()
 
 
-class ListingPackagesTests:
+class TestListingPackages:
     """Test for the list_compute_packages method"""
 
     def test_list_compute_packages_is_set(self, dcc):
@@ -77,7 +77,7 @@ def test_list_compute_packages_result_nonempty(self, dcc):
         assert dcc.list_compute_packages() != set()
 
 
-class ResettingSettingsTests:
+class TestResettingSettings:
     """ " Test for the reset_active_settings method"""
 
     def test_reset_active_settings(self, dcc):
diff --git a/tests/divvytests/regression/test_write_script.py b/tests/divvytests/regression/test_write_script.py
index c5b071fbf..1159ee46a 100644
--- a/tests/divvytests/regression/test_write_script.py
+++ b/tests/divvytests/regression/test_write_script.py
@@ -3,7 +3,7 @@
 from copy import deepcopy
 import random
 import pytest
-from looper.divvy import ComputingConfiguration
+from looper.divvy import ComputingConfiguration, select_divvy_config
 from tests.divvytests.helpers import get_random_key
 
 __author__ = "Vince Reuter"
@@ -19,7 +19,8 @@
 )
 def test_write_script_is_effect_free(tmpdir, extras):
     """Writing script doesn't change computing configuration."""
-    cc = ComputingConfiguration()
+    dcc_filepath = select_divvy_config(None)
+    cc = ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
     compute1 = deepcopy(cc["compute_packages"])
     cc.write_script(tmpdir.join(get_random_key(20) + ".sh").strpath, extras)
     assert cc["compute_packages"] == compute1
diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py
index 6fa2c5ffa..f7795696a 100644
--- a/tests/divvytests/test_divvy_simple.py
+++ b/tests/divvytests/test_divvy_simple.py
@@ -4,6 +4,7 @@
 from collections import OrderedDict
 
 from yacman import YacAttMap
+from divvy import select_divvy_config
 
 # For interactive debugging:
 # import logmuse
@@ -12,7 +13,8 @@
 
 class TestPackageAtivation:
     def test_activate_package(self):
-        dcc = divvy.ComputingConfiguration()
+        dcc_filepath = select_divvy_config(None)
+        dcc = divvy.ComputingConfiguration().from_yaml_file(filepath=dcc_filepath)
         dcc.activate_package("default")
         t = dcc.compute["submission_template"]
         t2 = dcc["compute_packages"]["default"]["submission_template"]
@@ -25,7 +27,8 @@ def test_activate_package(self):
 
 class TestWriting:
     def test_write_script(self):
-        dcc = divvy.ComputingConfiguration()
+        dcc_filepath = select_divvy_config(None)
+        dcc = divvy.ComputingConfiguration().from_yaml_file(filepath=dcc_filepath)
         dcc
         dcc.activate_package("singularity_slurm")
         extra_vars = {

From 4c81572ee057f62f2e78582544bf865e879ad8ed Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 12 Feb 2024 18:55:06 -0500
Subject: [PATCH 081/225] make empty dcc actually empty

---
 tests/divvytests/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/divvytests/conftest.py b/tests/divvytests/conftest.py
index 7d1111f41..2fa0c9049 100644
--- a/tests/divvytests/conftest.py
+++ b/tests/divvytests/conftest.py
@@ -19,7 +19,7 @@
 @pytest.fixture
 def empty_dcc():
     """Provide the empty/default ComputingConfiguration object"""
-    return divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
+    return divvy.ComputingConfiguration()
 
 
 @pytest.fixture(params=FILES)

From 1c8a8ad138b1d5096a40118e1a9bfdd6ed8ca203 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 13 Feb 2024 13:03:56 -0500
Subject: [PATCH 082/225] Update pephubclient requirements per
 https://github.com/pepkit/looper/issues/453

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 3e81e07bf..6f8ca4d8a 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -4,7 +4,7 @@ eido>=0.2.1
 jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
-pephubclient>=0.1.2
+pephubclient>=0.4.0
 peppy>=0.40.0
 pipestat>=0.8.2a1
 pyyaml>=3.12

From 60051e3b78a41152150c97040db279f5bd6f4f41 Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:53:51 -0500
Subject: [PATCH 083/225] fix pydantic move. Fix #454

---
 looper/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index 3796cbc6f..cd0018cdb 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -16,7 +16,7 @@
 from peppy.const import *
 from ubiquerg import convert_value, expandpath, parse_registry_path
 from pephubclient.constants import RegistryPath
-from pydantic.error_wrappers import ValidationError
+from pydantic import ValidationError
 
 from .const import *
 from .exceptions import MisconfigurationException, RegistryPathException

From 8d029bf5d4308271faddfa5722f2d1c0c983988d Mon Sep 17 00:00:00 2001
From: nsheff <nsheff@users.noreply.github.com>
Date: Thu, 15 Feb 2024 20:12:02 -0500
Subject: [PATCH 084/225] add looper init idea. See #466

---
 looper_init.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 looper_init.py

diff --git a/looper_init.py b/looper_init.py
new file mode 100644
index 000000000..18c992d05
--- /dev/null
+++ b/looper_init.py
@@ -0,0 +1,66 @@
+# A simple utility, to be run in the root of a project, to prompt a user through
+# configuring a .looper.yaml file for a new project. To be used as `looper init`.
+
+import os
+cfg = {}
+
+print("This utility will walk you through creating a .looper.yaml file.")
+print("See `looper init --help` for details.")
+print("Use `looper run` afterwards to run the pipeline.")
+print("Press ^C at any time to quit.\n")
+
+looper_cfg_path = ".looper.yaml"  # not changeable
+
+if os.path.exists(looper_cfg_path):
+    print(f"File exists at '{looper_cfg_path}'. Delete it to re-initialize.")
+    raise SystemExit
+
+DEFAULTS = {  # What you get if you just press enter
+    "pep_config": "databio/example",
+    "output_dir": "results",
+    "piface_path": "pipeline_interface.yaml",
+    "project_name": os.path.basename(os.getcwd())
+}
+
+
+cfg["project_name"] = (
+    input(f"Project name: ({DEFAULTS['project_name']}) ") or DEFAULTS["project_name"]
+)
+
+cfg["pep_config"] = (
+    input(f"Registry path or file path to PEP: ({DEFAULTS['pep_config']}) ")
+    or DEFAULTS["pep_config"]
+)
+
+if not os.path.exists(cfg["pep_config"]):
+    print(f"Warning: PEP file does not exist at '{cfg['pep_config']}'")
+
+cfg["output_dir"] = (
+    input(f"Path to output directory: ({DEFAULTS['output_dir']}) ") or DEFAULTS["output_dir"]
+)
+
+# TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline
+# but this is not the only way you could configure things.
+
+piface_path = (
+    input("Path to sample pipeline interface: (pipeline_interface.yaml) ")
+    or DEFAULTS["piface_path"]
+)
+
+if not os.path.exists(piface_path):
+    print(f"Warning: file does not exist at {piface_path}")
+
+print(f"Writing config file to {looper_cfg_path}")
+print(f"PEP path: {cfg['pep_config']}")
+print(f"Pipeline interface path: {piface_path}")
+
+
+with open(looper_cfg_path, "w") as fp:
+    fp.write(
+        f"""\
+pep_config: {cfg['pep_config']}
+output_dir: {cfg['output_dir']}
+pipeline_interfaces:
+  sample: {piface_path}
+"""
+    )

From 0f8efe2d83adca43bdecd83f8e8fd5fd9951e19d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Feb 2024 09:21:15 -0500
Subject: [PATCH 085/225] Update write_pipestat_config doc strings and out put
 message https://github.com/pepkit/looper/issues/459

---
 looper/conductor.py | 7 +++++--
 looper_init.py      | 6 ++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 5301772a8..bf4f78580 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -85,11 +85,14 @@ def _get_yaml_path(namespaces, template_key, default_name_appendix="", filename=
 
 def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict):
     """
-    This is run at the project level, not at the sample level.
+    This writes a combined configuration file to be passed to a PipestatManager.
+    :param str looper_pipestat_config_path: path to the created pipestat configuration file
+    :param dict pipestat_config_dict: the dict containing key value pairs to be written to the pipestat configutation
+    return bool
     """
     with open(looper_pipestat_config_path, "w") as f:
         yaml.dump(pipestat_config_dict, f)
-    print(f"Initialized looper config file: {looper_pipestat_config_path}")
+    print(f"Initialized pipestat config file: {looper_pipestat_config_path}")
 
     return True
 
diff --git a/looper_init.py b/looper_init.py
index 18c992d05..9d7a3c5f3 100644
--- a/looper_init.py
+++ b/looper_init.py
@@ -2,6 +2,7 @@
 # configuring a .looper.yaml file for a new project. To be used as `looper init`.
 
 import os
+
 cfg = {}
 
 print("This utility will walk you through creating a .looper.yaml file.")
@@ -19,7 +20,7 @@
     "pep_config": "databio/example",
     "output_dir": "results",
     "piface_path": "pipeline_interface.yaml",
-    "project_name": os.path.basename(os.getcwd())
+    "project_name": os.path.basename(os.getcwd()),
 }
 
 
@@ -36,7 +37,8 @@
     print(f"Warning: PEP file does not exist at '{cfg['pep_config']}'")
 
 cfg["output_dir"] = (
-    input(f"Path to output directory: ({DEFAULTS['output_dir']}) ") or DEFAULTS["output_dir"]
+    input(f"Path to output directory: ({DEFAULTS['output_dir']}) ")
+    or DEFAULTS["output_dir"]
 )
 
 # TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline

From 199967b9d79a3800c2a9015637b2f2be23f38c15 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 20 Feb 2024 09:50:47 -0500
Subject: [PATCH 086/225] Add fix for accessing psm.pipeline_name
 https://github.com/pepkit/looper/issues/468

---
 looper/looper.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 51a9ee02a..f42a606ba 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -679,21 +679,21 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                 [
                     get_file_for_project(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         directory="reports",
                     ),
                     get_file_for_table(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         appendix="stats_summary.tsv",
                     ),
                     get_file_for_table(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         appendix="objs_summary.yaml",
                     ),
                     get_file_for_table(
-                        psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
+                        psm, pipeline_name=psm.pipeline_name, appendix="reports"
                     ),
                 ],
                 dry_run,
@@ -711,21 +711,21 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                     [
                         get_file_for_project(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             directory="reports",
                         ),
                         get_file_for_table(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             appendix="stats_summary.tsv",
                         ),
                         get_file_for_table(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             appendix="objs_summary.yaml",
                         ),
                         get_file_for_table(
-                            psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
+                            psm, pipeline_name=psm.pipeline_name, appendix="reports"
                         ),
                     ],
                     dry_run,

From 1c7d6ffd2191da875c1fa3dc839fa32952238281 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 21 Feb 2024 17:25:16 -0500
Subject: [PATCH 087/225] Begin skeleton to rethink testing #464

---
 tests/test_comprehensive.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 tests/test_comprehensive.py

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
new file mode 100644
index 000000000..fdd872681
--- /dev/null
+++ b/tests/test_comprehensive.py
@@ -0,0 +1,35 @@
+import pytest
+from peppy.const import *
+from yaml import dump
+
+from looper.const import *
+from looper.project import Project
+from tests.conftest import *
+from looper.utils import *
+from looper.cli_looper import main
+
+CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
+
+
+def test_comprehensive_looper_no_pipestat(prep_temp_pep):
+    tp = prep_temp_pep
+
+    x = test_args_expansion(tp, "run")
+    try:
+        main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
+
+def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
+    tp = prep_temp_pep_pipestat
+    cmd = "run"
+
+    x = [cmd, "-d", "--looper-config", tp]
+
+    try:
+        result = main(test_args=x)
+        if cmd == "run":
+            assert result["Pipestat compatible"] is True
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
\ No newline at end of file

From ad020838dfcc8b06e724e6b7338342fabd99af83 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:42:42 -0500
Subject: [PATCH 088/225] Add cloning hello looper and executing run #464

---
 requirements/requirements-test.txt |  1 +
 tests/test_comprehensive.py        | 32 +++++++++++++++++++++---------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt
index f02a8bc9d..87d100866 100644
--- a/requirements/requirements-test.txt
+++ b/requirements/requirements-test.txt
@@ -4,3 +4,4 @@ pytest
 pytest-cov
 pytest-remotedata
 veracitools
+GitPython
\ No newline at end of file
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index fdd872681..205d70643 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -7,9 +7,14 @@
 from tests.conftest import *
 from looper.utils import *
 from looper.cli_looper import main
+from tests.smoketests.test_run import is_connected
+from tempfile import TemporaryDirectory
+from git import Repo
 
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
+REPO_URL = "https://github.com/pepkit/hello_looper.git"
+
 
 def test_comprehensive_looper_no_pipestat(prep_temp_pep):
     tp = prep_temp_pep
@@ -21,15 +26,24 @@ def test_comprehensive_looper_no_pipestat(prep_temp_pep):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
-def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
-    tp = prep_temp_pep_pipestat
+@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
+def test_comprehensive_looper_pipestat():
+    """
+    This test clones the hello_looper repository and runs the looper config file in the pipestat sub-directory
+    """
+
     cmd = "run"
 
-    x = [cmd, "-d", "--looper-config", tp]
+    with TemporaryDirectory() as d:
+        repo = Repo.clone_from(REPO_URL, d)
 
-    try:
-        result = main(test_args=x)
-        if cmd == "run":
-            assert result["Pipestat compatible"] is True
-    except Exception:
-        raise pytest.fail("DID RAISE {0}".format(Exception))
\ No newline at end of file
+        path_to_looper_config = os.path.join(d, "pipestat", ".looper.yaml")
+
+        x = [cmd, "-d", "--looper-config", path_to_looper_config]
+
+        try:
+            result = main(test_args=x)
+            if cmd == "run":
+                assert result["Pipestat compatible"] is True
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))

From 6b2c599fe3af7b57da123be9e6b2e45d5e2b21d2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 22 Feb 2024 16:26:31 -0500
Subject: [PATCH 089/225] Attempt to change test directory to fix pipestat
 results reporting

---
 tests/test_comprehensive.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 205d70643..c2c90838f 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -37,7 +37,11 @@ def test_comprehensive_looper_pipestat():
     with TemporaryDirectory() as d:
         repo = Repo.clone_from(REPO_URL, d)
 
-        path_to_looper_config = os.path.join(d, "pipestat", ".looper.yaml")
+        # TODO executing either the .py or the .sh pipeline does not correctly report results because pytest is running from a different directory
+        pipestat_dir = os.path.join(d, "pipestat")
+        os.chdir(pipestat_dir)
+
+        path_to_looper_config = os.path.join(pipestat_dir, ".looper_pipestat_shell.yaml")
 
         x = [cmd, "-d", "--looper-config", path_to_looper_config]
 
@@ -47,3 +51,5 @@ def test_comprehensive_looper_pipestat():
                 assert result["Pipestat compatible"] is True
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        print(result)

From d4cfe2362ea2244382f8201aae342acf16b80651 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 26 Feb 2024 11:50:57 -0500
Subject: [PATCH 090/225] Modify derived attribute path before running samples
 #464

---
 tests/test_comprehensive.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index c2c90838f..395524a60 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -11,6 +11,8 @@
 from tempfile import TemporaryDirectory
 from git import Repo
 
+from yaml import dump, safe_load
+
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
 REPO_URL = "https://github.com/pepkit/hello_looper.git"
@@ -35,15 +37,29 @@ def test_comprehensive_looper_pipestat():
     cmd = "run"
 
     with TemporaryDirectory() as d:
-        repo = Repo.clone_from(REPO_URL, d)
-
-        # TODO executing either the .py or the .sh pipeline does not correctly report results because pytest is running from a different directory
+        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
         pipestat_dir = os.path.join(d, "pipestat")
-        os.chdir(pipestat_dir)
 
-        path_to_looper_config = os.path.join(pipestat_dir, ".looper_pipestat_shell.yaml")
+        path_to_looper_config = os.path.join(
+            pipestat_dir, ".looper_pipestat_shell.yaml"
+        )
+
+        # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+        pipestat_project_file = os.path.join(
+            d, "pipestat/project", "project_config.yaml"
+        )
+        with open(pipestat_project_file, "r") as f:
+            pipestat_project_data = safe_load(f)
+
+        pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+            os.path.join(pipestat_dir, "data/{sample_name}.txt")
+        )
+
+        with open(pipestat_project_file, "w") as f:
+            dump(pipestat_project_data, f)
 
-        x = [cmd, "-d", "--looper-config", path_to_looper_config]
+        # x = [cmd, "-d", "--looper-config", path_to_looper_config]
+        x = [cmd, "--looper-config", path_to_looper_config]
 
         try:
             result = main(test_args=x)

From cf74a6d751677899f0b5cbe03f60b1e67f07552f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 26 Feb 2024 13:53:02 -0500
Subject: [PATCH 091/225]  Add check, report, destroy, and table to tests, add
 returning debug values for table #464

---
 looper/cli_looper.py        |  4 +--
 looper/looper.py            |  6 +++-
 tests/test_comprehensive.py | 62 +++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index fd620a1ec..ab07628aa 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -761,13 +761,13 @@ def main(test_args=None):
         )
         if args.command == "table":
             if use_pipestat:
-                Tabulator(prj)(args)
+                return Tabulator(prj)(args)
             else:
                 raise PipestatConfigurationException("table")
 
         if args.command == "report":
             if use_pipestat:
-                Reporter(prj)(args)
+                return Reporter(prj)(args)
             else:
                 raise PipestatConfigurationException("report")
 
diff --git a/looper/looper.py b/looper/looper.py
index f42a606ba..20be8195f 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -545,6 +545,7 @@ class Reporter(Executor):
 
     def __call__(self, args):
         # initialize the report builder
+        self.debug = {}
         p = self.prj
         project_level = args.project
 
@@ -559,6 +560,8 @@ def __call__(self, args):
                     looper_samples=self.prj.samples, portable=portable
                 )
                 print(f"Report directory: {report_directory}")
+                self.debug["report_directory"] = report_directory
+            return self.debug
         else:
             for piface_source_samples in self.prj._samples_by_piface(
                 self.prj.piface_key
@@ -576,6 +579,8 @@ def __call__(self, args):
                         looper_samples=self.prj.samples, portable=portable
                     )
                     print(f"Report directory: {report_directory}")
+                    self.debug["report_directory"] = report_directory
+            return self.debug
 
 
 class Linker(Executor):
@@ -614,7 +619,6 @@ class Tabulator(Executor):
     """
 
     def __call__(self, args):
-        # p = self.prj
         project_level = args.project
         results = []
         if project_level:
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 395524a60..09fad1e0a 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -10,6 +10,7 @@
 from tests.smoketests.test_run import is_connected
 from tempfile import TemporaryDirectory
 from git import Repo
+from pipestat import PipestatManager
 
 from yaml import dump, safe_load
 
@@ -68,4 +69,65 @@ def test_comprehensive_looper_pipestat():
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+        # TODO TEST PROJECT LEVEL RUN
+        # Must add this to hello_looper for pipestat example
+
+        # TEST LOOPER CHECK
+
+        # looper cannot create flags, the pipeline or pipestat does that
+        # if you do not specify flag dir, pipestat places them in the same dir as config file
+        path_to_pipestat_config = os.path.join(
+            pipestat_dir, "looper_pipestat_config.yaml"
+        )
+        psm = PipestatManager(config_file=path_to_pipestat_config)
+        psm.set_status(record_identifier="frog_1", status_identifier="completed")
+        psm.set_status(record_identifier="frog_2", status_identifier="completed")
+
+        # Now use looper check to get statuses
+        x = ["check", "--looper-config", path_to_looper_config]
+
+        try:
+            result = main(test_args=x)
+            assert result["example_pipestat_pipeline"]["frog_1"] == "completed"
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        # TEST LOOPER REPORT
+
+        x = ["report", "--looper-config", path_to_looper_config]
+
+        try:
+            result = main(test_args=x)
+            assert "report_directory" in result
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        # TEST LOOPER Table
+
+        x = ["table", "--looper-config", path_to_looper_config]
+
+        try:
+            result = main(test_args=x)
+            assert "example_pipestat_pipeline_stats_summary.tsv" in result[0]
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        # TEST LOOPER DESTROY
+        # TODO add destroying individual samples via pipestat
+
+        x = [
+            "destroy",
+            "--looper-config",
+            path_to_looper_config,
+            "--force-yes",
+        ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
+
+        try:
+            result = main(test_args=x)
+            # assert "report_directory" in result
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        # TODO TEST LOOPER INSPECT  -> I believe this moved to Eido?
+
         print(result)

From 19a24c25213c6edec5bc51128a329ef53bd03821 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 26 Feb 2024 15:42:14 -0500
Subject: [PATCH 092/225] begin adding more pydantic args: rerun and runp #438

---
 looper/command_models/commands.py | 59 +++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index f9c494b3f..416d1823f 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -61,6 +61,59 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 )
 RunParserModel = RunParser.create_model()
 
+# RERUN
+RerunParser = Command(
+    "rerun",
+    MESSAGE_BY_SUBCOMMAND["rerun"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value,
+        ArgumentEnum.SETTINGS.value,
+    ],
+)
+RerunParserModel = RerunParser.create_model()
+
+# RUNP
+RunProjectParser = Command(
+    "runp",
+    MESSAGE_BY_SUBCOMMAND["runp"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value,
+        ArgumentEnum.SETTINGS.value,
+    ],
+)
+RunProjectParserModel = RunProjectParser.create_model()
+
+# TABLE
+# REPORT
+# DESTROY
+# CHECK
+# CLEAN
+# INSPECT
+# INIT
+# INIT-PIFACE
+# LINK
+
+
 SUPPORTED_COMMANDS = [RunParser]
 
 
@@ -73,6 +126,12 @@ class TopLevelParser(pydantic.BaseModel):
 
     # commands
     run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
+    rerun: Optional[RerunParserModel] = pydantic.Field(
+        description=RerunParser.description
+    )
+    runp: Optional[RunProjectParserModel] = pydantic.Field(
+        description=RunProjectParser.description
+    )
 
     # arguments
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()

From d3f550512ba11ede07af58dc80e983434b406f25 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:39:56 -0500
Subject: [PATCH 093/225] Add table, report, destroy, check #438

---
 looper/command_models/arguments.py | 24 +++++++++++++++
 looper/command_models/commands.py  | 48 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 66516852e..ab6c259f7 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -80,6 +80,30 @@ class ArgumentEnum(enum.Enum):
         default=(bool, False),
         description="Ignore run status flags",
     )
+    FORCE_YES = Argument(
+        name="force_yes",
+        default=(bool, False),
+        description="Provide upfront confirmation of destruction intent, to skip console query. Default=False",
+    )
+
+    DESCRIBE_CODES = Argument(
+        name="describe_codes",
+        default=(bool, False),
+        description="Show status codes description. Default=False",
+    )
+
+    ITEMIZED = Argument(
+        name="itemized",
+        default=(bool, False),
+        description="Show detailed overview of sample statuses. Default=False",
+    )
+
+    FLAGS = Argument(
+        name="flags",
+        default=(List, []),
+        description="Only check samples based on these status flags.",
+    )
+
     TIME_DELAY = Argument(
         name="time_delay",
         default=(int, 0),
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 416d1823f..70752f0a2 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -104,9 +104,45 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 RunProjectParserModel = RunProjectParser.create_model()
 
 # TABLE
+TableParser = Command(
+    "table",
+    MESSAGE_BY_SUBCOMMAND["table"],
+    [],
+)
+TableParserModel = TableParser.create_model()
+
+
 # REPORT
+ReportParser = Command(
+    "report",
+    MESSAGE_BY_SUBCOMMAND["report"],
+    [],
+)
+ReportParserModel = ReportParser.create_model()
+
 # DESTROY
+DestroyParser = Command(
+    "destroy",
+    MESSAGE_BY_SUBCOMMAND["destroy"],
+    [
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.FORCE_YES.value,
+    ],
+)
+DestroyParserModel = DestroyParser.create_model()
+
 # CHECK
+CheckParser = Command(
+    "check",
+    MESSAGE_BY_SUBCOMMAND["check"],
+    [
+        ArgumentEnum.DESCRIBE_CODES.value,
+        ArgumentEnum.ITEMIZED.value,
+        ArgumentEnum.FLAGS.value,
+    ],
+)
+CheckParserModel = CheckParser.create_model()
+
 # CLEAN
 # INSPECT
 # INIT
@@ -132,6 +168,18 @@ class TopLevelParser(pydantic.BaseModel):
     runp: Optional[RunProjectParserModel] = pydantic.Field(
         description=RunProjectParser.description
     )
+    table: Optional[TableParserModel] = pydantic.Field(
+        description=TableParser.description
+    )
+    report: Optional[ReportParserModel] = pydantic.Field(
+        description=ReportParser.description
+    )
+    destroy: Optional[DestroyParserModel] = pydantic.Field(
+        description=DestroyParser.description
+    )
+    check: Optional[CheckParserModel] = pydantic.Field(
+        description=CheckParser.description
+    )
 
     # arguments
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()

From ca77d58785e4fc8a3398653e7c86aa815f9df1fd Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Feb 2024 10:19:33 -0500
Subject: [PATCH 094/225] add remaining commands #438

---
 looper/command_models/commands.py | 47 +++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 70752f0a2..45c60f6ff 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -144,11 +144,47 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 CheckParserModel = CheckParser.create_model()
 
 # CLEAN
+CleanParser = Command(
+    "clean",
+    MESSAGE_BY_SUBCOMMAND["clean"],
+    [
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.FORCE_YES.value,
+    ],
+)
+CleanParserModel = CleanParser.create_model()
+
 # INSPECT
+# TODO Did this move to Eido?
+
 # INIT
+# TODO rename to `init-config` ?
+InitParser = Command(
+    "init",
+    MESSAGE_BY_SUBCOMMAND["init"],
+    [
+        # Original command has force flag which is technically a different flag, but we should just use FORCE_YES
+        ArgumentEnum.FORCE_YES.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+    ],
+)
+InitParserModel = InitParser.create_model()
+
 # INIT-PIFACE
-# LINK
+InitPifaceParser = Command(
+    "init-piface",
+    MESSAGE_BY_SUBCOMMAND["init-piface"],
+    [],
+)
+InitPifaceParserModel = InitPifaceParser.create_model()
 
+# LINK
+LinkParser = Command(
+    "link",
+    MESSAGE_BY_SUBCOMMAND["link"],
+    [],
+)
+LinkParserModel = LinkParser.create_model()
 
 SUPPORTED_COMMANDS = [RunParser]
 
@@ -180,7 +216,14 @@ class TopLevelParser(pydantic.BaseModel):
     check: Optional[CheckParserModel] = pydantic.Field(
         description=CheckParser.description
     )
-
+    clean: Optional[CleanParserModel] = pydantic.Field(
+        description=CleanParser.description
+    )
+    init: Optional[InitParserModel] = pydantic.Field(description=InitParser.description)
+    init_piface: Optional[InitPifaceParserModel] = pydantic.Field(
+        description=InitPifaceParser.description
+    )
+    link: Optional[LinkParserModel] = pydantic.Field(description=LinkParser.description)
     # arguments
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()

From 10cce8095f72f347febeb30bbcecf5fd1526f3db Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 27 Feb 2024 11:29:15 -0500
Subject: [PATCH 095/225] check for none values during pydantic arg parsing
 #438

---
 looper/cli_pydantic.py            |  2 +-
 looper/command_models/commands.py | 14 +++++++++++++-
 looper/utils.py                   |  7 +++++--
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index d895b7ac3..891768431 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -57,7 +57,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
     subcommand_valued_args = [
         (arg, value)
         for arg, value in vars(args).items()
-        if arg and arg in supported_command_names
+        if arg and arg in supported_command_names and value is not None
     ]
     # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
     [(subcommand_name, subcommand_args)] = subcommand_valued_args
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 45c60f6ff..6d189814f 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -186,7 +186,19 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 )
 LinkParserModel = LinkParser.create_model()
 
-SUPPORTED_COMMANDS = [RunParser]
+SUPPORTED_COMMANDS = [
+    RunParser,
+    RerunParser,
+    RunProjectParser,
+    TableParser,
+    ReportParser,
+    DestroyParser,
+    CheckParser,
+    CleanParser,
+    InitParser,
+    InitPifaceParser,
+    LinkParser,
+]
 
 
 class TopLevelParser(pydantic.BaseModel):
diff --git a/looper/utils.py b/looper/utils.py
index 531ea01a6..e3bed6a27 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -300,8 +300,11 @@ def set_single_arg(argname, default_source_namespace, result_namespace):
             # this argument actually is a subcommand
             enriched_command_namespace = argparse.Namespace()
             command_namespace = getattr(parser_args, top_level_argname)
-            for argname in vars(command_namespace):
-                set_single_arg(argname, command_namespace, enriched_command_namespace)
+            if command_namespace:
+                for argname in vars(command_namespace):
+                    set_single_arg(
+                        argname, command_namespace, enriched_command_namespace
+                    )
             setattr(result, top_level_argname, enriched_command_namespace)
     return result
 

From 4ad60dbda230885deb3e80a63e77ecb3f2c04d6d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 09:59:52 -0500
Subject: [PATCH 096/225] get rerun to work

---
 looper/cli_looper.py   | 13 +++++++------
 looper/cli_pydantic.py |  5 +++--
 looper/looper.py       | 21 ++++++++++++++-------
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
index 06d3b4e1c..717dc504e 100644
--- a/looper/cli_looper.py
+++ b/looper/cli_looper.py
@@ -551,12 +551,12 @@ def _proc_resources_spec(args):
     :raise ValueError: if interpretation of the given specification as encoding
         of key-value pairs fails
     """
-    if (hasattr(args, "run") and args.run) or args.command in ("run",):
-        spec = getattr(args.run, "compute", None)
-        settings = args.run.settings
-    else:
-        spec = getattr(args, "compute", None)
-        settings = args.settings
+    # if (hasattr(args, "run") and args.run) or args.command in ("run",):
+    #     spec = getattr(args.run, "compute", None)
+    #     settings = args.run.settings
+    # else:
+    spec = getattr(args, "compute", None)
+    settings = args.settings
     try:
         settings_data = read_yaml_file(settings) or {}
     except yaml.YAMLError:
@@ -619,6 +619,7 @@ def add_command_hierarchy(command_args):
         setattr(args, args.command, command_namespace)
 
     if args.command == "run":
+        # if args.command == "run" or args.command == "rerun":
         # we only want to only move arguments to the `run` second-level namespace
         # that are in fact specific to the `run` subcommand
         run_args = [argument.name for argument in RunParser.arguments]
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 891768431..715252190 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -152,10 +152,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
         selector_flag=args.sel_flag,
         exclusion_flag=args.exc_flag,
     ) as prj:
-        if subcommand_name == "run":
+        if subcommand_name in ["run", "rerun"]:
             run = Runner(prj)
             try:
-                compute_kwargs = _proc_resources_spec(args)
+                # compute_kwargs = _proc_resources_spec(args)
+                compute_kwargs = _proc_resources_spec(subcommand_args)
                 return run(args, rerun=False, **compute_kwargs)
             except SampleFailedException:
                 sys.exit(1)
diff --git a/looper/looper.py b/looper/looper.py
index e8259215c..afe7c266d 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -395,17 +395,24 @@ def __call__(self, args, rerun=False, **compute_kwargs):
                 )
 
         submission_conductors = {}
+
         for piface in self.prj.pipeline_interfaces:
             conductor = SubmissionConductor(
                 pipeline_interface=piface,
                 prj=self.prj,
                 compute_variables=comp_vars,
-                delay=args.run.time_delay,
-                extra_args=args.run.command_extra,
-                extra_args_override=args.run.command_extra_override,
-                ignore_flags=args.run.ignore_flags,
-                max_cmds=args.run.lumpn,
-                max_size=args.run.lump,
+                delay=getattr(args.run, "time_delay", None)
+                or getattr(args.rerun, "time_delay", None),
+                extra_args=getattr(args.run, "command_extra", None)
+                or getattr(args.rerun, "command_extra", None),
+                extra_args_override=getattr(args.run, "command_extra_override", None)
+                or getattr(args.rerun, "command_extra_override", None),
+                ignore_flags=getattr(args.run, "ignore_flags", None)
+                or getattr(args.rerun, "ignore_flags", None),
+                max_cmds=getattr(args.run, "lumpn", None)
+                or getattr(args.rerun, "lumpn", None),
+                max_size=getattr(args.run, "lump", None)
+                or getattr(args.rerun, "lump", None),
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
@@ -486,7 +493,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
         )
         _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
         self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
-        if args.run.dry_run:
+        if getattr(args.run, "dry_run", None) or getattr(args.rerun, "dry_run", None):
             job_sub_total_if_real = job_sub_total
             job_sub_total = 0
             _LOGGER.info(

From 5614975fb6d02934ab278300e204d002e4825fca Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 10:15:57 -0500
Subject: [PATCH 097/225] get runp to work, tests are broken for the time being

---
 looper/cli_pydantic.py |  6 ++++++
 looper/looper.py       | 12 +++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 715252190..b8b137564 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -168,6 +168,12 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
                 )
                 raise
 
+        if subcommand_name in ["runp"]:
+            compute_kwargs = _proc_resources_spec(subcommand_args)
+            collate = Collator(prj)
+            collate(args, **compute_kwargs)
+            return collate.debug
+
 
 def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
diff --git a/looper/looper.py b/looper/looper.py
index afe7c266d..0873af597 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -341,13 +341,15 @@ def __call__(self, args, **compute_kwargs):
                 pipeline_interface=project_piface_object,
                 prj=self.prj,
                 compute_variables=compute_kwargs,
-                delay=args.time_delay,
-                extra_args=args.command_extra,
-                extra_args_override=args.command_extra_override,
-                ignore_flags=args.ignore_flags,
+                delay=getattr(args.runp, "time_delay", None),
+                extra_args=getattr(args.runp, "command_extra", None),
+                extra_args_override=getattr(args.runp, "command_extra_override", None),
+                ignore_flags=getattr(args.runp, "ignore_flags", None),
                 collate=True,
             )
-            if conductor.is_project_submittable(force=args.ignore_flags):
+            if conductor.is_project_submittable(
+                force=getattr(args.runp, "ignore_flags", None)
+            ):
                 conductor._pool = [None]
                 conductor.submit()
                 jobs += conductor.num_job_submissions

From c4c3fde93145a2f914bc87d192a0ef85dfd98036 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 10:32:03 -0500
Subject: [PATCH 098/225] add init functionality to new cli in cli_pydantic.py

---
 looper/cli_pydantic.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index b8b137564..cfe3fa691 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -43,6 +43,8 @@
     is_registry_path,
     read_looper_config_file,
     read_looper_dotfile,
+    initiate_looper_config,
+    init_generic_pipeline,
 )
 
 
@@ -62,6 +64,21 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
     # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
     [(subcommand_name, subcommand_args)] = subcommand_valued_args
 
+    if subcommand_name in ["init"]:
+        return int(
+            not initiate_looper_config(
+                dotfile_path(),
+                args.pep_config,
+                args.init.output_dir,
+                args.sample_pipeline_interfaces,
+                args.project_pipeline_interfaces,
+                args.init.force_yes,
+            )
+        )
+
+    # if subcommand_name in ["init_piface"]:
+    #     sys.exit(int(not init_generic_pipeline()))
+
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
     if args.config_file is None:

From 2ff51e1c9f0c1dd408f35c161118e7c153fb08b8 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 10:45:46 -0500
Subject: [PATCH 099/225] add init_piface

---
 looper/cli_pydantic.py            | 4 ++--
 looper/command_models/commands.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index cfe3fa691..7e58dcdd2 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -76,8 +76,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             )
         )
 
-    # if subcommand_name in ["init_piface"]:
-    #     sys.exit(int(not init_generic_pipeline()))
+    if subcommand_name in ["init_piface"]:
+        sys.exit(int(not init_generic_pipeline()))
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 6d189814f..c469c0db8 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -172,7 +172,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 
 # INIT-PIFACE
 InitPifaceParser = Command(
-    "init-piface",
+    "init_piface",
     MESSAGE_BY_SUBCOMMAND["init-piface"],
     [],
 )

From 492a4ed31f0884514ca8947fdbf975314bf934cb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 11:05:39 -0500
Subject: [PATCH 100/225] add working destroy to cli_pydantic.py

---
 looper/cli_pydantic.py |  3 +++
 looper/looper.py       | 30 ++++++++++++++++++------------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7e58dcdd2..2012dbff3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -191,6 +191,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             collate(args, **compute_kwargs)
             return collate.debug
 
+        if subcommand_name in ["destroy"]:
+            return Destroyer(prj)(args)
+
 
 def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
diff --git a/looper/looper.py b/looper/looper.py
index 0873af597..88b20b44a 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -255,11 +255,17 @@ def __call__(self, args, preview_flag=True):
         _LOGGER.info("Removing summary:")
         use_pipestat = (
             self.prj.pipestat_configured_project
-            if args.project
+            if getattr(
+                args, "project", None
+            )  # TODO this argument hasn't been added to the pydantic models.
             else self.prj.pipestat_configured
         )
         if use_pipestat:
-            destroy_summary(self.prj, args.dry_run, args.project)
+            destroy_summary(
+                self.prj,
+                getattr(args.destroy, "dry_run", None),
+                getattr(args, "project", None),
+            )
         else:
             _LOGGER.warning(
                 "Pipestat must be configured to destroy any created summaries."
@@ -269,11 +275,11 @@ def __call__(self, args, preview_flag=True):
             _LOGGER.info("Destroy complete.")
             return 0
 
-        if args.dry_run:
+        if getattr(args.destroy, "dry_run", None):
             _LOGGER.info("Dry run. No files destroyed.")
             return 0
 
-        if not args.force_yes and not query_yes_no(
+        if not getattr(args.destroy, "force_yes", None) and not query_yes_no(
             "Are you sure you want to permanently delete all pipeline "
             "results for this project?"
         ):
@@ -681,21 +687,21 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                 [
                     get_file_for_project(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         directory="reports",
                     ),
                     get_file_for_table(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         appendix="stats_summary.tsv",
                     ),
                     get_file_for_table(
                         psm,
-                        pipeline_name=psm["_pipeline_name"],
+                        pipeline_name=psm.pipeline_name,
                         appendix="objs_summary.yaml",
                     ),
                     get_file_for_table(
-                        psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
+                        psm, pipeline_name=psm.pipeline_name, appendix="reports"
                     ),
                 ],
                 dry_run,
@@ -713,21 +719,21 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                     [
                         get_file_for_project(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             directory="reports",
                         ),
                         get_file_for_table(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             appendix="stats_summary.tsv",
                         ),
                         get_file_for_table(
                             psm,
-                            pipeline_name=psm["_pipeline_name"],
+                            pipeline_name=psm.pipeline_name,
                             appendix="objs_summary.yaml",
                         ),
                         get_file_for_table(
-                            psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
+                            psm, pipeline_name=psm.pipeline_name, appendix="reports"
                         ),
                     ],
                     dry_run,

From 3981849b3d9df81d6613d2d0b7ee5991c19ca038 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 15:10:18 -0500
Subject: [PATCH 101/225] add working table command to cli_pydantic.py

---
 looper/cli_pydantic.py | 12 ++++++++++++
 looper/looper.py       |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 2012dbff3..aaa7786ac 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -194,6 +194,18 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
         if subcommand_name in ["destroy"]:
             return Destroyer(prj)(args)
 
+        use_pipestat = (
+            prj.pipestat_configured_project
+            if getattr(args, "project", None)
+            else prj.pipestat_configured
+        )
+
+        if subcommand_name in ["table"]:
+            if use_pipestat:
+                Tabulator(prj)(args)
+            else:
+                raise PipestatConfigurationException("table")
+
 
 def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
diff --git a/looper/looper.py b/looper/looper.py
index 88b20b44a..ffb10daa0 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -623,7 +623,7 @@ class Tabulator(Executor):
 
     def __call__(self, args):
         # p = self.prj
-        project_level = args.project
+        project_level = getattr(args, "project", None)
         results = []
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)

From 87d70bdac4fe237f3c80151513a0b6ccd9a22902 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 28 Feb 2024 15:25:50 -0500
Subject: [PATCH 102/225] add working report, link, check, clean commands to
 cli_pydantic.py

---
 looper/cli_pydantic.py            | 29 +++++++++++++++++++++++++++++
 looper/command_models/commands.py | 11 +++++++++++
 looper/const.py                   |  2 +-
 looper/looper.py                  | 15 ++++++++-------
 4 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index aaa7786ac..be8bab181 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -206,6 +206,35 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             else:
                 raise PipestatConfigurationException("table")
 
+        if subcommand_name in ["report"]:
+            if use_pipestat:
+                Reporter(prj)(args)
+            else:
+                raise PipestatConfigurationException("report")
+
+        if subcommand_name in ["link"]:
+            if use_pipestat:
+                Linker(prj)(args)
+            else:
+                raise PipestatConfigurationException("link")
+
+        if subcommand_name in ["check"]:
+            if use_pipestat:
+                return Checker(prj)(args)
+            else:
+                raise PipestatConfigurationException("check")
+
+        if subcommand_name in ["clean"]:
+            return Cleaner(prj)(args)
+
+        if subcommand_name in ["inspect"]:
+            from warnings import warn
+
+            warn(
+                "The inspect feature has moved to eido."
+                "Use `eido inspect` from now on.",
+            )
+
 
 def main() -> None:
     parser = pydantic_argparse.ArgumentParser(
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index c469c0db8..d4a24c085 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -156,6 +156,12 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 
 # INSPECT
 # TODO Did this move to Eido?
+InspectParser = Command(
+    "inspect",
+    MESSAGE_BY_SUBCOMMAND["inspect"],
+    [],
+)
+InspectParserModel = InspectParser.create_model()
 
 # INIT
 # TODO rename to `init-config` ?
@@ -236,6 +242,11 @@ class TopLevelParser(pydantic.BaseModel):
         description=InitPifaceParser.description
     )
     link: Optional[LinkParserModel] = pydantic.Field(description=LinkParser.description)
+
+    inspect: Optional[InspectParserModel] = pydantic.Field(
+        description=InspectParser.description
+    )
+
     # arguments
     settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
     pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
diff --git a/looper/const.py b/looper/const.py
index a866f2d84..4d7017567 100644
--- a/looper/const.py
+++ b/looper/const.py
@@ -263,7 +263,7 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
     "destroy": "Remove output files of the project.",
     "check": "Check flag status of current runs.",
     "clean": "Run clean scripts of already processed jobs.",
-    "inspect": "Print information about a project.",
+    "inspect": "Deprecated. Use `eido inspect` instead. Print information about a project.",
     "init": "Initialize looper config file.",
     "init-piface": "Initialize generic pipeline interface.",
     "link": "Create directory of symlinks for reported results.",
diff --git a/looper/looper.py b/looper/looper.py
index ffb10daa0..c67ee5a6d 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -6,6 +6,7 @@
 import abc
 import argparse
 import csv
+import glob
 import logging
 import subprocess
 import yaml
@@ -88,7 +89,7 @@ def __call__(self, args):
 
         # aggregate pipeline status data
         status = {}
-        if args.project:
+        if getattr(args, "project", None):
             psms = self.prj.get_pipestat_managers(project_level=True)
             for pipeline_name, psm in psms.items():
                 s = psm.get_status() or "unknown"
@@ -123,7 +124,7 @@ def __call__(self, args):
                     table.add_row(status_id, f"{status_count}/{len(status_list)}")
             console.print(table)
 
-        if args.itemized:
+        if getattr(args.check, "itemized", None):
             for pipeline_name, pipeline_status in status.items():
                 table_title = f"Pipeline: '{pipeline_name}'"
                 table = Table(
@@ -199,10 +200,10 @@ def __call__(self, args, preview_flag=True):
         if not preview_flag:
             _LOGGER.info("Clean complete.")
             return 0
-        if args.dry_run:
+        if getattr(args.clean, "dry_run", None):
             _LOGGER.info("Dry run. No files cleaned.")
             return 0
-        if not args.force_yes and not query_yes_no(
+        if not getattr(args.clean, "force_yes", None) and not query_yes_no(
             "Are you sure you want to permanently delete all "
             "intermediate pipeline results for this project?"
         ):
@@ -560,7 +561,7 @@ class Reporter(Executor):
     def __call__(self, args):
         # initialize the report builder
         p = self.prj
-        project_level = args.project
+        project_level = getattr(args, "project", None)
 
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)
@@ -592,8 +593,8 @@ class Linker(Executor):
     def __call__(self, args):
         # initialize the report builder
         p = self.prj
-        project_level = args.project
-        link_dir = args.output_dir
+        project_level = getattr(args, "project", None)
+        link_dir = getattr(args, "output_dir", None)
 
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)

From 24bdbbdd3cdb1302fb1d2aa0e68eddf1978bd328 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 10:47:48 -0500
Subject: [PATCH 103/225] switch to using cli_pydantic only

---
 looper/__main__.py | 2 +-
 setup.py           | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/looper/__main__.py b/looper/__main__.py
index 5ec266e80..3e9816554 100644
--- a/looper/__main__.py
+++ b/looper/__main__.py
@@ -1,6 +1,6 @@
 import sys
 
-from .cli_looper import main
+from .cli_pydantic import main
 from .cli_divvy import main as divvy_main
 
 if __name__ == "__main__":
diff --git a/setup.py b/setup.py
index c05314fe6..db8d94595 100644
--- a/setup.py
+++ b/setup.py
@@ -79,9 +79,8 @@ def get_static(name, condition=None):
     license="BSD2",
     entry_points={
         "console_scripts": [
-            "looper = looper.__main__:main",
+            "looper = looper.cli_pydantic:main",
             "divvy = looper.__main__:divvy_main",
-            "looper-pydantic-argparse = looper.cli_pydantic:main",
         ],
     },
     scripts=scripts,

From bdd5ee248792abf703df52a4f49df163116c9004 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 11:34:48 -0500
Subject: [PATCH 104/225] fix bug with getattr being False with default of 0

---
 looper/cli_pydantic.py | 10 +++++++---
 looper/looper.py       | 26 ++++++++++----------------
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index be8bab181..5793d4490 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -174,7 +174,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             try:
                 # compute_kwargs = _proc_resources_spec(args)
                 compute_kwargs = _proc_resources_spec(subcommand_args)
-                return run(args, rerun=False, **compute_kwargs)
+
+                # TODO Shouldn't top level args and subcommand args be accessible on the same object?
+                return run(
+                    subcommand_args, top_level_args=args, rerun=False, **compute_kwargs
+                )
             except SampleFailedException:
                 sys.exit(1)
             except IOError:
@@ -236,11 +240,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             )
 
 
-def main() -> None:
+def main(test_args=None) -> None:
     parser = pydantic_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
-        description="pydantic-argparse demo",
+        description="Looper Pydantic Argument Parser",
         add_help=True,
     )
     args = parser.parse_typed_args()
diff --git a/looper/looper.py b/looper/looper.py
index c67ee5a6d..f6c19b9a9 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -150,7 +150,7 @@ def __call__(self, args):
                     table.add_row(name, f"[{color}]{status_id}[/{color}]")
                 console.print(table)
 
-        if args.describe_codes:
+        if args.check.describe_codes:
             table = Table(
                 show_header=True,
                 header_style="bold magenta",
@@ -369,7 +369,7 @@ def __call__(self, args, **compute_kwargs):
 class Runner(Executor):
     """The true submitter of pipelines"""
 
-    def __call__(self, args, rerun=False, **compute_kwargs):
+    def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
         """
         Do the Sample submission.
 
@@ -410,18 +410,12 @@ def __call__(self, args, rerun=False, **compute_kwargs):
                 pipeline_interface=piface,
                 prj=self.prj,
                 compute_variables=comp_vars,
-                delay=getattr(args.run, "time_delay", None)
-                or getattr(args.rerun, "time_delay", None),
-                extra_args=getattr(args.run, "command_extra", None)
-                or getattr(args.rerun, "command_extra", None),
-                extra_args_override=getattr(args.run, "command_extra_override", None)
-                or getattr(args.rerun, "command_extra_override", None),
-                ignore_flags=getattr(args.run, "ignore_flags", None)
-                or getattr(args.rerun, "ignore_flags", None),
-                max_cmds=getattr(args.run, "lumpn", None)
-                or getattr(args.rerun, "lumpn", None),
-                max_size=getattr(args.run, "lump", None)
-                or getattr(args.rerun, "lump", None),
+                delay=getattr(args, "time_delay", None),
+                extra_args=getattr(args, "command_extra", None),
+                extra_args_override=getattr(args, "command_extra_override", None),
+                ignore_flags=getattr(args, "ignore_flags", None),
+                max_cmds=getattr(args, "lumpn", None),
+                max_size=getattr(args, "lump", None),
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
@@ -430,7 +424,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
             self.prj.pipestat_configured_project or self.prj.pipestat_configured
         )
 
-        for sample in select_samples(prj=self.prj, args=args):
+        for sample in select_samples(prj=self.prj, args=top_level_args):
             pl_fails = []
             skip_reasons = []
             sample_pifaces = self.prj.get_sample_piface(
@@ -502,7 +496,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
         )
         _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
         self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
-        if getattr(args.run, "dry_run", None) or getattr(args.rerun, "dry_run", None):
+        if getattr(args, "dry_run", None):
             job_sub_total_if_real = job_sub_total
             job_sub_total = 0
             _LOGGER.info(

From 89f0da76271e444e11da412edb29577a8641c4c7 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 13:43:38 -0500
Subject: [PATCH 105/225] fix test via test_args

---
 looper/cli_pydantic.py         | 11 +++++++----
 tests/smoketests/test_other.py |  4 ++--
 tests/smoketests/test_run.py   |  2 +-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 5793d4490..2466548d4 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -48,7 +48,7 @@
 )
 
 
-def run_looper(args: TopLevelParser, parser: ArgumentParser):
+def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     # here comes adapted `cli_looper.py` code
     global _LOGGER
 
@@ -106,7 +106,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser):
             "looper.databio.org/en/latest/looper-config"
         )
 
-    args = enrich_args_via_cfg(args, parser, False)
+    args = enrich_args_via_cfg(args, parser, test_args=test_args)
 
     # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
     if vars(args)[PROJECT_PL_ARG]:
@@ -247,8 +247,11 @@ def main(test_args=None) -> None:
         description="Looper Pydantic Argument Parser",
         add_help=True,
     )
-    args = parser.parse_typed_args()
-    run_looper(args, parser)
+    if test_args:
+        args = parser.parse_typed_args(args=test_args)
+    else:
+        args = parser.parse_typed_args()
+    return run_looper(args, parser, test_args=test_args)
 
 
 if __name__ == "__main__":
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index a724c7602..e0f484a5e 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -4,7 +4,7 @@
 from looper.const import FLAGS
 from looper.exceptions import PipestatConfigurationException
 from tests.conftest import *
-from looper.cli_looper import main
+from looper.cli_pydantic import main
 
 
 def _make_flags(cfg, type, pipeline_name):
@@ -55,7 +55,7 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags(tp, flag_id, pipeline_name)
 
-        x = ["check", "-d", "--looper-config", tp]
+        x = ["--looper-config", tp, "check"]
 
         try:
             results = main(test_args=x)
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index f34adf926..3f1c2cd42 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -6,7 +6,7 @@
 from looper.project import Project
 from tests.conftest import *
 from looper.utils import *
-from looper.cli_looper import main
+from looper.cli_pydantic import main
 
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 

From c165de379862a74335f46a687358d582885925ec Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 14:07:35 -0500
Subject: [PATCH 106/225] fix more tests, rethink test_args expansion

---
 tests/conftest.py              | 9 ++++++++-
 tests/smoketests/test_other.py | 6 +++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 29f601f4d..e4cce5e52 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -116,9 +116,16 @@ def test_args_expansion(pth=None, cmd=None, appendix=list(), dry=True) -> List[s
     :param bool dry: whether to append dry run flag
     :return list of strings to pass to looper.main for testing
     """
-    x = [cmd, "-d" if dry else ""]
+    # --looper-config .looper.yaml run --dry-run
+    #x = [cmd, "-d" if dry else ""]
+    x = []
     if pth:
+        x.append("--looper-config")
         x.append(pth)
+    if cmd:
+        x.append(cmd)
+    if dry:
+        x.append("--dry-run")
     x.extend(appendix)
     return x
 
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index e0f484a5e..4c7f7e829 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -27,7 +27,7 @@ class TestLooperPipestat:
     def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         "report, table, and check should fail if pipestat is NOT configured."
         tp = prep_temp_pep
-        x = test_args_expansion(tp, cmd)
+        x = test_args_expansion(tp, cmd, dry=False)
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
 
@@ -74,7 +74,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         _make_flags(tp, flag_id, pipeline_name)
         _make_flags(tp, FLAGS[1], pipeline_name)
 
-        x = ["check", "-d", "--looper-config", tp]
+        x = ["--looper-config", tp, "check"]
         # Multiple flag files SHOULD cause pipestat to throw an assertion error
         if flag_id != FLAGS[1]:
             with pytest.raises(AssertionError):
@@ -87,7 +87,7 @@ def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags(tp, flag_id, pipeline_name)
 
-        x = ["check", "-d", "--looper-config", tp]
+        x = ["--looper-config", tp, "check"]
         try:
             results = main(test_args=x)
             result_key = list(results.keys())[0]

From d59de5e6858ce6d69fb6e67a444bbbcd5a4a3229 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 15:00:11 -0500
Subject: [PATCH 107/225] lint

---
 tests/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index e4cce5e52..6552d6d13 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -117,7 +117,7 @@ def test_args_expansion(pth=None, cmd=None, appendix=list(), dry=True) -> List[s
     :return list of strings to pass to looper.main for testing
     """
     # --looper-config .looper.yaml run --dry-run
-    #x = [cmd, "-d" if dry else ""]
+    # x = [cmd, "-d" if dry else ""]
     x = []
     if pth:
         x.append("--looper-config")

From 973e660c72e17d860457a79164038ed0e8480685 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 15:11:12 -0500
Subject: [PATCH 108/225] re-add lumpj to pydantic arguments

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 2 ++
 looper/looper.py                   | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index ab6c259f7..ed4232fba 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -132,6 +132,11 @@ class ArgumentEnum(enum.Enum):
         default=(int, None),
         description="Number of commands to batch into one job",
     )
+    LUMPJ = Argument(
+        name="lumpj",
+        default=(int, None),
+        description="Lump samples into number of jobs.",
+    )
     LIMIT = Argument(
         name="limit", default=(int, None), description="Limit to n samples"
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index d4a24c085..ab676dfd5 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -52,6 +52,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
         ArgumentEnum.LUMP.value,
         ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LUMPJ.value,
         ArgumentEnum.DIVVY.value,
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
@@ -73,6 +74,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
         ArgumentEnum.LUMP.value,
         ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LUMPJ.value,
         ArgumentEnum.DIVVY.value,
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
diff --git a/looper/looper.py b/looper/looper.py
index ed9e39296..5472e61fa 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -416,7 +416,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
                 ignore_flags=getattr(args, "ignore_flags", None),
                 max_cmds=getattr(args, "lumpn", None),
                 max_size=getattr(args, "lump", None),
-                max_jobs=args.lump_j,
+                max_jobs=getattr(args, "lumpj", None),
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 

From 1e2e50b4c7cf5a4e3518c8a7474d8fcc9966acca Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 15:32:23 -0500
Subject: [PATCH 109/225] Fix portable argument for report

---
 looper/command_models/arguments.py | 5 +++++
 looper/command_models/commands.py  | 4 +++-
 looper/looper.py                   | 2 +-
 tests/smoketests/test_other.py     | 6 +++++-
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index ed4232fba..c938b8a2b 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -247,3 +247,8 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Path to pipestat files.",
     )
+    PORTABLE = Argument(
+        name="portable",
+        default=(bool, False),
+        description="Makes html report portable.",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index ab676dfd5..fc53ce8d5 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -118,7 +118,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 ReportParser = Command(
     "report",
     MESSAGE_BY_SUBCOMMAND["report"],
-    [],
+    [
+        ArgumentEnum.PORTABLE.value,
+    ],
 )
 ReportParserModel = ReportParser.create_model()
 
diff --git a/looper/looper.py b/looper/looper.py
index 5472e61fa..ae991b3d5 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -559,7 +559,7 @@ def __call__(self, args):
         p = self.prj
         project_level = getattr(args, "project", None)
 
-        portable = args.portable
+        portable = args.report.portable
 
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 4c7f7e829..58efd8ff6 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -35,7 +35,11 @@ def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
     def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
         tp = prep_temp_pep_pipestat
 
-        x = [cmd, "-d", "--looper-config", tp]
+        if cmd in ["run", "runp"]:
+            x = ["--looper-config", tp, cmd, "--dry-run"]
+        else:
+            # Not every command supports dry run
+            x = ["--looper-config", tp, cmd]
 
         try:
             result = main(test_args=x)

From e92468217c81fa9702b7835af8c8f23fa15aa271 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 29 Feb 2024 16:38:18 -0500
Subject: [PATCH 110/225] Add tests skipping for tests using outdated pep

---
 tests/smoketests/test_other.py |  8 ++++++--
 tests/smoketests/test_run.py   | 14 ++++++++------
 tests/test_clean.py            |  2 +-
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 58efd8ff6..1e3749729 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -23,11 +23,15 @@ def _make_flags(cfg, type, pipeline_name):
 
 
 class TestLooperPipestat:
+
+    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     @pytest.mark.parametrize("cmd", ["report", "table", "check"])
     def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         "report, table, and check should fail if pipestat is NOT configured."
-        tp = prep_temp_pep
-        x = test_args_expansion(tp, cmd, dry=False)
+        #tp = prep_temp_pep
+        dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
+        #x = test_args_expansion(tp, cmd, dry=False)
+        x = ["--looper-config", dot_file_path, cmd]
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
 
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 02f16586b..14e45a97f 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -10,7 +10,7 @@
 
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 def test_cli(prep_temp_pep):
     tp = prep_temp_pep
 
@@ -82,6 +82,7 @@ def test_cmd_extra_cli(self, prep_temp_pep, cmd, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_in_all_files(subs_list, arg[1])
 
+    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
         tp = prep_temp_pep
@@ -97,7 +98,7 @@ def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
         """Verify looper runs in a basic case and return code is 0"""
@@ -314,7 +315,7 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_not_in_any_files(subs_list, arg)
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunpBehavior:
     def test_looper_runp_basic(self, prep_temp_pep):
         """Verify looper runps in a basic case and return code is 0"""
@@ -363,7 +364,7 @@ def test_cmd_extra_project(self, prep_temp_pep, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_in_all_files(subs_list, arg)
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunPreSubmissionHooks:
     def test_looper_basic_plugin(self, prep_temp_pep):
         tp = prep_temp_pep
@@ -422,7 +423,7 @@ def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, "test.txt", 3)
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunSubmissionScript:
     def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
         tp = prep_temp_pep
@@ -474,7 +475,7 @@ def test_looper_limiting(self, prep_temp_pep):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, ".sub", 4)
 
-
+@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperCompute:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_looper_respects_pkg_selection(self, prep_temp_pep, cmd):
@@ -566,6 +567,7 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
 
 
 class TestLooperConfig:
+    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
         tp = prep_temp_pep
diff --git a/tests/test_clean.py b/tests/test_clean.py
index 17a1fa9d0..6324b7dff 100644
--- a/tests/test_clean.py
+++ b/tests/test_clean.py
@@ -23,7 +23,7 @@ def build_namespace(**kwargs):
     ]
 ]
 
-
+@pytest.mark.skip(reason="Test needs to be rewritten")
 @pytest.mark.parametrize(["args", "preview"], DRYRUN_OR_NOT_PREVIEW)
 def test_cleaner_does_not_crash(args, preview, prep_temp_pep):
     prj = Project(prep_temp_pep)

From 066e661bf2b55ad0a7d120147e7f81c23b629913 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 1 Mar 2024 10:32:08 -0500
Subject: [PATCH 111/225] Switch to pydantic2-argparse

---
 looper/cli_pydantic.py             | 6 +++---
 looper/command_models/arguments.py | 2 +-
 looper/command_models/commands.py  | 2 +-
 requirements/requirements-all.txt  | 2 +-
 tests/smoketests/test_other.py     | 4 ++--
 tests/smoketests/test_run.py       | 6 ++++++
 tests/test_clean.py                | 1 +
 7 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 2466548d4..018d2420d 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -21,10 +21,10 @@
 import sys
 
 import logmuse
-import pydantic_argparse
+import pydantic2_argparse
 import yaml
 from pephubclient import PEPHubClient
-from pydantic_argparse.argparse.parser import ArgumentParser
+from pydantic2_argparse.argparse.parser import ArgumentParser
 
 from divvy import select_divvy_config
 
@@ -241,7 +241,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
 
 def main(test_args=None) -> None:
-    parser = pydantic_argparse.ArgumentParser(
+    parser = pydantic2_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
         description="Looper Pydantic Argument Parser",
diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index c938b8a2b..356d0478f 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -7,7 +7,7 @@
 from copy import copy
 from typing import Any, List
 
-import pydantic
+import pydantic.v1 as pydantic
 
 
 class Argument(pydantic.fields.FieldInfo):
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index fc53ce8d5..85652d83e 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -5,7 +5,7 @@
 from dataclasses import dataclass
 from typing import List, Optional, Type
 
-import pydantic
+import pydantic.v1 as pydantic
 
 from ..const import MESSAGE_BY_SUBCOMMAND
 from .arguments import Argument, ArgumentEnum
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index a79a89653..5d7d5eb23 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -11,4 +11,4 @@ pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2
 yacman==0.9.3
-pydantic-argparse>=0.8.0
\ No newline at end of file
+pydantic2-argparse>=0.9.2
\ No newline at end of file
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 1e3749729..7d42c1027 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -28,9 +28,9 @@ class TestLooperPipestat:
     @pytest.mark.parametrize("cmd", ["report", "table", "check"])
     def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         "report, table, and check should fail if pipestat is NOT configured."
-        #tp = prep_temp_pep
+        # tp = prep_temp_pep
         dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
-        #x = test_args_expansion(tp, cmd, dry=False)
+        # x = test_args_expansion(tp, cmd, dry=False)
         x = ["--looper-config", dot_file_path, cmd]
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 14e45a97f..ce8672da1 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -10,6 +10,7 @@
 
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 def test_cli(prep_temp_pep):
     tp = prep_temp_pep
@@ -98,6 +99,7 @@ def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
@@ -315,6 +317,7 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_not_in_any_files(subs_list, arg)
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunpBehavior:
     def test_looper_runp_basic(self, prep_temp_pep):
@@ -364,6 +367,7 @@ def test_cmd_extra_project(self, prep_temp_pep, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_in_all_files(subs_list, arg)
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunPreSubmissionHooks:
     def test_looper_basic_plugin(self, prep_temp_pep):
@@ -423,6 +427,7 @@ def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, "test.txt", 3)
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunSubmissionScript:
     def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
@@ -475,6 +480,7 @@ def test_looper_limiting(self, prep_temp_pep):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, ".sub", 4)
 
+
 @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperCompute:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
diff --git a/tests/test_clean.py b/tests/test_clean.py
index 6324b7dff..ab464e47a 100644
--- a/tests/test_clean.py
+++ b/tests/test_clean.py
@@ -23,6 +23,7 @@ def build_namespace(**kwargs):
     ]
 ]
 
+
 @pytest.mark.skip(reason="Test needs to be rewritten")
 @pytest.mark.parametrize(["args", "preview"], DRYRUN_OR_NOT_PREVIEW)
 def test_cleaner_does_not_crash(args, preview, prep_temp_pep):

From eaafb64e597235e7eb785213efeb0af047c620a6 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 1 Mar 2024 11:15:29 -0500
Subject: [PATCH 112/225] Fix comprehensive looper-pipestat test

---
 looper/cli_pydantic.py      |  4 ++--
 looper/looper.py            |  2 +-
 tests/test_comprehensive.py | 13 ++++++-------
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 018d2420d..c3cf11670 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -206,13 +206,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
         if subcommand_name in ["table"]:
             if use_pipestat:
-                Tabulator(prj)(args)
+                return Tabulator(prj)(args)
             else:
                 raise PipestatConfigurationException("table")
 
         if subcommand_name in ["report"]:
             if use_pipestat:
-                Reporter(prj)(args)
+                return Reporter(prj)(args)
             else:
                 raise PipestatConfigurationException("report")
 
diff --git a/looper/looper.py b/looper/looper.py
index ae991b3d5..04411168f 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -251,7 +251,7 @@ def __call__(self, args, preview_flag=True):
                 # Preview: Don't actually delete, just show files.
                 _LOGGER.info(str(sample_output_folder))
             else:
-                _remove_or_dry_run(sample_output_folder, args.dry_run)
+                _remove_or_dry_run(sample_output_folder, args.destroy.dry_run)
 
         _LOGGER.info("Removing summary:")
         use_pipestat = (
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 09fad1e0a..35493b870 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -6,7 +6,7 @@
 from looper.project import Project
 from tests.conftest import *
 from looper.utils import *
-from looper.cli_looper import main
+from looper.cli_pydantic import main
 from tests.smoketests.test_run import is_connected
 from tempfile import TemporaryDirectory
 from git import Repo
@@ -60,7 +60,7 @@ def test_comprehensive_looper_pipestat():
             dump(pipestat_project_data, f)
 
         # x = [cmd, "-d", "--looper-config", path_to_looper_config]
-        x = [cmd, "--looper-config", path_to_looper_config]
+        x = ["--looper-config", path_to_looper_config, cmd]
 
         try:
             result = main(test_args=x)
@@ -84,7 +84,7 @@ def test_comprehensive_looper_pipestat():
         psm.set_status(record_identifier="frog_2", status_identifier="completed")
 
         # Now use looper check to get statuses
-        x = ["check", "--looper-config", path_to_looper_config]
+        x = ["--looper-config", path_to_looper_config, "check"]
 
         try:
             result = main(test_args=x)
@@ -94,7 +94,7 @@ def test_comprehensive_looper_pipestat():
 
         # TEST LOOPER REPORT
 
-        x = ["report", "--looper-config", path_to_looper_config]
+        x = ["--looper-config", path_to_looper_config, "report"]
 
         try:
             result = main(test_args=x)
@@ -104,7 +104,7 @@ def test_comprehensive_looper_pipestat():
 
         # TEST LOOPER Table
 
-        x = ["table", "--looper-config", path_to_looper_config]
+        x = ["--looper-config", path_to_looper_config, "table"]
 
         try:
             result = main(test_args=x)
@@ -116,15 +116,14 @@ def test_comprehensive_looper_pipestat():
         # TODO add destroying individual samples via pipestat
 
         x = [
-            "destroy",
             "--looper-config",
             path_to_looper_config,
+            "destroy",
             "--force-yes",
         ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
 
         try:
             result = main(test_args=x)
-            # assert "report_directory" in result
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 

From 5287863027598a7e18be2e5cb50155a759903c95 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 1 Mar 2024 11:47:05 -0500
Subject: [PATCH 113/225] Fix comprehensive looper-basic test

---
 tests/smoketests/test_run.py |  2 ++
 tests/test_comprehensive.py  | 35 +++++++++++++++++++++++++++--------
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index ce8672da1..ce78799de 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -37,6 +37,7 @@ def is_connected():
 
 class TestLooperBothRuns:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
+    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     def test_looper_cfg_invalid(self, cmd):
         """Verify looper does not accept invalid cfg paths"""
 
@@ -63,6 +64,7 @@ def test_looper_cfg_required(self, cmd):
             ["--command-extra", CMD_STRS[3]],
         ],
     )
+    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     def test_cmd_extra_cli(self, prep_temp_pep, cmd, arg):
         """
         Argument passing functionality works only for the above
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 35493b870..b35c58e6e 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -19,14 +19,33 @@
 REPO_URL = "https://github.com/pepkit/hello_looper.git"
 
 
-def test_comprehensive_looper_no_pipestat(prep_temp_pep):
-    tp = prep_temp_pep
-
-    x = test_args_expansion(tp, "run")
-    try:
-        main(test_args=x)
-    except Exception:
-        raise pytest.fail("DID RAISE {0}".format(Exception))
+@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
+def test_comprehensive_looper_no_pipestat():
+
+    with TemporaryDirectory() as d:
+        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
+        basic_dir = os.path.join(d, "basic")
+
+        path_to_looper_config = os.path.join(basic_dir, ".looper.yaml")
+
+        # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+        basic_project_file = os.path.join(d, "basic/project", "project_config.yaml")
+        with open(basic_project_file, "r") as f:
+            pipestat_project_data = safe_load(f)
+
+        pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+            os.path.join(basic_dir, "data/{sample_name}.txt")
+        )
+
+        with open(basic_project_file, "w") as f:
+            dump(pipestat_project_data, f)
+
+        x = ["--looper-config", path_to_looper_config, "run"]
+
+        try:
+            main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")

From 22497113e3e327e237d0b49f66ab1377194060bb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 1 Mar 2024 14:04:06 -0500
Subject: [PATCH 114/225] attempt to fix selecting multiple flags, does not
 work

---
 looper/command_models/arguments.py |  2 +-
 looper/command_models/commands.py  |  6 ++++--
 tests/smoketests/test_other.py     | 16 +++++++++++++---
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 356d0478f..45e5a159c 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -200,7 +200,7 @@ class ArgumentEnum(enum.Enum):
         name="sel_flag", default=(str, ""), description="Sample selection flag"
     )
     EXC_FLAG = Argument(
-        name="exc_flag", default=(str, ""), description="Sample exclusion flag"
+        name="exc_flag", default=(List, []), description="Sample exclusion flag"
     )
     SKIP_FILE_CHECKS = Argument(
         name="skip_file_checks",
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 85652d83e..f4a488c15 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -3,7 +3,7 @@
 """
 
 from dataclasses import dataclass
-from typing import List, Optional, Type
+from typing import List, Optional, Type, Union
 
 import pydantic.v1 as pydantic
 
@@ -270,7 +270,9 @@ class TopLevelParser(pydantic.BaseModel):
     sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
     sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
     sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
-    exc_flag: Optional[str] = ArgumentEnum.EXC_FLAG.value.with_reduced_default()
+    exc_flag: Optional[Union[str, List[str]]] = (
+        ArgumentEnum.EXC_FLAG.value.with_reduced_default()
+    )
     # arguments for logging
     silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
     verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 7d42c1027..c83af1a69 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -130,7 +130,7 @@ def test_selecting_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "-d", "--looper-config", tp, "--sel-flag", "failed"]
+        x = ["--looper-config", tp, "--sel-flag", "failed", "run", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -165,7 +165,7 @@ def test_excluding_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "-d", "--looper-config", tp, "--exc-flag", "failed"]
+        x = ["--looper-config", tp, "--exc-flag", "failed", "run", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -201,7 +201,17 @@ def test_excluding_multi_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "-d", "--looper-config", tp, "--exc-flag", "failed", "running"]
+        # x = ["--looper-config", "--exc-flag", "['failed','running']", tp, "run", "--dry-run"]
+
+        x = [
+            "--looper-config",
+            tp,
+            "--exc-flag",
+            "failed",
+            "running",
+            "run",
+            "--dry-run",
+        ]
 
         try:
             results = main(test_args=x)

From 9fdf85a518db8c429ef3a88206a122ac9cbeac01 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 4 Mar 2024 14:11:06 -0500
Subject: [PATCH 115/225] Add exc_flag and sel_flag to Run command as a poc
 #438

---
 looper/cli_pydantic.py            |  8 ++++----
 looper/command_models/commands.py | 10 ++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index c3cf11670..44d860042 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -118,8 +118,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         else None
     )
     # Ignore flags if user is selecting or excluding on flags:
-    if args.sel_flag or args.exc_flag:
-        args.ignore_flags = True
+    if subcommand_args.sel_flag or subcommand_args.exc_flag:
+        subcommand_args.ignore_flags = True
 
     # Initialize project
     if is_registry_path(args.config_file):
@@ -166,8 +166,8 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         selector_attribute=args.sel_attr,
         selector_include=args.sel_incl,
         selector_exclude=args.sel_excl,
-        selector_flag=args.sel_flag,
-        exclusion_flag=args.exc_flag,
+        selector_flag=subcommand_args.sel_flag,
+        exclusion_flag=subcommand_args.exc_flag,
     ) as prj:
         if subcommand_name in ["run", "rerun"]:
             run = Runner(prj)
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index f4a488c15..278eb7585 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -58,6 +58,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
         ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
     ],
 )
 RunParserModel = RunParser.create_model()
@@ -269,10 +271,10 @@ class TopLevelParser(pydantic.BaseModel):
     sel_attr: Optional[str] = ArgumentEnum.SEL_ATTR.value.with_reduced_default()
     sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
     sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
-    sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
-    exc_flag: Optional[Union[str, List[str]]] = (
-        ArgumentEnum.EXC_FLAG.value.with_reduced_default()
-    )
+    # sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
+    # exc_flag: Optional[Union[str, List[str]]] = (
+    #     ArgumentEnum.EXC_FLAG.value.with_reduced_default()
+    # )
     # arguments for logging
     silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
     verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()

From debb45c6aea56f28f3a87af74e3e1ef6307927a1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 4 Mar 2024 15:27:20 -0500
Subject: [PATCH 116/225] Move all optional arguments to be under each command
 as appropriate  #438

---
 looper/cli_pydantic.py            |   6 +-
 looper/command_models/commands.py | 157 +++++++++++++++++++++++++-----
 looper/looper.py                  |   2 +-
 3 files changed, 136 insertions(+), 29 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 44d860042..d9aba0df3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -163,9 +163,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
     with ProjectContext(
         prj=p,
-        selector_attribute=args.sel_attr,
-        selector_include=args.sel_incl,
-        selector_exclude=args.sel_excl,
+        selector_attribute=subcommand_args.sel_attr,
+        selector_include=subcommand_args.sel_incl,
+        selector_exclude=subcommand_args.sel_excl,
         selector_flag=subcommand_args.sel_flag,
         exclusion_flag=subcommand_args.exc_flag,
     ) as prj:
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 278eb7585..615a82d2a 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -60,6 +60,23 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 RunParserModel = RunParser.create_model()
@@ -82,6 +99,25 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
         ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 RerunParserModel = RerunParser.create_model()
@@ -103,6 +139,25 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
         ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 RunProjectParserModel = RunProjectParser.create_model()
@@ -111,7 +166,15 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 TableParser = Command(
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
-    [],
+    [
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
+    ],
 )
 TableParserModel = TableParser.create_model()
 
@@ -122,6 +185,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 ReportParserModel = ReportParser.create_model()
@@ -133,6 +203,14 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 DestroyParserModel = DestroyParser.create_model()
@@ -145,6 +223,14 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.DESCRIBE_CODES.value,
         ArgumentEnum.ITEMIZED.value,
         ArgumentEnum.FLAGS.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 CheckParserModel = CheckParser.create_model()
@@ -156,6 +242,18 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SETTINGS.value,
+        ArgumentEnum.AMEND.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
     ],
 )
 CleanParserModel = CleanParser.create_model()
@@ -194,7 +292,15 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 LinkParser = Command(
     "link",
     MESSAGE_BY_SUBCOMMAND["link"],
-    [],
+    [
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SILENT.value,
+        ArgumentEnum.VERBOSITY.value,
+        ArgumentEnum.LOGDEV.value,
+    ],
 )
 LinkParserModel = LinkParser.create_model()
 
@@ -254,31 +360,32 @@ class TopLevelParser(pydantic.BaseModel):
     )
 
     # arguments
-    settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
-    pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
-    output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
-    config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    looper_config: Optional[str] = (
-        ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
-    )
-    sample_pipeline_interfaces: Optional[List[str]] = (
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
-    )
-    project_pipeline_interfaces: Optional[List[str]] = (
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
-    )
-    amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
-    sel_attr: Optional[str] = ArgumentEnum.SEL_ATTR.value.with_reduced_default()
-    sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
-    sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
+    # settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
+    # pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
+    # output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
+    # config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
+    # looper_config: Optional[str] = (
+    #     ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
+    # )
+    # sample_pipeline_interfaces: Optional[List[str]] = (
+    #     ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
+    # )
+    # project_pipeline_interfaces: Optional[List[str]] = (
+    #     ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
+    # )
+    # amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
+    # sel_attr: Optional[str] = ArgumentEnum.SEL_ATTR.value.with_reduced_default()
+    # sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
+    # sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
     # sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
     # exc_flag: Optional[Union[str, List[str]]] = (
     #     ArgumentEnum.EXC_FLAG.value.with_reduced_default()
     # )
+
     # arguments for logging
-    silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
-    verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
-    logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
-    pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()
-    limit: Optional[int] = ArgumentEnum.LIMIT.value.with_reduced_default()
-    skip: Optional[int] = ArgumentEnum.SKIP.value.with_reduced_default()
+    # silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
+    # verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
+    # logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
+    # pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()
+    # limit: Optional[int] = ArgumentEnum.LIMIT.value.with_reduced_default()
+    # skip: Optional[int] = ArgumentEnum.SKIP.value.with_reduced_default()
diff --git a/looper/looper.py b/looper/looper.py
index 04411168f..4dd91f0c8 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -425,7 +425,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
             self.prj.pipestat_configured_project or self.prj.pipestat_configured
         )
 
-        for sample in select_samples(prj=self.prj, args=top_level_args):
+        for sample in select_samples(prj=self.prj, args=args):
             pl_fails = []
             skip_reasons = []
             sample_pifaces = self.prj.get_sample_piface(

From 0589a03d9ebaefc5700c4621c1b67307641b2fcd Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:55:23 -0500
Subject: [PATCH 117/225] Re-add optional commands to ensure manual tests pass,
 refactor #438

---
 looper/cli_pydantic.py            | 62 +++++++++++----------
 looper/command_models/commands.py | 92 +++++++++++++++++++++----------
 looper/looper.py                  | 18 +++---
 3 files changed, 105 insertions(+), 67 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index d9aba0df3..ca5f649b6 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -68,11 +68,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         return int(
             not initiate_looper_config(
                 dotfile_path(),
-                args.pep_config,
-                args.init.output_dir,
-                args.sample_pipeline_interfaces,
-                args.project_pipeline_interfaces,
-                args.init.force_yes,
+                subcommand_args.pep_config,
+                subcommand_args.output_dir,
+                subcommand_args.sample_pipeline_interfaces,
+                subcommand_args.project_pipeline_interfaces,
+                subcommand_args.force_yes,
             )
         )
 
@@ -81,17 +81,19 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
 
-    if args.config_file is None:
+    if subcommand_args.config_file is None:
         looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
         try:
-            if args.looper_config:
-                looper_config_dict = read_looper_config_file(args.looper_config)
+            if subcommand_args.looper_config:
+                looper_config_dict = read_looper_config_file(
+                    subcommand_args.looper_config
+                )
             else:
                 looper_config_dict = read_looper_dotfile()
                 _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
 
             for looper_config_key, looper_config_item in looper_config_dict.items():
-                setattr(args, looper_config_key, looper_config_item)
+                setattr(subcommand_args, looper_config_key, looper_config_item)
 
         except OSError:
             parser.print_help(sys.stderr)
@@ -106,11 +108,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             "looper.databio.org/en/latest/looper-config"
         )
 
-    args = enrich_args_via_cfg(args, parser, test_args=test_args)
+    subcommand_args = enrich_args_via_cfg(subcommand_args, parser, test_args=test_args)
 
     # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
-    if vars(args)[PROJECT_PL_ARG]:
-        args.pipeline_interfaces = vars(args)[PROJECT_PL_ARG]
+    if vars(subcommand_args)[PROJECT_PL_ARG]:
+        subcommand_args.pipeline_interfaces = vars(subcommand_args)[PROJECT_PL_ARG]
 
     divcfg = (
         select_divvy_config(filepath=subcommand_args.divvy)
@@ -122,17 +124,19 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         subcommand_args.ignore_flags = True
 
     # Initialize project
-    if is_registry_path(args.config_file):
-        if vars(args)[SAMPLE_PL_ARG]:
+    if is_registry_path(subcommand_args.config_file):
+        if vars(subcommand_args)[SAMPLE_PL_ARG]:
             p = Project(
-                amendments=args.amend,
+                amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
                 project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=args.config_file
+                    registry_path=subcommand_args.config_file
                 ),
                 **{
-                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                    attr: getattr(subcommand_args, attr)
+                    for attr in CLI_PROJ_ATTRS
+                    if attr in subcommand_args
                 },
             )
         else:
@@ -142,12 +146,14 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     else:
         try:
             p = Project(
-                cfg=args.config_file,
-                amendments=args.amend,
+                cfg=subcommand_args.config_file,
+                amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
                 **{
-                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                    attr: getattr(subcommand_args, attr)
+                    for attr in CLI_PROJ_ATTRS
+                    if attr in subcommand_args
                 },
             )
         except yaml.parser.ParserError as e:
@@ -192,44 +198,44 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         if subcommand_name in ["runp"]:
             compute_kwargs = _proc_resources_spec(subcommand_args)
             collate = Collator(prj)
-            collate(args, **compute_kwargs)
+            collate(subcommand_args, **compute_kwargs)
             return collate.debug
 
         if subcommand_name in ["destroy"]:
-            return Destroyer(prj)(args)
+            return Destroyer(prj)(subcommand_args)
 
         use_pipestat = (
             prj.pipestat_configured_project
-            if getattr(args, "project", None)
+            if getattr(subcommand_args, "project", None)
             else prj.pipestat_configured
         )
 
         if subcommand_name in ["table"]:
             if use_pipestat:
-                return Tabulator(prj)(args)
+                return Tabulator(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("table")
 
         if subcommand_name in ["report"]:
             if use_pipestat:
-                return Reporter(prj)(args)
+                return Reporter(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("report")
 
         if subcommand_name in ["link"]:
             if use_pipestat:
-                Linker(prj)(args)
+                Linker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("link")
 
         if subcommand_name in ["check"]:
             if use_pipestat:
-                return Checker(prj)(args)
+                return Checker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("check")
 
         if subcommand_name in ["clean"]:
-            return Cleaner(prj)(args)
+            return Cleaner(prj)(subcommand_args)
 
         if subcommand_name in ["inspect"]:
             from warnings import warn
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 615a82d2a..279084aa2 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -74,9 +74,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
     ],
 )
 RunParserModel = RunParser.create_model()
@@ -115,9 +112,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
     ],
 )
 RerunParserModel = RerunParser.create_model()
@@ -155,9 +149,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
     ],
 )
 RunProjectParserModel = RunProjectParser.create_model()
@@ -171,9 +162,17 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.OUTPUT_DIR.value,
         ArgumentEnum.CONFIG_FILE.value,
         ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+        ArgumentEnum.AMEND.value,
     ],
 )
 TableParserModel = TableParser.create_model()
@@ -189,9 +188,17 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.OUTPUT_DIR.value,
         ArgumentEnum.CONFIG_FILE.value,
         ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
+        ArgumentEnum.AMEND.value,
     ],
 )
 ReportParserModel = ReportParser.create_model()
@@ -208,9 +215,16 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.CONFIG_FILE.value,
         ArgumentEnum.LOOPER_CONFIG.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
     ],
 )
 DestroyParserModel = DestroyParser.create_model()
@@ -228,9 +242,16 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.CONFIG_FILE.value,
         ArgumentEnum.LOOPER_CONFIG.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
     ],
 )
 CheckParserModel = CheckParser.create_model()
@@ -251,9 +272,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
     ],
 )
 CleanParserModel = CleanParser.create_model()
@@ -276,6 +301,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         # Original command has force flag which is technically a different flag, but we should just use FORCE_YES
         ArgumentEnum.FORCE_YES.value,
         ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
     ],
 )
 InitParserModel = InitParser.create_model()
@@ -297,9 +325,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.OUTPUT_DIR.value,
         ArgumentEnum.CONFIG_FILE.value,
         ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SILENT.value,
-        ArgumentEnum.VERBOSITY.value,
-        ArgumentEnum.LOGDEV.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
     ],
 )
 LinkParserModel = LinkParser.create_model()
@@ -383,9 +415,9 @@ class TopLevelParser(pydantic.BaseModel):
     # )
 
     # arguments for logging
-    # silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
-    # verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
-    # logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
+    silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
+    verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
+    logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
     # pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()
     # limit: Optional[int] = ArgumentEnum.LIMIT.value.with_reduced_default()
     # skip: Optional[int] = ArgumentEnum.SKIP.value.with_reduced_default()
diff --git a/looper/looper.py b/looper/looper.py
index 4dd91f0c8..4acfefe26 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -264,7 +264,7 @@ def __call__(self, args, preview_flag=True):
         if use_pipestat:
             destroy_summary(
                 self.prj,
-                getattr(args.destroy, "dry_run", None),
+                getattr(args, "dry_run", None),
                 getattr(args, "project", None),
             )
         else:
@@ -276,11 +276,11 @@ def __call__(self, args, preview_flag=True):
             _LOGGER.info("Destroy complete.")
             return 0
 
-        if getattr(args.destroy, "dry_run", None):
+        if getattr(args, "dry_run", None):
             _LOGGER.info("Dry run. No files destroyed.")
             return 0
 
-        if not getattr(args.destroy, "force_yes", None) and not query_yes_no(
+        if not getattr(args, "force_yes", None) and not query_yes_no(
             "Are you sure you want to permanently delete all pipeline "
             "results for this project?"
         ):
@@ -348,14 +348,14 @@ def __call__(self, args, **compute_kwargs):
                 pipeline_interface=project_piface_object,
                 prj=self.prj,
                 compute_variables=compute_kwargs,
-                delay=getattr(args.runp, "time_delay", None),
-                extra_args=getattr(args.runp, "command_extra", None),
-                extra_args_override=getattr(args.runp, "command_extra_override", None),
-                ignore_flags=getattr(args.runp, "ignore_flags", None),
+                delay=getattr(args, "time_delay", None),
+                extra_args=getattr(args, "command_extra", None),
+                extra_args_override=getattr(args, "command_extra_override", None),
+                ignore_flags=getattr(args, "ignore_flags", None),
                 collate=True,
             )
             if conductor.is_project_submittable(
-                force=getattr(args.runp, "ignore_flags", None)
+                force=getattr(args, "ignore_flags", None)
             ):
                 conductor._pool = [None]
                 conductor.submit()
@@ -559,7 +559,7 @@ def __call__(self, args):
         p = self.prj
         project_level = getattr(args, "project", None)
 
-        portable = args.report.portable
+        portable = args.portable
 
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)

From 83950e98e24f771c41d769cb8e488bcd6b8cf57f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 11:13:13 -0500
Subject: [PATCH 118/225] re-arrange arguments to be consistent

---
 looper/command_models/commands.py | 82 +++++++++++++++----------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 279084aa2..f772d8d1d 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -158,13 +158,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
     [
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
         ArgumentEnum.SEL_ATTR.value,
@@ -172,6 +165,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SEL_EXCL.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.AMEND.value,
     ],
 )
@@ -184,13 +184,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
         ArgumentEnum.SEL_ATTR.value,
@@ -198,6 +191,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SEL_EXCL.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.AMEND.value,
     ],
 )
@@ -210,14 +210,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.AMEND.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
         ArgumentEnum.SEL_ATTR.value,
@@ -225,6 +217,14 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SEL_EXCL.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.AMEND.value,
     ],
 )
 DestroyParserModel = DestroyParser.create_model()
@@ -237,14 +237,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.DESCRIBE_CODES.value,
         ArgumentEnum.ITEMIZED.value,
         ArgumentEnum.FLAGS.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.AMEND.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
         ArgumentEnum.SEL_ATTR.value,
@@ -252,6 +244,14 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SEL_EXCL.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PIPESTAT.value,
+        ArgumentEnum.AMEND.value,
     ],
 )
 CheckParserModel = CheckParser.create_model()
@@ -263,6 +263,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
+        ArgumentEnum.EXC_FLAG.value,
+        ArgumentEnum.SEL_FLAG.value,
+        ArgumentEnum.SEL_ATTR.value,
+        ArgumentEnum.SEL_INCL.value,
+        ArgumentEnum.SEL_EXCL.value,
+        ArgumentEnum.LIMIT.value,
+        ArgumentEnum.SKIP.value,
         ArgumentEnum.PEP_CONFIG.value,
         ArgumentEnum.OUTPUT_DIR.value,
         ArgumentEnum.CONFIG_FILE.value,
@@ -272,13 +279,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PIPESTAT.value,
         ArgumentEnum.SETTINGS.value,
         ArgumentEnum.AMEND.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
     ],
 )
 CleanParserModel = CleanParser.create_model()
@@ -321,10 +321,6 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     "link",
     MESSAGE_BY_SUBCOMMAND["link"],
     [
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
         ArgumentEnum.EXC_FLAG.value,
         ArgumentEnum.SEL_FLAG.value,
         ArgumentEnum.SEL_ATTR.value,
@@ -332,6 +328,10 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SEL_EXCL.value,
         ArgumentEnum.LIMIT.value,
         ArgumentEnum.SKIP.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.CONFIG_FILE.value,
+        ArgumentEnum.LOOPER_CONFIG.value,
     ],
 )
 LinkParserModel = LinkParser.create_model()

From ebc2eb786ff527b5bd45d17f6f16139fd0abe503 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 11:16:58 -0500
Subject: [PATCH 119/225] remove unused code

---
 looper/command_models/commands.py | 29 ++---------------------------
 1 file changed, 2 insertions(+), 27 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index f772d8d1d..da9803233 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -391,33 +391,8 @@ class TopLevelParser(pydantic.BaseModel):
         description=InspectParser.description
     )
 
-    # arguments
-    # settings: Optional[str] = ArgumentEnum.SETTINGS.value.with_reduced_default()
-    # pep_config: Optional[str] = ArgumentEnum.PEP_CONFIG.value.with_reduced_default()
-    # output_dir: Optional[str] = ArgumentEnum.OUTPUT_DIR.value.with_reduced_default()
-    # config_file: Optional[str] = ArgumentEnum.CONFIG_FILE.value.with_reduced_default()
-    # looper_config: Optional[str] = (
-    #     ArgumentEnum.LOOPER_CONFIG.value.with_reduced_default()
-    # )
-    # sample_pipeline_interfaces: Optional[List[str]] = (
-    #     ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value.with_reduced_default()
-    # )
-    # project_pipeline_interfaces: Optional[List[str]] = (
-    #     ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value.with_reduced_default()
-    # )
-    # amend: Optional[List[str]] = ArgumentEnum.AMEND.value.with_reduced_default()
-    # sel_attr: Optional[str] = ArgumentEnum.SEL_ATTR.value.with_reduced_default()
-    # sel_incl: Optional[str] = ArgumentEnum.SEL_INCL.value.with_reduced_default()
-    # sel_excl: Optional[str] = ArgumentEnum.SEL_EXCL.value.with_reduced_default()
-    # sel_flag: Optional[str] = ArgumentEnum.SEL_FLAG.value.with_reduced_default()
-    # exc_flag: Optional[Union[str, List[str]]] = (
-    #     ArgumentEnum.EXC_FLAG.value.with_reduced_default()
-    # )
-
-    # arguments for logging
+    # Additional arguments for logging, added to ALL commands
+    # These must be used before the command
     silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
     verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
     logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()
-    # pipestat: Optional[str] = ArgumentEnum.PIPESTAT.value.with_reduced_default()
-    # limit: Optional[int] = ArgumentEnum.LIMIT.value.with_reduced_default()
-    # skip: Optional[int] = ArgumentEnum.SKIP.value.with_reduced_default()

From b447b5cc5eae2df86c24493647749e6dd3b6bfef Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 11:26:42 -0500
Subject: [PATCH 120/225] fix comprehensive tests per refactor

---
 looper/looper.py            |  6 +++---
 tests/test_comprehensive.py | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 4acfefe26..82adb9144 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -124,7 +124,7 @@ def __call__(self, args):
                     table.add_row(status_id, f"{status_count}/{len(status_list)}")
             console.print(table)
 
-        if getattr(args.check, "itemized", None):
+        if getattr(args, "itemized", None):
             for pipeline_name, pipeline_status in status.items():
                 table_title = f"Pipeline: '{pipeline_name}'"
                 table = Table(
@@ -150,7 +150,7 @@ def __call__(self, args):
                     table.add_row(name, f"[{color}]{status_id}[/{color}]")
                 console.print(table)
 
-        if args.check.describe_codes:
+        if args.describe_codes:
             table = Table(
                 show_header=True,
                 header_style="bold magenta",
@@ -251,7 +251,7 @@ def __call__(self, args, preview_flag=True):
                 # Preview: Don't actually delete, just show files.
                 _LOGGER.info(str(sample_output_folder))
             else:
-                _remove_or_dry_run(sample_output_folder, args.destroy.dry_run)
+                _remove_or_dry_run(sample_output_folder, args.dry_run)
 
         _LOGGER.info("Removing summary:")
         use_pipestat = (
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index b35c58e6e..e26671536 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -40,7 +40,7 @@ def test_comprehensive_looper_no_pipestat():
         with open(basic_project_file, "w") as f:
             dump(pipestat_project_data, f)
 
-        x = ["--looper-config", path_to_looper_config, "run"]
+        x = ["run", "--looper-config", path_to_looper_config]
 
         try:
             main(test_args=x)
@@ -79,7 +79,7 @@ def test_comprehensive_looper_pipestat():
             dump(pipestat_project_data, f)
 
         # x = [cmd, "-d", "--looper-config", path_to_looper_config]
-        x = ["--looper-config", path_to_looper_config, cmd]
+        x = [cmd, "--looper-config", path_to_looper_config]
 
         try:
             result = main(test_args=x)
@@ -103,7 +103,7 @@ def test_comprehensive_looper_pipestat():
         psm.set_status(record_identifier="frog_2", status_identifier="completed")
 
         # Now use looper check to get statuses
-        x = ["--looper-config", path_to_looper_config, "check"]
+        x = ["check", "--looper-config", path_to_looper_config]
 
         try:
             result = main(test_args=x)
@@ -113,7 +113,7 @@ def test_comprehensive_looper_pipestat():
 
         # TEST LOOPER REPORT
 
-        x = ["--looper-config", path_to_looper_config, "report"]
+        x = ["report", "--looper-config", path_to_looper_config]
 
         try:
             result = main(test_args=x)
@@ -123,7 +123,7 @@ def test_comprehensive_looper_pipestat():
 
         # TEST LOOPER Table
 
-        x = ["--looper-config", path_to_looper_config, "table"]
+        x = ["table", "--looper-config", path_to_looper_config]
 
         try:
             result = main(test_args=x)
@@ -135,9 +135,9 @@ def test_comprehensive_looper_pipestat():
         # TODO add destroying individual samples via pipestat
 
         x = [
+            "destroy",
             "--looper-config",
             path_to_looper_config,
-            "destroy",
             "--force-yes",
         ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
 

From 1219451be78086a5dafe1cd454e4d882f830f880 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 11:38:50 -0500
Subject: [PATCH 121/225] fix broken tests, change SEL_FLAG and SEL_INCL to
 default to lists

---
 looper/command_models/arguments.py |  4 ++--
 tests/smoketests/test_other.py     | 36 ++++++++++++++++++------------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 45e5a159c..2c8d9717b 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -188,7 +188,7 @@ class ArgumentEnum(enum.Enum):
     )
     SEL_INCL = Argument(
         name="sel_incl",
-        default=(str, ""),
+        default=(List, []),
         description="Include only samples with these values",
     )
     SEL_EXCL = Argument(
@@ -197,7 +197,7 @@ class ArgumentEnum(enum.Enum):
         description="Exclude samples with these values",
     )
     SEL_FLAG = Argument(
-        name="sel_flag", default=(str, ""), description="Sample selection flag"
+        name="sel_flag", default=(List, []), description="Sample selection flag"
     )
     EXC_FLAG = Argument(
         name="exc_flag", default=(List, []), description="Sample exclusion flag"
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index c83af1a69..3dcdd8076 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -31,7 +31,7 @@ def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         # tp = prep_temp_pep
         dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
         # x = test_args_expansion(tp, cmd, dry=False)
-        x = ["--looper-config", dot_file_path, cmd]
+        x = [cmd, "--looper-config", dot_file_path]
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
 
@@ -40,10 +40,10 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
         tp = prep_temp_pep_pipestat
 
         if cmd in ["run", "runp"]:
-            x = ["--looper-config", tp, cmd, "--dry-run"]
+            x = [cmd, "--looper-config", tp, "--dry-run"]
         else:
             # Not every command supports dry run
-            x = ["--looper-config", tp, cmd]
+            x = [cmd, "--looper-config", tp]
 
         try:
             result = main(test_args=x)
@@ -63,7 +63,7 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags(tp, flag_id, pipeline_name)
 
-        x = ["--looper-config", tp, "check"]
+        x = ["check", "--looper-config", tp]
 
         try:
             results = main(test_args=x)
@@ -82,7 +82,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         _make_flags(tp, flag_id, pipeline_name)
         _make_flags(tp, FLAGS[1], pipeline_name)
 
-        x = ["--looper-config", tp, "check"]
+        x = ["check", "--looper-config", tp]
         # Multiple flag files SHOULD cause pipestat to throw an assertion error
         if flag_id != FLAGS[1]:
             with pytest.raises(AssertionError):
@@ -95,7 +95,7 @@ def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         tp = prep_temp_pep_pipestat
         _make_flags(tp, flag_id, pipeline_name)
 
-        x = ["--looper-config", tp, "check"]
+        x = ["check", "--looper-config", tp]
         try:
             results = main(test_args=x)
             result_key = list(results.keys())[0]
@@ -130,7 +130,7 @@ def test_selecting_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["--looper-config", tp, "--sel-flag", "failed", "run", "--dry-run"]
+        x = ["run", "--looper-config", tp, "--sel-flag", "failed", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -165,7 +165,7 @@ def test_excluding_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["--looper-config", tp, "--exc-flag", "failed", "run", "--dry-run"]
+        x = ["run", "--looper-config", tp, "--exc-flag", "failed", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -204,12 +204,12 @@ def test_excluding_multi_flags_works(
         # x = ["--looper-config", "--exc-flag", "['failed','running']", tp, "run", "--dry-run"]
 
         x = [
+            "run",
             "--looper-config",
             tp,
             "--exc-flag",
             "failed",
             "running",
-            "run",
             "--dry-run",
         ]
 
@@ -247,7 +247,15 @@ def test_selecting_multi_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "-d", "--looper-config", tp, "--sel-flag", "failed", "running"]
+        x = [
+            "run",
+            "--dry-run",
+            "--looper-config",
+            tp,
+            "--sel-flag",
+            "failed",
+            "running",
+        ]
 
         try:
             results = main(test_args=x)
@@ -285,7 +293,7 @@ def test_selecting_attr_and_flags_works(
 
         x = [
             "run",
-            "-d",
+            "--dry-run",
             "--looper-config",
             tp,
             "--sel-flag",
@@ -332,7 +340,7 @@ def test_excluding_attr_and_flags_works(
 
         x = [
             "run",
-            "-d",
+            "--dry-run",
             "--looper-config",
             tp,
             "--exc-flag",
@@ -380,7 +388,7 @@ def test_excluding_toggle_attr(
 
         x = [
             "run",
-            "-d",
+            "--dry-run",
             "--looper-config",
             tp,
             "--sel-attr",
@@ -427,7 +435,7 @@ def test_including_toggle_attr(
 
         x = [
             "run",
-            "-d",
+            "--dry-run",
             "--looper-config",
             tp,
             "--sel-attr",

From 6c6150045919ce547011dafcdb9634ffc4232f73 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 16:00:37 -0500
Subject: [PATCH 122/225] refactor to reduce code duplication

---
 looper/command_models/commands.py | 196 +++++++-----------------------
 1 file changed, 46 insertions(+), 150 deletions(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index da9803233..5c5876541 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -41,6 +41,26 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         return pydantic.create_model(self.name, **arguments)
 
 
+SHARED_ARGUMENTS = [
+    ArgumentEnum.SETTINGS.value,
+    ArgumentEnum.EXC_FLAG.value,
+    ArgumentEnum.SEL_FLAG.value,
+    ArgumentEnum.SEL_ATTR.value,
+    ArgumentEnum.SEL_INCL.value,
+    ArgumentEnum.SEL_EXCL.value,
+    ArgumentEnum.LIMIT.value,
+    ArgumentEnum.SKIP.value,
+    ArgumentEnum.PEP_CONFIG.value,
+    ArgumentEnum.OUTPUT_DIR.value,
+    ArgumentEnum.CONFIG_FILE.value,
+    ArgumentEnum.LOOPER_CONFIG.value,
+    ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+    ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+    ArgumentEnum.PIPESTAT.value,
+    ArgumentEnum.SETTINGS.value,
+    ArgumentEnum.AMEND.value,
+]
+
 RunParser = Command(
     "run",
     MESSAGE_BY_SUBCOMMAND["run"],
@@ -57,26 +77,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-RunParserModel = RunParser.create_model()
 
 # RERUN
 RerunParser = Command(
@@ -95,26 +97,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-RerunParserModel = RerunParser.create_model()
 
 # RUNP
 RunProjectParser = Command(
@@ -132,50 +116,15 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-RunProjectParserModel = RunProjectParser.create_model()
 
 # TABLE
 TableParser = Command(
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
-    [
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.AMEND.value,
-    ],
+    [],
 )
-TableParserModel = TableParser.create_model()
 
 
 # REPORT
@@ -184,24 +133,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-ReportParserModel = ReportParser.create_model()
 
 # DESTROY
 DestroyParser = Command(
@@ -210,24 +143,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-DestroyParserModel = DestroyParser.create_model()
 
 # CHECK
 CheckParser = Command(
@@ -237,24 +154,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.DESCRIBE_CODES.value,
         ArgumentEnum.ITEMIZED.value,
         ArgumentEnum.FLAGS.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-CheckParserModel = CheckParser.create_model()
 
 # CLEAN
 CleanParser = Command(
@@ -263,25 +164,8 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     [
         ArgumentEnum.DRY_RUN.value,
         ArgumentEnum.FORCE_YES.value,
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
-        ArgumentEnum.PIPESTAT.value,
-        ArgumentEnum.SETTINGS.value,
-        ArgumentEnum.AMEND.value,
     ],
 )
-CleanParserModel = CleanParser.create_model()
 
 # INSPECT
 # TODO Did this move to Eido?
@@ -320,22 +204,34 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 LinkParser = Command(
     "link",
     MESSAGE_BY_SUBCOMMAND["link"],
-    [
-        ArgumentEnum.EXC_FLAG.value,
-        ArgumentEnum.SEL_FLAG.value,
-        ArgumentEnum.SEL_ATTR.value,
-        ArgumentEnum.SEL_INCL.value,
-        ArgumentEnum.SEL_EXCL.value,
-        ArgumentEnum.LIMIT.value,
-        ArgumentEnum.SKIP.value,
-        ArgumentEnum.PEP_CONFIG.value,
-        ArgumentEnum.OUTPUT_DIR.value,
-        ArgumentEnum.CONFIG_FILE.value,
-        ArgumentEnum.LOOPER_CONFIG.value,
-    ],
+    [],
 )
+
+
+# Add shared arguments for all commands that use them
+for arg in SHARED_ARGUMENTS:
+    RunParser.arguments.append(arg)
+    RerunParser.arguments.append(arg)
+    RunProjectParser.arguments.append(arg)
+    ReportParser.arguments.append(arg)
+    DestroyParser.arguments.append(arg)
+    CheckParser.arguments.append(arg)
+    CleanParser.arguments.append(arg)
+    TableParser.arguments.append(arg)
+    LinkParser.arguments.append(arg)
+
+# Create all Models
+RunParserModel = RunParser.create_model()
+RerunParserModel = RerunParser.create_model()
+RunProjectParserModel = RunProjectParser.create_model()
+ReportParserModel = ReportParser.create_model()
+DestroyParserModel = DestroyParser.create_model()
+CheckParserModel = CheckParser.create_model()
+CleanParserModel = CleanParser.create_model()
+TableParserModel = TableParser.create_model()
 LinkParserModel = LinkParser.create_model()
 
+
 SUPPORTED_COMMANDS = [
     RunParser,
     RerunParser,

From f1eac9cf893f0ff52f508749e9e876eca59baff1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 5 Mar 2024 16:17:16 -0500
Subject: [PATCH 123/225] remove original looper cli

---
 looper/cli_looper.py                    | 867 ------------------------
 looper/cli_pydantic.py                  |  45 +-
 tests/smoketests/test_cli_validation.py |   2 +-
 3 files changed, 45 insertions(+), 869 deletions(-)
 delete mode 100644 looper/cli_looper.py

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
deleted file mode 100644
index 199686215..000000000
--- a/looper/cli_looper.py
+++ /dev/null
@@ -1,867 +0,0 @@
-import argparse
-import logmuse
-import os
-import sys
-import yaml
-
-from eido import inspect_project
-from pephubclient import PEPHubClient
-from typing import Tuple, List
-from ubiquerg import VersionInHelpParser
-
-from . import __version__
-from .command_models.commands import RunParser
-from .const import *
-from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
-from .exceptions import *
-from .looper import *
-from .parser_types import *
-from .project import Project, ProjectContext
-from .utils import (
-    dotfile_path,
-    enrich_args_via_cfg,
-    is_registry_path,
-    read_looper_dotfile,
-    read_looper_config_file,
-    read_yaml_file,
-    initiate_looper_config,
-    init_generic_pipeline,
-)
-
-
-class _StoreBoolActionType(argparse.Action):
-    """
-    Enables the storage of a boolean const and custom type definition needed
-    for systematic html interface generation. To get the _StoreTrueAction
-    output use default=False in the add_argument function
-    and default=True to get _StoreFalseAction output.
-    """
-
-    def __init__(self, option_strings, dest, type, default, required=False, help=None):
-        super(_StoreBoolActionType, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=0,
-            const=not default,
-            default=default,
-            type=type,
-            required=required,
-            help=help,
-        )
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        setattr(namespace, self.dest, self.const)
-
-
-def build_parser():
-    """
-    Building argument parser.
-
-    :return argparse.ArgumentParser
-    """
-    # Main looper program help text messages
-    banner = "%(prog)s - A project job submission engine and project manager."
-    additional_description = (
-        "For subcommand-specific options, " "type: '%(prog)s <subcommand> -h'"
-    )
-    additional_description += "\nhttps://github.com/pepkit/looper"
-
-    parser = VersionInHelpParser(
-        prog="looper",
-        description=banner,
-        epilog=additional_description,
-        version=__version__,
-    )
-
-    aux_parser = VersionInHelpParser(
-        prog="looper",
-        description=banner,
-        epilog=additional_description,
-        version=__version__,
-    )
-    result = []
-    for parser in [parser, aux_parser]:
-        # Logging control
-        parser.add_argument(
-            "--logfile",
-            help="Optional output file for looper logs " "(default: %(default)s)",
-        )
-        parser.add_argument("--logging-level", help=argparse.SUPPRESS)
-        parser.add_argument(
-            "--dbg",
-            action="store_true",
-            help="Turn on debug mode (default: %(default)s)",
-        )
-
-        parser = logmuse.add_logging_options(parser)
-        subparsers = parser.add_subparsers(dest="command")
-
-        def add_subparser(cmd):
-            message = MESSAGE_BY_SUBCOMMAND[cmd]
-            return subparsers.add_parser(
-                cmd,
-                description=message,
-                help=message,
-                formatter_class=lambda prog: argparse.HelpFormatter(
-                    prog, max_help_position=37, width=90
-                ),
-            )
-
-        # Run and rerun command
-        run_subparser = add_subparser("run")
-        rerun_subparser = add_subparser("rerun")
-        collate_subparser = add_subparser("runp")
-        table_subparser = add_subparser("table")
-        report_subparser = add_subparser("report")
-        destroy_subparser = add_subparser("destroy")
-        check_subparser = add_subparser("check")
-        clean_subparser = add_subparser("clean")
-        inspect_subparser = add_subparser("inspect")
-        init_subparser = add_subparser("init")
-        init_piface = add_subparser("init-piface")
-        link_subparser = add_subparser("link")
-
-        # Flag arguments
-        ####################################################################
-        for subparser in [run_subparser, rerun_subparser, collate_subparser]:
-            subparser.add_argument(
-                "-i",
-                "--ignore-flags",
-                default=False,
-                action=_StoreBoolActionType,
-                type=html_checkbox(checked=False),
-                help="Ignore run status flags? Default=False",
-            )
-
-        for subparser in [
-            run_subparser,
-            rerun_subparser,
-            destroy_subparser,
-            clean_subparser,
-            collate_subparser,
-        ]:
-            subparser.add_argument(
-                "-d",
-                "--dry-run",
-                action=_StoreBoolActionType,
-                default=False,
-                type=html_checkbox(checked=False),
-                help="Don't actually submit the jobs.  Default=False",
-            )
-
-        # Parameter arguments
-        ####################################################################
-        for subparser in [run_subparser, rerun_subparser, collate_subparser]:
-            subparser.add_argument(
-                "-t",
-                "--time-delay",
-                metavar="S",
-                type=html_range(min_val=0, max_val=30, value=0),
-                default=0,
-                help="Time delay in seconds between job submissions",
-            )
-
-            subparser.add_argument(
-                "-x",
-                "--command-extra",
-                default="",
-                metavar="S",
-                help="String to append to every command",
-            )
-            subparser.add_argument(
-                "-y",
-                "--command-extra-override",
-                metavar="S",
-                default="",
-                help="Same as command-extra, but overrides values in PEP",
-            )
-            subparser.add_argument(
-                "-f",
-                "--skip-file-checks",
-                action=_StoreBoolActionType,
-                default=False,
-                type=html_checkbox(checked=False),
-                help="Do not perform input file checks",
-            )
-
-            divvy_group = subparser.add_argument_group(
-                "divvy arguments", "Configure divvy to change computing settings"
-            )
-            divvy_group.add_argument(
-                "--divvy",
-                default=None,
-                metavar="DIVCFG",
-                help="Path to divvy configuration file. Default=$DIVCFG env "
-                "variable. Currently: {}".format(
-                    os.getenv("DIVCFG", None) or "not set"
-                ),
-            )
-            divvy_group.add_argument(
-                "-p",
-                "--package",
-                metavar="P",
-                help="Name of computing resource package to use",
-            )
-            divvy_group.add_argument(
-                "-s",
-                "--settings",
-                default="",
-                metavar="S",
-                help="Path to a YAML settings file with compute settings",
-            )
-            divvy_group.add_argument(
-                "-c",
-                "--compute",
-                metavar="K",
-                nargs="+",
-                help="List of key-value pairs (k1=v1)",
-            )
-
-        for subparser in [run_subparser, rerun_subparser]:
-            subparser.add_argument(
-                "-u",
-                "--lump-s",
-                default=None,
-                metavar="X",
-                type=html_range(min_val=0, max_val=100, step=0.1, value=0),
-                help="Lump by size: total input file size (GB) to batch into one job",
-            )
-            subparser.add_argument(
-                "-n",
-                "--lump-n",
-                default=None,
-                metavar="N",
-                type=html_range(min_val=1, max_val="num_samples", value=1),
-                help="Lump by number: number of samples to batch into one job",
-            )
-            subparser.add_argument(
-                "-j",
-                "--lump-j",
-                default=None,
-                metavar="J",
-                type=int,
-                help="Lump samples into number of jobs.",
-            )
-
-        check_subparser.add_argument(
-            "--describe-codes",
-            help="Show status codes description",
-            action="store_true",
-            default=False,
-        )
-
-        check_subparser.add_argument(
-            "--itemized",
-            help="Show a detailed, by sample statuses",
-            action="store_true",
-            default=False,
-        )
-
-        check_subparser.add_argument(
-            "-f",
-            "--flags",
-            nargs="*",
-            default=FLAGS,
-            type=html_select(choices=FLAGS),
-            metavar="F",
-            help="Check on only these flags/status values",
-        )
-
-        for subparser in [destroy_subparser, clean_subparser]:
-            subparser.add_argument(
-                "--force-yes",
-                action=_StoreBoolActionType,
-                default=False,
-                type=html_checkbox(checked=False),
-                help="Provide upfront confirmation of destruction intent, "
-                "to skip console query.  Default=False",
-            )
-
-        init_subparser.add_argument(
-            "pep_config", help="Project configuration file (PEP)"
-        )
-
-        init_subparser.add_argument(
-            "-f", "--force", help="Force overwrite", action="store_true", default=False
-        )
-
-        init_subparser.add_argument(
-            "-o",
-            "--output-dir",
-            dest="output_dir",
-            metavar="DIR",
-            default=None,
-            type=str,
-        )
-
-        init_subparser.add_argument(
-            "-S",
-            "--sample-pipeline-interfaces",
-            dest=SAMPLE_PL_ARG,
-            metavar="YAML",
-            default=None,
-            nargs="+",
-            type=str,
-            help="Path to looper sample config file",
-        )
-        init_subparser.add_argument(
-            "-P",
-            "--project-pipeline-interfaces",
-            dest=PROJECT_PL_ARG,
-            metavar="YAML",
-            default=None,
-            nargs="+",
-            type=str,
-            help="Path to looper project config file",
-        )
-
-        # TODO: add  ouput dir, sample, project pifaces
-
-        init_subparser.add_argument(
-            "-p",
-            "--piface",
-            help="Generates generic pipeline interface",
-            action="store_true",
-            default=False,
-        )
-
-        # Common arguments
-        for subparser in [
-            run_subparser,
-            rerun_subparser,
-            table_subparser,
-            report_subparser,
-            destroy_subparser,
-            check_subparser,
-            clean_subparser,
-            collate_subparser,
-            inspect_subparser,
-            link_subparser,
-        ]:
-            subparser.add_argument(
-                "config_file",
-                nargs="?",
-                default=None,
-                help="Project configuration file (YAML) or pephub registry path.",
-            )
-            subparser.add_argument(
-                "--looper-config",
-                required=False,
-                default=None,
-                type=str,
-                help="Looper configuration file (YAML)",
-            )
-            # help="Path to the looper config file"
-            subparser.add_argument(
-                "-S",
-                "--sample-pipeline-interfaces",
-                dest=SAMPLE_PL_ARG,
-                metavar="YAML",
-                default=None,
-                nargs="+",
-                type=str,
-                help="Path to looper sample config file",
-            )
-            subparser.add_argument(
-                "-P",
-                "--project-pipeline-interfaces",
-                dest=PROJECT_PL_ARG,
-                metavar="YAML",
-                default=None,
-                nargs="+",
-                type=str,
-                help="Path to looper project config file",
-            )
-            # help="Path to the output directory"
-            subparser.add_argument(
-                "-o",
-                "--output-dir",
-                dest="output_dir",
-                metavar="DIR",
-                default=None,
-                type=str,
-                help=argparse.SUPPRESS,
-            )
-            # "Submission subdirectory name"
-            subparser.add_argument(
-                "--submission-subdir", metavar="DIR", help=argparse.SUPPRESS
-            )
-            # "Results subdirectory name"
-            subparser.add_argument(
-                "--results-subdir", metavar="DIR", help=argparse.SUPPRESS
-            )
-            # "Sample attribute for pipeline interface sources"
-            subparser.add_argument(
-                "--pipeline-interfaces-key", metavar="K", help=argparse.SUPPRESS
-            )
-            # "Paths to pipeline interface files"
-            subparser.add_argument(
-                "--pipeline-interfaces",
-                metavar="P",
-                nargs="+",
-                action="append",
-                help=argparse.SUPPRESS,
-            )
-
-        for subparser in [
-            run_subparser,
-            rerun_subparser,
-            table_subparser,
-            report_subparser,
-            destroy_subparser,
-            check_subparser,
-            clean_subparser,
-            collate_subparser,
-            inspect_subparser,
-            link_subparser,
-        ]:
-            fetch_samples_group = subparser.add_argument_group(
-                "sample selection arguments",
-                "Specify samples to include or exclude based on sample attribute values",
-            )
-            fetch_samples_group.add_argument(
-                "-l",
-                "--limit",
-                default=None,
-                metavar="N",
-                type=html_range(min_val=1, max_val="num_samples", value="num_samples"),
-                help="Limit to n samples",
-            )
-            fetch_samples_group.add_argument(
-                "-k",
-                "--skip",
-                default=None,
-                metavar="N",
-                type=html_range(min_val=1, max_val="num_samples", value="num_samples"),
-                help="Skip samples by numerical index",
-            )
-
-            fetch_samples_group.add_argument(
-                f"--{SAMPLE_SELECTION_ATTRIBUTE_OPTNAME}",
-                default="toggle",
-                metavar="ATTR",
-                help="Attribute for sample exclusion OR inclusion",
-            )
-
-            protocols = fetch_samples_group.add_mutually_exclusive_group()
-            protocols.add_argument(
-                f"--{SAMPLE_EXCLUSION_OPTNAME}",
-                nargs="*",
-                metavar="E",
-                help="Exclude samples with these values",
-            )
-            protocols.add_argument(
-                f"--{SAMPLE_INCLUSION_OPTNAME}",
-                nargs="*",
-                metavar="I",
-                help="Include only samples with these values",
-            )
-            fetch_samples_group.add_argument(
-                f"--{SAMPLE_SELECTION_FLAG_OPTNAME}",
-                default=None,
-                nargs="*",
-                metavar="SELFLAG",
-                help="Include samples with this flag status, e.g. completed",
-            )
-
-            fetch_samples_group.add_argument(
-                f"--{SAMPLE_EXCLUSION_FLAG_OPTNAME}",
-                default=None,
-                nargs="*",
-                metavar="EXCFLAG",
-                help="Exclude samples with this flag status, e.g. completed",
-            )
-
-            subparser.add_argument(
-                "-a",
-                "--amend",
-                nargs="+",
-                metavar="A",
-                help="List of amendments to activate",
-            )
-        for subparser in [
-            report_subparser,
-            table_subparser,
-            check_subparser,
-            destroy_subparser,
-            link_subparser,
-        ]:
-            subparser.add_argument(
-                "--project",
-                help="Process project-level pipelines",
-                action="store_true",
-                default=False,
-            )
-        inspect_subparser.add_argument(
-            "--sample-names",
-            help="Names of the samples to inspect",
-            nargs="*",
-            default=None,
-        )
-
-        inspect_subparser.add_argument(
-            "--attr-limit",
-            help="Number of attributes to display",
-            type=int,
-        )
-        parser.add_argument(
-            "--commands",
-            action="version",
-            version="{}".format(" ".join(subparsers.choices.keys())),
-        )
-
-        report_subparser.add_argument(
-            "--portable",
-            help="Makes html report portable.",
-            action="store_true",
-        )
-
-        result.append(parser)
-    return result
-
-
-def opt_attr_pair(name: str) -> Tuple[str, str]:
-    return f"--{name}", name.replace("-", "_")
-
-
-def validate_post_parse(args: argparse.Namespace) -> List[str]:
-    problems = []
-    used_exclusives = [
-        opt
-        for opt, attr in map(
-            opt_attr_pair,
-            [
-                "skip",
-                "limit",
-                SAMPLE_EXCLUSION_OPTNAME,
-                SAMPLE_INCLUSION_OPTNAME,
-            ],
-        )
-        # Depending on the subcommand used, the above options might either be in
-        # the top-level namespace or in the subcommand namespace (the latter due
-        # to a `modify_args_namespace()`)
-        if getattr(
-            args, attr, None
-        )  # or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
-    ]
-    if len(used_exclusives) > 1:
-        problems.append(
-            f"Used multiple mutually exclusive options: {', '.join(used_exclusives)}"
-        )
-    return problems
-
-
-def _proc_resources_spec(args):
-    """
-    Process CLI-sources compute setting specification. There are two sources
-    of compute settings in the CLI alone:
-        * YAML file (--settings argument)
-        * itemized compute settings (--compute argument)
-
-    The itemized compute specification is given priority
-
-    :param argparse.Namespace: arguments namespace
-    :return Mapping[str, str]: binding between resource setting name and value
-    :raise ValueError: if interpretation of the given specification as encoding
-        of key-value pairs fails
-    """
-    # if (hasattr(args, "run") and args.run) or args.command in ("run",):
-    #     spec = getattr(args.run, "compute", None)
-    #     settings = args.run.settings
-    # else:
-    spec = getattr(args, "compute", None)
-    settings = args.settings
-    try:
-        settings_data = read_yaml_file(settings) or {}
-    except yaml.YAMLError:
-        _LOGGER.warning(
-            "Settings file ({}) does not follow YAML format,"
-            " disregarding".format(settings)
-        )
-        settings_data = {}
-    if not spec:
-        return settings_data
-    pairs = [(kv, kv.split("=")) for kv in spec]
-    bads = []
-    for orig, pair in pairs:
-        try:
-            k, v = pair
-        except ValueError:
-            bads.append(orig)
-        else:
-            settings_data[k] = v
-    if bads:
-        raise ValueError(
-            "Could not correctly parse itemized compute specification. "
-            "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
-        )
-    return settings_data
-
-
-def make_hierarchical_if_needed(args):
-    """
-    Make an `argparse` namespace hierarchic for commands that support it
-
-    In the course of introducing `pydantic` models as ground truths, some logic pertaining
-    to the `run` command in `looper/looper.py` was changed in order to do accurately reflect
-    which arguments are top-level arguments and which arguments are specific to the `run`
-    command.
-
-    If the command in the given arguments is 'run', this function creates a sub-namespace
-    named 'run' and moves selected arguments specified in RUN_ARGS to the 'run' namespace.
-    The selected arguments are also removed from the original namespace.
-    This thus morphes the original namespace into the hierarchy the `run` command logic
-    expects downstream.
-
-    :param argparse.Namespace: The argparse namespace containing program arguments.
-    :return argparse.Namespace: The modified, partially hierarchical argparse namespace.
-    """
-
-    def add_command_hierarchy(command_args):
-        # make a new namespace that will be the resulting second-level namespace
-        command_namespace = argparse.Namespace()
-        for arg in vars(args):
-            if arg in command_args:
-                setattr(command_namespace, arg, getattr(args, arg))
-
-        # remove arguments that have been moved into the second-level namespace
-        # from the top-level namespace
-        for arg in command_args:
-            if hasattr(args, arg):
-                delattr(args, arg)
-
-        setattr(args, args.command, command_namespace)
-
-    if args.command == "run":
-        # if args.command == "run" or args.command == "rerun":
-        # we only want to only move arguments to the `run` second-level namespace
-        # that are in fact specific to the `run` subcommand
-        run_args = [argument.name for argument in RunParser.arguments]
-        add_command_hierarchy(run_args)
-
-    return args
-
-
-def main(test_args=None):
-    """Primary workflow"""
-    global _LOGGER
-
-    parser, aux_parser = build_parser()
-    aux_parser.suppress_defaults()
-
-    if test_args:
-        args, remaining_args = parser.parse_known_args(args=test_args)
-    else:
-        args, remaining_args = parser.parse_known_args()
-
-    args = make_hierarchical_if_needed(args)
-    cli_use_errors = validate_post_parse(args)
-    if cli_use_errors:
-        parser.print_help(sys.stderr)
-        parser.error(
-            f"{len(cli_use_errors)} CLI use problem(s): {', '.join(cli_use_errors)}"
-        )
-    if args.command is None:
-        parser.print_help(sys.stderr)
-        sys.exit(1)
-
-    if args.command == "init":
-        return int(
-            not initiate_looper_config(
-                dotfile_path(),
-                args.pep_config,
-                args.output_dir,
-                args.sample_pipeline_interfaces,
-                args.project_pipeline_interfaces,
-                args.force,
-            )
-        )
-
-    if args.command == "init-piface":
-        sys.exit(int(not init_generic_pipeline()))
-
-    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
-    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, args.command))
-
-    if "config_file" in vars(args):
-        if args.config_file is None:
-            looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
-            try:
-                if args.looper_config:
-                    looper_config_dict = read_looper_config_file(args.looper_config)
-                else:
-                    looper_config_dict = read_looper_dotfile()
-                    _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
-
-                for looper_config_key, looper_config_item in looper_config_dict.items():
-                    setattr(args, looper_config_key, looper_config_item)
-
-            except OSError:
-                parser.print_help(sys.stderr)
-                _LOGGER.warning(
-                    f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
-                )
-                sys.exit(1)
-        else:
-            _LOGGER.warning(
-                "This PEP configures looper through the project config. This approach is deprecated and will "
-                "be removed in future versions. Please use a looper config file. For more information see "
-                "looper.databio.org/en/latest/looper-config"
-            )
-
-    args = enrich_args_via_cfg(args, aux_parser, test_args)
-    # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
-    if vars(args)[PROJECT_PL_ARG]:
-        args.pipeline_interfaces = vars(args)[PROJECT_PL_ARG]
-
-    if len(remaining_args) > 0:
-        _LOGGER.warning(
-            "Unrecognized arguments: {}".format(
-                " ".join([str(x) for x in remaining_args])
-            )
-        )
-
-    if args.command == "run":
-        divcfg = (
-            select_divvy_config(filepath=args.run.divvy)
-            if hasattr(args.run, "divvy")
-            else None
-        )
-    else:
-        divcfg = (
-            select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
-        )
-
-    # Ignore flags if user is selecting or excluding on flags:
-    if args.sel_flag or args.exc_flag:
-        args.ignore_flags = True
-
-    # Initialize project
-    if is_registry_path(args.config_file):
-        if vars(args)[SAMPLE_PL_ARG]:
-            p = Project(
-                amendments=args.amend,
-                divcfg_path=divcfg,
-                runp=args.command == "runp",
-                project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=args.config_file
-                ),
-                **{
-                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
-                },
-            )
-        else:
-            raise MisconfigurationException(
-                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
-            )
-    else:
-        try:
-            project_args = {
-                attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
-            }
-            if args.command == "run":
-                project_args.update(
-                    **{
-                        attr: getattr(args.run, attr)
-                        for attr in CLI_PROJ_ATTRS
-                        if attr in args.run
-                    }
-                )
-            p = Project(
-                cfg=args.config_file,
-                amendments=args.amend,
-                divcfg_path=divcfg,
-                runp=args.command == "runp",
-                **project_args,
-            )
-        except yaml.parser.ParserError as e:
-            _LOGGER.error(f"Project config parse failed -- {e}")
-            sys.exit(1)
-
-    selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
-    if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
-        _LOGGER.info(
-            "Failed to activate '{}' computing package. "
-            "Using the default one".format(selected_compute_pkg)
-        )
-
-    with ProjectContext(
-        prj=p,
-        selector_attribute=args.sel_attr,
-        selector_include=args.sel_incl,
-        selector_exclude=args.sel_excl,
-        selector_flag=args.sel_flag,
-        exclusion_flag=args.exc_flag,
-    ) as prj:
-        if args.command in ["run", "rerun"]:
-            run = Runner(prj)
-            try:
-                compute_kwargs = _proc_resources_spec(args)
-                return run(args, rerun=(args.command == "rerun"), **compute_kwargs)
-            except SampleFailedException:
-                sys.exit(1)
-            except IOError:
-                _LOGGER.error(
-                    "{} pipeline_interfaces: '{}'".format(
-                        prj.__class__.__name__, prj.pipeline_interface_sources
-                    )
-                )
-                raise
-
-        if args.command == "runp":
-            compute_kwargs = _proc_resources_spec(args)
-            collate = Collator(prj)
-            collate(args, **compute_kwargs)
-            return collate.debug
-
-        if args.command == "destroy":
-            return Destroyer(prj)(args)
-
-        # pipestat support introduces breaking changes and pipelines run
-        # with no pipestat reporting would not be compatible with
-        # commands: table, report and check. Therefore we plan maintain
-        # the old implementations for a couple of releases.
-        # if hasattr(args, "project"):
-        #     use_pipestat = (
-        #         prj.pipestat_configured_project
-        #         if args.project
-        #         else prj.pipestat_configured
-        #     )
-        use_pipestat = (
-            prj.pipestat_configured_project if args.project else prj.pipestat_configured
-        )
-        if args.command == "table":
-            if use_pipestat:
-                return Tabulator(prj)(args)
-            else:
-                raise PipestatConfigurationException("table")
-
-        if args.command == "report":
-            if use_pipestat:
-                return Reporter(prj)(args)
-            else:
-                raise PipestatConfigurationException("report")
-
-        if args.command == "link":
-            if use_pipestat:
-                Linker(prj)(args)
-            else:
-                raise PipestatConfigurationException("link")
-
-        if args.command == "check":
-            if use_pipestat:
-                return Checker(prj)(args)
-            else:
-                raise PipestatConfigurationException("check")
-
-        if args.command == "clean":
-            return Cleaner(prj)(args)
-
-        if args.command == "inspect":
-            inspect_project(p, args.sample_names, args.attr_limit)
-            from warnings import warn
-
-            warn(
-                "The inspect feature has moved to eido and will be removed in the future release of looper. "
-                "Use `eido inspect` from now on.",
-            )
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index ca5f649b6..683fbb0d5 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -29,7 +29,6 @@
 from divvy import select_divvy_config
 
 from . import __version__
-from .cli_looper import _proc_resources_spec
 from .command_models.commands import SUPPORTED_COMMANDS, TopLevelParser
 from .const import *
 from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
@@ -45,6 +44,7 @@
     read_looper_dotfile,
     initiate_looper_config,
     init_generic_pipeline,
+    read_yaml_file,
 )
 
 
@@ -260,5 +260,48 @@ def main(test_args=None) -> None:
     return run_looper(args, parser, test_args=test_args)
 
 
+def _proc_resources_spec(args):
+    """
+    Process CLI-sources compute setting specification. There are two sources
+    of compute settings in the CLI alone:
+        * YAML file (--settings argument)
+        * itemized compute settings (--compute argument)
+
+    The itemized compute specification is given priority
+
+    :param argparse.Namespace: arguments namespace
+    :return Mapping[str, str]: binding between resource setting name and value
+    :raise ValueError: if interpretation of the given specification as encoding
+        of key-value pairs fails
+    """
+    spec = getattr(args, "compute", None)
+    settings = args.settings
+    try:
+        settings_data = read_yaml_file(settings) or {}
+    except yaml.YAMLError:
+        _LOGGER.warning(
+            "Settings file ({}) does not follow YAML format,"
+            " disregarding".format(settings)
+        )
+        settings_data = {}
+    if not spec:
+        return settings_data
+    pairs = [(kv, kv.split("=")) for kv in spec]
+    bads = []
+    for orig, pair in pairs:
+        try:
+            k, v = pair
+        except ValueError:
+            bads.append(orig)
+        else:
+            settings_data[k] = v
+    if bads:
+        raise ValueError(
+            "Could not correctly parse itemized compute specification. "
+            "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
+        )
+    return settings_data
+
+
 if __name__ == "__main__":
     main()
diff --git a/tests/smoketests/test_cli_validation.py b/tests/smoketests/test_cli_validation.py
index be3ea91ee..82e6b4eb1 100644
--- a/tests/smoketests/test_cli_validation.py
+++ b/tests/smoketests/test_cli_validation.py
@@ -10,7 +10,7 @@
     SAMPLE_INCLUSION_OPTNAME,
 )
 from tests.conftest import print_standard_stream, subp_exec, test_args_expansion
-from looper.cli_looper import main
+from looper.cli_pydantic import main
 
 
 SUBCOMMANDS_WHICH_SUPPORT_SKIP_XOR_LIMIT = ["run", "destroy"]

From a9c38f75654a599082d8f63f41b58afb565a29a5 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 6 Mar 2024 15:41:57 -0500
Subject: [PATCH 124/225] implement temp_pep from hello_looper repo, begin
 fixing tests

---
 looper/cli_pydantic.py       |   2 +-
 looper/utils.py              |  11 +--
 tests/conftest.py            | 136 +++++++++++------------------------
 tests/smoketests/test_run.py |  55 +++++---------
 tests/test_comprehensive.py  |   6 +-
 5 files changed, 69 insertions(+), 141 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 683fbb0d5..52fac86fb 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -108,7 +108,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             "looper.databio.org/en/latest/looper-config"
         )
 
-    subcommand_args = enrich_args_via_cfg(subcommand_args, parser, test_args=test_args)
+    subcommand_args = enrich_args_via_cfg(subcommand_name, subcommand_args, parser, test_args=test_args)
 
     # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
     if vars(subcommand_args)[PROJECT_PL_ARG]:
diff --git a/looper/utils.py b/looper/utils.py
index b76d1be8a..b2ca2f26a 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -251,19 +251,20 @@ def read_yaml_file(filepath):
     return data
 
 
-def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
+def enrich_args_via_cfg(subcommand_name, parser_args, aux_parser, test_args=None):
     """
     Read in a looper dotfile and set arguments.
 
     Priority order: CLI > dotfile/config > parser default
 
+    :param subcommand name: the name of the command used
     :param argparse.Namespace parser_args: parsed args by the original parser
     :param argparse.Namespace aux_parser: parsed args by the a parser
         with defaults suppressed
     :return argparse.Namespace: selected argument values
     """
     cfg_args_all = (
-        _get_subcommand_args(parser_args)
+        _get_subcommand_args(subcommand_name, parser_args)
         if os.path.exists(parser_args.config_file)
         else dict()
     )
@@ -309,7 +310,7 @@ def set_single_arg(argname, default_source_namespace, result_namespace):
     return result
 
 
-def _get_subcommand_args(parser_args):
+def _get_subcommand_args(subcommand_name, parser_args):
     """
     Get the union of values for the subcommand arguments from
     Project.looper, Project.looper.cli.<subcommand> and Project.looper.cli.all.
@@ -341,8 +342,8 @@ def _get_subcommand_args(parser_args):
                 else dict()
             )
             args.update(
-                cfg_args[parser_args.command] or dict()
-                if parser_args.command in cfg_args
+                cfg_args[subcommand_name] or dict()
+                if subcommand_name in cfg_args
                 else dict()
             )
         except (TypeError, KeyError, AttributeError, ValueError) as e:
diff --git a/tests/conftest.py b/tests/conftest.py
index 6552d6d13..11d3d312c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,6 +11,9 @@
 from yaml import dump, safe_load
 
 from looper.const import *
+from git import Repo
+
+REPO_URL = "https://github.com/pepkit/hello_looper.git"
 
 CFG = "project_config.yaml"
 PIPESTAT_CONFIG = "global_pipestat_config.yaml"
@@ -43,7 +46,24 @@ def get_outdir(pth):
     """
     with open(pth, "r") as conf_file:
         config_data = safe_load(conf_file)
-    return config_data[LOOPER_KEY][OUTDIR_KEY]
+
+    output_path = config_data[OUTDIR_KEY]
+    dirname = os.path.dirname(pth)
+
+
+    return os.path.join(dirname, output_path)
+
+def get_project_config_path(looper_config_pth):
+    """
+    Get project config file path from a config file path since they are relative
+
+    :param str pth:
+    :return str: output directory
+    """
+    dirname = os.path.dirname(looper_config_pth)
+
+
+    return os.path.join(dirname, "project/project_config.yaml")
 
 
 def _assert_content_in_files(fs: Union[str, Iterable[str]], query: str, negate: bool):
@@ -119,11 +139,11 @@ def test_args_expansion(pth=None, cmd=None, appendix=list(), dry=True) -> List[s
     # --looper-config .looper.yaml run --dry-run
     # x = [cmd, "-d" if dry else ""]
     x = []
+    if cmd:
+        x.append(cmd)
     if pth:
         x.append("--looper-config")
         x.append(pth)
-    if cmd:
-        x.append(cmd)
     if dry:
         x.append("--dry-run")
     x.extend(appendix)
@@ -176,62 +196,29 @@ def example_pep_piface_path_cfg(example_pep_piface_path):
 
 
 @pytest.fixture
-def prep_temp_pep(example_pep_piface_path):
-    # temp dir
-    td = tempfile.mkdtemp()
-    out_td = os.path.join(td, "output")
-    # ori paths
+def prep_temp_pep():
+    d = tempfile.mkdtemp()
+    repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
 
-    cfg_path = os.path.join(example_pep_piface_path, CFG)
-    output_schema_path = os.path.join(example_pep_piface_path, OS)
-    sample_table_path = os.path.join(example_pep_piface_path, ST)
-    piface1p_path = os.path.join(example_pep_piface_path, PIP.format("1"))
-    piface2p_path = os.path.join(example_pep_piface_path, PIP.format("2"))
-    piface1s_path = os.path.join(example_pep_piface_path, PIS.format("1"))
-    piface2s_path = os.path.join(example_pep_piface_path, PIS.format("2"))
+    advanced_dir = os.path.join(d, "advanced")
 
-    res_proj_path = os.path.join(example_pep_piface_path, RES.format("project"))
-    res_samp_path = os.path.join(example_pep_piface_path, RES.format("sample"))
-    # temp copies
-    temp_path_cfg = os.path.join(td, CFG)
-    temp_path_output_schema = os.path.join(td, OS)
-    temp_path_sample_table = os.path.join(td, ST)
-    temp_path_piface1s = os.path.join(td, PIS.format("1"))
-    temp_path_piface2s = os.path.join(td, PIS.format("2"))
-    temp_path_piface1p = os.path.join(td, PIP.format("1"))
-    temp_path_piface2p = os.path.join(td, PIP.format("2"))
-    temp_path_res_proj = os.path.join(td, RES.format("project"))
-    temp_path_res_samp = os.path.join(td, RES.format("sample"))
-    # copying
-    copyfile(cfg_path, temp_path_cfg)
-    copyfile(sample_table_path, temp_path_sample_table)
-    copyfile(piface1s_path, temp_path_piface1s)
-    copyfile(piface2s_path, temp_path_piface2s)
-    copyfile(piface1p_path, temp_path_piface1p)
-    copyfile(piface2p_path, temp_path_piface2p)
-    copyfile(output_schema_path, temp_path_output_schema)
-    copyfile(res_proj_path, temp_path_res_proj)
-    copyfile(res_samp_path, temp_path_res_samp)
-    # modififactions
-    from yaml import dump, safe_load
+    path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
-    with open(temp_path_cfg, "r") as f:
-        piface_data = safe_load(f)
-    piface_data[LOOPER_KEY][OUTDIR_KEY] = out_td
-    piface_data[LOOPER_KEY][CLI_KEY] = {}
-    piface_data[LOOPER_KEY][CLI_KEY]["runp"] = {}
-    piface_data[LOOPER_KEY][CLI_KEY]["runp"][PIPELINE_INTERFACES_KEY] = [
-        temp_path_piface1p,
-        temp_path_piface2p,
-    ]
-    piface_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = [
-        temp_path_piface1s,
-        temp_path_piface2s,
-    ]
-    with open(temp_path_cfg, "w") as f:
-        dump(piface_data, f)
+    # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+    #advanced_project_file = os.path.join(d, "advanced/project", "project_config.yaml")
+
+    # with open(advanced_project_file, "r") as f:
+    #     advanced_project_data = safe_load(f)
+    #
+    # advanced_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+    #     os.path.join(advanced_dir, "data/{sample_name}.txt")
+    # )
+    #
+    # with open(advanced_project_file, "w") as f:
+    #     dump(advanced_project_data, f)
 
-    return temp_path_cfg
+
+    return path_to_looper_config
 
 
 @pytest.fixture
@@ -255,45 +242,6 @@ def prep_temp_config_with_pep(example_pep_piface_path):
     return peppy.Project(temp_path_cfg).to_dict(extended=True), temp_path_piface1s
 
 
-@pytest.fixture
-def prepare_pep_with_dot_file(prep_temp_pep):
-    pep_config = prep_temp_pep
-    with open(pep_config) as f:
-        pep_data = safe_load(f)
-
-    output_dir = pep_data["looper"]["output_dir"]
-    project_piface = pep_data["looper"]["cli"]["runp"]["pipeline_interfaces"]
-    sample_piface = pep_data["sample_modifiers"]["append"]["pipeline_interfaces"]
-
-    pep_data.pop("looper")
-    pep_data["sample_modifiers"].pop("append")
-
-    with open(pep_config, "w") as f:
-        config = dump(pep_data, f)
-
-    looper_config = {
-        "pep_config": pep_config,
-        "output_dir": output_dir,
-        "pipeline_interfaces": {
-            "sample": sample_piface,
-            "project": project_piface,
-        },
-    }
-
-    # looper_config_path = os.path.join(os.path.dirname(pep_config), "looper_config.yaml")
-    #
-    # with open(looper_config_path, "w") as f:
-    #     config = dump(looper_config, f)
-    #
-    # looper_dot_file_content = {"looper_config": looper_config_path}
-
-    dot_file_path = ".looper.yaml"
-    with open(dot_file_path, "w") as f:
-        config = dump(looper_config, f)
-
-    return dot_file_path
-
-
 @pytest.fixture
 def prep_temp_pep_pipestat(example_pep_piface_path):
     # TODO this should be combined with the other prep_temp_pep
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index ce78799de..c0b08aced 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -11,11 +11,10 @@
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 def test_cli(prep_temp_pep):
     tp = prep_temp_pep
 
-    x = test_args_expansion(tp, "run")
+    x = ["run", "--looper-config", tp, "--dry-run"]
     try:
         main(test_args=x)
     except Exception:
@@ -37,13 +36,13 @@ def is_connected():
 
 class TestLooperBothRuns:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
-    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     def test_looper_cfg_invalid(self, cmd):
         """Verify looper does not accept invalid cfg paths"""
 
-        x = test_args_expansion("jdfskfds/dsjfklds/dsjklsf.yaml", cmd)
-        with pytest.raises(OSError):
-            main(test_args=x)
+        x = test_args_expansion(cmd, "--looper-config", "jdfskfds/dsjfklds/dsjklsf.yaml")
+        with pytest.raises(SystemExit):
+            result = main(test_args=x)
+            print(result)
 
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_looper_cfg_required(self, cmd):
@@ -54,7 +53,7 @@ def test_looper_cfg_required(self, cmd):
             ff = main(test_args=x)
             print(ff)
 
-    @pytest.mark.parametrize("cmd", ["run", "runp"])
+    @pytest.mark.parametrize("cmd", ["run","runp"])
     @pytest.mark.parametrize(
         "arg",
         [
@@ -64,7 +63,6 @@ def test_looper_cfg_required(self, cmd):
             ["--command-extra", CMD_STRS[3]],
         ],
     )
-    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     def test_cmd_extra_cli(self, prep_temp_pep, cmd, arg):
         """
         Argument passing functionality works only for the above
@@ -85,24 +83,16 @@ def test_cmd_extra_cli(self, prep_temp_pep, cmd, arg):
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_in_all_files(subs_list, arg[1])
 
-    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
         tp = prep_temp_pep
 
         x = test_args_expansion(tp, cmd, ["--unknown-arg", "4"])
-        try:
+        with pytest.raises(SystemExit):
             main(test_args=x)
-            sd = os.path.join(get_outdir(tp), "submission")
-            subs_list = [
-                os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")
-            ]
-            assert_content_not_in_any_files(subs_list, "--unknown-arg")
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
+
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
         """Verify looper runs in a basic case and return code is 0"""
@@ -124,13 +114,13 @@ def test_looper_multi_pipeline(self, prep_temp_pep):
 
     def test_looper_single_pipeline(self, prep_temp_pep):
         tp = prep_temp_pep
+
         with mod_yaml_data(tp) as config_data:
-            pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
-                pifaces[1]
+            pifaces = config_data[PIPELINE_INTERFACES_KEY]
+            config_data[PIPELINE_INTERFACES_KEY]["sample"] = (
+                pifaces["sample"][1]
             )
+            del config_data[PIPELINE_INTERFACES_KEY]["project"]
 
         x = test_args_expansion(tp, "run")
         try:
@@ -142,11 +132,9 @@ def test_looper_single_pipeline(self, prep_temp_pep):
     def test_looper_var_templates(self, prep_temp_pep):
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
-            pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
-                pifaces[1]
+            pifaces = config_data[PIPELINE_INTERFACES_KEY]
+            config_data[PIPELINE_INTERFACES_KEY]["sample"] = (
+                pifaces["sample"][1]
             )
         x = test_args_expansion(tp, "run")
         try:
@@ -592,17 +580,6 @@ def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
         except Exception as err:
             raise pytest.fail(f"DID RAISE {err}")
 
-    def test_correct_execution_of_config(self, prepare_pep_with_dot_file):
-        """
-        Test executing dot file and looper_config
-        """
-        dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
-        x = test_args_expansion("", "run")
-        try:
-            main(test_args=x)
-        except Exception as err:
-            raise pytest.fail(f"DID RAISE {err}")
-        os.remove(dot_file_path)
 
 
 class TestLooperPEPhub:
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index e26671536..cb77b078c 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -16,7 +16,7 @@
 
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
-REPO_URL = "https://github.com/pepkit/hello_looper.git"
+
 
 
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
@@ -43,10 +43,12 @@ def test_comprehensive_looper_no_pipestat():
         x = ["run", "--looper-config", path_to_looper_config]
 
         try:
-            main(test_args=x)
+            results = main(test_args=x)
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+        print(results)
+
 
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
 def test_comprehensive_looper_pipestat():

From 153ec4c3ff4f46052ce6862c20b82f721971cfa2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 6 Mar 2024 17:02:11 -0500
Subject: [PATCH 125/225] lint

---
 looper/cli_pydantic.py       |  4 +++-
 tests/conftest.py            |  6 ++----
 tests/smoketests/test_run.py | 16 ++++++----------
 tests/test_comprehensive.py  |  2 --
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 52fac86fb..f4a191306 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -108,7 +108,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             "looper.databio.org/en/latest/looper-config"
         )
 
-    subcommand_args = enrich_args_via_cfg(subcommand_name, subcommand_args, parser, test_args=test_args)
+    subcommand_args = enrich_args_via_cfg(
+        subcommand_name, subcommand_args, parser, test_args=test_args
+    )
 
     # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
     if vars(subcommand_args)[PROJECT_PL_ARG]:
diff --git a/tests/conftest.py b/tests/conftest.py
index 11d3d312c..a7f502e4e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -50,9 +50,9 @@ def get_outdir(pth):
     output_path = config_data[OUTDIR_KEY]
     dirname = os.path.dirname(pth)
 
-
     return os.path.join(dirname, output_path)
 
+
 def get_project_config_path(looper_config_pth):
     """
     Get project config file path from a config file path since they are relative
@@ -62,7 +62,6 @@ def get_project_config_path(looper_config_pth):
     """
     dirname = os.path.dirname(looper_config_pth)
 
-
     return os.path.join(dirname, "project/project_config.yaml")
 
 
@@ -205,7 +204,7 @@ def prep_temp_pep():
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
-    #advanced_project_file = os.path.join(d, "advanced/project", "project_config.yaml")
+    # advanced_project_file = os.path.join(d, "advanced/project", "project_config.yaml")
 
     # with open(advanced_project_file, "r") as f:
     #     advanced_project_data = safe_load(f)
@@ -217,7 +216,6 @@ def prep_temp_pep():
     # with open(advanced_project_file, "w") as f:
     #     dump(advanced_project_data, f)
 
-
     return path_to_looper_config
 
 
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index c0b08aced..1b33e7806 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -39,7 +39,9 @@ class TestLooperBothRuns:
     def test_looper_cfg_invalid(self, cmd):
         """Verify looper does not accept invalid cfg paths"""
 
-        x = test_args_expansion(cmd, "--looper-config", "jdfskfds/dsjfklds/dsjklsf.yaml")
+        x = test_args_expansion(
+            cmd, "--looper-config", "jdfskfds/dsjfklds/dsjklsf.yaml"
+        )
         with pytest.raises(SystemExit):
             result = main(test_args=x)
             print(result)
@@ -53,7 +55,7 @@ def test_looper_cfg_required(self, cmd):
             ff = main(test_args=x)
             print(ff)
 
-    @pytest.mark.parametrize("cmd", ["run","runp"])
+    @pytest.mark.parametrize("cmd", ["run", "runp"])
     @pytest.mark.parametrize(
         "arg",
         [
@@ -92,7 +94,6 @@ def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
             main(test_args=x)
 
 
-
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
         """Verify looper runs in a basic case and return code is 0"""
@@ -117,9 +118,7 @@ def test_looper_single_pipeline(self, prep_temp_pep):
 
         with mod_yaml_data(tp) as config_data:
             pifaces = config_data[PIPELINE_INTERFACES_KEY]
-            config_data[PIPELINE_INTERFACES_KEY]["sample"] = (
-                pifaces["sample"][1]
-            )
+            config_data[PIPELINE_INTERFACES_KEY]["sample"] = pifaces["sample"][1]
             del config_data[PIPELINE_INTERFACES_KEY]["project"]
 
         x = test_args_expansion(tp, "run")
@@ -133,9 +132,7 @@ def test_looper_var_templates(self, prep_temp_pep):
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
             pifaces = config_data[PIPELINE_INTERFACES_KEY]
-            config_data[PIPELINE_INTERFACES_KEY]["sample"] = (
-                pifaces["sample"][1]
-            )
+            config_data[PIPELINE_INTERFACES_KEY]["sample"] = pifaces["sample"][1]
         x = test_args_expansion(tp, "run")
         try:
             # Test that {looper.piface_dir} is correctly rendered to a path which will show up in the final .sub file
@@ -581,7 +578,6 @@ def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
             raise pytest.fail(f"DID RAISE {err}")
 
 
-
 class TestLooperPEPhub:
     @pytest.mark.parametrize(
         "pep_path",
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index cb77b078c..e0d454e76 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -17,8 +17,6 @@
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
 
-
-
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
 def test_comprehensive_looper_no_pipestat():
 

From e3dce56eb5c081405ba04854927095997b13adc9 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 09:41:03 -0500
Subject: [PATCH 126/225] re-add validating and checking for mutually exclusive
 commands post parse

---
 looper/cli_pydantic.py | 44 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index f4a191306..67597d915 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -47,6 +47,39 @@
     read_yaml_file,
 )
 
+from typing import List, Tuple
+
+
+def opt_attr_pair(name: str) -> Tuple[str, str]:
+    return f"--{name}", name.replace("-", "_")
+
+
+def validate_post_parse(args: argparse.Namespace) -> List[str]:
+    problems = []
+    used_exclusives = [
+        opt
+        for opt, attr in map(
+            opt_attr_pair,
+            [
+                "skip",
+                "limit",
+                SAMPLE_EXCLUSION_OPTNAME,
+                SAMPLE_INCLUSION_OPTNAME,
+            ],
+        )
+        # Depending on the subcommand used, the above options might either be in
+        # the top-level namespace or in the subcommand namespace (the latter due
+        # to a `modify_args_namespace()`)
+        if getattr(
+            args, attr, None
+        )  # or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
+    ]
+    if len(used_exclusives) > 1:
+        problems.append(
+            f"Used multiple mutually exclusive options: {', '.join(used_exclusives)}"
+        )
+    return problems
+
 
 def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     # here comes adapted `cli_looper.py` code
@@ -64,6 +97,17 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
     [(subcommand_name, subcommand_args)] = subcommand_valued_args
 
+    cli_use_errors = validate_post_parse(subcommand_args)
+    if cli_use_errors:
+        parser.print_help(sys.stderr)
+        parser.error(
+            f"{len(cli_use_errors)} CLI use problem(s): {', '.join(cli_use_errors)}"
+        )
+
+    if subcommand_name is None:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
     if subcommand_name in ["init"]:
         return int(
             not initiate_looper_config(

From 6c9a78ef6a43386e2c05c3a9f495ae23dd71a3b3 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 10:11:24 -0500
Subject: [PATCH 127/225] fix more tests within test_run

---
 tests/smoketests/test_run.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 1b33e7806..75bd9f0d4 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -130,13 +130,12 @@ def test_looper_single_pipeline(self, prep_temp_pep):
 
     def test_looper_var_templates(self, prep_temp_pep):
         tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            pifaces = config_data[PIPELINE_INTERFACES_KEY]
-            config_data[PIPELINE_INTERFACES_KEY]["sample"] = pifaces["sample"][1]
         x = test_args_expansion(tp, "run")
+        x.pop(-1)  # remove the --dry-run argument for this specific test
+
         try:
             # Test that {looper.piface_dir} is correctly rendered to a path which will show up in the final .sub file
-            main(test_args=x)
+            results = main(test_args=x)
             sd = os.path.join(get_outdir(tp), "submission")
             subs_list = [
                 os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")
@@ -148,8 +147,10 @@ def test_looper_var_templates(self, prep_temp_pep):
     def test_looper_cli_pipeline(self, prep_temp_pep):
         """CLI-specified pipelines overwrite ones from config"""
         tp = prep_temp_pep
-        pi_pth = os.path.join(os.path.dirname(tp), PIS.format("1"))
-        x = test_args_expansion(tp, "run", ["--pipeline-interfaces", pi_pth])
+        with mod_yaml_data(tp) as config_data:
+            pifaces = config_data[PIPELINE_INTERFACES_KEY]
+            pi_pth = pifaces["sample"][1]
+        x = test_args_expansion(tp, "run", ["--sample-pipeline-interfaces", pi_pth])
         try:
             result = main(test_args=x)
 
@@ -164,12 +165,12 @@ def test_looper_no_pipeline(self, prep_temp_pep):
         """
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
-            del config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY]
+            del config_data[PIPELINE_INTERFACES_KEY]
         x = test_args_expansion(tp, "run")
         try:
             result = main(test_args=x)
-
             assert result[DEBUG_JOBS] == 0
+            assert "No pipeline interfaces defined" in list(result.keys())
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
@@ -179,9 +180,7 @@ def test_looper_pipeline_not_found(self, prep_temp_pep):
         """
         tp = prep_temp_pep
         with mod_yaml_data(tp) as config_data:
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = [
-                "bogus"
-            ]
+            config_data[PIPELINE_INTERFACES_KEY]["sample"] = ["bogus"]
         x = test_args_expansion(tp, "run")
         try:
             result = main(test_args=x)

From 1f09f70e29822ae1cdbdd2adf0d77bada55260cc Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 10:24:12 -0500
Subject: [PATCH 128/225] remove redundant test

---
 tests/smoketests/test_run.py | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 75bd9f0d4..8b79b668d 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -190,31 +190,6 @@ def test_looper_pipeline_not_found(self, prep_temp_pep):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    def test_looper_pipeline_invalid(self, prep_temp_pep):
-        """
-        Pipeline is ignored when does not validate successfully
-        against a schema
-        """
-        tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][
-                PIPELINE_INTERFACES_KEY
-            ]
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = (
-                pifaces[1]
-            )
-        piface_path = os.path.join(os.path.dirname(tp), pifaces[1])
-        with mod_yaml_data(piface_path) as piface_data:
-            del piface_data["pipeline_name"]
-        x = test_args_expansion(tp, "run")
-        try:
-            result = main(test_args=x)
-
-            assert result[DEBUG_JOBS] == 0
-            assert "No pipeline interfaces defined" in result.keys()
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
-
     def test_looper_sample_attr_missing(self, prep_temp_pep):
         """
         Piface is ignored when it does not exist

From 6f5ae2482ea1faff0d6d5546b81d8e330095f50b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 11:30:28 -0500
Subject: [PATCH 129/225] finish TestLooperRunBehavior fixes

---
 tests/smoketests/test_run.py | 48 +++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 8b79b668d..afb55456b 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -94,6 +94,7 @@ def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
             main(test_args=x)
 
 
+@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
         """Verify looper runs in a basic case and return code is 0"""
@@ -190,35 +191,24 @@ def test_looper_pipeline_not_found(self, prep_temp_pep):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    def test_looper_sample_attr_missing(self, prep_temp_pep):
-        """
-        Piface is ignored when it does not exist
-        """
-        tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            del config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["attr"]
-        x = test_args_expansion(tp, "run")
-        try:
-            result = main(test_args=x)
-
-            assert result[DEBUG_JOBS] == 0
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
-
-    @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
     def test_looper_sample_name_whitespace(self, prep_temp_pep):
         """
         Piface is ignored when it does not exist
         """
         tp = prep_temp_pep
+
         imply_whitespace = [
             {
                 IMPLIED_IF_KEY: {"sample_name": "sample1"},
                 IMPLIED_THEN_KEY: {"sample_name": "sample whitespace"},
             }
         ]
-        with mod_yaml_data(tp) as config_data:
-            config_data[SAMPLE_MODS_KEY][IMPLIED_KEY] = imply_whitespace
+
+        project_config_path = get_project_config_path(tp)
+
+        with mod_yaml_data(project_config_path) as project_config_data:
+            project_config_data[SAMPLE_MODS_KEY][IMPLIED_KEY] = imply_whitespace
+
         x = test_args_expansion(tp, "run")
         with pytest.raises(Exception):
             result = main(test_args=x)
@@ -230,12 +220,16 @@ def test_looper_toggle(self, prep_temp_pep):
         If all samples have toggle attr set to 0, no jobs are submitted
         """
         tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][SAMPLE_TOGGLE_ATTR] = 0
+        project_config_path = get_project_config_path(tp)
+
+        with mod_yaml_data(project_config_path) as project_config_data:
+            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][SAMPLE_TOGGLE_ATTR] = 0
+
         x = test_args_expansion(tp, "run")
+        x.pop(-1)  # remove dry run for this test
+
         try:
             result = main(test_args=x)
-
             assert result[DEBUG_JOBS] == 0
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
@@ -247,9 +241,10 @@ def test_cmd_extra_sample(self, prep_temp_pep, arg):
         appended to the pipelinecommand
         """
         tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+        project_config_path = get_project_config_path(tp)
 
+        with mod_yaml_data(project_config_path) as project_config_data:
+            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
         x = test_args_expansion(tp, "run")
         try:
             main(test_args=x)
@@ -267,8 +262,11 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         pipeline command
         """
         tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+        project_config_path = get_project_config_path(tp)
+
+        with mod_yaml_data(project_config_path) as project_config_data:
+            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+
         x = test_args_expansion(tp, "run", ["--command-extra-override='different'"])
         try:
             main(test_args=x)

From a3eb5eb22413cd00aab5278f2aff6c29b8c2110a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 14:54:31 -0500
Subject: [PATCH 130/225] finish TestLooperRunPBehavior fixes

---
 tests/smoketests/test_run.py | 27 ++++++++++++++++++++-------
 tests/test_comprehensive.py  | 19 +++++++++++++++++++
 2 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index afb55456b..13f9b14d3 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -277,7 +277,7 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         assert_content_not_in_any_files(subs_list, arg)
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
+# @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunpBehavior:
     def test_looper_runp_basic(self, prep_temp_pep):
         """Verify looper runps in a basic case and return code is 0"""
@@ -299,12 +299,17 @@ def test_looper_multi_pipeline(self, prep_temp_pep):
 
     def test_looper_single_pipeline(self, prep_temp_pep):
         tp = prep_temp_pep
+
         with mod_yaml_data(tp) as config_data:
-            piface_path = os.path.join(os.path.dirname(tp), PIP.format("1"))
-            config_data[LOOPER_KEY][CLI_KEY]["runp"][
-                PIPELINE_INTERFACES_KEY
-            ] = piface_path
+            # Modifying in this way due to https://github.com/pepkit/looper/issues/474
+            config_data[PIPELINE_INTERFACES_KEY]["project"] = os.path.join(
+                os.path.dirname(tp), "pipeline/pipeline_interface1_project.yaml"
+            )
+            del config_data[PIPELINE_INTERFACES_KEY]["sample"]
+
+        print(tp)
         x = test_args_expansion(tp, "runp")
+        x.pop(-1)  # remove the --dry-run argument for this specific test
         try:
             result = main(test_args=x)
             assert result[DEBUG_JOBS] != 2
@@ -312,12 +317,20 @@ def test_looper_single_pipeline(self, prep_temp_pep):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+    @pytest.mark.skip(reason="Functionality broken")
     @pytest.mark.parametrize("arg", CMD_STRS)
     def test_cmd_extra_project(self, prep_temp_pep, arg):
+
+        # Test is currently broken, see https://github.com/pepkit/looper/issues/475
+
         tp = prep_temp_pep
-        with mod_yaml_data(tp) as config_data:
-            config_data[LOOPER_KEY]["command_extra"] = arg
+
+        project_config_path = get_project_config_path(tp)
+
+        with mod_yaml_data(project_config_path) as project_config_data:
+            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
         x = test_args_expansion(tp, "runp")
+
         try:
             main(test_args=x)
         except Exception:
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index e0d454e76..0ccd288c4 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -17,6 +17,25 @@
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
 
+@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
+def test_comprehensive_advanced_looper_no_pipestat():
+
+    with TemporaryDirectory() as d:
+        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
+        basic_dir = os.path.join(d, "advanced")
+
+        path_to_looper_config = os.path.join(basic_dir, ".looper.yaml")
+
+        x = ["run", "--looper-config", path_to_looper_config]
+
+        try:
+            results = main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        print(results)
+
+
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
 def test_comprehensive_looper_no_pipestat():
 

From 0f8055f35fb3cc1f8c81c0c56ebf8fc2e19b1af4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 15:00:02 -0500
Subject: [PATCH 131/225] fix Compute Tests

---
 tests/smoketests/test_run.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 13f9b14d3..7265c1937 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -277,7 +277,6 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg):
         assert_content_not_in_any_files(subs_list, arg)
 
 
-# @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunpBehavior:
     def test_looper_runp_basic(self, prep_temp_pep):
         """Verify looper runps in a basic case and return code is 0"""
@@ -453,7 +452,6 @@ def test_looper_limiting(self, prep_temp_pep):
         verify_filecount_in_dir(sd, ".sub", 4)
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperCompute:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_looper_respects_pkg_selection(self, prep_temp_pep, cmd):
@@ -512,7 +510,7 @@ def test_cli_yaml_settings_passes_settings(self, prep_temp_pep, cmd):
             dump({"mem": "testin_mem"}, sf)
 
         x = test_args_expansion(
-            tp, cmd, ["--settings", settings_file_path, "-p", "slurm"]
+            tp, cmd, ["--settings", settings_file_path, "--package", "slurm"]
         )
         try:
             main(test_args=x)
@@ -532,7 +530,14 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
         x = test_args_expansion(
             tp,
             cmd,
-            ["--settings", settings_file_path, "--compute", "mem=10", "-p", "slurm"],
+            [
+                "--settings",
+                settings_file_path,
+                "--compute",
+                "mem=10",
+                "--package",
+                "slurm",
+            ],
         )
         try:
             main(test_args=x)

From 5333436c211d68ff648078f830e9d97d835c02ca Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 16:11:10 -0500
Subject: [PATCH 132/225] fix TestLooperRunPreSubmissionHooks

---
 tests/smoketests/test_run.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 7265c1937..513cb1bdc 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -1,3 +1,5 @@
+import os.path
+
 import pytest
 from peppy.const import *
 from yaml import dump
@@ -339,7 +341,6 @@ def test_cmd_extra_project(self, prep_temp_pep, arg):
         assert_content_in_all_files(subs_list, arg)
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunPreSubmissionHooks:
     def test_looper_basic_plugin(self, prep_temp_pep):
         tp = prep_temp_pep
@@ -362,13 +363,16 @@ def test_looper_basic_plugin(self, prep_temp_pep):
     @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
     def test_looper_other_plugins(self, prep_temp_pep, plugin, appendix):
         tp = prep_temp_pep
-        for path in {
-            piface.pipe_iface_file for piface in Project(tp).pipeline_interfaces
-        }:
-            with mod_yaml_data(path) as piface_data:
-                piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_PY_FUN_KEY] = [plugin]
+        pep_dir = os.path.dirname(tp)
+        pipeline_interface1 = os.path.join(
+            pep_dir, "pipeline/pipeline_interface1_sample.yaml"
+        )
+
+        with mod_yaml_data(pipeline_interface1) as piface_data:
+            piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_PY_FUN_KEY] = [plugin]
 
         x = test_args_expansion(tp, "run")
+        x.pop(-1)
         try:
             main(test_args=x)
         except Exception as err:
@@ -385,11 +389,13 @@ def test_looper_other_plugins(self, prep_temp_pep, plugin, appendix):
     )
     def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         tp = prep_temp_pep
-        for path in {
-            piface.pipe_iface_file for piface in Project(tp).pipeline_interfaces
-        }:
-            with mod_yaml_data(path) as piface_data:
-                piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_CMD_KEY] = [cmd]
+        pep_dir = os.path.dirname(tp)
+        pipeline_interface1 = os.path.join(
+            pep_dir, "pipeline/pipeline_interface1_sample.yaml"
+        )
+
+        with mod_yaml_data(pipeline_interface1) as piface_data:
+            piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_CMD_KEY] = [cmd]
         x = test_args_expansion(tp, "run")
         try:
             main(test_args=x)

From b8e2f8ec465a8b25fedd2e71098aa15056bdb27a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 7 Mar 2024 16:26:26 -0500
Subject: [PATCH 133/225] fix config tests

---
 tests/smoketests/test_run.py | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 513cb1bdc..6701f56af 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -405,13 +405,11 @@ def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         verify_filecount_in_dir(sd, "test.txt", 3)
 
 
-@pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
 class TestLooperRunSubmissionScript:
     def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
         tp = prep_temp_pep
-        with open(tp, "r") as conf_file:
-            config_data = safe_load(conf_file)
-        outdir = config_data[LOOPER_KEY][OUTDIR_KEY]
+
+        outdir = get_outdir(tp)
         x = test_args_expansion(tp, "run")
         try:
             main(test_args=x)
@@ -422,7 +420,7 @@ def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
 
     def test_looper_lumping(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lump-n", "2"])
+        x = test_args_expansion(tp, "run", ["--lumpn", "2"])
         try:
             main(test_args=x)
         except Exception:
@@ -432,7 +430,7 @@ def test_looper_lumping(self, prep_temp_pep):
 
     def test_looper_lumping_jobs(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lump-j", "1"])
+        x = test_args_expansion(tp, "run", ["--lumpj", "1"])
         try:
             main(test_args=x)
         except Exception:
@@ -442,7 +440,7 @@ def test_looper_lumping_jobs(self, prep_temp_pep):
 
     def test_looper_lumping_jobs_negative(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lump-j", "-1"])
+        x = test_args_expansion(tp, "run", ["--lumpj", "-1"])
 
         with pytest.raises(ValueError):
             main(test_args=x)
@@ -556,22 +554,15 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
 
 
 class TestLooperConfig:
-    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
-    @pytest.mark.parametrize("cmd", ["run", "runp"])
-    def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
+
+    def test_init_config_file(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "init")
+        x = ["init", "--force-yes"]
         try:
             result = main(test_args=x)
         except Exception as err:
             raise pytest.fail(f"DID RAISE: {err}")
         assert result == 0
-        assert_content_in_all_files(dotfile_path, tp)
-        x = test_args_expansion(tp, cmd)
-        try:
-            result = main(test_args=x)
-        except Exception as err:
-            raise pytest.fail(f"DID RAISE {err}")
 
 
 class TestLooperPEPhub:

From 14cdfa07a61beb0275b33ac78a9add5c2f97ea9a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 10:32:21 -0400
Subject: [PATCH 134/225] add shell script to download hello_looper branch

---
 .gitignore                |  1 +
 tests/update_test_data.sh | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 tests/update_test_data.sh

diff --git a/.gitignore b/.gitignore
index 584209944..74d56a6f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,3 +81,4 @@ __pycache__/
 *ipynb_checkpoints*
 hello_looper-master*
 /pipeline/
+/tests/data/hello_looper-dev_derive/
diff --git a/tests/update_test_data.sh b/tests/update_test_data.sh
new file mode 100644
index 000000000..b5c49073c
--- /dev/null
+++ b/tests/update_test_data.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+branch='dev_derive'
+
+wget https://github.com/pepkit/hello_looper/archive/refs/heads/${branch}.zip
+mv ${branch}.zip data/
+cd data/
+rm -rf hello_looper-${branch}
+unzip ${branch}.zip
+rm ${branch}.zip
\ No newline at end of file

From 61cea90d4dfaed7cdb1fb529e8f77edc916a13bf Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 10:50:31 -0400
Subject: [PATCH 135/225] replace cloning repo with copying local hello_looper
 copy

---
 tests/conftest.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index a7f502e4e..7fb567acc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,7 +1,8 @@
+import shutil
 from contextlib import contextmanager
 import os
 import subprocess
-from shutil import copyfile, rmtree
+from shutil import copyfile, rmtree, copytree
 import tempfile
 from typing import *
 
@@ -11,7 +12,6 @@
 from yaml import dump, safe_load
 
 from looper.const import *
-from git import Repo
 
 REPO_URL = "https://github.com/pepkit/hello_looper.git"
 
@@ -195,9 +195,15 @@ def example_pep_piface_path_cfg(example_pep_piface_path):
 
 
 @pytest.fixture
-def prep_temp_pep():
+def prep_temp_pep(example_pep_piface_path):
+
+
+    # Get Path to local copy of hello_looper
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev_derive")
+
+    # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
-    repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
     advanced_dir = os.path.join(d, "advanced")
 

From faba7c8065d7fa05e09bd881c4f38a0af7b014e1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 11:16:50 -0400
Subject: [PATCH 136/225] remove skips if not connected, add
 prep_temp_pep_basic, and refactor comprehensive test

---
 looper/looper.py             |  4 +--
 tests/conftest.py            | 35 ++++++++++++---------
 tests/smoketests/test_run.py |  2 --
 tests/test_clean.py          |  1 -
 tests/test_comprehensive.py  | 61 +++++++++++++++---------------------
 5 files changed, 48 insertions(+), 55 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 82adb9144..42d5510c0 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -200,10 +200,10 @@ def __call__(self, args, preview_flag=True):
         if not preview_flag:
             _LOGGER.info("Clean complete.")
             return 0
-        if getattr(args.clean, "dry_run", None):
+        if getattr(args, "dry_run", None):
             _LOGGER.info("Dry run. No files cleaned.")
             return 0
-        if not getattr(args.clean, "force_yes", None) and not query_yes_no(
+        if not getattr(args, "force_yes", None) and not query_yes_no(
             "Are you sure you want to permanently delete all "
             "intermediate pipeline results for this project?"
         ):
diff --git a/tests/conftest.py b/tests/conftest.py
index 7fb567acc..d2fa061f8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -197,30 +197,35 @@ def example_pep_piface_path_cfg(example_pep_piface_path):
 @pytest.fixture
 def prep_temp_pep(example_pep_piface_path):
 
-
     # Get Path to local copy of hello_looper
-    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev_derive")
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
     advanced_dir = os.path.join(d, "advanced")
-
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
-    # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
-    # advanced_project_file = os.path.join(d, "advanced/project", "project_config.yaml")
-
-    # with open(advanced_project_file, "r") as f:
-    #     advanced_project_data = safe_load(f)
-    #
-    # advanced_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
-    #     os.path.join(advanced_dir, "data/{sample_name}.txt")
-    # )
-    #
-    # with open(advanced_project_file, "w") as f:
-    #     dump(advanced_project_data, f)
+    return path_to_looper_config
+
+
+@pytest.fixture
+def prep_temp_pep_basic(example_pep_piface_path):
+
+    # Get Path to local copy of hello_looper
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
+
+    # Make local temp copy of hello_looper
+    d = tempfile.mkdtemp()
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
+
+    advanced_dir = os.path.join(d, "basic")
+    path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
 
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 6701f56af..9ea4156c2 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -96,7 +96,6 @@ def test_unrecognized_args_not_passing(self, prep_temp_pep, cmd):
             main(test_args=x)
 
 
-@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
 class TestLooperRunBehavior:
     def test_looper_run_basic(self, prep_temp_pep):
         """Verify looper runs in a basic case and return code is 0"""
@@ -360,7 +359,6 @@ def test_looper_basic_plugin(self, prep_temp_pep):
             ("looper.write_sample_yaml_cwl", "cwl.yaml"),
         ],
     )
-    @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
     def test_looper_other_plugins(self, prep_temp_pep, plugin, appendix):
         tp = prep_temp_pep
         pep_dir = os.path.dirname(tp)
diff --git a/tests/test_clean.py b/tests/test_clean.py
index ab464e47a..17a1fa9d0 100644
--- a/tests/test_clean.py
+++ b/tests/test_clean.py
@@ -24,7 +24,6 @@ def build_namespace(**kwargs):
 ]
 
 
-@pytest.mark.skip(reason="Test needs to be rewritten")
 @pytest.mark.parametrize(["args", "preview"], DRYRUN_OR_NOT_PREVIEW)
 def test_cleaner_does_not_crash(args, preview, prep_temp_pep):
     prj = Project(prep_temp_pep)
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 0ccd288c4..bec3fc9ba 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -17,54 +17,45 @@
 CMD_STRS = ["string", " --string", " --sjhsjd 212", "7867#$@#$cc@@"]
 
 
-@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
-def test_comprehensive_advanced_looper_no_pipestat():
+def test_comprehensive_advanced_looper_no_pipestat(prep_temp_pep):
 
-    with TemporaryDirectory() as d:
-        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
-        basic_dir = os.path.join(d, "advanced")
+    path_to_looper_config = prep_temp_pep
 
-        path_to_looper_config = os.path.join(basic_dir, ".looper.yaml")
+    x = ["run", "--looper-config", path_to_looper_config]
 
-        x = ["run", "--looper-config", path_to_looper_config]
+    try:
+        results = main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        try:
-            results = main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    print(results)
 
-        print(results)
 
+def test_comprehensive_looper_no_pipestat(prep_temp_pep_basic):
 
-@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
-def test_comprehensive_looper_no_pipestat():
+    path_to_looper_config = prep_temp_pep_basic
+    basic_dir = os.path.dirname(path_to_looper_config)
 
-    with TemporaryDirectory() as d:
-        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
-        basic_dir = os.path.join(d, "basic")
+    # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+    basic_project_file = os.path.join(basic_dir, "project", "project_config.yaml")
+    with open(basic_project_file, "r") as f:
+        basic_project_data = safe_load(f)
 
-        path_to_looper_config = os.path.join(basic_dir, ".looper.yaml")
+    basic_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+        os.path.join(basic_dir, "data/{sample_name}.txt")
+    )
 
-        # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
-        basic_project_file = os.path.join(d, "basic/project", "project_config.yaml")
-        with open(basic_project_file, "r") as f:
-            pipestat_project_data = safe_load(f)
+    with open(basic_project_file, "w") as f:
+        dump(basic_project_data, f)
 
-        pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
-            os.path.join(basic_dir, "data/{sample_name}.txt")
-        )
-
-        with open(basic_project_file, "w") as f:
-            dump(pipestat_project_data, f)
+    x = ["run", "--looper-config", path_to_looper_config]
 
-        x = ["run", "--looper-config", path_to_looper_config]
-
-        try:
-            results = main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    try:
+        results = main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        print(results)
+    print(results)
 
 
 @pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")

From fabe28322cf8785da8745fd64b5dacdbebb2337c Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 13:31:27 -0400
Subject: [PATCH 137/225] fix check tests with new pipestat PEP

---
 tests/conftest.py              | 61 ++++++----------------------------
 tests/smoketests/test_other.py | 31 ++++++++++-------
 2 files changed, 29 insertions(+), 63 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index d2fa061f8..22a6f8efa 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -253,59 +253,18 @@ def prep_temp_config_with_pep(example_pep_piface_path):
 
 @pytest.fixture
 def prep_temp_pep_pipestat(example_pep_piface_path):
-    # TODO this should be combined with the other prep_temp_pep
-    # temp dir
-    td = tempfile.mkdtemp()
-    out_td = os.path.join(td, "output")
-    # ori paths
 
-    cfg_path = os.path.join(example_pep_piface_path, LOOPER_CFG)
-    project_cfg_pipestat_path = os.path.join(
-        example_pep_piface_path, PROJECT_CFG_PIPESTAT
-    )
-    output_schema_path = os.path.join(example_pep_piface_path, PIPESTAT_OS)
+    # Get Path to local copy of hello_looper
 
-    sample_table_path = os.path.join(example_pep_piface_path, ST)
-    piface1s_path = os.path.join(example_pep_piface_path, PIPESTAT_PI)
-    piface1p_path = os.path.join(example_pep_piface_path, PIPESTAT_PI_PRJ)
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
 
-    res_proj_path = os.path.join(example_pep_piface_path, RES.format("project"))
-    res_samp_path = os.path.join(example_pep_piface_path, RES.format("sample"))
-    # temp copies
-    temp_path_cfg = os.path.join(td, LOOPER_CFG)
-    temp_path_project_cfg_pipestat = os.path.join(td, PROJECT_CFG_PIPESTAT)
-    temp_path_output_schema = os.path.join(td, PIPESTAT_OS)
+    # Make local temp copy of hello_looper
+    d = tempfile.mkdtemp()
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    temp_path_sample_table = os.path.join(td, ST)
-    temp_path_piface1s = os.path.join(td, PIPESTAT_PI)
-    temp_path_piface1p = os.path.join(td, PIPESTAT_PI_PRJ)
-    temp_path_res_proj = os.path.join(td, RES.format("project"))
-    temp_path_res_samp = os.path.join(td, RES.format("sample"))
-    # copying
-    copyfile(cfg_path, temp_path_cfg)
-    copyfile(project_cfg_pipestat_path, temp_path_project_cfg_pipestat)
+    advanced_dir = os.path.join(d, "pipestat")
+    path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
-    copyfile(sample_table_path, temp_path_sample_table)
-    copyfile(piface1s_path, temp_path_piface1s)
-    copyfile(piface1p_path, temp_path_piface1p)
-    copyfile(output_schema_path, temp_path_output_schema)
-    copyfile(res_proj_path, temp_path_res_proj)
-    copyfile(res_samp_path, temp_path_res_samp)
-    # modifications
-    from yaml import dump, safe_load
-
-    with open(temp_path_cfg, "r") as f:
-        piface_data = safe_load(f)
-    piface_data[LOOPER_KEY][OUTDIR_KEY] = out_td
-    piface_data[LOOPER_KEY][CLI_KEY] = {}
-    piface_data[LOOPER_KEY][CLI_KEY]["runp"] = {}
-    piface_data[LOOPER_KEY][CLI_KEY]["runp"][PIPELINE_INTERFACES_KEY] = [
-        temp_path_piface1p,
-    ]
-    piface_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = [
-        temp_path_piface1s,
-    ]
-    with open(temp_path_cfg, "w") as f:
-        dump(piface_data, f)
-
-    return temp_path_cfg
+    return path_to_looper_config
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 3dcdd8076..a90104fb2 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -1,3 +1,5 @@
+import os.path
+
 import pytest
 from peppy import Project
 
@@ -8,11 +10,19 @@
 
 
 def _make_flags(cfg, type, pipeline_name):
-    p = Project(cfg)
-    out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
-    print(p.samples)
+
+    # get flag dir from .looper.yaml
+    with open(cfg, "r") as f:
+        looper_cfg_data = safe_load(f)
+        flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+    flag_dir = os.path.join(os.path.dirname(cfg), flag_dir)
+    # get samples from the project config via Peppy
+    project_config_path = get_project_config_path(cfg)
+    p = Project(project_config_path)
+
     for s in p.samples:
-        sf = os.path.join(out_dir, "results_pipeline")
+        sf = flag_dir
         if not os.path.exists(sf):
             os.makedirs(sf)
         flag_path = os.path.join(
@@ -24,14 +34,11 @@ def _make_flags(cfg, type, pipeline_name):
 
 class TestLooperPipestat:
 
-    @pytest.mark.skip(reason="prep_temp_pep needs to be rewritten")
     @pytest.mark.parametrize("cmd", ["report", "table", "check"])
     def test_fail_no_pipestat_config(self, prep_temp_pep, cmd):
         "report, table, and check should fail if pipestat is NOT configured."
-        # tp = prep_temp_pep
-        dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
-        # x = test_args_expansion(tp, cmd, dry=False)
-        x = [cmd, "--looper-config", dot_file_path]
+        tp = prep_temp_pep
+        x = [cmd, "--looper-config", tp]
         with pytest.raises(PipestatConfigurationException):
             main(test_args=x)
 
@@ -56,7 +63,7 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
 class TestLooperCheck:
     @pytest.mark.parametrize("flag_id", FLAGS)
     @pytest.mark.parametrize(
-        "pipeline_name", ["test_pipe"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works"""
@@ -75,7 +82,7 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
     @pytest.mark.parametrize("flag_id", FLAGS)
-    @pytest.mark.parametrize("pipeline_name", ["test_pipe"])
+    @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"])
     def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works when multiple flags are created"""
         tp = prep_temp_pep_pipestat
@@ -89,7 +96,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
                 main(test_args=x)
 
     @pytest.mark.parametrize("flag_id", ["3333", "tonieflag", "bogus", "ms"])
-    @pytest.mark.parametrize("pipeline_name", ["test_pipe"])
+    @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"])
     def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works when bogus flags are created"""
         tp = prep_temp_pep_pipestat

From 8f725067c7f8bf0ca058036a9c88d6a2344f3af0 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 14:37:16 -0400
Subject: [PATCH 138/225] fix Selector tests, add advanced pipestat fixture

---
 tests/conftest.py              |  19 ++++
 tests/smoketests/test_other.py | 185 ++++++++++++++++++++++-----------
 2 files changed, 145 insertions(+), 59 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 22a6f8efa..692f6e2ff 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -268,3 +268,22 @@ def prep_temp_pep_pipestat(example_pep_piface_path):
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
+
+
+@pytest.fixture
+def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
+
+    # Get Path to local copy of hello_looper
+
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
+
+    # Make local temp copy of hello_looper
+    d = tempfile.mkdtemp()
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
+
+    advanced_dir = os.path.join(d, "advanced")
+    path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml")
+
+    return path_to_looper_config
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index a90104fb2..55009af67 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -7,6 +7,7 @@
 from looper.exceptions import PipestatConfigurationException
 from tests.conftest import *
 from looper.cli_pydantic import main
+import pandas as pd
 
 
 def _make_flags(cfg, type, pipeline_name):
@@ -116,18 +117,26 @@ def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
 class TestSelector:
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_selecting_flags_works(
         self, prep_temp_pep_pipestat, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
+        """Verify selecting on a single flag"""
         tp = prep_temp_pep_pipestat
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
+
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -137,7 +146,7 @@ def test_selecting_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "--looper-config", tp, "--sel-flag", "failed", "--dry-run"]
+        x = ["run", "--looper-config", tp, "--sel-flag", "completed", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -150,19 +159,25 @@ def test_selecting_flags_works(
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_excluding_flags_works(
         self, prep_temp_pep_pipestat, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
+        """Verify that excluding a single flag works"""
         tp = prep_temp_pep_pipestat
-        # _make_flags(tp, flag_id, pipeline_name)
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -172,7 +187,7 @@ def test_excluding_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        x = ["run", "--looper-config", tp, "--exc-flag", "failed", "--dry-run"]
+        x = ["run", "--looper-config", tp, "--exc-flag", "running", "--dry-run"]
 
         try:
             results = main(test_args=x)
@@ -182,23 +197,30 @@ def test_excluding_flags_works(
         sd = os.path.join(get_outdir(tp), "submission")
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
 
-        assert len(subs_list) == 2
+        assert len(subs_list) == 1
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_excluding_multi_flags_works(
         self, prep_temp_pep_pipestat, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
+        """Verify excluding multi flags"""
         tp = prep_temp_pep_pipestat
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -208,14 +230,12 @@ def test_excluding_multi_flags_works(
                 f.write(FLAGS[count])
             count += 1
 
-        # x = ["--looper-config", "--exc-flag", "['failed','running']", tp, "run", "--dry-run"]
-
         x = [
             "run",
             "--looper-config",
             tp,
             "--exc-flag",
-            "failed",
+            "completed",
             "running",
             "--dry-run",
         ]
@@ -225,26 +245,31 @@ def test_excluding_multi_flags_works(
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+        # No submission directory will exist because both samples are excluded.
         sd = os.path.join(get_outdir(tp), "submission")
-        subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
-
-        assert len(subs_list) == 1
+        assert os.path.exists(sd) is False
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_selecting_multi_flags_works(
         self, prep_temp_pep_pipestat, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
+        """Verify selecting multiple flags"""
         tp = prep_temp_pep_pipestat
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -260,7 +285,7 @@ def test_selecting_multi_flags_works(
             "--looper-config",
             tp,
             "--sel-flag",
-            "failed",
+            "completed",
             "running",
         ]
 
@@ -276,19 +301,26 @@ def test_selecting_multi_flags_works(
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_selecting_attr_and_flags_works(
-        self, prep_temp_pep_pipestat, flag_id, pipeline_name
+        self, prep_temp_pep_pipestat_advanced, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
-        tp = prep_temp_pep_pipestat
+        """Verify selecting via attr and flags"""
+
+        tp = prep_temp_pep_pipestat_advanced
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -323,19 +355,25 @@ def test_selecting_attr_and_flags_works(
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_excluding_attr_and_flags_works(
-        self, prep_temp_pep_pipestat, flag_id, pipeline_name
+        self, prep_temp_pep_pipestat_advanced, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
-        tp = prep_temp_pep_pipestat
+        """Verify excluding via attr and flags"""
+        tp = prep_temp_pep_pipestat_advanced
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -371,19 +409,33 @@ def test_excluding_attr_and_flags_works(
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_excluding_toggle_attr(
-        self, prep_temp_pep_pipestat, flag_id, pipeline_name
+        self, prep_temp_pep_pipestat_advanced, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
-        tp = prep_temp_pep_pipestat
+        """Verify excluding based on toggle attr"""
+        tp = prep_temp_pep_pipestat_advanced
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+        # Manually add a toggle column to the PEP for this specific test
+        sample_csv = os.path.join(
+            os.path.dirname(project_config_path), "annotation_sheet.csv"
+        )
+        df = pd.read_csv(sample_csv)
+        df["toggle"] = 1
+        df.to_csv(sample_csv, index=False)
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(
@@ -418,19 +470,34 @@ def test_excluding_toggle_attr(
 
     @pytest.mark.parametrize("flag_id", ["completed"])
     @pytest.mark.parametrize(
-        "pipeline_name", ["PIPELINE1"]
+        "pipeline_name", ["example_pipestat_pipeline"]
     )  # This is given in the pipestat_output_schema.yaml
     def test_including_toggle_attr(
-        self, prep_temp_pep_pipestat, flag_id, pipeline_name
+        self, prep_temp_pep_pipestat_advanced, flag_id, pipeline_name
     ):
-        """Verify that checking works"""
-        tp = prep_temp_pep_pipestat
+        """Verify including based on toggle attr"""
+
+        tp = prep_temp_pep_pipestat_advanced
+        project_config_path = get_project_config_path(tp)
+        p = Project(project_config_path)
+
+        # get flag dir from .looper.yaml
+        with open(tp, "r") as f:
+            looper_cfg_data = safe_load(f)
+            flag_dir = looper_cfg_data[PIPESTAT_KEY]["flag_file_dir"]
+
+        # Manually add a toggle column to the PEP for this specific test
+        sample_csv = os.path.join(
+            os.path.dirname(project_config_path), "annotation_sheet.csv"
+        )
+        df = pd.read_csv(sample_csv)
+        df["toggle"] = 1
+        df.to_csv(sample_csv, index=False)
 
-        p = Project(tp)
-        out_dir = p[CONFIG_KEY][LOOPER_KEY][OUTDIR_KEY]
+        flag_dir = os.path.join(os.path.dirname(tp), flag_dir)
         count = 0
         for s in p.samples:
-            sf = os.path.join(out_dir, "results_pipeline")
+            sf = flag_dir
             if not os.path.exists(sf):
                 os.makedirs(sf)
             flag_path = os.path.join(

From cded639bfe6645cba01a1f4932c6d2890dbf031c Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 14:39:42 -0400
Subject: [PATCH 139/225] fix test_looper_cfg_required

---
 tests/smoketests/test_run.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 9ea4156c2..bf68b39e0 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -53,7 +53,8 @@ def test_looper_cfg_required(self, cmd):
         """Verify looper does not accept invalid cfg paths"""
 
         x = test_args_expansion("", cmd)
-        with pytest.raises(SystemExit):
+
+        with pytest.raises(ValueError):
             ff = main(test_args=x)
             print(ff)
 

From 30c54a183dd84fa5c901629d83d3ee2ad4731c95 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 11 Mar 2024 17:44:23 -0400
Subject: [PATCH 140/225] use local pep for pipestat comprehensive test

---
 tests/test_comprehensive.py | 144 +++++++++++++++---------------------
 1 file changed, 61 insertions(+), 83 deletions(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index bec3fc9ba..a1560289c 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -9,7 +9,6 @@
 from looper.cli_pydantic import main
 from tests.smoketests.test_run import is_connected
 from tempfile import TemporaryDirectory
-from git import Repo
 from pipestat import PipestatManager
 
 from yaml import dump, safe_load
@@ -28,8 +27,6 @@ def test_comprehensive_advanced_looper_no_pipestat(prep_temp_pep):
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    print(results)
-
 
 def test_comprehensive_looper_no_pipestat(prep_temp_pep_basic):
 
@@ -55,107 +52,88 @@ def test_comprehensive_looper_no_pipestat(prep_temp_pep_basic):
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    print(results)
-
 
-@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
-def test_comprehensive_looper_pipestat():
-    """
-    This test clones the hello_looper repository and runs the looper config file in the pipestat sub-directory
-    """
+def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     cmd = "run"
 
-    with TemporaryDirectory() as d:
-        repo = Repo.clone_from(REPO_URL, d, branch="dev_derive")
-        pipestat_dir = os.path.join(d, "pipestat")
-
-        path_to_looper_config = os.path.join(
-            pipestat_dir, ".looper_pipestat_shell.yaml"
-        )
+    path_to_looper_config = prep_temp_pep_pipestat
+    pipestat_dir = os.path.dirname(path_to_looper_config)
 
-        # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
-        pipestat_project_file = os.path.join(
-            d, "pipestat/project", "project_config.yaml"
-        )
-        with open(pipestat_project_file, "r") as f:
-            pipestat_project_data = safe_load(f)
-
-        pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
-            os.path.join(pipestat_dir, "data/{sample_name}.txt")
-        )
+    # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
+    pipestat_project_file = get_project_config_path(path_to_looper_config)
 
-        with open(pipestat_project_file, "w") as f:
-            dump(pipestat_project_data, f)
+    with open(pipestat_project_file, "r") as f:
+        pipestat_project_data = safe_load(f)
 
-        # x = [cmd, "-d", "--looper-config", path_to_looper_config]
-        x = [cmd, "--looper-config", path_to_looper_config]
+    pipestat_project_data["sample_modifiers"]["derive"]["sources"]["source1"] = (
+        os.path.join(pipestat_dir, "data/{sample_name}.txt")
+    )
 
-        try:
-            result = main(test_args=x)
-            if cmd == "run":
-                assert result["Pipestat compatible"] is True
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    with open(pipestat_project_file, "w") as f:
+        dump(pipestat_project_data, f)
 
-        # TODO TEST PROJECT LEVEL RUN
-        # Must add this to hello_looper for pipestat example
+    x = [cmd, "--looper-config", path_to_looper_config]
 
-        # TEST LOOPER CHECK
+    try:
+        result = main(test_args=x)
+        if cmd == "run":
+            assert result["Pipestat compatible"] is True
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        # looper cannot create flags, the pipeline or pipestat does that
-        # if you do not specify flag dir, pipestat places them in the same dir as config file
-        path_to_pipestat_config = os.path.join(
-            pipestat_dir, "looper_pipestat_config.yaml"
-        )
-        psm = PipestatManager(config_file=path_to_pipestat_config)
-        psm.set_status(record_identifier="frog_1", status_identifier="completed")
-        psm.set_status(record_identifier="frog_2", status_identifier="completed")
+    # TODO TEST PROJECT LEVEL RUN
+    # Must add this to hello_looper for pipestat example
 
-        # Now use looper check to get statuses
-        x = ["check", "--looper-config", path_to_looper_config]
+    # TEST LOOPER CHECK
 
-        try:
-            result = main(test_args=x)
-            assert result["example_pipestat_pipeline"]["frog_1"] == "completed"
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    # looper cannot create flags, the pipeline or pipestat does that
+    # if you do not specify flag dir, pipestat places them in the same dir as config file
+    path_to_pipestat_config = os.path.join(pipestat_dir, "looper_pipestat_config.yaml")
+    psm = PipestatManager(config_file=path_to_pipestat_config)
+    psm.set_status(record_identifier="frog_1", status_identifier="completed")
+    psm.set_status(record_identifier="frog_2", status_identifier="completed")
 
-        # TEST LOOPER REPORT
+    # Now use looper check to get statuses
+    x = ["check", "--looper-config", path_to_looper_config]
 
-        x = ["report", "--looper-config", path_to_looper_config]
+    try:
+        result = main(test_args=x)
+        assert result["example_pipestat_pipeline"]["frog_1"] == "completed"
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        try:
-            result = main(test_args=x)
-            assert "report_directory" in result
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    # TEST LOOPER REPORT
 
-        # TEST LOOPER Table
+    x = ["report", "--looper-config", path_to_looper_config]
 
-        x = ["table", "--looper-config", path_to_looper_config]
+    try:
+        result = main(test_args=x)
+        assert "report_directory" in result
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        try:
-            result = main(test_args=x)
-            assert "example_pipestat_pipeline_stats_summary.tsv" in result[0]
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    # TEST LOOPER Table
 
-        # TEST LOOPER DESTROY
-        # TODO add destroying individual samples via pipestat
+    x = ["table", "--looper-config", path_to_looper_config]
 
-        x = [
-            "destroy",
-            "--looper-config",
-            path_to_looper_config,
-            "--force-yes",
-        ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
+    try:
+        result = main(test_args=x)
+        assert "example_pipestat_pipeline_stats_summary.tsv" in result[0]
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
 
-        try:
-            result = main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+    # TEST LOOPER DESTROY
+    # TODO add destroying individual samples via pipestat
 
-        # TODO TEST LOOPER INSPECT  -> I believe this moved to Eido?
+    x = [
+        "destroy",
+        "--looper-config",
+        path_to_looper_config,
+        "--force-yes",
+    ]  # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!"
 
-        print(result)
+    try:
+        result = main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))

From 4125b78bd97c03159ab622e350544f54c618b737 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 15:19:43 -0400
Subject: [PATCH 141/225] Update tests/conftest.py

Co-authored-by: Nathan Sheffield <nsheff@users.noreply.github.com>
---
 tests/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 692f6e2ff..ac8f71a2f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -55,7 +55,7 @@ def get_outdir(pth):
 
 def get_project_config_path(looper_config_pth):
     """
-    Get project config file path from a config file path since they are relative
+    Get project config file path from a looper config file path, since they are relative
 
     :param str pth:
     :return str: output directory

From 54f0b1fd4b8123348bdb78d91358098b8a33507e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 15:35:51 -0400
Subject: [PATCH 142/225] add todo

---
 looper/cli_pydantic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 67597d915..9658f9134 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -81,6 +81,8 @@ def validate_post_parse(args: argparse.Namespace) -> List[str]:
     return problems
 
 
+# TODO rename to run_looper_via_cli for running lloper as a python library:
+#  https://github.com/pepkit/looper/pull/472#discussion_r1521970763
 def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     # here comes adapted `cli_looper.py` code
     global _LOGGER

From 5da3fc916ba291b6d5cc2aab7e692782e3205a1f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 15:42:14 -0400
Subject: [PATCH 143/225] use == instead of in for assessing subcommands

---
 looper/cli_pydantic.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 9658f9134..f688772ee 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -110,7 +110,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         parser.print_help(sys.stderr)
         sys.exit(1)
 
-    if subcommand_name in ["init"]:
+    if subcommand_name == "init":
         return int(
             not initiate_looper_config(
                 dotfile_path(),
@@ -122,7 +122,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             )
         )
 
-    if subcommand_name in ["init_piface"]:
+    if subcommand_name == "init_piface":
         sys.exit(int(not init_generic_pipeline()))
 
     _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
@@ -243,13 +243,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 )
                 raise
 
-        if subcommand_name in ["runp"]:
+        if subcommand_name == "runp":
             compute_kwargs = _proc_resources_spec(subcommand_args)
             collate = Collator(prj)
             collate(subcommand_args, **compute_kwargs)
             return collate.debug
 
-        if subcommand_name in ["destroy"]:
+        if subcommand_name == "destroy":
             return Destroyer(prj)(subcommand_args)
 
         use_pipestat = (
@@ -258,34 +258,34 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             else prj.pipestat_configured
         )
 
-        if subcommand_name in ["table"]:
+        if subcommand_name == "table":
             if use_pipestat:
                 return Tabulator(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("table")
 
-        if subcommand_name in ["report"]:
+        if subcommand_name == "report":
             if use_pipestat:
                 return Reporter(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("report")
 
-        if subcommand_name in ["link"]:
+        if subcommand_name == "link":
             if use_pipestat:
                 Linker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("link")
 
-        if subcommand_name in ["check"]:
+        if subcommand_name == "check":
             if use_pipestat:
                 return Checker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("check")
 
-        if subcommand_name in ["clean"]:
+        if subcommand_name == "clean":
             return Cleaner(prj)(subcommand_args)
 
-        if subcommand_name in ["inspect"]:
+        if subcommand_name == "inspect":
             from warnings import warn
 
             warn(

From 3d277c8b9716e3aa502152d3fe926fab83394e4e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 15:52:23 -0400
Subject: [PATCH 144/225] add one line function explanations

---
 looper/cli_pydantic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index f688772ee..fac891727 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -51,10 +51,12 @@
 
 
 def opt_attr_pair(name: str) -> Tuple[str, str]:
+    """Takes argument as attribute and returns as tuple of top-level or subcommand used."""
     return f"--{name}", name.replace("-", "_")
 
 
 def validate_post_parse(args: argparse.Namespace) -> List[str]:
+    """Checks if user is attempting to use mutually exclusive options."""
     problems = []
     used_exclusives = [
         opt

From 98ef5f2e1ce7020502ee2bf905560b6918d37ecb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 16:30:23 -0400
Subject: [PATCH 145/225] re-add inspect command and information

---
 looper/cli_pydantic.py         | 10 ++++------
 looper/const.py                |  2 +-
 tests/smoketests/test_other.py | 13 +++++++++++++
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index fac891727..467aa5294 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -23,6 +23,7 @@
 import logmuse
 import pydantic2_argparse
 import yaml
+from eido import inspect_project
 from pephubclient import PEPHubClient
 from pydantic2_argparse.argparse.parser import ArgumentParser
 
@@ -288,12 +289,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             return Cleaner(prj)(subcommand_args)
 
         if subcommand_name == "inspect":
-            from warnings import warn
-
-            warn(
-                "The inspect feature has moved to eido."
-                "Use `eido inspect` from now on.",
-            )
+            # Inspects project from eido
+            inspect_project(p, args.sample_names, args.attr_limit)
+            # TODO add inspecting looper config: https://github.com/pepkit/looper/issues/462
 
 
 def main(test_args=None) -> None:
diff --git a/looper/const.py b/looper/const.py
index 4d7017567..a866f2d84 100644
--- a/looper/const.py
+++ b/looper/const.py
@@ -263,7 +263,7 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
     "destroy": "Remove output files of the project.",
     "check": "Check flag status of current runs.",
     "clean": "Run clean scripts of already processed jobs.",
-    "inspect": "Deprecated. Use `eido inspect` instead. Print information about a project.",
+    "inspect": "Print information about a project.",
     "init": "Initialize looper config file.",
     "init-piface": "Initialize generic pipeline interface.",
     "link": "Create directory of symlinks for reported results.",
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 55009af67..6be068c42 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -527,3 +527,16 @@ def test_including_toggle_attr(
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
 
         assert len(subs_list) == 3
+
+
+@pytest.mark.skip(reason="Functionality not implemented.")
+class TestLooperInspect:
+    @pytest.mark.parametrize("cmd", ["inspect"])
+    def test_inspect_config(self, prep_temp_pep, cmd):
+        "Checks inspect command"
+        tp = prep_temp_pep
+        x = [cmd, "--looper-config", tp]
+        try:
+            results = main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))

From 953d6d9ad3bcdd0c4a2778e28b03b4316678a123 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 12 Mar 2024 16:41:38 -0400
Subject: [PATCH 146/225] revert lumpj, lumpn to lump-j and lump-n

---
 looper/command_models/arguments.py | 4 ++--
 looper/looper.py                   | 4 ++--
 tests/smoketests/test_run.py       | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 2c8d9717b..6cab9eafe 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -128,12 +128,12 @@ class ArgumentEnum(enum.Enum):
         description="Total input file size (GB) to batch into one job",
     )
     LUMPN = Argument(
-        name="lumpn",
+        name="lump_n",
         default=(int, None),
         description="Number of commands to batch into one job",
     )
     LUMPJ = Argument(
-        name="lumpj",
+        name="lump_j",
         default=(int, None),
         description="Lump samples into number of jobs.",
     )
diff --git a/looper/looper.py b/looper/looper.py
index 42d5510c0..53004c919 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -414,9 +414,9 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
                 extra_args=getattr(args, "command_extra", None),
                 extra_args_override=getattr(args, "command_extra_override", None),
                 ignore_flags=getattr(args, "ignore_flags", None),
-                max_cmds=getattr(args, "lumpn", None),
+                max_cmds=getattr(args, "lump_n", None),
                 max_size=getattr(args, "lump", None),
-                max_jobs=getattr(args, "lumpj", None),
+                max_jobs=getattr(args, "lump_j", None),
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index bf68b39e0..ee1d54cc6 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -419,7 +419,7 @@ def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
 
     def test_looper_lumping(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lumpn", "2"])
+        x = test_args_expansion(tp, "run", ["--lump-n", "2"])
         try:
             main(test_args=x)
         except Exception:
@@ -429,7 +429,7 @@ def test_looper_lumping(self, prep_temp_pep):
 
     def test_looper_lumping_jobs(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lumpj", "1"])
+        x = test_args_expansion(tp, "run", ["--lump-j", "1"])
         try:
             main(test_args=x)
         except Exception:
@@ -439,7 +439,7 @@ def test_looper_lumping_jobs(self, prep_temp_pep):
 
     def test_looper_lumping_jobs_negative(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lumpj", "-1"])
+        x = test_args_expansion(tp, "run", ["--lump-j", "-1"])
 
         with pytest.raises(ValueError):
             main(test_args=x)

From 23827a7104b4134d11fc634b43f85ab7bf8cbb94 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 09:09:12 -0400
Subject: [PATCH 147/225] add local hello_looper to ensure tests do not fail on
 github

---
 .gitignore                                    |  2 +-
 tests/data/hello_looper-dev_derive/README.md  | 18 +++++++++++++
 .../advanced/.looper_advanced_pipestat.yaml   |  9 +++++++
 .../advanced/pipeline/output_schema.yaml      | 27 +++++++++++++++++++
 .../pipeline/pipeline_interface1_project.yaml | 11 ++++++++
 .../pipeline/pipeline_interface1_sample.yaml  | 15 +++++++++++
 .../pipeline/pipeline_interface2_project.yaml | 13 +++++++++
 .../pipeline/pipeline_interface2_sample.yaml  | 16 +++++++++++
 .../pipeline/pipestat_output_schema.yaml      |  5 ++++
 .../pipestat_pipeline_interface1_sample.yaml  | 15 +++++++++++
 .../pipestat_pipeline_interface2_sample.yaml  | 17 ++++++++++++
 .../advanced/pipeline/readData.R              | 10 +++++++
 .../advanced/pipeline/resources-project.tsv   |  6 +++++
 .../advanced/pipeline/resources-sample.tsv    |  7 +++++
 .../advanced/project/annotation_sheet.csv     |  4 +++
 .../advanced/project/project_config.yaml      | 12 +++++++++
 .../basic/.looper_project.yaml                |  4 +++
 .../basic/data/frog_1.txt                     |  4 +++
 .../basic/data/frog_2.txt                     |  7 +++++
 .../basic/pipeline/count_lines.sh             |  3 +++
 .../basic/pipeline/pipeline_interface.yaml    |  6 +++++
 .../pipeline/pipeline_interface_project.yaml  |  6 +++++
 .../basic/project/project_config.yaml         |  7 +++++
 .../basic/project/sample_annotation.csv       |  3 +++
 .../csv/data/frog1_data.txt                   |  4 +++
 .../csv/data/frog2_data.txt                   |  7 +++++
 .../csv/pipeline/count_lines.sh               |  3 +++
 .../csv/pipeline/pipeline_interface.yaml      |  6 +++++
 .../pipeline/pipeline_interface_project.yaml  |  6 +++++
 .../csv/project/sample_annotation.csv         |  3 +++
 .../pephub/data/frog1_data.txt                |  4 +++
 .../pephub/data/frog2_data.txt                |  7 +++++
 .../pephub/pipeline/count_lines.sh            |  3 +++
 .../pephub/pipeline/pipeline_interface.yaml   |  6 +++++
 .../pipeline/pipeline_interface_project.yaml  |  6 +++++
 .../pipestat/.looper_pipestat_shell.yaml      |  7 +++++
 .../pipestat/data/frog_1.txt                  |  4 +++
 .../pipestat/data/frog_2.txt                  |  7 +++++
 .../pipestat/looper_pipestat_config.yaml      |  7 +++++
 .../pipestat/pipeline_pipestat/count_lines.py | 26 ++++++++++++++++++
 .../pipeline_pipestat/count_lines_pipestat.sh |  4 +++
 .../pipeline_pipestat/pipeline_interface.yaml |  5 ++++
 .../pipeline_interface_project.yaml           |  8 ++++++
 .../pipeline_interface_shell.yaml             |  5 ++++
 .../pipestat_output_schema.yaml               |  5 ++++
 .../pipestat/project/project_config.yaml      |  7 +++++
 .../pipestat/project/sample_annotation.csv    |  3 +++
 47 files changed, 369 insertions(+), 1 deletion(-)
 create mode 100644 tests/data/hello_looper-dev_derive/README.md
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/.looper_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/data/frog_1.txt
 create mode 100644 tests/data/hello_looper-dev_derive/basic/data/frog_2.txt
 create mode 100755 tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh
 create mode 100644 tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/project/project_config.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv
 create mode 100644 tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt
 create mode 100644 tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt
 create mode 100755 tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh
 create mode 100644 tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv
 create mode 100644 tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt
 create mode 100644 tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt
 create mode 100755 tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh
 create mode 100644 tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml
 create mode 100755 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py
 create mode 100755 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv

diff --git a/.gitignore b/.gitignore
index 74d56a6f3..0940500f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,4 +81,4 @@ __pycache__/
 *ipynb_checkpoints*
 hello_looper-master*
 /pipeline/
-/tests/data/hello_looper-dev_derive/
+/tests/data/hello_looper-dev_derive/.gitignore
diff --git a/tests/data/hello_looper-dev_derive/README.md b/tests/data/hello_looper-dev_derive/README.md
new file mode 100644
index 000000000..5ba0385fe
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/README.md
@@ -0,0 +1,18 @@
+# Hello World! example for looper
+
+This repository provides minimal working examples for the [looper pipeline submission engine](http://pep.databio.org/looper).
+
+This repository contains examples
+
+1. `/basic` - A basic example pipeline and project.
+2. `/pephub` - Example of how to point looper to PEPhub.
+3. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
+4. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
+
+Each example contains:
+
+1. A looper config file (`.looper.yaml`).
+2. Sample data plus metadata in PEP format (or pointer to PEPhub).
+3. A looper-compatible pipeline.
+
+Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/)
diff --git a/tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml
new file mode 100644
index 000000000..74da1a3fb
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml
@@ -0,0 +1,9 @@
+pep_config: project/project_config.yaml
+output_dir: "results"
+pipeline_interfaces:
+  sample:
+    - ../pipeline/pipestat_pipeline_interface1_sample.yaml
+    - ../pipeline/pipestat_pipeline_interface2_sample.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml
new file mode 100644
index 000000000..8bc1f6f8e
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml
@@ -0,0 +1,27 @@
+description: Sample objects produced by test pipeline.
+properties:
+  samples:
+    type: array
+    items:
+      type: object
+      properties:
+        test_property:
+          type: string
+          description: "Test sample property"
+          path: "~/sample/{sample_name}_file.txt"
+        test_property1:
+          type: string
+          description: "Test sample property"
+          path: "~/sample/{sample_name}_file1.txt"
+  test_property:
+    type: image
+    title: "Test title"
+    description: "Test project property"
+    thumbnail_path: "~/test_{name}.png"
+    path: "~/test_{name}.pdf"
+  test_property1:
+    type: image
+    title: "Test title1"
+    description: "Test project property1"
+    thumbnail_path: "~/test_{name}.png"
+    path: "~/test_{name}1.pdf"
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml
new file mode 100644
index 000000000..cddc14b76
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml
@@ -0,0 +1,11 @@
+pipeline_name: PIPELINE1
+pipeline_type: project
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/col_pipeline1.py"
+command_template: >
+  {pipeline.var_templates.path} --project-name {project.name}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml
new file mode 100644
index 000000000..43638d923
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml
@@ -0,0 +1,15 @@
+pipeline_name: PIPELINE1
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml
new file mode 100644
index 000000000..7c4a42238
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml
@@ -0,0 +1,13 @@
+pipeline_name: OTHER_PIPELINE2
+pipeline_type: project
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/col_pipeline2.py"
+command_template: >
+  {pipeline.var_templates.path} --project-name {project.name}
+compute:
+  size_dependent_variables: resources-project.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml
new file mode 100644
index 000000000..987f7873d
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml
@@ -0,0 +1,16 @@
+pipeline_name: OTHER_PIPELINE2
+pipeline_type: sample
+output_schema: output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+compute:
+  size_dependent_variables: resources-sample.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml
new file mode 100644
index 000000000..d6b05c2ac
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml
@@ -0,0 +1,5 @@
+pipeline_name: example_pipestat_pipeline
+samples:
+  number_of_lines:
+    type: integer
+    description: "Number of lines in the input file."
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
new file mode 100644
index 000000000..ff40c411a
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
@@ -0,0 +1,15 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: pipestat_output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/pipeline1.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
new file mode 100644
index 000000000..79dcf50f8
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
@@ -0,0 +1,17 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+input_schema: https://schema.databio.org/pep/2.0.0.yaml
+output_schema: pipestat_output_schema.yaml
+var_templates:
+  path: "{looper.piface_dir}/pipelines/other_pipeline2.py"
+pre_submit:
+    python_functions:
+      - looper.write_sample_yaml
+command_template: >
+  {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr}
+compute:
+  size_dependent_variables: resources-sample.tsv
+
+bioconductor:
+  readFunName: readData
+  readFunPath: readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R b/tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R
new file mode 100644
index 000000000..89557a11b
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R
@@ -0,0 +1,10 @@
+readData = function(project, sampleName="sample1") {
+  lapply(getOutputsBySample(project, sampleName), function(x) {
+      lapply(x, function(x1){
+          message("Reading: ", basename(x1))
+      df = read.table(x1, stringsAsFactors=F)
+      colnames(df)[1:3] = c('chr', 'start', 'end')
+          GenomicRanges::GRanges(df)
+      })
+  })
+}
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv b/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv
new file mode 100644
index 000000000..4efd0f19c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv
@@ -0,0 +1,6 @@
+max_file_size	cores	mem	time
+0.05	1	12000	00-01:00:00
+0.5	1	16000	00-01:00:00
+1	1	16000	00-01:00:00
+10	1	16000	00-01:00:00
+NaN	1	32000	00-02:00:00
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv
new file mode 100644
index 000000000..20ec284b6
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv
@@ -0,0 +1,7 @@
+max_file_size	cores	mem	time
+0.001	1	8000	00-04:00:00
+0.05	2	12000	00-08:00:00
+0.5	4	16000	00-12:00:00
+1	8	16000	00-24:00:00
+10	16	32000	02-00:00:00
+NaN	32	32000	04-00:00:00
diff --git a/tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv b/tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv
new file mode 100644
index 000000000..2d0e1265c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv
@@ -0,0 +1,4 @@
+sample_name,protocol,data_source,SRR,Sample_geo_accession,read1,read2
+sample1,PROTO1,SRA,SRR5210416,GSM2471255,SRA_1,SRA_2
+sample2,PROTO1,SRA,SRR5210450,GSM2471300,SRA_1,SRA_2
+sample3,PROTO2,SRA,SRR5210398,GSM2471249,SRA_1,SRA_2
diff --git a/tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml b/tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml
new file mode 100644
index 000000000..54db02372
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml
@@ -0,0 +1,12 @@
+name: looper_advanced_test
+pep_version: "2.0.0"
+sample_table: annotation_sheet.csv
+
+sample_modifiers:
+  append:
+    attr: "val"
+  derive:
+    attributes: [read1, read2]
+    sources:
+      SRA_1: "{SRR}_1.fastq.gz"
+      SRA_2: "{SRR}_2.fastq.gz"
diff --git a/tests/data/hello_looper-dev_derive/basic/.looper_project.yaml b/tests/data/hello_looper-dev_derive/basic/.looper_project.yaml
new file mode 100644
index 000000000..b44ef03b7
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/.looper_project.yaml
@@ -0,0 +1,4 @@
+pep_config: project/project_config.yaml # local path to pep config
+output_dir: "results"
+pipeline_interfaces:
+  project: pipeline/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/data/frog_1.txt b/tests/data/hello_looper-dev_derive/basic/data/frog_1.txt
new file mode 100644
index 000000000..815c0cf7c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/data/frog_1.txt
@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/basic/data/frog_2.txt b/tests/data/hello_looper-dev_derive/basic/data/frog_2.txt
new file mode 100644
index 000000000..e6fdd5350
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/data/frog_2.txt
@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh b/tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh
new file mode 100755
index 000000000..71b887fe7
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml
new file mode 100644
index 000000000..732e69761
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml
new file mode 100644
index 000000000..9063c7d61
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev_derive/basic/project/project_config.yaml b/tests/data/hello_looper-dev_derive/basic/project/project_config.yaml
new file mode 100644
index 000000000..2ba1efdde
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/project/project_config.yaml
@@ -0,0 +1,7 @@
+pep_version: 2.0.0
+sample_table: sample_annotation.csv
+sample_modifiers:
+  derive:
+    attributes: [file]
+    sources:
+      source1: "data/{sample_name}.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv b/tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv
new file mode 100644
index 000000000..8a2a0565f
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv
@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,source1,1
+frog_2,anySampleType,source1,1
diff --git a/tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt b/tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt
new file mode 100644
index 000000000..815c0cf7c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt
@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt b/tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt
new file mode 100644
index 000000000..e6fdd5350
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt
@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh b/tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh
new file mode 100755
index 000000000..71b887fe7
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml
new file mode 100644
index 000000000..732e69761
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml
new file mode 100644
index 000000000..9063c7d61
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv b/tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv
new file mode 100644
index 000000000..05bf4d172
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv
@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,data/frog1_data.txt,1
+frog_2,anySampleType,data/frog2_data.txt,1
diff --git a/tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt b/tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt
new file mode 100644
index 000000000..815c0cf7c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt
@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt b/tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt
new file mode 100644
index 000000000..e6fdd5350
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt
@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh b/tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh
new file mode 100755
index 000000000..71b887fe7
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml
new file mode 100644
index 000000000..732e69761
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml
new file mode 100644
index 000000000..9063c7d61
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: project
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"
diff --git a/tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml
new file mode 100644
index 000000000..fb645a9bd
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml
@@ -0,0 +1,7 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  sample:  ./pipeline_pipestat/pipeline_interface_shell.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt
new file mode 100644
index 000000000..815c0cf7c
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt
@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt
new file mode 100644
index 000000000..e6fdd5350
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt
@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml b/tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml
new file mode 100644
index 000000000..0a04ac6f9
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml
@@ -0,0 +1,7 @@
+results_file_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results.yaml
+flag_file_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results/flags
+output_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results
+record_identifier: frog_2
+schema_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./pipeline_pipestat/pipestat_output_schema.yaml
+pipeline_name: test_pipe
+pipeline_type: sample
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py
new file mode 100755
index 000000000..97e866ee4
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py
@@ -0,0 +1,26 @@
+import pipestat
+import sys
+
+# Very simple pipeline that calls pipestat
+# takes arguments invoked during looper submission via command templates
+text_file = sys.argv[
+    1
+]  # this is the sample we wish to process by reading the number of lines
+sample_name = sys.argv[2]
+results_file = sys.argv[3]
+
+# Create pipestat manager and then report values
+psm = pipestat.PipestatManager(
+    schema_path="pipeline_pipestat/pipestat_output_schema.yaml",
+    results_file_path=results_file,
+    record_identifier=sample_name,
+)
+
+# Read text file and count lines
+with open(text_file, "r") as f:
+    result = {"number_of_lines": len(f.readlines())}
+
+# The results are defined in the pipestat output schema.
+psm.report(record_identifier=sample_name, values=result)
+
+# end of pipeline
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh
new file mode 100755
index 000000000..99f83f906
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml
new file mode 100644
index 000000000..1d26ac435
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml
@@ -0,0 +1,5 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+output_schema: pipestat_output_schema.yaml
+command_template: >
+  python {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file}
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
new file mode 100644
index 000000000..2237c2f39
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
@@ -0,0 +1,8 @@
+pipeline_name: example_pipestat_project_pipeline
+pipeline_type: project
+output_schema: pipestat_output_schema.yaml
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} "data/*.txt"
+
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
new file mode 100644
index 000000000..82df8b942
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
@@ -0,0 +1,5 @@
+pipeline_name: example_pipestat_pipeline
+pipeline_type: sample
+output_schema: pipestat_output_schema.yaml
+command_template: >
+  {looper.piface_dir}/count_lines_pipestat.sh {sample.file} {sample.sample_name} {pipestat.config_file}
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
new file mode 100644
index 000000000..d6b05c2ac
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
@@ -0,0 +1,5 @@
+pipeline_name: example_pipestat_pipeline
+samples:
+  number_of_lines:
+    type: integer
+    description: "Number of lines in the input file."
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml b/tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml
new file mode 100644
index 000000000..2ba1efdde
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml
@@ -0,0 +1,7 @@
+pep_version: 2.0.0
+sample_table: sample_annotation.csv
+sample_modifiers:
+  derive:
+    attributes: [file]
+    sources:
+      source1: "data/{sample_name}.txt"
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv b/tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv
new file mode 100644
index 000000000..8a2a0565f
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv
@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,source1,1
+frog_2,anySampleType,source1,1

From a81389057d1851f8a3d177347351f03ee7aa33a1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 09:21:07 -0400
Subject: [PATCH 148/225] adjust .gitignore to include relevant .looper.yaml
 files

---
 .gitignore                                             |  5 +++--
 tests/data/hello_looper-dev_derive/.looper.yaml        |  4 ++++
 .../data/hello_looper-dev_derive/advanced/.looper.yaml | 10 ++++++++++
 tests/data/hello_looper-dev_derive/basic/.looper.yaml  |  5 +++++
 tests/data/hello_looper-dev_derive/csv/.looper.yaml    |  5 +++++
 tests/data/hello_looper-dev_derive/pephub/.looper.yaml |  4 ++++
 .../data/hello_looper-dev_derive/pipestat/.looper.yaml |  8 ++++++++
 7 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 tests/data/hello_looper-dev_derive/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/advanced/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/basic/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/csv/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pephub/.looper.yaml
 create mode 100644 tests/data/hello_looper-dev_derive/pipestat/.looper.yaml

diff --git a/.gitignore b/.gitignore
index 0940500f6..a38fed409 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,7 +64,9 @@ open_pipelines/
 .coverage*
 .pytest_cache
 .vscode/
-.looper.yaml
+/tests/data/hello_looper-dev_derive/.gitignore
+/.looper.yaml
+/tests/smoketests/.looper.yaml
 
 # Reserved files for comparison
 *RESERVE*
@@ -81,4 +83,3 @@ __pycache__/
 *ipynb_checkpoints*
 hello_looper-master*
 /pipeline/
-/tests/data/hello_looper-dev_derive/.gitignore
diff --git a/tests/data/hello_looper-dev_derive/.looper.yaml b/tests/data/hello_looper-dev_derive/.looper.yaml
new file mode 100644
index 000000000..e812a1ea8
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/.looper.yaml
@@ -0,0 +1,4 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: "./results"
+pipeline_interfaces:
+  sample:  ../pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/.looper.yaml b/tests/data/hello_looper-dev_derive/advanced/.looper.yaml
new file mode 100644
index 000000000..d2c5797f8
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/advanced/.looper.yaml
@@ -0,0 +1,10 @@
+pep_config: project/project_config.yaml
+output_dir: "results"
+pipeline_interfaces:
+  sample:
+    - ../pipeline/pipeline_interface1_sample.yaml
+    - ../pipeline/pipeline_interface2_sample.yaml
+  project:
+    - ../pipeline/pipeline_interface1_project.yaml
+    - ../pipeline/pipeline_interface2_project.yaml
+
diff --git a/tests/data/hello_looper-dev_derive/basic/.looper.yaml b/tests/data/hello_looper-dev_derive/basic/.looper.yaml
new file mode 100644
index 000000000..19fac81d4
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/basic/.looper.yaml
@@ -0,0 +1,5 @@
+pep_config: project/project_config.yaml # local path to pep config
+# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
+output_dir: "results"
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/csv/.looper.yaml b/tests/data/hello_looper-dev_derive/csv/.looper.yaml
new file mode 100644
index 000000000..c88f0c9a5
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/csv/.looper.yaml
@@ -0,0 +1,5 @@
+pep_config: project/sample_annotation.csv # local path to CSV
+# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
+output_dir: "results"
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/pephub/.looper.yaml b/tests/data/hello_looper-dev_derive/pephub/.looper.yaml
new file mode 100644
index 000000000..00e60ded6
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pephub/.looper.yaml
@@ -0,0 +1,4 @@
+pep_config: pepkit/hello_looper:default # pephub registry path or local path
+output_dir: results
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/.looper.yaml b/tests/data/hello_looper-dev_derive/pipestat/.looper.yaml
new file mode 100644
index 000000000..852c6fa41
--- /dev/null
+++ b/tests/data/hello_looper-dev_derive/pipestat/.looper.yaml
@@ -0,0 +1,8 @@
+pep_config: ./project/project_config.yaml # pephub registry path or local path
+output_dir: ./results
+pipeline_interfaces:
+  sample:  ./pipeline_pipestat/pipeline_interface.yaml
+  project: ./pipeline_pipestat/pipeline_interface_project.yaml
+pipestat:
+  results_file_path: results.yaml
+  flag_file_dir: results/flags
\ No newline at end of file

From c5dcb40a55245666f5bf4ca70d050c0c0280d71d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 09:25:16 -0400
Subject: [PATCH 149/225] remove other .looper files from .gitignore

---
 .gitignore                    | 2 --
 .looper.yaml                  | 5 +++++
 tests/smoketests/.looper.yaml | 5 +++++
 3 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 .looper.yaml
 create mode 100644 tests/smoketests/.looper.yaml

diff --git a/.gitignore b/.gitignore
index a38fed409..76415cf6c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,8 +65,6 @@ open_pipelines/
 .pytest_cache
 .vscode/
 /tests/data/hello_looper-dev_derive/.gitignore
-/.looper.yaml
-/tests/smoketests/.looper.yaml
 
 # Reserved files for comparison
 *RESERVE*
diff --git a/.looper.yaml b/.looper.yaml
new file mode 100644
index 000000000..d4cfc108f
--- /dev/null
+++ b/.looper.yaml
@@ -0,0 +1,5 @@
+pep_config: example/pep/path
+output_dir: .
+pipeline_interfaces:
+  sample: []
+  project: []
diff --git a/tests/smoketests/.looper.yaml b/tests/smoketests/.looper.yaml
new file mode 100644
index 000000000..d4cfc108f
--- /dev/null
+++ b/tests/smoketests/.looper.yaml
@@ -0,0 +1,5 @@
+pep_config: example/pep/path
+output_dir: .
+pipeline_interfaces:
+  sample: []
+  project: []

From fe68c7d5588e58cead327f0f794e8ab27aa3427b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 09:57:46 -0400
Subject: [PATCH 150/225] re-add looper inspect command to cli_pydantic

---
 looper/cli_pydantic.py            |  5 ++++-
 looper/command_models/commands.py | 13 ++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 467aa5294..7e66f6066 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -290,7 +290,10 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
         if subcommand_name == "inspect":
             # Inspects project from eido
-            inspect_project(p, args.sample_names, args.attr_limit)
+            sample_names = []
+            for sample in p.samples:
+                sample_names.append(sample["sample_name"])
+            inspect_project(p, sample_names)
             # TODO add inspecting looper config: https://github.com/pepkit/looper/issues/462
 
 
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 5c5876541..653df6e2b 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -168,16 +168,14 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 )
 
 # INSPECT
-# TODO Did this move to Eido?
 InspectParser = Command(
     "inspect",
     MESSAGE_BY_SUBCOMMAND["inspect"],
     [],
 )
-InspectParserModel = InspectParser.create_model()
+
 
 # INIT
-# TODO rename to `init-config` ?
 InitParser = Command(
     "init",
     MESSAGE_BY_SUBCOMMAND["init"],
@@ -190,7 +188,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
     ],
 )
-InitParserModel = InitParser.create_model()
+
 
 # INIT-PIFACE
 InitPifaceParser = Command(
@@ -198,7 +196,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["init-piface"],
     [],
 )
-InitPifaceParserModel = InitPifaceParser.create_model()
+
 
 # LINK
 LinkParser = Command(
@@ -219,6 +217,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     CleanParser.arguments.append(arg)
     TableParser.arguments.append(arg)
     LinkParser.arguments.append(arg)
+    InspectParser.arguments.append(arg)
 
 # Create all Models
 RunParserModel = RunParser.create_model()
@@ -230,6 +229,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 CleanParserModel = CleanParser.create_model()
 TableParserModel = TableParser.create_model()
 LinkParserModel = LinkParser.create_model()
+InspectParserModel = InspectParser.create_model()
+InitParserModel = InitParser.create_model()
+InitPifaceParserModel = InitPifaceParser.create_model()
 
 
 SUPPORTED_COMMANDS = [
@@ -244,6 +246,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     InitParser,
     InitPifaceParser,
     LinkParser,
+    InspectParser,
 ]
 
 

From 0252578d62d35535323171a42c7fc08936cf46cb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:20:04 -0400
Subject: [PATCH 151/225] change from using eido inspect for PEP to inspecting
 looper_config_dict

---
 looper/cli_pydantic.py         | 12 ++++++------
 looper/utils.py                | 12 ++++++++++++
 tests/smoketests/test_other.py |  8 +++++++-
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 7e66f6066..8a3590719 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -46,6 +46,7 @@
     initiate_looper_config,
     init_generic_pipeline,
     read_yaml_file,
+    inspect_looper_config_file,
 )
 
 from typing import List, Tuple
@@ -289,12 +290,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             return Cleaner(prj)(subcommand_args)
 
         if subcommand_name == "inspect":
-            # Inspects project from eido
-            sample_names = []
-            for sample in p.samples:
-                sample_names.append(sample["sample_name"])
-            inspect_project(p, sample_names)
-            # TODO add inspecting looper config: https://github.com/pepkit/looper/issues/462
+            # Inspect looper config file
+            if looper_config_dict:
+                inspect_looper_config_file(looper_config_dict)
+            else:
+                _LOGGER.warning("No looper configuration was supplied.")
 
 
 def main(test_args=None) -> None:
diff --git a/looper/utils.py b/looper/utils.py
index b2ca2f26a..265d8e3cb 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -788,3 +788,15 @@ def write_submit_script(fp, content, data):
         with open(fp, "w") as f:
             f.write(content)
         return fp
+
+
+def inspect_looper_config_file(looper_config_dict) -> None:
+    """
+    Inspects looper config by printing it to terminal.
+    param dict looper_config_dict: dict representing looper_config
+
+    """
+    # Simply print this to terminal
+    print("LOOPER INSPECT")
+    for key, value in looper_config_dict.items():
+        print(f"{key} {value}")
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 6be068c42..772472533 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -529,7 +529,6 @@ def test_including_toggle_attr(
         assert len(subs_list) == 3
 
 
-@pytest.mark.skip(reason="Functionality not implemented.")
 class TestLooperInspect:
     @pytest.mark.parametrize("cmd", ["inspect"])
     def test_inspect_config(self, prep_temp_pep, cmd):
@@ -540,3 +539,10 @@ def test_inspect_config(self, prep_temp_pep, cmd):
             results = main(test_args=x)
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
+
+    @pytest.mark.parametrize("cmd", ["inspect"])
+    def test_inspect_no_config_found(self, cmd):
+        "Checks inspect command"
+        x = [cmd]
+        with pytest.raises(ValueError):
+            results = main(test_args=x)

From 76b2100aa7eebcd866384857033e65ca51fa88bf Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:25:33 -0400
Subject: [PATCH 152/225] add eido inspect back as well

---
 looper/cli_pydantic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 8a3590719..73b3e990f 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -290,6 +290,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             return Cleaner(prj)(subcommand_args)
 
         if subcommand_name == "inspect":
+            # Inspect PEP from Eido
+            sample_names = []
+            for sample in p.samples:
+                sample_names.append(sample["sample_name"])
+            inspect_project(p, sample_names)
             # Inspect looper config file
             if looper_config_dict:
                 inspect_looper_config_file(looper_config_dict)

From 20e94f69c6f4fe2efc25c79adf9a1a714e509f8e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:02:44 -0400
Subject: [PATCH 153/225] potential fix for #463

---
 looper/cli_pydantic.py         |  5 ++++-
 looper/conductor.py            |  5 +++--
 tests/smoketests/test_other.py | 15 +++++++++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 467aa5294..b0d0142a3 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -227,6 +227,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         exclusion_flag=subcommand_args.exc_flag,
     ) as prj:
         if subcommand_name in ["run", "rerun"]:
+            rerun = False
+            if subcommand_name == "rerun":
+                rerun = True
             run = Runner(prj)
             try:
                 # compute_kwargs = _proc_resources_spec(args)
@@ -234,7 +237,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
                 # TODO Shouldn't top level args and subcommand args be accessible on the same object?
                 return run(
-                    subcommand_args, top_level_args=args, rerun=False, **compute_kwargs
+                    subcommand_args, top_level_args=args, rerun=rerun, **compute_kwargs
                 )
             except SampleFailedException:
                 sys.exit(1)
diff --git a/looper/conductor.py b/looper/conductor.py
index bf4f78580..52e921173 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -309,6 +309,7 @@ def add_sample(self, sample, rerun=False):
         if sample_statuses:
             status_str = ", ".join(sample_statuses)
             failed_flag = any("failed" in x for x in sample_statuses)
+            waiting_flag = any("waiting" in x for x in sample_statuses)
             if self.ignore_flags:
                 msg = f"> Found existing status: {status_str}. Ignoring."
             else:  # this pipeline already has a status
@@ -318,11 +319,11 @@ def add_sample(self, sample, rerun=False):
                 use_this_sample = False
             if rerun:
                 # Rescue the sample if rerun requested, and failed flag is found
-                if failed_flag:
+                if failed_flag or waiting_flag:
                     msg = f"> Re-running failed sample. Status: {status_str}"
                     use_this_sample = True
                 else:
-                    msg = f"> Skipping sample because rerun requested, but no failed flag found. Status: {status_str}"
+                    msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found. Status: {status_str}"
                     use_this_sample = False
         if msg:
             _LOGGER.info(msg)
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 6be068c42..e01fccf28 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -61,6 +61,21 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+class TestLooperRerun:
+    @pytest.mark.parametrize("flag_id", FLAGS)
+    @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"])
+    def test_pipestat_rerun(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
+        """Verify that checking works when multiple flags are created"""
+        tp = prep_temp_pep_pipestat
+        _make_flags(tp, FLAGS[2], pipeline_name)
+
+        x = ["rerun", "--looper-config", tp]
+        try:
+            result = main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+
 class TestLooperCheck:
     @pytest.mark.parametrize("flag_id", FLAGS)
     @pytest.mark.parametrize(

From 2d8b46b8237229657ecd7b4e4949db2506579427 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 13:03:02 -0400
Subject: [PATCH 154/225] reduce to one-liner

---
 looper/cli_pydantic.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index b0d0142a3..6fa5a864c 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -227,9 +227,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         exclusion_flag=subcommand_args.exc_flag,
     ) as prj:
         if subcommand_name in ["run", "rerun"]:
-            rerun = False
-            if subcommand_name == "rerun":
-                rerun = True
+            rerun = subcommand_name == "rerun"
             run = Runner(prj)
             try:
                 # compute_kwargs = _proc_resources_spec(args)

From e0cd85dca456826d2e7d1ba9fb3ce4062ac7819d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 13:12:14 -0400
Subject: [PATCH 155/225] add assertion to test

---
 tests/smoketests/test_other.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index e01fccf28..c877ddedc 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -62,9 +62,8 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
 
 
 class TestLooperRerun:
-    @pytest.mark.parametrize("flag_id", FLAGS)
     @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"])
-    def test_pipestat_rerun(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
+    def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name):
         """Verify that checking works when multiple flags are created"""
         tp = prep_temp_pep_pipestat
         _make_flags(tp, FLAGS[2], pipeline_name)
@@ -75,6 +74,8 @@ def test_pipestat_rerun(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
+        assert result["Jobs submitted"] == 2
+
 
 class TestLooperCheck:
     @pytest.mark.parametrize("flag_id", FLAGS)

From 7a52dfd8b98420b4438a3656677ac75cc951fb79 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 13:16:14 -0400
Subject: [PATCH 156/225] add waiting status to rerun test

---
 tests/smoketests/test_other.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index c877ddedc..082afd3a9 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -62,11 +62,14 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
 
 
 class TestLooperRerun:
+    @pytest.mark.parametrize(
+        "flags", [FLAGS[2], FLAGS[3]]
+    )  # Waiting and Failed flags should work
     @pytest.mark.parametrize("pipeline_name", ["example_pipestat_pipeline"])
-    def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name):
-        """Verify that checking works when multiple flags are created"""
+    def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags):
+        """Verify that rerun works with either failed or waiting flags"""
         tp = prep_temp_pep_pipestat
-        _make_flags(tp, FLAGS[2], pipeline_name)
+        _make_flags(tp, flags, pipeline_name)
 
         x = ["rerun", "--looper-config", tp]
         try:

From 15a7a9fbb496cd0675657dcf6a9e885ad6cf153d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 14:59:56 -0400
Subject: [PATCH 157/225] fix for #467

---
 looper/conductor.py            |  7 +++--
 tests/smoketests/test_other.py | 56 ++++++++++++++++++++++++++++++----
 2 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 52e921173..dd59663b8 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -306,6 +306,9 @@ def add_sample(self, sample, rerun=False):
 
         use_this_sample = True  # default to running this sample
         msg = None
+        if rerun and sample_statuses == []:
+            msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found."
+            use_this_sample = False
         if sample_statuses:
             status_str = ", ".join(sample_statuses)
             failed_flag = any("failed" in x for x in sample_statuses)
@@ -314,13 +317,13 @@ def add_sample(self, sample, rerun=False):
                 msg = f"> Found existing status: {status_str}. Ignoring."
             else:  # this pipeline already has a status
                 msg = f"> Found existing status: {status_str}. Skipping sample."
-                if failed_flag:
+                if failed_flag and not rerun:
                     msg += " Use rerun to ignore failed status."  # help guidance
                 use_this_sample = False
             if rerun:
                 # Rescue the sample if rerun requested, and failed flag is found
                 if failed_flag or waiting_flag:
-                    msg = f"> Re-running failed sample. Status: {status_str}"
+                    msg = f"> Re-running sample. Status: {status_str}"
                     use_this_sample = True
                 else:
                     msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found. Status: {status_str}"
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 9b142980e..1ad41bbbd 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -10,7 +10,8 @@
 import pandas as pd
 
 
-def _make_flags(cfg, type, pipeline_name):
+def _make_flags_pipestat(cfg, type, pipeline_name):
+    """This makes flags for projects where pipestat is configured and used"""
 
     # get flag dir from .looper.yaml
     with open(cfg, "r") as f:
@@ -33,6 +34,31 @@ def _make_flags(cfg, type, pipeline_name):
             f.write(type)
 
 
+def _make_flags(cfg, type, pipeline_name):
+    """This makes flags for projects where pipestat is NOT configured"""
+
+    # get flag dir from .looper.yaml
+    with open(cfg, "r") as f:
+        looper_cfg_data = safe_load(f)
+        output_dir = looper_cfg_data[OUTDIR_KEY]
+
+    output_dir = os.path.join(os.path.dirname(cfg), output_dir)
+    # get samples from the project config via Peppy
+    project_config_path = get_project_config_path(cfg)
+    p = Project(project_config_path)
+
+    for s in p.samples:
+        # Make flags in sample subfolder, e.g /tmp/tmphqxdmxnl/advanced/results/results_pipeline/sample1
+        sf = os.path.join(output_dir, "results_pipeline", s.sample_name)
+        if not os.path.exists(sf):
+            os.makedirs(sf)
+        flag_path = os.path.join(
+            sf, pipeline_name + "_" + s.sample_name + "_" + type + ".flag"
+        )
+        with open(flag_path, "w") as f:
+            f.write(type)
+
+
 class TestLooperPipestat:
 
     @pytest.mark.parametrize("cmd", ["report", "table", "check"])
@@ -69,7 +95,7 @@ class TestLooperRerun:
     def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags):
         """Verify that rerun works with either failed or waiting flags"""
         tp = prep_temp_pep_pipestat
-        _make_flags(tp, flags, pipeline_name)
+        _make_flags_pipestat(tp, flags, pipeline_name)
 
         x = ["rerun", "--looper-config", tp]
         try:
@@ -79,6 +105,24 @@ def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags):
 
         assert result["Jobs submitted"] == 2
 
+    @pytest.mark.parametrize(
+        "flags", [FLAGS[2], FLAGS[3]]
+    )  # Waiting and Failed flags should work
+    @pytest.mark.parametrize("pipeline_name", ["PIPELINE1"])
+    def test_rerun_no_pipestat(self, prep_temp_pep, pipeline_name, flags):
+        """Verify that rerun works with either failed or waiting flags"""
+        tp = prep_temp_pep
+        _make_flags(tp, flags, pipeline_name)
+
+        x = ["rerun", "--looper-config", tp]
+        try:
+            result = main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+
+        # Only 3 failed flags exist for PIPELINE1, so only 3 samples should be submitted
+        assert result["Jobs submitted"] == 3
+
 
 class TestLooperCheck:
     @pytest.mark.parametrize("flag_id", FLAGS)
@@ -88,7 +132,7 @@ class TestLooperCheck:
     def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works"""
         tp = prep_temp_pep_pipestat
-        _make_flags(tp, flag_id, pipeline_name)
+        _make_flags_pipestat(tp, flag_id, pipeline_name)
 
         x = ["check", "--looper-config", tp]
 
@@ -106,8 +150,8 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
     def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works when multiple flags are created"""
         tp = prep_temp_pep_pipestat
-        _make_flags(tp, flag_id, pipeline_name)
-        _make_flags(tp, FLAGS[1], pipeline_name)
+        _make_flags_pipestat(tp, flag_id, pipeline_name)
+        _make_flags_pipestat(tp, FLAGS[1], pipeline_name)
 
         x = ["check", "--looper-config", tp]
         # Multiple flag files SHOULD cause pipestat to throw an assertion error
@@ -120,7 +164,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
     def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name):
         """Verify that checking works when bogus flags are created"""
         tp = prep_temp_pep_pipestat
-        _make_flags(tp, flag_id, pipeline_name)
+        _make_flags_pipestat(tp, flag_id, pipeline_name)
 
         x = ["check", "--looper-config", tp]
         try:

From e99b5d0b7726feb0cb61a8b70bb7d225d259f346 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 15:13:39 -0400
Subject: [PATCH 158/225] change looper_pipestat_config.yaml to debug output
 instead fo simple print https://github.com/pepkit/looper/issues/459

---
 looper/conductor.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index dd59663b8..0ebf554b9 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -92,7 +92,9 @@ def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict):
     """
     with open(looper_pipestat_config_path, "w") as f:
         yaml.dump(pipestat_config_dict, f)
-    print(f"Initialized pipestat config file: {looper_pipestat_config_path}")
+    _LOGGER.debug(
+        msg=f"Initialized pipestat config file: {looper_pipestat_config_path}"
+    )
 
     return True
 

From c089082512be5a82efa2f049fb06ad7569ad8b56 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 16:36:49 -0400
Subject: [PATCH 159/225] fix for https://github.com/pepkit/looper/issues/470

---
 looper/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index 265d8e3cb..aaee23336 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -95,7 +95,9 @@ def fetch_sample_flags(prj, sample, pl_name, flag_dir=None):
     return [
         x
         for x in folder_contents
-        if os.path.splitext(x)[1] == ".flag" and os.path.basename(x).startswith(pl_name)
+        if os.path.splitext(x)[1] == ".flag"
+        and os.path.basename(x).startswith(pl_name)
+        and sample.sample_name in x
     ]
 
 

From 08824daf468089f0b19dcce1bd6352dd70cd5de4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 13 Mar 2024 17:25:58 -0400
Subject: [PATCH 160/225] first attempt at
 https://github.com/pepkit/looper/issues/469

---
 looper/looper.py | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 53004c919..a120eebbe 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -242,35 +242,37 @@ def __call__(self, args, preview_flag=True):
         :param bool preview_flag: whether to halt before actually removing files
         """
 
-        _LOGGER.info("Removing results:")
-
-        for sample in select_samples(prj=self.prj, args=args):
-            _LOGGER.info(self.counter.show(sample.sample_name))
-            sample_output_folder = sample_folder(self.prj, sample)
-            if preview_flag:
-                # Preview: Don't actually delete, just show files.
-                _LOGGER.info(str(sample_output_folder))
-            else:
-                _remove_or_dry_run(sample_output_folder, args.dry_run)
-
-        _LOGGER.info("Removing summary:")
         use_pipestat = (
             self.prj.pipestat_configured_project
-            if getattr(
-                args, "project", None
-            )  # TODO this argument hasn't been added to the pydantic models.
+            if getattr(args, "project", None)
             else self.prj.pipestat_configured
         )
+
         if use_pipestat:
+            _LOGGER.info("Removing summary:")
             destroy_summary(
                 self.prj,
                 getattr(args, "dry_run", None),
                 getattr(args, "project", None),
             )
-        else:
-            _LOGGER.warning(
-                "Pipestat must be configured to destroy any created summaries."
-            )
+
+        _LOGGER.info("Removing results:")
+
+        for sample in select_samples(prj=self.prj, args=args):
+            _LOGGER.info(self.counter.show(sample.sample_name))
+            sample_output_folder = sample_folder(self.prj, sample)
+            if preview_flag:
+                # Preview: Don't actually delete, just show files.
+                _LOGGER.info(str(sample_output_folder))
+            else:
+                if use_pipestat:
+                    psms = self.prj.get_pipestat_managers(
+                        sample_name=sample.sample_name
+                    )
+                    for pipeline_name, psm in psms.items():
+                        psm.remove(record_identifier=sample.sample_name)
+                else:
+                    _remove_or_dry_run(sample_output_folder, args.dry_run)
 
         if not preview_flag:
             _LOGGER.info("Destroy complete.")

From fd05c04e75e53992a63ef4d1e0fef63020aefede Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 14 Mar 2024 12:14:52 -0400
Subject: [PATCH 161/225] correct retrieving reports directory, use pipestat
 rm_record for filtered samples when destroying
 https://github.com/pepkit/looper/issues/469

---
 looper/looper.py                  | 22 ++++++++--------------
 requirements/requirements-all.txt |  2 +-
 2 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index a120eebbe..defe11597 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -46,7 +46,6 @@
     sample_folder,
 )
 from pipestat.reports import get_file_for_table
-from pipestat.reports import get_file_for_project
 
 _PKGNAME = "looper"
 _LOGGER = logging.getLogger(_PKGNAME)
@@ -270,7 +269,9 @@ def __call__(self, args, preview_flag=True):
                         sample_name=sample.sample_name
                     )
                     for pipeline_name, psm in psms.items():
-                        psm.remove(record_identifier=sample.sample_name)
+                        psm.backend.remove_record(
+                            record_identifier=sample.sample_name, rm_record=True
+                        )
                 else:
                     _remove_or_dry_run(sample_output_folder, args.dry_run)
 
@@ -694,10 +695,8 @@ def destroy_summary(prj, dry_run=False, project_level=False):
         for name, psm in psms.items():
             _remove_or_dry_run(
                 [
-                    get_file_for_project(
-                        psm,
-                        pipeline_name=psm.pipeline_name,
-                        directory="reports",
+                    get_file_for_table(
+                        psm, pipeline_name=psm.pipeline_name, directory="reports"
                     ),
                     get_file_for_table(
                         psm,
@@ -709,9 +708,6 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                         pipeline_name=psm.pipeline_name,
                         appendix="objs_summary.yaml",
                     ),
-                    get_file_for_table(
-                        psm, pipeline_name=psm.pipeline_name, appendix="reports"
-                    ),
                 ],
                 dry_run,
             )
@@ -726,10 +722,8 @@ def destroy_summary(prj, dry_run=False, project_level=False):
             for name, psm in psms.items():
                 _remove_or_dry_run(
                     [
-                        get_file_for_project(
-                            psm,
-                            pipeline_name=psm.pipeline_name,
-                            directory="reports",
+                        get_file_for_table(
+                            psm, pipeline_name=psm.pipeline_name, directory="reports"
                         ),
                         get_file_for_table(
                             psm,
@@ -742,7 +736,7 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                             appendix="objs_summary.yaml",
                         ),
                         get_file_for_table(
-                            psm, pipeline_name=psm.pipeline_name, appendix="reports"
+                            psm, pipeline_name="", directory="aggregate_results.yaml"
                         ),
                     ],
                     dry_run,
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 5d7d5eb23..108303465 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
 peppy>=0.40.0
-pipestat>=0.8.2a1
+pipestat>=0.8.3a1
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2

From 2952f1a63cc4aff2df8ce19d6a23228568474784 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 14 Mar 2024 12:46:47 -0400
Subject: [PATCH 162/225] correct destroy bug, gather aggregate_results
 https://github.com/pepkit/looper/issues/469

---
 looper/looper.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index defe11597..b044ef1d9 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -251,8 +251,8 @@ def __call__(self, args, preview_flag=True):
             _LOGGER.info("Removing summary:")
             destroy_summary(
                 self.prj,
-                getattr(args, "dry_run", None),
-                getattr(args, "project", None),
+                dry_run=preview_flag,
+                project_level=getattr(args, "project", None),
             )
 
         _LOGGER.info("Removing results:")
@@ -708,6 +708,9 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                         pipeline_name=psm.pipeline_name,
                         appendix="objs_summary.yaml",
                     ),
+                    os.path.join(
+                        os.path.dirname(psm.config_path), "aggregate_results.yaml"
+                    ),
                 ],
                 dry_run,
             )
@@ -735,8 +738,8 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                             pipeline_name=psm.pipeline_name,
                             appendix="objs_summary.yaml",
                         ),
-                        get_file_for_table(
-                            psm, pipeline_name="", directory="aggregate_results.yaml"
+                        os.path.join(
+                            os.path.dirname(psm.config_path), "aggregate_results.yaml"
                         ),
                     ],
                     dry_run,

From bd175533df8e5ccdd45e5ca7bf12ae6ecabcc614 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 14 Mar 2024 13:07:07 -0400
Subject: [PATCH 163/225] add checks to destroy test
 https://github.com/pepkit/looper/issues/469

---
 tests/test_comprehensive.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index a1560289c..a40a0f41d 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -1,3 +1,5 @@
+import os.path
+
 import pytest
 from peppy.const import *
 from yaml import dump
@@ -10,6 +12,7 @@
 from tests.smoketests.test_run import is_connected
 from tempfile import TemporaryDirectory
 from pipestat import PipestatManager
+from pipestat.exceptions import RecordNotFoundError
 
 from yaml import dump, safe_load
 
@@ -137,3 +140,9 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
         result = main(test_args=x)
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
+
+    sd = os.path.dirname(path_to_looper_config)
+    tsv_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".tsv")]
+    assert len(tsv_list) == 0
+    with pytest.raises(RecordNotFoundError):
+        retrieved_result = psm.retrieve_one(record_identifier="frog_2")

From 25f438d52a7cbeeafccb9987df9b2509db39e2ab Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 14 Mar 2024 14:14:34 -0400
Subject: [PATCH 164/225] add simple tests, and return false if path ends with
 csv for is_registry_path #456

---
 looper/utils.py              |  2 +-
 tests/conftest.py            | 18 ++++++++++++++++++
 tests/smoketests/test_run.py | 21 +++++++++++++++++++++
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/looper/utils.py b/looper/utils.py
index aaee23336..1ba6af9e4 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -605,7 +605,7 @@ def is_registry_path(input_string: str) -> bool:
     :return bool: True if input is a registry path
     """
     try:
-        if input_string.endswith(".yaml"):
+        if input_string.endswith(".yaml") or input_string.endswith(".csv"):
             return False
     except AttributeError:
         raise RegistryPathException(
diff --git a/tests/conftest.py b/tests/conftest.py
index ac8f71a2f..0293bd807 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -230,6 +230,24 @@ def prep_temp_pep_basic(example_pep_piface_path):
     return path_to_looper_config
 
 
+@pytest.fixture
+def prep_temp_pep_csv(example_pep_piface_path):
+
+    # Get Path to local copy of hello_looper
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
+
+    # Make local temp copy of hello_looper
+    d = tempfile.mkdtemp()
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
+
+    advanced_dir = os.path.join(d, "csv")
+    path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
+
+    return path_to_looper_config
+
+
 @pytest.fixture
 def prep_temp_config_with_pep(example_pep_piface_path):
     # temp dir
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index ee1d54cc6..18c10f98e 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -23,6 +23,16 @@ def test_cli(prep_temp_pep):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+def test_running_csv_pep(prep_temp_pep_csv):
+    tp = prep_temp_pep_csv
+
+    x = ["run", "--looper-config", tp, "--dry-run"]
+    try:
+        main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
+
 def is_connected():
     """Determines if local machine can connect to the internet."""
     import socket
@@ -597,3 +607,14 @@ def test_init_project_using_dict(self, prep_temp_config_with_pep):
         )
 
         assert len(init_project.pipeline_interfaces) == 3
+
+    def test_init_project_using_csv(self, prep_temp_pep_csv):
+        """Verify looper runs using pephub in a basic case and return code is 0"""
+        tp = prep_temp_pep_csv
+        with mod_yaml_data(tp) as config_data:
+            pep_config_csv = config_data["pep_config"]
+
+        pep_config_csv = os.path.join(os.path.dirname(tp), pep_config_csv)
+        init_project = Project(cfg=pep_config_csv)
+
+        assert len(init_project.samples) == 2

From 199ba63f0dfa8fe2ddb305b59a4e70ffd17a11fa Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 19 Mar 2024 13:50:08 -0400
Subject: [PATCH 165/225] skip simple test since functionality is broken

---
 tests/smoketests/test_run.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 18c10f98e..932dfbe37 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -23,6 +23,7 @@ def test_cli(prep_temp_pep):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+@pytest.mark.skip(reason="PEP via CSV is currently broken.")
 def test_running_csv_pep(prep_temp_pep_csv):
     tp = prep_temp_pep_csv
 

From b6aeda05b0718604784149cdbdf44e4e3288dcc9 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 19 Mar 2024 15:51:42 -0400
Subject: [PATCH 166/225] skip simple test since functionality is broken

---
 looper/utils.py              | 15 +++++++++++++++
 tests/smoketests/test_run.py |  9 +++++++++
 2 files changed, 24 insertions(+)

diff --git a/looper/utils.py b/looper/utils.py
index 1ba6af9e4..8c233243a 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -598,6 +598,21 @@ def dotfile_path(directory=os.getcwd(), must_exist=False):
         cur_dir = parent_dir
 
 
+def is_PEP_file_type(input_string: str) -> bool:
+    """
+    Determines if the provided path is actually a file type that Looper can use for loading PEP
+    """
+
+    PEP_FILE_TYPES = ["yaml", "csv"]
+
+    parsed_path = parse_registry_path(input_string)
+
+    if parsed_path["subitem"] in PEP_FILE_TYPES:
+        return True
+    else:
+        return False
+
+
 def is_registry_path(input_string: str) -> bool:
     """
     Check if input is a registry path to pephub
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 932dfbe37..fc8650fd0 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -34,6 +34,15 @@ def test_running_csv_pep(prep_temp_pep_csv):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+@pytest.mark.parametrize(
+    "path", ["something/example.yaml", "somethingelse/example2.csv"]
+)
+def test_is_PEP_file_type(path):
+
+    result = is_PEP_file_type(path)
+    assert result == True
+
+
 def is_connected():
     """Determines if local machine can connect to the internet."""
     import socket

From 11517edf4b6cb7638a34dca8bcf69450341f809f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 19 Mar 2024 15:51:42 -0400
Subject: [PATCH 167/225] add checking for pep file type and simple test

---
 looper/utils.py              | 15 +++++++++++++++
 tests/smoketests/test_run.py |  9 +++++++++
 2 files changed, 24 insertions(+)

diff --git a/looper/utils.py b/looper/utils.py
index 1ba6af9e4..8c233243a 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -598,6 +598,21 @@ def dotfile_path(directory=os.getcwd(), must_exist=False):
         cur_dir = parent_dir
 
 
+def is_PEP_file_type(input_string: str) -> bool:
+    """
+    Determines if the provided path is actually a file type that Looper can use for loading PEP
+    """
+
+    PEP_FILE_TYPES = ["yaml", "csv"]
+
+    parsed_path = parse_registry_path(input_string)
+
+    if parsed_path["subitem"] in PEP_FILE_TYPES:
+        return True
+    else:
+        return False
+
+
 def is_registry_path(input_string: str) -> bool:
     """
     Check if input is a registry path to pephub
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 932dfbe37..fc8650fd0 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -34,6 +34,15 @@ def test_running_csv_pep(prep_temp_pep_csv):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+@pytest.mark.parametrize(
+    "path", ["something/example.yaml", "somethingelse/example2.csv"]
+)
+def test_is_PEP_file_type(path):
+
+    result = is_PEP_file_type(path)
+    assert result == True
+
+
 def is_connected():
     """Determines if local machine can connect to the internet."""
     import socket

From 089f9ee8bada38c7fdd15ff0a2e531d341751cc5 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 19 Mar 2024 16:12:24 -0400
Subject: [PATCH 168/225] add check for PEP file types first THEN attempt
 registry path, some tests broken due to exceptions

---
 looper/cli_pydantic.py | 36 +++++++++++++++++++++---------------
 looper/utils.py        |  8 ++------
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 1f7808606..80beaeda4 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -47,6 +47,7 @@
     init_generic_pipeline,
     read_yaml_file,
     inspect_looper_config_file,
+    is_PEP_file_type,
 )
 
 from typing import List, Tuple
@@ -176,41 +177,46 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         subcommand_args.ignore_flags = True
 
     # Initialize project
-    if is_registry_path(subcommand_args.config_file):
-        if vars(subcommand_args)[SAMPLE_PL_ARG]:
+    if is_PEP_file_type(subcommand_args.config_file) and os.path.exists(subcommand_args.config_file):
+        try:
             p = Project(
+                cfg=subcommand_args.config_file,
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
-                project_dict=PEPHubClient()._load_raw_pep(
-                    registry_path=subcommand_args.config_file
-                ),
                 **{
                     attr: getattr(subcommand_args, attr)
                     for attr in CLI_PROJ_ATTRS
                     if attr in subcommand_args
                 },
             )
-        else:
-            raise MisconfigurationException(
-                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
-            )
-    else:
-        try:
+        except yaml.parser.ParserError as e:
+            _LOGGER.error(f"Project config parse failed -- {e}")
+            sys.exit(1)
+    elif is_registry_path(subcommand_args.config_file):
+        if vars(subcommand_args)[SAMPLE_PL_ARG]:
             p = Project(
-                cfg=subcommand_args.config_file,
                 amendments=subcommand_args.amend,
                 divcfg_path=divcfg,
                 runp=subcommand_name == "runp",
+                project_dict=PEPHubClient()._load_raw_pep(
+                    registry_path=subcommand_args.config_file
+                ),
                 **{
                     attr: getattr(subcommand_args, attr)
                     for attr in CLI_PROJ_ATTRS
                     if attr in subcommand_args
                 },
             )
-        except yaml.parser.ParserError as e:
-            _LOGGER.error(f"Project config parse failed -- {e}")
-            sys.exit(1)
+        else:
+            raise MisconfigurationException(
+                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
+            )
+    else:
+        raise MisconfigurationException(
+            f"Cannot load PEP. Check file path or registry path to pep."
+        )
+
 
     selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
     if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
diff --git a/looper/utils.py b/looper/utils.py
index 8c233243a..fe2e78175 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -605,12 +605,8 @@ def is_PEP_file_type(input_string: str) -> bool:
 
     PEP_FILE_TYPES = ["yaml", "csv"]
 
-    parsed_path = parse_registry_path(input_string)
-
-    if parsed_path["subitem"] in PEP_FILE_TYPES:
-        return True
-    else:
-        return False
+    res = list(filter(input_string.endswith, PEP_FILE_TYPES)) != []
+    return res
 
 
 def is_registry_path(input_string: str) -> bool:

From f300c838f4f8bcf36dc448311994c4555caf0f2a Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 20 Mar 2024 08:29:35 -0400
Subject: [PATCH 169/225] fix tests by changing expected Exception

---
 looper/cli_pydantic.py         | 5 +++--
 tests/smoketests/test_other.py | 5 ++---
 tests/smoketests/test_run.py   | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 80beaeda4..ef2a5809b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -177,7 +177,9 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         subcommand_args.ignore_flags = True
 
     # Initialize project
-    if is_PEP_file_type(subcommand_args.config_file) and os.path.exists(subcommand_args.config_file):
+    if is_PEP_file_type(subcommand_args.config_file) and os.path.exists(
+        subcommand_args.config_file
+    ):
         try:
             p = Project(
                 cfg=subcommand_args.config_file,
@@ -217,7 +219,6 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
             f"Cannot load PEP. Check file path or registry path to pep."
         )
 
-
     selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
     if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
         _LOGGER.info(
diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py
index 1ad41bbbd..2527f4f25 100644
--- a/tests/smoketests/test_other.py
+++ b/tests/smoketests/test_other.py
@@ -3,8 +3,7 @@
 import pytest
 from peppy import Project
 
-from looper.const import FLAGS
-from looper.exceptions import PipestatConfigurationException
+from looper.exceptions import PipestatConfigurationException, MisconfigurationException
 from tests.conftest import *
 from looper.cli_pydantic import main
 import pandas as pd
@@ -607,5 +606,5 @@ def test_inspect_config(self, prep_temp_pep, cmd):
     def test_inspect_no_config_found(self, cmd):
         "Checks inspect command"
         x = [cmd]
-        with pytest.raises(ValueError):
+        with pytest.raises(MisconfigurationException):
             results = main(test_args=x)
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index fc8650fd0..4c865dee9 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -74,7 +74,7 @@ def test_looper_cfg_required(self, cmd):
 
         x = test_args_expansion("", cmd)
 
-        with pytest.raises(ValueError):
+        with pytest.raises(MisconfigurationException):
             ff = main(test_args=x)
             print(ff)
 

From a3833919993fc9672d5bd6611de5e7ea683c5563 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 20 Mar 2024 09:16:37 -0400
Subject: [PATCH 170/225] add basic pephub test

---
 tests/conftest.py           | 19 +++++++++++++++++++
 tests/test_comprehensive.py | 13 +++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 0293bd807..294c23ec5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -305,3 +305,22 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
     path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml")
 
     return path_to_looper_config
+
+
+@pytest.fixture
+def prep_temp_pep_pephub(example_pep_piface_path):
+
+    # Get Path to local copy of hello_looper
+
+    hello_looper_dir_path = os.path.join(
+        example_pep_piface_path, "hello_looper-dev_derive"
+    )
+
+    # Make local temp copy of hello_looper
+    d = tempfile.mkdtemp()
+    shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
+
+    advanced_dir = os.path.join(d, "pephub")
+    path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
+
+    return path_to_looper_config
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index a40a0f41d..421e6736e 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -146,3 +146,16 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     assert len(tsv_list) == 0
     with pytest.raises(RecordNotFoundError):
         retrieved_result = psm.retrieve_one(record_identifier="frog_2")
+
+
+def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
+    """Basic test to determine if Looper can run a PEP from PEPHub"""
+
+    path_to_looper_config = prep_temp_pep_pephub
+
+    x = ["run", "--looper-config", path_to_looper_config]
+
+    try:
+        results = main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))

From fad869f8984beccb207871429b66360d25aedd7c Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 20 Mar 2024 09:55:09 -0400
Subject: [PATCH 171/225] refactor remove unnecessary tests

---
 looper/cli_pydantic.py       |  4 ++--
 looper/utils.py              | 27 +++++++++++++++------------
 tests/smoketests/test_run.py | 15 +--------------
 3 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index ef2a5809b..f9c510098 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -40,7 +40,7 @@
 from .utils import (
     dotfile_path,
     enrich_args_via_cfg,
-    is_registry_path,
+    is_pephub_registry_path,
     read_looper_config_file,
     read_looper_dotfile,
     initiate_looper_config,
@@ -195,7 +195,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         except yaml.parser.ParserError as e:
             _LOGGER.error(f"Project config parse failed -- {e}")
             sys.exit(1)
-    elif is_registry_path(subcommand_args.config_file):
+    elif is_pephub_registry_path(subcommand_args.config_file):
         if vars(subcommand_args)[SAMPLE_PL_ARG]:
             p = Project(
                 amendments=subcommand_args.amend,
diff --git a/looper/utils.py b/looper/utils.py
index fe2e78175..849395454 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -472,7 +472,7 @@ def initiate_looper_config(
         return False
 
     if pep_path:
-        if is_registry_path(pep_path):
+        if is_pephub_registry_path(pep_path):
             pass
         else:
             pep_path = expandpath(pep_path)
@@ -562,10 +562,20 @@ def read_looper_config_file(looper_config_path: str) -> dict:
     for k, v in return_dict.items():
         if isinstance(v, str):
             v = expandpath(v)
-            if not os.path.isabs(v) and not is_registry_path(v):
-                return_dict[k] = os.path.join(config_dir_path, v)
-            else:
+            # TODO this is messy because is_pephub_registry needs to fail on anything NOT a pephub registry path
+            # https://github.com/pepkit/ubiquerg/issues/43
+            if is_PEP_file_type(v):
+                if not os.path.isabs(v):
+                    return_dict[k] = os.path.join(config_dir_path, v)
+                else:
+                    return_dict[k] = v
+            elif is_pephub_registry_path(v):
                 return_dict[k] = v
+            else:
+                if not os.path.isabs(v):
+                    return_dict[k] = os.path.join(config_dir_path, v)
+                else:
+                    return_dict[k] = v
 
     return return_dict
 
@@ -609,19 +619,12 @@ def is_PEP_file_type(input_string: str) -> bool:
     return res
 
 
-def is_registry_path(input_string: str) -> bool:
+def is_pephub_registry_path(input_string: str) -> bool:
     """
     Check if input is a registry path to pephub
     :param str input_string: path to the PEP (or registry path)
     :return bool: True if input is a registry path
     """
-    try:
-        if input_string.endswith(".yaml") or input_string.endswith(".csv"):
-            return False
-    except AttributeError:
-        raise RegistryPathException(
-            msg=f"Malformed registry path. Unable to parse {input_string} as a registry path."
-        )
     try:
         registry_path = RegistryPath(**parse_registry_path(input_string))
     except (ValidationError, TypeError):
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 4c865dee9..48c7acb96 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -594,20 +594,7 @@ class TestLooperPEPhub:
         ],
     )
     def test_pephub_registry_path_recognition(self, pep_path):
-        assert is_registry_path(pep_path) is True
-
-    @pytest.mark.parametrize(
-        "pep_path",
-        [
-            "some/path/to/pep.yaml",
-            "different/path.yaml",
-            "default/path/to/file/without/yaml",
-            "file_in_folder.yaml",
-            "not_yaml_file",
-        ],
-    )
-    def test_config_recognition(self, pep_path):
-        assert is_registry_path(pep_path) is False
+        assert is_pephub_registry_path(pep_path) is True
 
     def test_init_project_using_dict(self, prep_temp_config_with_pep):
         """Verify looper runs using pephub in a basic case and return code is 0"""

From e3f1681ed0eda7ab661ce1bc5006068722b5ac37 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 20 Mar 2024 12:31:01 -0400
Subject: [PATCH 172/225] potential fix for attribute error when inferring
 project name from a csv #484

---
 looper/project.py            | 6 ++++++
 tests/smoketests/test_run.py | 1 -
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/looper/project.py b/looper/project.py
index c67f1ce08..fba207e9c 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -126,6 +126,12 @@ def __init__(self, cfg=None, amendments=None, divcfg_path=None, **kwargs):
 
         self[EXTRA_KEY] = {}
 
+        try:
+            # For loading PEPs via CSV, Peppy cannot infer project name.
+            name = self.name
+        except NotImplementedError:
+            self.name = None
+
         # add sample pipeline interface to the project
         if kwargs.get(SAMPLE_PL_ARG):
             self.set_sample_piface(kwargs.get(SAMPLE_PL_ARG))
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 48c7acb96..3706b7b78 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -23,7 +23,6 @@ def test_cli(prep_temp_pep):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
-@pytest.mark.skip(reason="PEP via CSV is currently broken.")
 def test_running_csv_pep(prep_temp_pep_csv):
     tp = prep_temp_pep_csv
 

From fd588e71d1fbc835f6997578fd0ef472efa61910 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 21 Mar 2024 10:56:04 -0400
Subject: [PATCH 173/225] Fix for #474

---
 looper/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/looper/utils.py b/looper/utils.py
index 849395454..fc6671722 100644
--- a/looper/utils.py
+++ b/looper/utils.py
@@ -560,6 +560,9 @@ def read_looper_config_file(looper_config_path: str) -> dict:
 
     # Expand paths in case ENV variables are used
     for k, v in return_dict.items():
+        if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG:
+            # Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474
+            pass
         if isinstance(v, str):
             v = expandpath(v)
             # TODO this is messy because is_pephub_registry needs to fail on anything NOT a pephub registry path

From 32697a0635d4193aee09e1c288db84501e3db5c9 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 21 Mar 2024 10:59:56 -0400
Subject: [PATCH 174/225] add test skipping for pephub test

---
 tests/test_comprehensive.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 421e6736e..f03f06922 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -148,6 +148,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
         retrieved_result = psm.retrieve_one(record_identifier="frog_2")
 
 
+@pytest.mark.skipif(not is_connected(), reason="This test needs internet access.")
 def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
     """Basic test to determine if Looper can run a PEP from PEPHub"""
 

From c85fa8dcc0d2bc75085615d698efe89e626668f9 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 21 Mar 2024 12:05:06 -0400
Subject: [PATCH 175/225] change pulling tests from hello looper dev branch
 instead of dev-deriv

---
 .gitignore                                         |  2 +-
 tests/conftest.py                                  | 14 +++++++-------
 .../.looper.yaml                                   |  0
 .../README.md                                      | 10 ++++++----
 .../advanced/.looper.yaml                          |  0
 .../advanced/.looper_advanced_pipestat.yaml        |  0
 .../advanced/pipeline/output_schema.yaml           |  0
 .../pipeline/pipeline_interface1_project.yaml      |  0
 .../pipeline/pipeline_interface1_sample.yaml       |  0
 .../pipeline/pipeline_interface2_project.yaml      |  0
 .../pipeline/pipeline_interface2_sample.yaml       |  0
 .../advanced/pipeline/pipestat_output_schema.yaml  |  0
 .../pipestat_pipeline_interface1_sample.yaml       |  0
 .../pipestat_pipeline_interface2_sample.yaml       |  0
 .../advanced/pipeline/readData.R                   |  0
 .../advanced/pipeline/resources-project.tsv        |  0
 .../advanced/pipeline/resources-sample.tsv         |  0
 .../advanced/project/annotation_sheet.csv          |  0
 .../advanced/project/project_config.yaml           |  0
 .../csv/.looper.yaml                               |  0
 .../csv/data/frog1_data.txt                        |  0
 .../csv/data/frog2_data.txt                        |  0
 .../csv}/pipeline/count_lines.sh                   |  0
 .../csv}/pipeline/pipeline_interface.yaml          |  0
 .../csv}/pipeline/pipeline_interface_project.yaml  |  0
 .../csv/project/sample_annotation.csv              |  0
 .../intermediate}/.looper.yaml                     |  0
 .../intermediate}/.looper_project.yaml             |  0
 .../intermediate}/data/frog_1.txt                  |  0
 .../intermediate}/data/frog_2.txt                  |  0
 .../intermediate}/pipeline/count_lines.sh          |  0
 .../intermediate}/pipeline/pipeline_interface.yaml |  0
 .../pipeline/pipeline_interface_project.yaml       |  0
 .../intermediate}/project/project_config.yaml      |  0
 .../intermediate}/project/sample_annotation.csv    |  0
 tests/data/hello_looper-dev/minimal/.looper.yaml   |  5 +++++
 .../minimal}/data/frog_1.txt                       |  0
 .../minimal}/data/frog_2.txt                       |  0
 .../minimal}/pipeline/count_lines.sh               |  0
 .../minimal}/pipeline/pipeline_interface.yaml      |  0
 .../minimal/project/project_config.yaml            |  2 ++
 .../minimal/project/sample_annotation.csv          |  3 +++
 .../pephub/.looper.yaml                            |  0
 .../pephub/data/frog1_data.txt                     |  0
 .../pephub/data/frog2_data.txt                     |  0
 .../pephub/pipeline/count_lines.sh                 |  3 +++
 .../pephub/pipeline/pipeline_interface.yaml        |  6 ++++++
 .../pipeline/pipeline_interface_project.yaml       |  0
 .../pipestat/.looper.yaml                          |  0
 .../pipestat/.looper_pipestat_shell.yaml           |  0
 .../data/hello_looper-dev/pipestat/data/frog_1.txt |  4 ++++
 .../data/hello_looper-dev/pipestat/data/frog_2.txt |  7 +++++++
 .../pipestat/looper_pipestat_config.yaml           |  0
 .../pipestat/pipeline_pipestat/count_lines.py      |  0
 .../pipeline_pipestat/count_lines_pipestat.sh      |  0
 .../pipeline_pipestat/pipeline_interface.yaml      |  0
 .../pipeline_interface_project.yaml                |  0
 .../pipeline_interface_shell.yaml                  |  0
 .../pipeline_pipestat/pipestat_output_schema.yaml  |  0
 .../pipestat/project/project_config.yaml           |  0
 .../pipestat/project/sample_annotation.csv         |  0
 tests/update_test_data.sh                          |  2 +-
 62 files changed, 45 insertions(+), 13 deletions(-)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/README.md (56%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/.looper_advanced_pipestat.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/output_schema.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipeline_interface1_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipeline_interface1_sample.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipeline_interface2_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipeline_interface2_sample.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipestat_output_schema.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/readData.R (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/resources-project.tsv (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/pipeline/resources-sample.tsv (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/project/annotation_sheet.csv (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/advanced/project/project_config.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/csv/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/csv/data/frog1_data.txt (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/csv/data/frog2_data.txt (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/csv}/pipeline/count_lines.sh (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/csv}/pipeline/pipeline_interface.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/csv}/pipeline/pipeline_interface_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/csv/project/sample_annotation.csv (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/.looper_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/data/frog_1.txt (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/data/frog_2.txt (100%)
 rename tests/data/{hello_looper-dev_derive/csv => hello_looper-dev/intermediate}/pipeline/count_lines.sh (100%)
 rename tests/data/{hello_looper-dev_derive/csv => hello_looper-dev/intermediate}/pipeline/pipeline_interface.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/csv => hello_looper-dev/intermediate}/pipeline/pipeline_interface_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/project/project_config.yaml (100%)
 rename tests/data/{hello_looper-dev_derive/basic => hello_looper-dev/intermediate}/project/sample_annotation.csv (100%)
 create mode 100644 tests/data/hello_looper-dev/minimal/.looper.yaml
 rename tests/data/{hello_looper-dev_derive/pipestat => hello_looper-dev/minimal}/data/frog_1.txt (100%)
 rename tests/data/{hello_looper-dev_derive/pipestat => hello_looper-dev/minimal}/data/frog_2.txt (100%)
 rename tests/data/{hello_looper-dev_derive/pephub => hello_looper-dev/minimal}/pipeline/count_lines.sh (100%)
 rename tests/data/{hello_looper-dev_derive/pephub => hello_looper-dev/minimal}/pipeline/pipeline_interface.yaml (100%)
 create mode 100644 tests/data/hello_looper-dev/minimal/project/project_config.yaml
 create mode 100644 tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pephub/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pephub/data/frog1_data.txt (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pephub/data/frog2_data.txt (100%)
 create mode 100755 tests/data/hello_looper-dev/pephub/pipeline/count_lines.sh
 create mode 100644 tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pephub/pipeline/pipeline_interface_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/.looper.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/.looper_pipestat_shell.yaml (100%)
 create mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_1.txt
 create mode 100644 tests/data/hello_looper-dev/pipestat/data/frog_2.txt
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/looper_pipestat_config.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/count_lines.py (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/count_lines_pipestat.sh (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/pipeline_interface.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/pipeline_interface_project.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/pipeline_pipestat/pipestat_output_schema.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/project/project_config.yaml (100%)
 rename tests/data/{hello_looper-dev_derive => hello_looper-dev}/pipestat/project/sample_annotation.csv (100%)

diff --git a/.gitignore b/.gitignore
index 76415cf6c..2bf77713d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,7 +64,7 @@ open_pipelines/
 .coverage*
 .pytest_cache
 .vscode/
-/tests/data/hello_looper-dev_derive/.gitignore
+/tests/data/hello_looper-dev/.gitignore
 
 # Reserved files for comparison
 *RESERVE*
diff --git a/tests/conftest.py b/tests/conftest.py
index 294c23ec5..b8ecdf369 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -199,7 +199,7 @@ def prep_temp_pep(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
@@ -217,14 +217,14 @@ def prep_temp_pep_basic(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
     shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True)
 
-    advanced_dir = os.path.join(d, "basic")
+    advanced_dir = os.path.join(d, "intermediate")
     path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml")
 
     return path_to_looper_config
@@ -235,7 +235,7 @@ def prep_temp_pep_csv(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
@@ -275,7 +275,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path):
     # Get Path to local copy of hello_looper
 
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
@@ -294,7 +294,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
     # Get Path to local copy of hello_looper
 
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
@@ -313,7 +313,7 @@ def prep_temp_pep_pephub(example_pep_piface_path):
     # Get Path to local copy of hello_looper
 
     hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev_derive"
+        example_pep_piface_path, "hello_looper-dev"
     )
 
     # Make local temp copy of hello_looper
diff --git a/tests/data/hello_looper-dev_derive/.looper.yaml b/tests/data/hello_looper-dev/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/.looper.yaml
rename to tests/data/hello_looper-dev/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/README.md b/tests/data/hello_looper-dev/README.md
similarity index 56%
rename from tests/data/hello_looper-dev_derive/README.md
rename to tests/data/hello_looper-dev/README.md
index 5ba0385fe..6c213b1a9 100644
--- a/tests/data/hello_looper-dev_derive/README.md
+++ b/tests/data/hello_looper-dev/README.md
@@ -4,10 +4,12 @@ This repository provides minimal working examples for the [looper pipeline submi
 
 This repository contains examples
 
-1. `/basic` - A basic example pipeline and project.
-2. `/pephub` - Example of how to point looper to PEPhub.
-3. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
-4. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
+1. `/minimal` - A basic example pipeline and project.
+2. `/intermediate` - An intermediate example pipeline and project with a couple extra options.
+3. `/advanced` - A more advanced example, showcasing the capabilities of Looper.
+4. `/pephub` - Example of how to point looper to PEPhub.
+5. `/pipestat` - Example of a pipeline that uses pipestat for recording results.
+6. `/csv` - How to use a pipeline with a CSV sample table (no YAML config)
 
 Each example contains:
 
diff --git a/tests/data/hello_looper-dev_derive/advanced/.looper.yaml b/tests/data/hello_looper-dev/advanced/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/.looper.yaml
rename to tests/data/hello_looper-dev/advanced/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/.looper_advanced_pipestat.yaml
rename to tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml b/tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/output_schema.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_project.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface1_sample.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_project.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipeline_interface2_sample.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_output_schema.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
rename to tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R b/tests/data/hello_looper-dev/advanced/pipeline/readData.R
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/readData.R
rename to tests/data/hello_looper-dev/advanced/pipeline/readData.R
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv b/tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/resources-project.tsv
rename to tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv
diff --git a/tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/pipeline/resources-sample.tsv
rename to tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv
diff --git a/tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv b/tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/project/annotation_sheet.csv
rename to tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv
diff --git a/tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml b/tests/data/hello_looper-dev/advanced/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/advanced/project/project_config.yaml
rename to tests/data/hello_looper-dev/advanced/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev_derive/csv/.looper.yaml b/tests/data/hello_looper-dev/csv/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/.looper.yaml
rename to tests/data/hello_looper-dev/csv/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt b/tests/data/hello_looper-dev/csv/data/frog1_data.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/data/frog1_data.txt
rename to tests/data/hello_looper-dev/csv/data/frog1_data.txt
diff --git a/tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt b/tests/data/hello_looper-dev/csv/data/frog2_data.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/data/frog2_data.txt
rename to tests/data/hello_looper-dev/csv/data/frog2_data.txt
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh b/tests/data/hello_looper-dev/csv/pipeline/count_lines.sh
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/csv/pipeline/count_lines.sh
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/pipeline/pipeline_interface_project.yaml
rename to tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv b/tests/data/hello_looper-dev/csv/project/sample_annotation.csv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/project/sample_annotation.csv
rename to tests/data/hello_looper-dev/csv/project/sample_annotation.csv
diff --git a/tests/data/hello_looper-dev_derive/basic/.looper.yaml b/tests/data/hello_looper-dev/intermediate/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/.looper.yaml
rename to tests/data/hello_looper-dev/intermediate/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/.looper_project.yaml b/tests/data/hello_looper-dev/intermediate/.looper_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/.looper_project.yaml
rename to tests/data/hello_looper-dev/intermediate/.looper_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/data/frog_1.txt b/tests/data/hello_looper-dev/intermediate/data/frog_1.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/data/frog_1.txt
rename to tests/data/hello_looper-dev/intermediate/data/frog_1.txt
diff --git a/tests/data/hello_looper-dev_derive/basic/data/frog_2.txt b/tests/data/hello_looper-dev/intermediate/data/frog_2.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/data/frog_2.txt
rename to tests/data/hello_looper-dev/intermediate/data/frog_2.txt
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh b/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/csv/pipeline/pipeline_interface_project.yaml
rename to tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/project/project_config.yaml b/tests/data/hello_looper-dev/intermediate/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/project/project_config.yaml
rename to tests/data/hello_looper-dev/intermediate/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv b/tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/basic/project/sample_annotation.csv
rename to tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv
diff --git a/tests/data/hello_looper-dev/minimal/.looper.yaml b/tests/data/hello_looper-dev/minimal/.looper.yaml
new file mode 100644
index 000000000..19fac81d4
--- /dev/null
+++ b/tests/data/hello_looper-dev/minimal/.looper.yaml
@@ -0,0 +1,5 @@
+pep_config: project/project_config.yaml # local path to pep config
+# pep_config: pepkit/hello_looper:default  # you can also use a pephub registry path
+output_dir: "results"
+pipeline_interfaces:
+  sample: pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev/minimal/data/frog_1.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/data/frog_1.txt
rename to tests/data/hello_looper-dev/minimal/data/frog_1.txt
diff --git a/tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev/minimal/data/frog_2.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/data/frog_2.txt
rename to tests/data/hello_looper-dev/minimal/data/frog_2.txt
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh b/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/pipeline/count_lines.sh
rename to tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev/minimal/project/project_config.yaml b/tests/data/hello_looper-dev/minimal/project/project_config.yaml
new file mode 100644
index 000000000..5456cca30
--- /dev/null
+++ b/tests/data/hello_looper-dev/minimal/project/project_config.yaml
@@ -0,0 +1,2 @@
+pep_version: 2.0.0
+sample_table: sample_annotation.csv
\ No newline at end of file
diff --git a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv b/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
new file mode 100644
index 000000000..97f223700
--- /dev/null
+++ b/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv
@@ -0,0 +1,3 @@
+sample_name,library,file,toggle
+frog_1,anySampleType,data/frog_1.txt,1
+frog_2,anySampleType,data/frog_2.txt,1
diff --git a/tests/data/hello_looper-dev_derive/pephub/.looper.yaml b/tests/data/hello_looper-dev/pephub/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/.looper.yaml
rename to tests/data/hello_looper-dev/pephub/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt b/tests/data/hello_looper-dev/pephub/data/frog1_data.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/data/frog1_data.txt
rename to tests/data/hello_looper-dev/pephub/data/frog1_data.txt
diff --git a/tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt b/tests/data/hello_looper-dev/pephub/data/frog2_data.txt
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/data/frog2_data.txt
rename to tests/data/hello_looper-dev/pephub/data/frog2_data.txt
diff --git a/tests/data/hello_looper-dev/pephub/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pephub/pipeline/count_lines.sh
new file mode 100755
index 000000000..71b887fe7
--- /dev/null
+++ b/tests/data/hello_looper-dev/pephub/pipeline/count_lines.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+echo "Number of lines: $linecount"
diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
new file mode 100644
index 000000000..732e69761
--- /dev/null
+++ b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml
@@ -0,0 +1,6 @@
+pipeline_name: count_lines
+pipeline_type: sample
+var_templates:
+  pipeline: '{looper.piface_dir}/count_lines.sh'
+command_template: >
+  {pipeline.var_templates.pipeline} {sample.file}
diff --git a/tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pephub/pipeline/pipeline_interface_project.yaml
rename to tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/.looper.yaml b/tests/data/hello_looper-dev/pipestat/.looper.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/.looper.yaml
rename to tests/data/hello_looper-dev/pipestat/.looper.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/.looper_pipestat_shell.yaml
rename to tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml
diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev/pipestat/data/frog_1.txt
new file mode 100644
index 000000000..815c0cf7c
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat/data/frog_1.txt
@@ -0,0 +1,4 @@
+ribbit
+ribbit
+ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev/pipestat/data/frog_2.txt
new file mode 100644
index 000000000..e6fdd5350
--- /dev/null
+++ b/tests/data/hello_looper-dev/pipestat/data/frog_2.txt
@@ -0,0 +1,7 @@
+ribbit
+ribbit
+ribbit
+
+ribbit, ribbit
+ribbit, ribbit
+CROAK!
diff --git a/tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml b/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/looper_pipestat_config.yaml
rename to tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines.py
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/count_lines_pipestat.sh
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface.yaml
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
rename to tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml b/tests/data/hello_looper-dev/pipestat/project/project_config.yaml
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/project/project_config.yaml
rename to tests/data/hello_looper-dev/pipestat/project/project_config.yaml
diff --git a/tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv b/tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv
similarity index 100%
rename from tests/data/hello_looper-dev_derive/pipestat/project/sample_annotation.csv
rename to tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv
diff --git a/tests/update_test_data.sh b/tests/update_test_data.sh
index b5c49073c..ece3c1ea8 100644
--- a/tests/update_test_data.sh
+++ b/tests/update_test_data.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-branch='dev_derive'
+branch='dev'
 
 wget https://github.com/pepkit/hello_looper/archive/refs/heads/${branch}.zip
 mv ${branch}.zip data/

From ac94facc3ddca0141b429f15f87c7194614632df Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 21 Mar 2024 13:58:47 -0400
Subject: [PATCH 176/225] lint

---
 tests/conftest.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index b8ecdf369..ef2176feb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -198,9 +198,7 @@ def example_pep_piface_path_cfg(example_pep_piface_path):
 def prep_temp_pep(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
@@ -216,9 +214,7 @@ def prep_temp_pep(example_pep_piface_path):
 def prep_temp_pep_basic(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
@@ -234,9 +230,7 @@ def prep_temp_pep_basic(example_pep_piface_path):
 def prep_temp_pep_csv(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
@@ -274,9 +268,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
 
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
@@ -293,9 +285,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
 
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()
@@ -312,9 +302,7 @@ def prep_temp_pep_pephub(example_pep_piface_path):
 
     # Get Path to local copy of hello_looper
 
-    hello_looper_dir_path = os.path.join(
-        example_pep_piface_path, "hello_looper-dev"
-    )
+    hello_looper_dir_path = os.path.join(example_pep_piface_path, "hello_looper-dev")
 
     # Make local temp copy of hello_looper
     d = tempfile.mkdtemp()

From 2f0946dfb0fe35987a2e11f8f7e355c9bff562e2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 21 Mar 2024 14:17:11 -0400
Subject: [PATCH 177/225] fix test related to #475

---
 tests/smoketests/test_run.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 3706b7b78..828e210a8 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -337,18 +337,16 @@ def test_looper_single_pipeline(self, prep_temp_pep):
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
 
-    @pytest.mark.skip(reason="Functionality broken")
     @pytest.mark.parametrize("arg", CMD_STRS)
     def test_cmd_extra_project(self, prep_temp_pep, arg):
 
-        # Test is currently broken, see https://github.com/pepkit/looper/issues/475
-
         tp = prep_temp_pep
 
         project_config_path = get_project_config_path(tp)
 
         with mod_yaml_data(project_config_path) as project_config_data:
-            project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg
+            project_config_data["looper"] = {}
+            project_config_data["looper"]["command_extra"] = arg
         x = test_args_expansion(tp, "runp")
 
         try:

From ac8cd2b6cdb1d44a507cffdee32e76b184cedeb4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 27 Mar 2024 17:27:49 -0400
Subject: [PATCH 178/225] update version and changelog for 1.8.0

---
 docs/changelog.md  | 22 ++++++++++++++++++++++
 looper/_version.py |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 9095091b3..d9c190384 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,6 +2,28 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [1.8.0] -- 2024-03-29
+
+### Added
+- looper destroy now destroys individual results when pipestat is configured: https://github.com/pepkit/looper/issues/469
+- comprehensive smoketests: https://github.com/pepkit/looper/issues/464
+- allow rerun to work on both failed or waiting flags: https://github.com/pepkit/looper/issues/463
+
+### Changed
+- Migrated `argparse` CLI definition to a pydantic basis for all commands. See: https://github.com/pepkit/looper/issues/438
+- during project load, check if PEP file path is a file first, then check if it is a registry path: https://github.com/pepkit/looper/issues/456 
+- Looper now uses FutureYamlConfigManager due to the yacman refactor v0.9.3: https://github.com/pepkit/looper/issues/452
+
+### Fixed
+- inferring project name when loading PEP from csv: https://github.com/pepkit/looper/issues/484
+- fix inconsistency resolving pipeline interface paths if multiple paths are supplied: https://github.com/pepkit/looper/issues/474
+- fix bug with checking for completed flags: https://github.com/pepkit/looper/issues/470
+- fix looper destroy not properly destroying all related files: https://github.com/pepkit/looper/issues/468
+- looper rerun now only runs failed jobs as intended: https://github.com/pepkit/looper/issues/467 
+- looper inspect now inspects the looper config: https://github.com/pepkit/looper/issues/462
+- Load PEP from CSV: https://github.com/pepkit/looper/issues/456
+
+
 ## [1.7.0] -- 2024-01-26
 
 ### Added
diff --git a/looper/_version.py b/looper/_version.py
index 14d9d2f58..29654eec0 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1 +1 @@
-__version__ = "1.7.0"
+__version__ = "1.8.0"

From 13499b5d3be5c9647de48a48ffd60b9a0520eac1 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 13 May 2024 11:28:03 -0400
Subject: [PATCH 179/225] test workaround for shortform arguments

---
 looper/cli_pydantic.py       | 45 ++++++++++++++++++++++++++++++++++--
 tests/smoketests/test_run.py | 10 ++++++++
 tests/test_comprehensive.py  |  1 +
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index f9c510098..58798711b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -53,6 +53,22 @@
 from typing import List, Tuple
 
 
+run_arguments_dict = {
+    "-d": "--dry-run",
+    "-l": "--limit",
+    "-k": "--skip",
+    "-o": "--output-dir",
+    "-S": "--sample-pipeline-interfaces",
+    "-P": "--project-pipeline-interfaces",
+    "-p": "--piface",
+    "-i": "--ignore-flags",
+    "-t": "--time-delay",
+    "-x": "--command-extra",
+    "-y": "--command-extra-override",
+    "-f": "--skip-file-checks",
+}
+
+
 def opt_attr_pair(name: str) -> Tuple[str, str]:
     """Takes argument as attribute and returns as tuple of top-level or subcommand used."""
     return f"--{name}", name.replace("-", "_")
@@ -310,6 +326,27 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 _LOGGER.warning("No looper configuration was supplied.")
 
 
+def create_command_string(args, command):
+    """
+    This is a workaround for short form arguments not being supported by the pydantic argparse package
+    """
+    arguments_dict = {}
+
+    # Must determine argument dict based on command since there is overlap in shortform keys...
+    if command in ["run", "runp", "rerun"]:
+        arguments_dict = run_arguments_dict
+
+    modified_command_string = []
+    for arg in args:
+        replacement = arguments_dict.get(arg)
+        modified_command_string.append(replacement if replacement else arg)
+
+    if command not in modified_command_string:
+        modified_command_string.insert(command)
+
+    return modified_command_string
+
+
 def main(test_args=None) -> None:
     parser = pydantic2_argparse.ArgumentParser(
         model=TopLevelParser,
@@ -318,9 +355,13 @@ def main(test_args=None) -> None:
         add_help=True,
     )
     if test_args:
-        args = parser.parse_typed_args(args=test_args)
+        command_string = create_command_string(args=test_args, command=test_args[0])
     else:
-        args = parser.parse_typed_args()
+        sys_args = sys.argv[1:]
+        command_string = create_command_string(args=sys_args, command=sys_args[0])
+
+    args = parser.parse_typed_args(args=command_string)
+
     return run_looper(args, parser, test_args=test_args)
 
 
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 828e210a8..dda6499d0 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -23,6 +23,16 @@ def test_cli(prep_temp_pep):
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
 
+def test_cli_shortform(prep_temp_pep):
+    tp = prep_temp_pep
+
+    x = ["run", "--looper-config", tp, "-d"]
+    try:
+        main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
+
 def test_running_csv_pep(prep_temp_pep_csv):
     tp = prep_temp_pep_csv
 
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index f03f06922..523857832 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -149,6 +149,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
 
 @pytest.mark.skipif(not is_connected(), reason="This test needs internet access.")
+@pytest.mark.skip("Test broken")
 def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
     """Basic test to determine if Looper can run a PEP from PEPHub"""
 

From d1b93f1b7e11bfcbc105d3901f08a56049849f2c Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 13 May 2024 12:04:02 -0400
Subject: [PATCH 180/225] add more arguments dictionaries, doc strings,

---
 looper/cli_pydantic.py | 69 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 65 insertions(+), 4 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 58798711b..e5fad4515 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -55,6 +55,7 @@
 
 run_arguments_dict = {
     "-d": "--dry-run",
+    "-c": "--compute",
     "-l": "--limit",
     "-k": "--skip",
     "-o": "--output-dir",
@@ -66,6 +67,47 @@
     "-x": "--command-extra",
     "-y": "--command-extra-override",
     "-f": "--skip-file-checks",
+    "-u": "--lump-s",
+    "-n": "--lump-n",
+    "-j": "--lump-j",
+}
+
+init_arguments_dict = {
+    "-d": "--dry-run",
+    "-c": "--compute",
+    "-l": "--limit",
+    "-k": "--skip",
+    "-o": "--output-dir",
+    "-S": "--sample-pipeline-interfaces",
+    "-P": "--project-pipeline-interfaces",
+    "-p": "--piface",
+    "-i": "--ignore-flags",
+    "-t": "--time-delay",
+    "-x": "--command-extra",
+    "-y": "--command-extra-override",
+    "-f": "--force",
+    "-u": "--lump-s",
+    "-n": "--lump-n",
+    "-j": "--lump-j",
+}
+
+check_arguments_dict = {
+    "-d": "--dry-run",
+    "-c": "--compute",
+    "-l": "--limit",
+    "-k": "--skip",
+    "-o": "--output-dir",
+    "-S": "--sample-pipeline-interfaces",
+    "-P": "--project-pipeline-interfaces",
+    "-p": "--piface",
+    "-i": "--ignore-flags",
+    "-t": "--time-delay",
+    "-x": "--command-extra",
+    "-y": "--command-extra-override",
+    "-f": "--flags",
+    "-u": "--lump-s",
+    "-n": "--lump-n",
+    "-j": "--lump-j",
 }
 
 
@@ -326,22 +368,41 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 _LOGGER.warning("No looper configuration was supplied.")
 
 
-def create_command_string(args, command):
+def create_command_string(args: list[str], command: str):
     """
-    This is a workaround for short form arguments not being supported by the pydantic argparse package
+    This is a workaround for short form arguments not being supported by the pydantic argparse package.
+    :param args: list[str] taken from test_args or sys.argv
+    :param command: str  high level command
+    :return modified_command_string: list[str] that has been modified, replacing shortforms with longforms
     """
     arguments_dict = {}
 
-    # Must determine argument dict based on command since there is overlap in shortform keys...
-    if command in ["run", "runp", "rerun"]:
+    # Must determine argument dict based on command since there is overlap in shortform keys for a couple of commands
+    if command in [
+        "run",
+        "runp",
+        "rerun",
+        "table",
+        "report",
+        "destroy",
+        "clean",
+        "inspect",
+        "link",
+    ]:
         arguments_dict = run_arguments_dict
+    if command in ["init"]:
+        arguments_dict = init_arguments_dict
+    if command in ["check"]:
+        arguments_dict = check_arguments_dict
 
     modified_command_string = []
     for arg in args:
+        # Replace shortform with long form based on the dictionary
         replacement = arguments_dict.get(arg)
         modified_command_string.append(replacement if replacement else arg)
 
     if command not in modified_command_string:
+        # required when using sys.argv during normal usage i.e. not test_args
         modified_command_string.insert(command)
 
     return modified_command_string

From c35dcb0b604cffc74dab5c12ba1a3f23b2021351 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 13 May 2024 13:05:13 -0400
Subject: [PATCH 181/225] revert skipping 1 test for pephub

---
 tests/test_comprehensive.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 523857832..f03f06922 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -149,7 +149,6 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
 
 @pytest.mark.skipif(not is_connected(), reason="This test needs internet access.")
-@pytest.mark.skip("Test broken")
 def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
     """Basic test to determine if Looper can run a PEP from PEPHub"""
 

From 1cf1f9b790f3919085f90502bb8dd5ee46f33cf3 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 10:55:08 -0400
Subject: [PATCH 182/225] new approach to add short arguments by adding them to
 the parser object immediately after parser creation

---
 looper/cli_pydantic.py            | 114 +++---------------------------
 looper/command_models/commands.py |  42 +++++++++++
 tests/smoketests/test_run.py      |  13 ++++
 3 files changed, 65 insertions(+), 104 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index e5fad4515..e9b0f7dad 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -30,7 +30,11 @@
 from divvy import select_divvy_config
 
 from . import __version__
-from .command_models.commands import SUPPORTED_COMMANDS, TopLevelParser
+from .command_models.commands import (
+    SUPPORTED_COMMANDS,
+    TopLevelParser,
+    add_short_arguments,
+)
 from .const import *
 from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
 from .exceptions import *
@@ -53,64 +57,6 @@
 from typing import List, Tuple
 
 
-run_arguments_dict = {
-    "-d": "--dry-run",
-    "-c": "--compute",
-    "-l": "--limit",
-    "-k": "--skip",
-    "-o": "--output-dir",
-    "-S": "--sample-pipeline-interfaces",
-    "-P": "--project-pipeline-interfaces",
-    "-p": "--piface",
-    "-i": "--ignore-flags",
-    "-t": "--time-delay",
-    "-x": "--command-extra",
-    "-y": "--command-extra-override",
-    "-f": "--skip-file-checks",
-    "-u": "--lump-s",
-    "-n": "--lump-n",
-    "-j": "--lump-j",
-}
-
-init_arguments_dict = {
-    "-d": "--dry-run",
-    "-c": "--compute",
-    "-l": "--limit",
-    "-k": "--skip",
-    "-o": "--output-dir",
-    "-S": "--sample-pipeline-interfaces",
-    "-P": "--project-pipeline-interfaces",
-    "-p": "--piface",
-    "-i": "--ignore-flags",
-    "-t": "--time-delay",
-    "-x": "--command-extra",
-    "-y": "--command-extra-override",
-    "-f": "--force",
-    "-u": "--lump-s",
-    "-n": "--lump-n",
-    "-j": "--lump-j",
-}
-
-check_arguments_dict = {
-    "-d": "--dry-run",
-    "-c": "--compute",
-    "-l": "--limit",
-    "-k": "--skip",
-    "-o": "--output-dir",
-    "-S": "--sample-pipeline-interfaces",
-    "-P": "--project-pipeline-interfaces",
-    "-p": "--piface",
-    "-i": "--ignore-flags",
-    "-t": "--time-delay",
-    "-x": "--command-extra",
-    "-y": "--command-extra-override",
-    "-f": "--flags",
-    "-u": "--lump-s",
-    "-n": "--lump-n",
-    "-j": "--lump-j",
-}
-
-
 def opt_attr_pair(name: str) -> Tuple[str, str]:
     """Takes argument as attribute and returns as tuple of top-level or subcommand used."""
     return f"--{name}", name.replace("-", "_")
@@ -368,46 +314,6 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
                 _LOGGER.warning("No looper configuration was supplied.")
 
 
-def create_command_string(args: list[str], command: str):
-    """
-    This is a workaround for short form arguments not being supported by the pydantic argparse package.
-    :param args: list[str] taken from test_args or sys.argv
-    :param command: str  high level command
-    :return modified_command_string: list[str] that has been modified, replacing shortforms with longforms
-    """
-    arguments_dict = {}
-
-    # Must determine argument dict based on command since there is overlap in shortform keys for a couple of commands
-    if command in [
-        "run",
-        "runp",
-        "rerun",
-        "table",
-        "report",
-        "destroy",
-        "clean",
-        "inspect",
-        "link",
-    ]:
-        arguments_dict = run_arguments_dict
-    if command in ["init"]:
-        arguments_dict = init_arguments_dict
-    if command in ["check"]:
-        arguments_dict = check_arguments_dict
-
-    modified_command_string = []
-    for arg in args:
-        # Replace shortform with long form based on the dictionary
-        replacement = arguments_dict.get(arg)
-        modified_command_string.append(replacement if replacement else arg)
-
-    if command not in modified_command_string:
-        # required when using sys.argv during normal usage i.e. not test_args
-        modified_command_string.insert(command)
-
-    return modified_command_string
-
-
 def main(test_args=None) -> None:
     parser = pydantic2_argparse.ArgumentParser(
         model=TopLevelParser,
@@ -415,13 +321,13 @@ def main(test_args=None) -> None:
         description="Looper Pydantic Argument Parser",
         add_help=True,
     )
+
+    parser = add_short_arguments(parser)
+
     if test_args:
-        command_string = create_command_string(args=test_args, command=test_args[0])
+        args = parser.parse_typed_args(args=test_args)
     else:
-        sys_args = sys.argv[1:]
-        command_string = create_command_string(args=sys_args, command=sys_args[0])
-
-    args = parser.parse_typed_args(args=command_string)
+        args = parser.parse_typed_args()
 
     return run_looper(args, parser, test_args=test_args)
 
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 653df6e2b..3dc4c0d46 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -234,6 +234,48 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 InitPifaceParserModel = InitPifaceParser.create_model()
 
 
+def add_short_arguments(parser):
+    """
+    This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
+    This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
+    """
+    # Loop through commands, add relevant short arguments
+
+    short_arguments_dict = {
+        "--dry-run": "-d",
+        "--limit": "-l",
+        "--compute": "-c",
+        "--skip": "-k",
+        "--output-dir": "-o",
+        "--sample-pipeline-interfaces": "-S",
+        "--project-pipeline-interfaces": "-P",
+        "--piface": "-p",
+        "--ignore-flags": "-i",
+        "--time-delay": "-t",
+        "--command-extra": "-x",
+        "--command-extra-override": "-y",
+        "--lump-s": "-u",
+        "--lump-n": "-n",
+        "--lump-j": "-j",
+        "--skip-file-checks": "-f",
+        "--force": "-f",
+        "--flags": "-f",
+    }
+
+    for cmd in parser._subcommands.choices.keys():
+        for long_key, short_key in short_arguments_dict.items():
+            if long_key in parser._subcommands.choices[cmd]._option_string_actions:
+                argument = parser._subcommands.choices[cmd]._option_string_actions[
+                    long_key
+                ]
+                argument.option_strings = (short_key, long_key)
+                parser._subcommands.choices[cmd]._option_string_actions[
+                    short_key
+                ] = argument
+
+    return parser
+
+
 SUPPORTED_COMMANDS = [
     RunParser,
     RerunParser,
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index dda6499d0..05231f594 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -32,6 +32,19 @@ def test_cli_shortform(prep_temp_pep):
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
+    x = ["run", "--looper-config", tp, "-d", "-l", "2"]
+    try:
+        main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
+    tp = prep_temp_pep
+    x = ["run", "--looper-config", tp, "-d", "-n", "2"]
+    try:
+        main(test_args=x)
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
 
 def test_running_csv_pep(prep_temp_pep_csv):
     tp = prep_temp_pep_csv

From 833c41c0097e4197db1cf6ad77a0799c63ead4db Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 11:27:45 -0400
Subject: [PATCH 183/225] fix force argument, add clarification for overlapping
 short form arguments

---
 looper/command_models/commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index 3dc4c0d46..cce321d01 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -240,6 +240,7 @@ def add_short_arguments(parser):
     This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
     """
     # Loop through commands, add relevant short arguments
+    # Note there are three long form arguments that have 'f' as a short form. However, they are used on 3 separate commands.
 
     short_arguments_dict = {
         "--dry-run": "-d",
@@ -258,7 +259,7 @@ def add_short_arguments(parser):
         "--lump-n": "-n",
         "--lump-j": "-j",
         "--skip-file-checks": "-f",
-        "--force": "-f",
+        "--force-yes": "-f",
         "--flags": "-f",
     }
 

From eca5c646dc4466799923b2f1995d8abbbef0c530 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 11:37:11 -0400
Subject: [PATCH 184/225] add docstrings

---
 looper/command_models/commands.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index cce321d01..dd0dc81c8 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -9,6 +9,7 @@
 
 from ..const import MESSAGE_BY_SUBCOMMAND
 from .arguments import Argument, ArgumentEnum
+from pydantic2_argparse import ArgumentParser
 
 
 @dataclass
@@ -234,10 +235,13 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 InitPifaceParserModel = InitPifaceParser.create_model()
 
 
-def add_short_arguments(parser):
+def add_short_arguments(parser: ArgumentParser) -> ArgumentParser:
     """
     This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
     This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
+
+    :param ArgumentParser
+    :return ArgumentParser
     """
     # Loop through commands, add relevant short arguments
     # Note there are three long form arguments that have 'f' as a short form. However, they are used on 3 separate commands.

From db8f06401183a087d4d550009168daeccf082e62 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 14:37:57 -0400
Subject: [PATCH 185/225] use alias on ArgumentEnum to do key value replacement
 for short args

---
 looper/command_models/arguments.py | 34 ++++++++++++++++---
 looper/command_models/commands.py  | 52 +++++++++++-------------------
 2 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 6cab9eafe..75855511c 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -33,9 +33,13 @@ class Argument(pydantic.fields.FieldInfo):
         `FieldInfo`. These are passed along as they are.
     """
 
-    def __init__(self, name: str, default: Any, description: str, **kwargs) -> None:
+    def __init__(
+        self, name: str, default: Any, description: str, alias: str = None, **kwargs
+    ) -> None:
         self._name = name
-        super().__init__(default=default, description=description, **kwargs)
+        super().__init__(
+            default=default, description=description, alias=alias, **kwargs
+        )
         self._validate()
 
     @property
@@ -77,11 +81,13 @@ class ArgumentEnum(enum.Enum):
 
     IGNORE_FLAGS = Argument(
         name="ignore_flags",
+        alias="-i",
         default=(bool, False),
         description="Ignore run status flags",
     )
     FORCE_YES = Argument(
         name="force_yes",
+        alias="-f",
         default=(bool, False),
         description="Provide upfront confirmation of destruction intent, to skip console query. Default=False",
     )
@@ -100,48 +106,61 @@ class ArgumentEnum(enum.Enum):
 
     FLAGS = Argument(
         name="flags",
+        alias="-f",
         default=(List, []),
         description="Only check samples based on these status flags.",
     )
 
     TIME_DELAY = Argument(
         name="time_delay",
+        alias="-t",
         default=(int, 0),
         description="Time delay in seconds between job submissions (min: 0, max: 30)",
     )
     DRY_RUN = Argument(
-        name="dry_run", default=(bool, False), description="Don't actually submit jobs"
+        name="dry_run",
+        alias="-d",
+        default=(bool, False),
+        description="Don't actually submit jobs",
     )
     COMMAND_EXTRA = Argument(
         name="command_extra",
+        alias="-x",
         default=(str, ""),
         description="String to append to every command",
     )
     COMMAND_EXTRA_OVERRIDE = Argument(
         name="command_extra_override",
+        alias="-y",
         default=(str, ""),
         description="Same as command-extra, but overrides values in PEP",
     )
     LUMP = Argument(
         name="lump",
+        alias="-u",
         default=(float, None),
         description="Total input file size (GB) to batch into one job",
     )
     LUMPN = Argument(
         name="lump_n",
+        alias="-n",
         default=(int, None),
         description="Number of commands to batch into one job",
     )
     LUMPJ = Argument(
         name="lump_j",
+        alias="-j",
         default=(int, None),
         description="Lump samples into number of jobs.",
     )
     LIMIT = Argument(
-        name="limit", default=(int, None), description="Limit to n samples"
+        name="limit", alias="-l", default=(int, None), description="Limit to n samples"
     )
     SKIP = Argument(
-        name="skip", default=(int, None), description="Skip samples by numerical index"
+        name="skip",
+        alias="-k",
+        default=(int, None),
+        description="Skip samples by numerical index",
     )
     CONFIG_FILE = Argument(
         name="config_file",
@@ -165,16 +184,19 @@ class ArgumentEnum(enum.Enum):
     )
     OUTPUT_DIR = Argument(
         name="output_dir",
+        alias="-o",
         default=(str, None),
         description="Output directory",
     )
     SAMPLE_PIPELINE_INTERFACES = Argument(
         name="sample_pipeline_interfaces",
+        alias="-S",
         default=(List, []),
         description="Paths to looper sample config files",
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
         name="project_pipeline_interfaces",
+        alias="-P",
         default=(List, []),
         description="Paths to looper project config files",
     )
@@ -204,6 +226,7 @@ class ArgumentEnum(enum.Enum):
     )
     SKIP_FILE_CHECKS = Argument(
         name="skip_file_checks",
+        alias="-f",
         default=(bool, False),
         description="Do not perform input file checks",
     )
@@ -214,6 +237,7 @@ class ArgumentEnum(enum.Enum):
     )
     COMPUTE = Argument(
         name="compute",
+        alias="-c",
         default=(List, []),
         description="List of key-value pairs (k1=v1)",
     )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index dd0dc81c8..b46132655 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -240,43 +240,27 @@ def add_short_arguments(parser: ArgumentParser) -> ArgumentParser:
     This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
     This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
 
-    :param ArgumentParser
-    :return ArgumentParser
+    :param ArgumentParser: parser before adding short arguments
+    :return ArgumentParser: parser after short arguments have been added
     """
-    # Loop through commands, add relevant short arguments
-    # Note there are three long form arguments that have 'f' as a short form. However, they are used on 3 separate commands.
-
-    short_arguments_dict = {
-        "--dry-run": "-d",
-        "--limit": "-l",
-        "--compute": "-c",
-        "--skip": "-k",
-        "--output-dir": "-o",
-        "--sample-pipeline-interfaces": "-S",
-        "--project-pipeline-interfaces": "-P",
-        "--piface": "-p",
-        "--ignore-flags": "-i",
-        "--time-delay": "-t",
-        "--command-extra": "-x",
-        "--command-extra-override": "-y",
-        "--lump-s": "-u",
-        "--lump-n": "-n",
-        "--lump-j": "-j",
-        "--skip-file-checks": "-f",
-        "--force-yes": "-f",
-        "--flags": "-f",
-    }
 
     for cmd in parser._subcommands.choices.keys():
-        for long_key, short_key in short_arguments_dict.items():
-            if long_key in parser._subcommands.choices[cmd]._option_string_actions:
-                argument = parser._subcommands.choices[cmd]._option_string_actions[
-                    long_key
-                ]
-                argument.option_strings = (short_key, long_key)
-                parser._subcommands.choices[cmd]._option_string_actions[
-                    short_key
-                ] = argument
+
+        for argument_enum in list(ArgumentEnum):
+            # First check there is an alias for the argument otherwise skip
+            if argument_enum.value.alias:
+                short_key = argument_enum.value.alias
+                long_key = "--" + argument_enum.value.name.replace(
+                    "_", "-"
+                )  # We must do this because the ArgumentEnum names are transformed during parser creation
+                if long_key in parser._subcommands.choices[cmd]._option_string_actions:
+                    argument = parser._subcommands.choices[cmd]._option_string_actions[
+                        long_key
+                    ]
+                    argument.option_strings = (short_key, long_key)
+                    parser._subcommands.choices[cmd]._option_string_actions[
+                        short_key
+                    ] = argument
 
     return parser
 

From 6bcc2cdf275cb6d8fa8c39fc38a2635ed5361ead Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 15:29:20 -0400
Subject: [PATCH 186/225] pass ArgumentEnums to function to make it explicit
 and not use lexical scoping

---
 looper/cli_pydantic.py            | 5 ++++-
 looper/command_models/commands.py | 9 +++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index e9b0f7dad..924a6c7eb 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -30,6 +30,9 @@
 from divvy import select_divvy_config
 
 from . import __version__
+
+from .command_models.arguments import ArgumentEnum
+
 from .command_models.commands import (
     SUPPORTED_COMMANDS,
     TopLevelParser,
@@ -322,7 +325,7 @@ def main(test_args=None) -> None:
         add_help=True,
     )
 
-    parser = add_short_arguments(parser)
+    parser = add_short_arguments(parser, ArgumentEnum)
 
     if test_args:
         args = parser.parse_typed_args(args=test_args)
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index b46132655..b040d9316 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -235,18 +235,19 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 InitPifaceParserModel = InitPifaceParser.create_model()
 
 
-def add_short_arguments(parser: ArgumentParser) -> ArgumentParser:
+def add_short_arguments(parser: ArgumentParser, argument_enums: Type[ArgumentEnum]) -> ArgumentParser:
     """
     This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
     This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
 
-    :param ArgumentParser: parser before adding short arguments
-    :return ArgumentParser: parser after short arguments have been added
+    :param ArgumentParser parser: parser before adding short arguments
+    :param Type[ArgumentEnum] argument_enums:  enumeration of arguments that contain names and aliases
+    :return ArgumentParser parser: parser after short arguments have been added
     """
 
     for cmd in parser._subcommands.choices.keys():
 
-        for argument_enum in list(ArgumentEnum):
+        for argument_enum in list(argument_enums):
             # First check there is an alias for the argument otherwise skip
             if argument_enum.value.alias:
                 short_key = argument_enum.value.alias

From 206e47bd0a16856b78a75bd89dc238b6f0f4a524 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 15 May 2024 15:30:31 -0400
Subject: [PATCH 187/225] lint

---
 looper/command_models/commands.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index b040d9316..e9025d909 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -235,7 +235,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 InitPifaceParserModel = InitPifaceParser.create_model()
 
 
-def add_short_arguments(parser: ArgumentParser, argument_enums: Type[ArgumentEnum]) -> ArgumentParser:
+def add_short_arguments(
+    parser: ArgumentParser, argument_enums: Type[ArgumentEnum]
+) -> ArgumentParser:
     """
     This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
     This is a workaround as pydantic-argparse does not currently support this during initial parser creation.

From 74f9cf8456bc41f2ad82b468b9e99f06d28cbc6f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 20 May 2024 13:05:36 -0400
Subject: [PATCH 188/225] update black gha to stable branch

---
 .github/workflows/black.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
index 04792190b..aec1766ed 100644
--- a/.github/workflows/black.yml
+++ b/.github/workflows/black.yml
@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - uses: psf/black@20.8b1
+      - uses: psf/black@stable
         with:
             options: "--check --diff --color --verbose"
             jupyter: true

From 389967231963ee00020baf93b5cc66288fc32745 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 20 May 2024 13:20:32 -0400
Subject: [PATCH 189/225] lint jupyter notebooks

---
 docs_jupyter/debug_divvy.ipynb    |  1 +
 docs_jupyter/tutorial_divvy.ipynb | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs_jupyter/debug_divvy.ipynb b/docs_jupyter/debug_divvy.ipynb
index 5614b4db0..050581e69 100644
--- a/docs_jupyter/debug_divvy.ipynb
+++ b/docs_jupyter/debug_divvy.ipynb
@@ -27,6 +27,7 @@
    ],
    "source": [
     "import divvy\n",
+    "\n",
     "divvy.setup_divvy_logger(\"DEBUG\", devmode=True)"
    ]
   }
diff --git a/docs_jupyter/tutorial_divvy.ipynb b/docs_jupyter/tutorial_divvy.ipynb
index 2a1f8b844..a9a3c044d 100644
--- a/docs_jupyter/tutorial_divvy.ipynb
+++ b/docs_jupyter/tutorial_divvy.ipynb
@@ -22,7 +22,8 @@
    "outputs": [],
    "source": [
     "import divvy\n",
-    "dcc = divvy.ComputingConfiguration()\n"
+    "\n",
+    "dcc = divvy.ComputingConfiguration()"
    ]
   },
   {
@@ -116,7 +117,9 @@
     }
    ],
    "source": [
-    "dcc.write_script(\"test_local.sub\", {\"code\": \"run-this-command\", \"logfile\": \"logfile.txt\"})"
+    "dcc.write_script(\n",
+    "    \"test_local.sub\", {\"code\": \"run-this-command\", \"logfile\": \"logfile.txt\"}\n",
+    ")"
    ]
   },
   {
@@ -276,7 +279,7 @@
     }
    ],
    "source": [
-    "s = dcc.write_script(\"test_script.sub\", {\"code\":\"yellow\"})"
+    "s = dcc.write_script(\"test_script.sub\", {\"code\": \"yellow\"})"
    ]
   },
   {
@@ -341,7 +344,9 @@
     }
    ],
    "source": [
-    "s = dcc.write_script(\"test_script.sub\", [{\"code\":\"red\"}, {\"code\": \"yellow\", \"time\": \"now\"}])"
+    "s = dcc.write_script(\n",
+    "    \"test_script.sub\", [{\"code\": \"red\"}, {\"code\": \"yellow\", \"time\": \"now\"}]\n",
+    ")"
    ]
   },
   {

From 5bb45ec297897afe5e3ecbb2871182861f629253 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 21 May 2024 16:29:45 -0400
Subject: [PATCH 190/225] skip pephub test comprehensive for now

---
 tests/test_comprehensive.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index f03f06922..5026eb538 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -149,9 +149,10 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
 
 @pytest.mark.skipif(not is_connected(), reason="This test needs internet access.")
+@pytest.mark.skip(reason="user must be logged into pephub otherwise this will fail.")
 def test_comprehensive_looper_pephub(prep_temp_pep_pephub):
     """Basic test to determine if Looper can run a PEP from PEPHub"""
-
+    # TODO need to add way to check if user is logged into pephub and then run test otherwise skip
     path_to_looper_config = prep_temp_pep_pephub
 
     x = ["run", "--looper-config", path_to_looper_config]

From ba8c3db9a2d767e91a0c66f43c94ea8dcad246af Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 22 May 2024 16:29:19 -0400
Subject: [PATCH 191/225] first pass at creating pipestat config and
 associating psms with pifaces

---
 looper/__init__.py           |   1 +
 looper/cli_pydantic.py       |   4 ++
 looper/conductor.py          |   7 ++
 looper/plugins.py            |   9 +++
 looper/project.py            | 122 +++++++++++++++++++++++++++++++++++
 tests/smoketests/test_run.py |  29 +++++++++
 6 files changed, 172 insertions(+)

diff --git a/looper/__init__.py b/looper/__init__.py
index fe751d02d..5db46a828 100644
--- a/looper/__init__.py
+++ b/looper/__init__.py
@@ -25,6 +25,7 @@
     write_sample_yaml_cwl,
     write_sample_yaml_prj,
     write_custom_template,
+    # write_local_pipestat_config,
 )
 from .const import *
 from .pipeline_interface import PipelineInterface
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 924a6c7eb..f25e6ebd5 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -241,6 +241,10 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         selector_flag=subcommand_args.sel_flag,
         exclusion_flag=subcommand_args.exc_flag,
     ) as prj:
+
+        # Check at the beginning if user wants to use pipestat and pipestat is configurable
+        is_pipestat_configured = prj._check_if_pipestat_configured_2()
+
         if subcommand_name in ["run", "rerun"]:
             rerun = subcommand_name == "rerun"
             run = Runner(prj)
diff --git a/looper/conductor.py b/looper/conductor.py
index 0ebf554b9..84ee24963 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -90,6 +90,13 @@ def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict):
     :param dict pipestat_config_dict: the dict containing key value pairs to be written to the pipestat configutation
     return bool
     """
+
+    if not os.path.exists(os.path.dirname(looper_pipestat_config_path)):
+        try:
+            os.makedirs(os.path.dirname(looper_pipestat_config_path))
+        except FileExistsError:
+            pass
+
     with open(looper_pipestat_config_path, "w") as f:
         yaml.dump(pipestat_config_dict, f)
     _LOGGER.debug(
diff --git a/looper/plugins.py b/looper/plugins.py
index dc34283e0..b0c4a246c 100644
--- a/looper/plugins.py
+++ b/looper/plugins.py
@@ -158,3 +158,12 @@ def write_sample_yaml(namespaces):
     )
     sample.to_yaml(sample["sample_yaml_path"], add_prj_ref=False)
     return {"sample": sample}
+
+
+# def write_local_pipestat_config(namespaces):
+#
+#     config_path = ""
+#
+#     print(config_path)
+#
+#     return config_path
diff --git a/looper/project.py b/looper/project.py
index fba207e9c..1df8f447d 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -468,6 +468,128 @@ def get_pipestat_managers(self, sample_name=None, project_level=False):
             for pipeline_name, pipestat_vars in pipestat_configs.items()
         }
 
+    def _check_if_pipestat_configured_2(self):
+
+        # First check if pipestat key is in looper_config, if not return false
+
+        if PIPESTAT_KEY not in self[EXTRA_KEY]:
+            return False
+        else:
+            # If pipestat key is available assume user desires pipestat usage
+            # This should return True OR raise an exception at this point.
+            return self._get_pipestat_configuration2()
+
+    def _get_pipestat_configuration2(self):
+
+        # First check if it already exists
+        print("DEBUG!")
+
+        for piface in self.pipeline_interfaces:
+            print(piface)
+            # first check if this piface has a psm?
+
+            if not self._check_for_existing_pipestat_config(piface):
+                self._create_pipestat_config(piface)
+
+        return True
+
+    def _check_for_existing_pipestat_config(self, piface):
+        """
+
+        config files should be in looper output directory and named as:
+
+        pipestat_config_pipelinename.yaml
+
+        """
+
+        # Cannot do much if we cannot retrieve the pipeline_name
+        try:
+            pipeline_name = piface.data["pipeline_name"]
+        except KeyError:
+            raise Exception(
+                "To use pipestat, a pipeline_name must be set in the pipeline interface."
+            )
+
+        config_file_name = f"pipestat_config_{pipeline_name}.yaml"
+        output_dir = expandpath(self.output_dir)
+
+        config_file_path = os.path.join(
+            # os.path.dirname(output_dir), config_file_name
+            output_dir,
+            config_file_name,
+        )
+
+        return os.path.exists(config_file_path)
+
+    def _create_pipestat_config(self, piface):
+        """
+        Each piface needs its own config file and associated psm
+        """
+
+        if PIPESTAT_KEY in self[EXTRA_KEY]:
+            pipestat_config_dict = self[EXTRA_KEY][PIPESTAT_KEY]
+        else:
+            _LOGGER.debug(
+                f"'{PIPESTAT_KEY}' not found in '{LOOPER_KEY}' section of the "
+                f"project configuration file."
+            )
+            # We cannot use pipestat without it being defined in the looper config file.
+            raise ValueError
+
+        # Expand paths in the event ENV variables were used in config files
+        output_dir = expandpath(self.output_dir)
+
+        pipestat_config_dict.update({"output_dir": output_dir})
+
+        if "output_schema" in piface.data:
+            schema_path = expandpath(piface.data["output_schema"])
+            if not os.path.isabs(schema_path):
+                # Get path relative to the pipeline_interface
+                schema_path = os.path.join(
+                    os.path.dirname(piface.pipe_iface_file), schema_path
+                )
+            pipestat_config_dict.update({"schema_path": schema_path})
+        if "pipeline_name" in piface.data:
+            pipeline_name = piface.data["pipeline_name"]
+            pipestat_config_dict.update({"pipeline_name": piface.data["pipeline_name"]})
+        if "pipeline_type" in piface.data:
+            pipestat_config_dict.update({"pipeline_type": piface.data["pipeline_type"]})
+
+        try:
+            # TODO if user gives non-absolute path should we force results to be in a pipeline folder?
+            # TODO otherwise pipelines could write to the same results file!
+            results_file_path = expandpath(pipestat_config_dict["results_file_path"])
+            if not os.path.exists(os.path.dirname(results_file_path)):
+                results_file_path = os.path.join(
+                    os.path.dirname(output_dir), results_file_path
+                )
+            pipestat_config_dict.update({"results_file_path": results_file_path})
+        except KeyError:
+            results_file_path = None
+
+        try:
+            flag_file_dir = expandpath(pipestat_config_dict["flag_file_dir"])
+            if not os.path.isabs(flag_file_dir):
+                flag_file_dir = os.path.join(os.path.dirname(output_dir), flag_file_dir)
+            pipestat_config_dict.update({"flag_file_dir": flag_file_dir})
+        except KeyError:
+            flag_file_dir = None
+
+        # Pipestat_dict_ is now updated from all sources and can be written to a yaml.
+        pipestat_config_path = os.path.join(
+            # os.path.dirname(output_dir), f"pipestat_config_{pipeline_name}.yaml"
+            output_dir,
+            f"pipestat_config_{pipeline_name}.yaml",
+        )
+
+        # Two end goals, create a config file
+        write_pipestat_config(pipestat_config_path, pipestat_config_dict)
+
+        # piface['psm'] = PipestatManager(config_file=pipestat_config_path)
+        piface.psm = PipestatManager(config_file=pipestat_config_path)
+
+        return None
+
     def _check_if_pipestat_configured(self, project_level=False):
         """
         A helper method determining whether pipestat configuration is complete
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index 05231f594..a94fa9b8a 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -443,6 +443,35 @@ def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, "test.txt", 3)
 
+    # @pytest.mark.parametrize(
+    #     "plugin,appendix",
+    #     [
+    #         ("looper.write_local_pipestat_config", "submission.yaml"),
+    #     ],
+    # )
+    # def test_looper_pipestat_plugins(self, prep_temp_pep_pipestat, plugin, appendix):
+    #     # tp = prep_temp_pep
+    #     tp = prep_temp_pep_pipestat
+    #     pep_dir = os.path.dirname(tp)
+    #     pipeline_interface1 = os.path.join(
+    #         pep_dir, "pipeline_pipestat/pipeline_interface.yaml"
+    #     )
+    #
+    #     with mod_yaml_data(pipeline_interface1) as piface_data:
+    #         piface_data.update({PRE_SUBMIT_HOOK_KEY: {}})
+    #         piface_data[PRE_SUBMIT_HOOK_KEY].update({PRE_SUBMIT_PY_FUN_KEY: {}})
+    #         piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_PY_FUN_KEY] = [plugin]
+    #
+    #     # x = test_args_expansion(tp, "run")
+    #     x = ["run", "--looper-config", tp, "--dry-run"]
+    #     # x.pop(-1)
+    #     try:
+    #         main(test_args=x)
+    #     except Exception as err:
+    #         raise pytest.fail(f"DID RAISE {err}")
+    #     sd = os.path.join(get_outdir(tp), "submission")
+    #     verify_filecount_in_dir(sd, appendix, 3)
+
 
 class TestLooperRunSubmissionScript:
     def test_looper_run_produces_submission_scripts(self, prep_temp_pep):

From 1ce86fb143d45a8e9eafc4483eff83fe0c4c94f7 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 10:37:51 -0400
Subject: [PATCH 192/225] fix bug with pipestat key present but value being
 None

---
 looper/project.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/looper/project.py b/looper/project.py
index 1df8f447d..132d35bb0 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -474,10 +474,13 @@ def _check_if_pipestat_configured_2(self):
 
         if PIPESTAT_KEY not in self[EXTRA_KEY]:
             return False
-        else:
-            # If pipestat key is available assume user desires pipestat usage
-            # This should return True OR raise an exception at this point.
-            return self._get_pipestat_configuration2()
+        elif PIPESTAT_KEY in self[EXTRA_KEY]:
+            if self[EXTRA_KEY][PIPESTAT_KEY] is None:
+                return False
+            else:
+                # If pipestat key is available assume user desires pipestat usage
+                # This should return True OR raise an exception at this point.
+                return self._get_pipestat_configuration2()
 
     def _get_pipestat_configuration2(self):
 

From 8040e299c5c763d273cee4f648d6d13836868fbb Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 10:48:47 -0400
Subject: [PATCH 193/225] use_pipestat = is_pipestat_configured

---
 looper/cli_pydantic.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index f25e6ebd5..cb7e9e90e 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -275,11 +275,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         if subcommand_name == "destroy":
             return Destroyer(prj)(subcommand_args)
 
-        use_pipestat = (
-            prj.pipestat_configured_project
-            if getattr(subcommand_args, "project", None)
-            else prj.pipestat_configured
-        )
+        # use_pipestat = (
+        #     prj.pipestat_configured_project
+        #     if getattr(subcommand_args, "project", None)
+        #     else prj.pipestat_configured
+        # )
+
+        use_pipestat = is_pipestat_configured
 
         if subcommand_name == "table":
             if use_pipestat:

From 396ea40f61b00ecd935dfc6cb0390ca70906abaa Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 11:45:10 -0400
Subject: [PATCH 194/225] ensure a pipestatmanager is created if a config file
 does already exist

---
 looper/conductor.py |  6 +++---
 looper/looper.py    |  6 ++----
 looper/project.py   | 13 ++++++++++---
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 84ee24963..813270a18 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -300,12 +300,12 @@ def add_sample(self, sample, rerun=False):
             )
         )
         if self.prj.pipestat_configured:
-            psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
-            sample_statuses = psms[self.pl_name].get_status(
+            # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
+            sample_statuses = self.pl_iface.psm.get_status(
                 record_identifier=sample.sample_name
             )
             if sample_statuses == "failed" and rerun is True:
-                psms[self.pl_name].set_status(
+                self.pl_iface.psm.set_status(
                     record_identifier=sample.sample_name, status_identifier="waiting"
                 )
                 sample_statuses = "waiting"
diff --git a/looper/looper.py b/looper/looper.py
index b044ef1d9..bb5508f93 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -423,10 +423,8 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
-        _LOGGER.info(f"Pipestat compatible: {self.prj.pipestat_configured_project}")
-        self.debug["Pipestat compatible"] = (
-            self.prj.pipestat_configured_project or self.prj.pipestat_configured
-        )
+        _LOGGER.info(f"Pipestat compatible: {self.prj.pipestat_configured}")
+        self.debug["Pipestat compatible"] = self.prj.pipestat_configured
 
         for sample in select_samples(prj=self.prj, args=args):
             pl_fails = []
diff --git a/looper/project.py b/looper/project.py
index 132d35bb0..3b4ba1224 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -340,7 +340,7 @@ def pipestat_configured(self):
 
         :return bool: whether pipestat configuration is complete
         """
-        return self._check_if_pipestat_configured()
+        return self._check_if_pipestat_configured_2()
 
     @cached_property
     def pipestat_configured_project(self):
@@ -491,8 +491,12 @@ def _get_pipestat_configuration2(self):
             print(piface)
             # first check if this piface has a psm?
 
-            if not self._check_for_existing_pipestat_config(piface):
+            pipestat_config_path = self._check_for_existing_pipestat_config(piface)
+
+            if not pipestat_config_path:
                 self._create_pipestat_config(piface)
+            else:
+                piface.psm = PipestatManager(config_file=pipestat_config_path)
 
         return True
 
@@ -522,7 +526,10 @@ def _check_for_existing_pipestat_config(self, piface):
             config_file_name,
         )
 
-        return os.path.exists(config_file_path)
+        if os.path.exists(config_file_path):
+            return config_file_path
+        else:
+            return None
 
     def _create_pipestat_config(self, piface):
         """

From 970f18be882ec30617c06844dba03851a6de5947 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 12:11:24 -0400
Subject: [PATCH 195/225] modify comprehensive tests to have correct paths

---
 tests/test_comprehensive.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 5026eb538..65566d578 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -66,6 +66,8 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
     pipestat_project_file = get_project_config_path(path_to_looper_config)
 
+    pipestat_pipeline_interface_file = os.path.join(pipestat_dir, "pipeline_pipestat/pipeline_interface.yaml")
+
     with open(pipestat_project_file, "r") as f:
         pipestat_project_data = safe_load(f)
 
@@ -73,6 +75,11 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
         os.path.join(pipestat_dir, "data/{sample_name}.txt")
     )
 
+    with open(pipestat_pipeline_interface_file, "r") as f:
+        pipestat_piface_data = safe_load(f)
+
+    pipeline_name =  pipestat_piface_data["pipeline_name"]
+
     with open(pipestat_project_file, "w") as f:
         dump(pipestat_project_data, f)
 
@@ -92,7 +99,8 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     # looper cannot create flags, the pipeline or pipestat does that
     # if you do not specify flag dir, pipestat places them in the same dir as config file
-    path_to_pipestat_config = os.path.join(pipestat_dir, "looper_pipestat_config.yaml")
+    path_to_pipestat_config = os.path.join(pipestat_dir, f"results/pipestat_config_{pipeline_name}.yaml")
+    # pipestat_config_example_pipestat_pipeline.yaml
     psm = PipestatManager(config_file=path_to_pipestat_config)
     psm.set_status(record_identifier="frog_1", status_identifier="completed")
     psm.set_status(record_identifier="frog_2", status_identifier="completed")

From 99c2e24c703dcb47b0cd762eb3659ce05689e5fe Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 13:36:22 -0400
Subject: [PATCH 196/225] refactor Checker code

---
 looper/looper.py            | 55 +++++++++++++++++++++++++------------
 tests/test_comprehensive.py | 10 +++++--
 2 files changed, 45 insertions(+), 20 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index bb5508f93..9788a45d0 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -88,21 +88,41 @@ def __call__(self, args):
 
         # aggregate pipeline status data
         status = {}
+
+        psms = {}
         if getattr(args, "project", None):
-            psms = self.prj.get_pipestat_managers(project_level=True)
-            for pipeline_name, psm in psms.items():
-                s = psm.get_status() or "unknown"
-                status.setdefault(pipeline_name, {})
-                status[pipeline_name][self.prj.name] = s
-                _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}")
+            # psms = self.prj.get_pipestat_managers(project_level=True)
+            # for pipeline_name, psm in psms.items():
+            #     s = psm.get_status() or "unknown"
+            #     status.setdefault(pipeline_name, {})
+            #     status[pipeline_name][self.prj.name] = s
+            #     _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}")
+
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "project":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    s = piface.psm.get_status() or "unknown"
+                    status.setdefault(piface.psm.pipeline_name, {})
+                    status[piface.psm.pipeline_name][self.prj.name] = s
+                    _LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}")
+
         else:
             for sample in self.prj.samples:
-                psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
-                for pipeline_name, psm in psms.items():
-                    s = psm.get_status(record_identifier=sample.sample_name)
-                    status.setdefault(pipeline_name, {})
-                    status[pipeline_name][sample.sample_name] = s
-                    _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}")
+                for piface in sample.project.pipeline_interfaces:
+                    if piface.psm.pipeline_type == "sample":
+                        psms[piface.psm.pipeline_name] = piface.psm
+                        s = piface.psm.get_status(record_identifier=sample.sample_name)
+                        status.setdefault(piface.psm.pipeline_name, {})
+                        status[piface.psm.pipeline_name][sample.sample_name] = s
+                        _LOGGER.debug(
+                            f"{sample.sample_name} ({piface.psm.pipeline_name}): {s}"
+                        )
+                # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
+                # for pipeline_name, psm in psms.items():
+                #     s = psm.get_status(record_identifier=sample.sample_name)
+                #     status.setdefault(pipeline_name, {})
+                #     status[pipeline_name][sample.sample_name] = s
+                #     _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}")
 
         console = Console()
 
@@ -116,7 +136,7 @@ def __call__(self, args):
             )
             table.add_column(f"Status", justify="center")
             table.add_column("Jobs count/total jobs", justify="center")
-            for status_id in psm.status_schema.keys():
+            for status_id in psms[pipeline_name].status_schema.keys():
                 status_list = list(pipeline_status.values())
                 if status_id in status_list:
                     status_count = status_list.count(status_id)
@@ -141,7 +161,7 @@ def __call__(self, args):
                 for name, status_id in pipeline_status.items():
                     try:
                         color = Color.from_rgb(
-                            *psm.status_schema[status_id]["color"]
+                            *psms[pipeline_name].status_schema[status_id]["color"]
                         ).name
                     except KeyError:
                         color = "#bcbcbc"
@@ -150,16 +170,17 @@ def __call__(self, args):
                 console.print(table)
 
         if args.describe_codes:
+            # TODO this needs to be redone because it only takes the last psm in the list and gets status code and descriptions
             table = Table(
                 show_header=True,
                 header_style="bold magenta",
                 title=f"Status codes description",
-                width=len(psm.status_schema_source) + 20,
-                caption=f"Descriptions source: {psm.status_schema_source}",
+                width=len(psms[pipeline_name].status_schema_source) + 20,
+                caption=f"Descriptions source: {psms[pipeline_name].status_schema_source}",
             )
             table.add_column("Status code", justify="center")
             table.add_column("Description", justify="left")
-            for status, status_obj in psm.status_schema.items():
+            for status, status_obj in psms[pipeline_name].status_schema.items():
                 if "description" in status_obj:
                     desc = status_obj["description"]
                 else:
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 65566d578..eb22fb608 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -66,7 +66,9 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     # open up the project config and replace the derived attributes with the path to the data. In a way, this simulates using the environment variables.
     pipestat_project_file = get_project_config_path(path_to_looper_config)
 
-    pipestat_pipeline_interface_file = os.path.join(pipestat_dir, "pipeline_pipestat/pipeline_interface.yaml")
+    pipestat_pipeline_interface_file = os.path.join(
+        pipestat_dir, "pipeline_pipestat/pipeline_interface.yaml"
+    )
 
     with open(pipestat_project_file, "r") as f:
         pipestat_project_data = safe_load(f)
@@ -78,7 +80,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     with open(pipestat_pipeline_interface_file, "r") as f:
         pipestat_piface_data = safe_load(f)
 
-    pipeline_name =  pipestat_piface_data["pipeline_name"]
+    pipeline_name = pipestat_piface_data["pipeline_name"]
 
     with open(pipestat_project_file, "w") as f:
         dump(pipestat_project_data, f)
@@ -99,7 +101,9 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     # looper cannot create flags, the pipeline or pipestat does that
     # if you do not specify flag dir, pipestat places them in the same dir as config file
-    path_to_pipestat_config = os.path.join(pipestat_dir, f"results/pipestat_config_{pipeline_name}.yaml")
+    path_to_pipestat_config = os.path.join(
+        pipestat_dir, f"results/pipestat_config_{pipeline_name}.yaml"
+    )
     # pipestat_config_example_pipestat_pipeline.yaml
     psm = PipestatManager(config_file=path_to_pipestat_config)
     psm.set_status(record_identifier="frog_1", status_identifier="completed")

From f54f85daaec6286e528e285b60638d4ff6ca8f37 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 14:02:42 -0400
Subject: [PATCH 197/225] refactor Reporter code

---
 looper/looper.py | 58 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 9788a45d0..734eb1c48 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -583,35 +583,53 @@ def __call__(self, args):
 
         portable = args.portable
 
+        psms = {}
+
         if project_level:
-            psms = self.prj.get_pipestat_managers(project_level=True)
-            print(psms)
-            for name, psm in psms.items():
-                # Summarize will generate the static HTML Report Function
-                report_directory = psm.summarize(
-                    looper_samples=self.prj.samples, portable=portable
-                )
+            # psms = self.prj.get_pipestat_managers(project_level=True)
+            # print(psms)
+            # for name, psm in psms.items():
+            #     # Summarize will generate the static HTML Report Function
+            #     report_directory = psm.summarize(
+            #         looper_samples=self.prj.samples, portable=portable
+            #     )
+
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "project":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    report_directory = piface.psm.summarize(
+                        looper_samples=self.prj.samples, portable=portable
+                    )
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
         else:
-            for piface_source_samples in self.prj._samples_by_piface(
-                self.prj.piface_key
-            ).values():
-                # For each piface_key, we have a list of samples, but we only need one sample from the list to
-                # call the related pipestat manager object which will pull ALL samples when using psm.summarize
-                first_sample_name = list(piface_source_samples)[0]
-                psms = self.prj.get_pipestat_managers(
-                    sample_name=first_sample_name, project_level=False
-                )
-                print(psms)
-                for name, psm in psms.items():
-                    # Summarize will generate the static HTML Report Function
-                    report_directory = psm.summarize(
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "sample":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    report_directory = piface.psm.summarize(
                         looper_samples=self.prj.samples, portable=portable
                     )
                     print(f"Report directory: {report_directory}")
                     self.debug["report_directory"] = report_directory
+
+            # for piface_source_samples in self.prj._samples_by_piface(
+            #     self.prj.piface_key
+            # ).values():
+            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
+            #     # call the related pipestat manager object which will pull ALL samples when using psm.summarize
+            #     first_sample_name = list(piface_source_samples)[0]
+            #     psms = self.prj.get_pipestat_managers(
+            #         sample_name=first_sample_name, project_level=False
+            #     )
+            #     print(psms)
+            #     for name, psm in psms.items():
+            #         # Summarize will generate the static HTML Report Function
+            #         report_directory = psm.summarize(
+            #             looper_samples=self.prj.samples, portable=portable
+            #         )
+            #         print(f"Report directory: {report_directory}")
+            #         self.debug["report_directory"] = report_directory
             return self.debug
 
 

From 946b408f319e2472e5c41dab6355b389862618aa Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 14:17:22 -0400
Subject: [PATCH 198/225] refactor Linker and Tabulator

---
 looper/looper.py | 79 ++++++++++++++++++++++++++++++------------------
 1 file changed, 50 insertions(+), 29 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 734eb1c48..fa8785ff2 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -642,23 +642,35 @@ def __call__(self, args):
         project_level = getattr(args, "project", None)
         link_dir = getattr(args, "output_dir", None)
 
+        psms = {}
+
         if project_level:
-            psms = self.prj.get_pipestat_managers(project_level=True)
-            for name, psm in psms.items():
-                linked_results_path = psm.link(link_dir=link_dir)
-                print(f"Linked directory: {linked_results_path}")
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "project":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    linked_results_path = piface.psm.link(link_dir=link_dir)
+                    print(f"Linked directory: {linked_results_path}")
+            # psms = self.prj.get_pipestat_managers(project_level=True)
+            # for name, psm in psms.items():
+            #     linked_results_path = psm.link(link_dir=link_dir)
+            #     print(f"Linked directory: {linked_results_path}")
         else:
-            for piface_source_samples in self.prj._samples_by_piface(
-                self.prj.piface_key
-            ).values():
-                # For each piface_key, we have a list of samples, but we only need one sample from the list to
-                # call the related pipestat manager object which will pull ALL samples when using psm.summarize
-                first_sample_name = list(piface_source_samples)[0]
-                psms = self.prj.get_pipestat_managers(
-                    sample_name=first_sample_name, project_level=False
-                )
-                for name, psm in psms.items():
-                    linked_results_path = psm.link(link_dir=link_dir)
+            # for piface_source_samples in self.prj._samples_by_piface(
+            #     self.prj.piface_key
+            # ).values():
+            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
+            #     # call the related pipestat manager object which will pull ALL samples when using psm.summarize
+            #     first_sample_name = list(piface_source_samples)[0]
+            #     psms = self.prj.get_pipestat_managers(
+            #         sample_name=first_sample_name, project_level=False
+            #     )
+            #     for name, psm in psms.items():
+            #         linked_results_path = psm.link(link_dir=link_dir)
+            #         print(f"Linked directory: {linked_results_path}")
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "sample":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    linked_results_path = piface.psm.link(link_dir=link_dir)
                     print(f"Linked directory: {linked_results_path}")
 
 
@@ -672,22 +684,31 @@ def __call__(self, args):
         # p = self.prj
         project_level = getattr(args, "project", None)
         results = []
+        psms = {}
         if project_level:
-            psms = self.prj.get_pipestat_managers(project_level=True)
-            for name, psm in psms.items():
-                results = psm.table()
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "project":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    results = piface.psm.table()
+            # psms = self.prj.get_pipestat_managers(project_level=True)
+            # for name, psm in psms.items():
+            #     results = psm.table()
         else:
-            for piface_source_samples in self.prj._samples_by_piface(
-                self.prj.piface_key
-            ).values():
-                # For each piface_key, we have a list of samples, but we only need one sample from the list to
-                # call the related pipestat manager object which will pull ALL samples when using psm.table
-                first_sample_name = list(piface_source_samples)[0]
-                psms = self.prj.get_pipestat_managers(
-                    sample_name=first_sample_name, project_level=False
-                )
-                for name, psm in psms.items():
-                    results = psm.table()
+            # for piface_source_samples in self.prj._samples_by_piface(
+            #     self.prj.piface_key
+            # ).values():
+            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
+            #     # call the related pipestat manager object which will pull ALL samples when using psm.table
+            #     first_sample_name = list(piface_source_samples)[0]
+            #     psms = self.prj.get_pipestat_managers(
+            #         sample_name=first_sample_name, project_level=False
+            #     )
+            #     for name, psm in psms.items():
+            #         results = psm.table()
+            for piface in self.prj.pipeline_interfaces:
+                if piface.psm.pipeline_type == "sample":
+                    psms[piface.psm.pipeline_name] = piface.psm
+                    results = piface.psm.table()
         # Results contains paths to stats and object summaries.
         return results
 

From 9989c9ff893e44a088a7d431633e7d278db697e3 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 14:24:52 -0400
Subject: [PATCH 199/225] refactor Destroyer

---
 looper/looper.py | 67 ++++++++++++++++++++++++++----------------------
 1 file changed, 37 insertions(+), 30 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index fa8785ff2..3145c90f6 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -263,7 +263,7 @@ def __call__(self, args, preview_flag=True):
         """
 
         use_pipestat = (
-            self.prj.pipestat_configured_project
+            self.prj.pipestat_configured
             if getattr(args, "project", None)
             else self.prj.pipestat_configured
         )
@@ -748,8 +748,12 @@ def destroy_summary(prj, dry_run=False, project_level=False):
     This function is for use with pipestat configured projects.
     """
 
+    psms = {}
     if project_level:
-        psms = prj.get_pipestat_managers(project_level=True)
+        for piface in prj.pipeline_interfaces:
+            if piface.psm.pipeline_type == "project":
+                psms[piface.psm.pipeline_name] = piface.psm
+
         for name, psm in psms.items():
             _remove_or_dry_run(
                 [
@@ -773,35 +777,38 @@ def destroy_summary(prj, dry_run=False, project_level=False):
                 dry_run,
             )
     else:
-        for piface_source_samples in prj._samples_by_piface(prj.piface_key).values():
-            # For each piface_key, we have a list of samples, but we only need one sample from the list to
-            # call the related pipestat manager object which will pull ALL samples when using psm.table
-            first_sample_name = list(piface_source_samples)[0]
-            psms = prj.get_pipestat_managers(
-                sample_name=first_sample_name, project_level=False
+        for piface in prj.pipeline_interfaces:
+            if piface.psm.pipeline_type == "sample":
+                psms[piface.psm.pipeline_name] = piface.psm
+        # for piface_source_samples in prj._samples_by_piface(prj.piface_key).values():
+        #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
+        #     # call the related pipestat manager object which will pull ALL samples when using psm.table
+        #     first_sample_name = list(piface_source_samples)[0]
+        #     psms = prj.get_pipestat_managers(
+        #         sample_name=first_sample_name, project_level=False
+        #     )
+        for name, psm in psms.items():
+            _remove_or_dry_run(
+                [
+                    get_file_for_table(
+                        psm, pipeline_name=psm.pipeline_name, directory="reports"
+                    ),
+                    get_file_for_table(
+                        psm,
+                        pipeline_name=psm.pipeline_name,
+                        appendix="stats_summary.tsv",
+                    ),
+                    get_file_for_table(
+                        psm,
+                        pipeline_name=psm.pipeline_name,
+                        appendix="objs_summary.yaml",
+                    ),
+                    os.path.join(
+                        os.path.dirname(psm.config_path), "aggregate_results.yaml"
+                    ),
+                ],
+                dry_run,
             )
-            for name, psm in psms.items():
-                _remove_or_dry_run(
-                    [
-                        get_file_for_table(
-                            psm, pipeline_name=psm.pipeline_name, directory="reports"
-                        ),
-                        get_file_for_table(
-                            psm,
-                            pipeline_name=psm.pipeline_name,
-                            appendix="stats_summary.tsv",
-                        ),
-                        get_file_for_table(
-                            psm,
-                            pipeline_name=psm.pipeline_name,
-                            appendix="objs_summary.yaml",
-                        ),
-                        os.path.join(
-                            os.path.dirname(psm.config_path), "aggregate_results.yaml"
-                        ),
-                    ],
-                    dry_run,
-                )
 
 
 class LooperCounter(object):

From 08e4fa0f74eadb36a783b22e5c9f6400ea6d8b52 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 23 May 2024 18:00:40 -0400
Subject: [PATCH 200/225] attempt refactor for obtaining project level pifaces
 and associated psms

---
 looper/conductor.py |  6 +++++-
 looper/looper.py    | 30 +++++++++++++++---------------
 looper/project.py   | 40 ++++++++++++++++++++++++++++------------
 3 files changed, 48 insertions(+), 28 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 813270a18..f54fe63db 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -273,8 +273,12 @@ def is_project_submittable(self, force=False):
 
         :param bool frorce: whether to force the project submission (ignore status/flags)
         """
+        psms = {}
         if self.prj.pipestat_configured_project:
-            psm = self.prj.get_pipestat_managers(project_level=True)[self.pl_name]
+            for piface in self.prj.project_pipeline_interfaces:
+                if piface.psm.pipeline_type == "project":
+                    psms[piface.psm.pipeline_name] = piface.psm
+            psm = psms[self.pl_name]
             status = psm.get_status()
             if not force and status is not None:
                 _LOGGER.info(f"> Skipping project. Determined status: {status}")
diff --git a/looper/looper.py b/looper/looper.py
index 3145c90f6..1cad8ff48 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -339,7 +339,7 @@ def __call__(self, args, **compute_kwargs):
         """
         jobs = 0
         self.debug = {}
-        project_pifaces = self.prj.project_pipeline_interface_sources
+        project_pifaces = self.prj.project_pipeline_interfaces
         if not project_pifaces:
             raise MisconfigurationException(
                 "Looper requires a pointer to at least one project pipeline. "
@@ -349,27 +349,27 @@ def __call__(self, args, **compute_kwargs):
             )
         self.counter = LooperCounter(len(project_pifaces))
         for project_piface in project_pifaces:
-            try:
-                project_piface_object = PipelineInterface(
-                    project_piface, pipeline_type="project"
-                )
-            except (IOError, ValidationError) as e:
-                _LOGGER.warning(
-                    "Ignoring invalid pipeline interface source: {}. "
-                    "Caught exception: {}".format(
-                        project_piface, getattr(e, "message", repr(e))
-                    )
-                )
-                continue
+            # try:
+            #     project_piface_object = PipelineInterface(
+            #         project_piface, pipeline_type="project"
+            #     )
+            # except (IOError, ValidationError) as e:
+            #     _LOGGER.warning(
+            #         "Ignoring invalid pipeline interface source: {}. "
+            #         "Caught exception: {}".format(
+            #             project_piface, getattr(e, "message", repr(e))
+            #         )
+            #     )
+            #     continue
             _LOGGER.info(
                 self.counter.show(
                     name=self.prj.name,
                     type="project",
-                    pipeline_name=project_piface_object.pipeline_name,
+                    pipeline_name=project_piface.pipeline_name,
                 )
             )
             conductor = SubmissionConductor(
-                pipeline_interface=project_piface_object,
+                pipeline_interface=project_piface,
                 prj=self.prj,
                 compute_variables=compute_kwargs,
                 delay=getattr(args, "time_delay", None),
diff --git a/looper/project.py b/looper/project.py
index 3b4ba1224..ff89f5b2f 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -349,7 +349,7 @@ def pipestat_configured_project(self):
 
         :return bool: whether pipestat configuration is complete
         """
-        return self._check_if_pipestat_configured(project_level=True)
+        return self._check_if_pipestat_configured_2(pipeline_type="project")
 
     def get_sample_piface(self, sample_name):
         """
@@ -468,7 +468,7 @@ def get_pipestat_managers(self, sample_name=None, project_level=False):
             for pipeline_name, pipestat_vars in pipestat_configs.items()
         }
 
-    def _check_if_pipestat_configured_2(self):
+    def _check_if_pipestat_configured_2(self, pipeline_type="sample"):
 
         # First check if pipestat key is in looper_config, if not return false
 
@@ -480,23 +480,39 @@ def _check_if_pipestat_configured_2(self):
             else:
                 # If pipestat key is available assume user desires pipestat usage
                 # This should return True OR raise an exception at this point.
-                return self._get_pipestat_configuration2()
+                return self._get_pipestat_configuration2(pipeline_type)
 
-    def _get_pipestat_configuration2(self):
+    def _get_pipestat_configuration2(self, pipeline_type="sample"):
 
         # First check if it already exists
         print("DEBUG!")
 
-        for piface in self.pipeline_interfaces:
-            print(piface)
-            # first check if this piface has a psm?
+        if pipeline_type == "sample":
+            for piface in self.pipeline_interfaces:
+                print(piface)
+                # first check if this piface has a psm?
 
-            pipestat_config_path = self._check_for_existing_pipestat_config(piface)
+                pipestat_config_path = self._check_for_existing_pipestat_config(piface)
 
-            if not pipestat_config_path:
-                self._create_pipestat_config(piface)
-            else:
-                piface.psm = PipestatManager(config_file=pipestat_config_path)
+                if not pipestat_config_path:
+                    self._create_pipestat_config(piface)
+                else:
+                    piface.psm = PipestatManager(config_file=pipestat_config_path)
+
+        elif pipeline_type == "project":
+            for prj_piface in self.project_pipeline_interfaces:
+                pipestat_config_path = self._check_for_existing_pipestat_config(
+                    prj_piface
+                )
+
+                if not pipestat_config_path:
+                    self._create_pipestat_config(prj_piface)
+                else:
+                    prj_piface.psm = PipestatManager(config_file=pipestat_config_path)
+        else:
+            _LOGGER.error(
+                msg="No pipeline type specified during pipestat configuration"
+            )
 
         return True
 

From 25d5d6af48d1f84406ed437a2feb554449789ef2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 24 May 2024 14:49:54 -0400
Subject: [PATCH 201/225] bump pipestat req to 0.9.2a1

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 108303465..ab0eff0f2 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
 peppy>=0.40.0
-pipestat>=0.8.3a1
+pipestat>=0.9.2a1
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2

From e1b3aefb05d29b08683bc7628e162c49a5620f33 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 24 May 2024 14:59:20 -0400
Subject: [PATCH 202/225] set multi_pipelines to True so that sample and
 project pipelines can be reported to the same results.yaml file.

---
 looper/project.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/looper/project.py b/looper/project.py
index ff89f5b2f..6cd3ceee4 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -497,7 +497,9 @@ def _get_pipestat_configuration2(self, pipeline_type="sample"):
                 if not pipestat_config_path:
                     self._create_pipestat_config(piface)
                 else:
-                    piface.psm = PipestatManager(config_file=pipestat_config_path)
+                    piface.psm = PipestatManager(
+                        config_file=pipestat_config_path, multi_pipelines=True
+                    )
 
         elif pipeline_type == "project":
             for prj_piface in self.project_pipeline_interfaces:
@@ -508,7 +510,9 @@ def _get_pipestat_configuration2(self, pipeline_type="sample"):
                 if not pipestat_config_path:
                     self._create_pipestat_config(prj_piface)
                 else:
-                    prj_piface.psm = PipestatManager(config_file=pipestat_config_path)
+                    prj_piface.psm = PipestatManager(
+                        config_file=pipestat_config_path, multi_pipelines=True
+                    )
         else:
             _LOGGER.error(
                 msg="No pipeline type specified during pipestat configuration"
@@ -612,7 +616,9 @@ def _create_pipestat_config(self, piface):
         write_pipestat_config(pipestat_config_path, pipestat_config_dict)
 
         # piface['psm'] = PipestatManager(config_file=pipestat_config_path)
-        piface.psm = PipestatManager(config_file=pipestat_config_path)
+        piface.psm = PipestatManager(
+            config_file=pipestat_config_path, multi_pipelines=True
+        )
 
         return None
 

From f63e1ca8eac81b3531fc9783606534a2599f92fc Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Fri, 24 May 2024 15:10:28 -0400
Subject: [PATCH 203/225] Add warning for mismatched pipeline names

---
 looper/project.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/looper/project.py b/looper/project.py
index 6cd3ceee4..ba0881a14 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -3,6 +3,8 @@
 import itertools
 import os
 
+from yaml import safe_load
+
 try:
     from functools import cached_property
 except ImportError:
@@ -579,12 +581,30 @@ def _create_pipestat_config(self, piface):
                     os.path.dirname(piface.pipe_iface_file), schema_path
                 )
             pipestat_config_dict.update({"schema_path": schema_path})
+            try:
+                with open(schema_path, "r") as f:
+                    output_schema_data = safe_load(f)
+                    output_schema_pipeline_name = output_schema_data[
+                        PIPELINE_INTERFACE_PIPELINE_NAME_KEY
+                    ]
+            except Exception:
+                output_schema_pipeline_name = None
+        else:
+            output_schema_pipeline_name = None
         if "pipeline_name" in piface.data:
             pipeline_name = piface.data["pipeline_name"]
             pipestat_config_dict.update({"pipeline_name": piface.data["pipeline_name"]})
+        else:
+            pipeline_name = None
         if "pipeline_type" in piface.data:
             pipestat_config_dict.update({"pipeline_type": piface.data["pipeline_type"]})
 
+        # Warn user if there is a mismatch in pipeline_names from sources!!!
+        if pipeline_name != output_schema_pipeline_name:
+            _LOGGER.warning(
+                msg=f"Pipeline name mismatch detected. Pipeline interface: {pipeline_name}  Output schema: {output_schema_pipeline_name}  Defaulting to pipeline_interface value."
+            )
+
         try:
             # TODO if user gives non-absolute path should we force results to be in a pipeline folder?
             # TODO otherwise pipelines could write to the same results file!

From 318b8fe9c9365f14e4f85da53f1da5690b583dc4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 28 May 2024 12:56:05 -0400
Subject: [PATCH 204/225] bump peppy req to 0.40.2 for issue #458

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index 108303465..5f0c846d3 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -5,7 +5,7 @@ jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
-peppy>=0.40.0
+peppy>=0.40.2
 pipestat>=0.8.3a1
 pyyaml>=3.12
 rich>=9.10.0

From a0e1167d21c2b78320d884b54b6ffc404fa198f2 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 28 May 2024 12:57:28 -0400
Subject: [PATCH 205/225] update changelog #458

---
 docs/changelog.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog.md b/docs/changelog.md
index d9c190384..9b0291898 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -22,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - looper rerun now only runs failed jobs as intended: https://github.com/pepkit/looper/issues/467 
 - looper inspect now inspects the looper config: https://github.com/pepkit/looper/issues/462
 - Load PEP from CSV: https://github.com/pepkit/looper/issues/456
+- looper now works with sample_table_index https://github.com/pepkit/looper/issues/458
 
 
 ## [1.7.0] -- 2024-01-26

From e95b62a833e0737f7da37c846a245b4d06aa0ab3 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 09:51:43 -0400
Subject: [PATCH 206/225] refactor _set_pipestat_namespace

---
 looper/conductor.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index f54fe63db..378e2c9a1 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -548,12 +548,7 @@ def _set_pipestat_namespace(
         :return yacman.YAMLConfigManager: pipestat namespace
         """
         try:
-            psms = (
-                self.prj.get_pipestat_managers(sample_name)
-                if sample_name
-                else self.prj.get_pipestat_managers(project_level=True)
-            )
-            psm = psms[self.pl_iface.pipeline_name]
+            psm = self.pl_iface.psm
         except (PipestatError, AttributeError) as e:
             # pipestat section faulty or not found in project.looper or sample
             # or project is missing required pipestat attributes

From 2e1d1368ce776225c1716c094f2b8d215a655392 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:19:19 -0400
Subject: [PATCH 207/225] remove unused code, finish Destroyer refactor

---
 looper/looper.py  |   8 +--
 looper/project.py | 151 ----------------------------------------------
 2 files changed, 4 insertions(+), 155 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 1cad8ff48..0827a0507 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -277,7 +277,7 @@ def __call__(self, args, preview_flag=True):
             )
 
         _LOGGER.info("Removing results:")
-
+        psms = {}
         for sample in select_samples(prj=self.prj, args=args):
             _LOGGER.info(self.counter.show(sample.sample_name))
             sample_output_folder = sample_folder(self.prj, sample)
@@ -286,9 +286,9 @@ def __call__(self, args, preview_flag=True):
                 _LOGGER.info(str(sample_output_folder))
             else:
                 if use_pipestat:
-                    psms = self.prj.get_pipestat_managers(
-                        sample_name=sample.sample_name
-                    )
+                    for piface in sample.project.pipeline_interfaces:
+                        if piface.psm.pipeline_type == "sample":
+                            psms[piface.psm.pipeline_name] = piface.psm
                     for pipeline_name, psm in psms.items():
                         psm.backend.remove_record(
                             record_identifier=sample.sample_name, rm_record=True
diff --git a/looper/project.py b/looper/project.py
index ba0881a14..0155e138a 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -449,27 +449,6 @@ def get_schemas(pifaces, schema_key=INPUT_SCHEMA_KEY):
                 schema_set.update([schema_file])
         return list(schema_set)
 
-    def get_pipestat_managers(self, sample_name=None, project_level=False):
-        """
-        Get a collection of pipestat managers for the selected sample or project.
-
-        The number of pipestat managers corresponds to the number of unique
-        output schemas in the pipeline interfaces specified by the sample or project.
-
-        :param str sample_name: sample name to get pipestat managers for
-        :param bool project_level: whether the project PipestatManagers
-            should be returned
-        :return dict[str, pipestat.PipestatManager]: a mapping of pipestat
-            managers by pipeline interface name
-        """
-        pipestat_configs = self._get_pipestat_configuration(
-            sample_name=sample_name, project_level=project_level
-        )
-        return {
-            pipeline_name: PipestatManager(**pipestat_vars)
-            for pipeline_name, pipestat_vars in pipestat_configs.items()
-        }
-
     def _check_if_pipestat_configured_2(self, pipeline_type="sample"):
 
         # First check if pipestat key is in looper_config, if not return false
@@ -642,136 +621,6 @@ def _create_pipestat_config(self, piface):
 
         return None
 
-    def _check_if_pipestat_configured(self, project_level=False):
-        """
-        A helper method determining whether pipestat configuration is complete
-
-        :param bool project_level: whether the project pipestat config should be checked
-        :return bool: whether pipestat configuration is complete
-        """
-        try:
-            if project_level:
-                pipestat_configured = self._get_pipestat_configuration(
-                    sample_name=None, project_level=project_level
-                )
-            else:
-                for s in self.samples:
-                    pipestat_configured = self._get_pipestat_configuration(
-                        sample_name=s.sample_name
-                    )
-        except Exception as e:
-            context = (
-                f"Project '{self.name}'"
-                if project_level
-                else f"Sample '{s.sample_name}'"
-            )
-            _LOGGER.debug(
-                f"Pipestat configuration incomplete for {context}; "
-                f"caught exception: {getattr(e, 'message', repr(e))}"
-            )
-            return False
-        else:
-            if pipestat_configured is not None and pipestat_configured != {}:
-                return True
-            else:
-                return False
-
-    def _get_pipestat_configuration(self, sample_name=None, project_level=False):
-        """
-        Get all required pipestat configuration variables from looper_config file
-        """
-
-        ret = {}
-        if not project_level and sample_name is None:
-            raise ValueError(
-                "Must provide the sample_name to determine the "
-                "sample to get the PipestatManagers for"
-            )
-
-        if PIPESTAT_KEY in self[EXTRA_KEY]:
-            pipestat_config_dict = self[EXTRA_KEY][PIPESTAT_KEY]
-        else:
-            _LOGGER.debug(
-                f"'{PIPESTAT_KEY}' not found in '{LOOPER_KEY}' section of the "
-                f"project configuration file."
-            )
-            # We cannot use pipestat without it being defined in the looper config file.
-            raise ValueError
-
-        # Expand paths in the event ENV variables were used in config files
-        output_dir = expandpath(self.output_dir)
-
-        # Get looper user configured items first and update the pipestat_config_dict
-        try:
-            results_file_path = expandpath(pipestat_config_dict["results_file_path"])
-            if not os.path.exists(os.path.dirname(results_file_path)):
-                results_file_path = os.path.join(
-                    os.path.dirname(output_dir), results_file_path
-                )
-            pipestat_config_dict.update({"results_file_path": results_file_path})
-        except KeyError:
-            results_file_path = None
-
-        try:
-            flag_file_dir = expandpath(pipestat_config_dict["flag_file_dir"])
-            if not os.path.isabs(flag_file_dir):
-                flag_file_dir = os.path.join(os.path.dirname(output_dir), flag_file_dir)
-            pipestat_config_dict.update({"flag_file_dir": flag_file_dir})
-        except KeyError:
-            flag_file_dir = None
-
-        if sample_name:
-            pipestat_config_dict.update({"record_identifier": sample_name})
-
-        if project_level and "project_name" in pipestat_config_dict:
-            pipestat_config_dict.update(
-                {"project_name": pipestat_config_dict["project_name"]}
-            )
-
-        if project_level and "{record_identifier}" in results_file_path:
-            # if project level and using {record_identifier}, pipestat needs some sort of record_identifier during creation
-            pipestat_config_dict.update(
-                {"record_identifier": "default_project_record_identifier"}
-            )
-
-        pipestat_config_dict.update({"output_dir": output_dir})
-
-        pifaces = (
-            self.project_pipeline_interfaces
-            if project_level
-            else self._interfaces_by_sample[sample_name]
-        )
-
-        for piface in pifaces:
-            # We must also obtain additional pipestat items from the pipeline author's piface
-            if "output_schema" in piface.data:
-                schema_path = expandpath(piface.data["output_schema"])
-                if not os.path.isabs(schema_path):
-                    # Get path relative to the pipeline_interface
-                    schema_path = os.path.join(
-                        os.path.dirname(piface.pipe_iface_file), schema_path
-                    )
-                pipestat_config_dict.update({"schema_path": schema_path})
-            if "pipeline_name" in piface.data:
-                pipestat_config_dict.update(
-                    {"pipeline_name": piface.data["pipeline_name"]}
-                )
-            if "pipeline_type" in piface.data:
-                pipestat_config_dict.update(
-                    {"pipeline_type": piface.data["pipeline_type"]}
-                )
-
-            # Pipestat_dict_ is now updated from all sources and can be written to a yaml.
-            looper_pipestat_config_path = os.path.join(
-                os.path.dirname(output_dir), "looper_pipestat_config.yaml"
-            )
-            write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict)
-
-            ret[piface.pipeline_name] = {
-                "config_file": looper_pipestat_config_path,
-            }
-        return ret
-
     def populate_pipeline_outputs(self):
         """
         Populate project and sample output attributes based on output schemas

From 55eb51fd21253147c68197d403c2c8cefa23adbe Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:24:16 -0400
Subject: [PATCH 208/225] refactor func names

---
 looper/cli_pydantic.py |  2 +-
 looper/project.py      | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index cb7e9e90e..5d2410277 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -243,7 +243,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     ) as prj:
 
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
-        is_pipestat_configured = prj._check_if_pipestat_configured_2()
+        is_pipestat_configured = prj._check_if_pipestat_configured()
 
         if subcommand_name in ["run", "rerun"]:
             rerun = subcommand_name == "rerun"
diff --git a/looper/project.py b/looper/project.py
index 0155e138a..0969572f8 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -342,7 +342,7 @@ def pipestat_configured(self):
 
         :return bool: whether pipestat configuration is complete
         """
-        return self._check_if_pipestat_configured_2()
+        return self._check_if_pipestat_configured()
 
     @cached_property
     def pipestat_configured_project(self):
@@ -351,7 +351,7 @@ def pipestat_configured_project(self):
 
         :return bool: whether pipestat configuration is complete
         """
-        return self._check_if_pipestat_configured_2(pipeline_type="project")
+        return self._check_if_pipestat_configured(pipeline_type="project")
 
     def get_sample_piface(self, sample_name):
         """
@@ -449,7 +449,7 @@ def get_schemas(pifaces, schema_key=INPUT_SCHEMA_KEY):
                 schema_set.update([schema_file])
         return list(schema_set)
 
-    def _check_if_pipestat_configured_2(self, pipeline_type="sample"):
+    def _check_if_pipestat_configured(self, pipeline_type="sample"):
 
         # First check if pipestat key is in looper_config, if not return false
 
@@ -461,12 +461,11 @@ def _check_if_pipestat_configured_2(self, pipeline_type="sample"):
             else:
                 # If pipestat key is available assume user desires pipestat usage
                 # This should return True OR raise an exception at this point.
-                return self._get_pipestat_configuration2(pipeline_type)
+                return self._get_pipestat_configuration(pipeline_type)
 
-    def _get_pipestat_configuration2(self, pipeline_type="sample"):
+    def _get_pipestat_configuration(self, pipeline_type="sample"):
 
         # First check if it already exists
-        print("DEBUG!")
 
         if pipeline_type == "sample":
             for piface in self.pipeline_interfaces:

From 60d807c85447cb57969a541a7d354845297570e5 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:26:54 -0400
Subject: [PATCH 209/225] remove unnecessary variable duplication

---
 looper/cli_pydantic.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 5d2410277..6f60d8b8b 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -275,34 +275,26 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         if subcommand_name == "destroy":
             return Destroyer(prj)(subcommand_args)
 
-        # use_pipestat = (
-        #     prj.pipestat_configured_project
-        #     if getattr(subcommand_args, "project", None)
-        #     else prj.pipestat_configured
-        # )
-
-        use_pipestat = is_pipestat_configured
-
         if subcommand_name == "table":
-            if use_pipestat:
+            if is_pipestat_configured:
                 return Tabulator(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("table")
 
         if subcommand_name == "report":
-            if use_pipestat:
+            if is_pipestat_configured:
                 return Reporter(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("report")
 
         if subcommand_name == "link":
-            if use_pipestat:
+            if is_pipestat_configured:
                 Linker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("link")
 
         if subcommand_name == "check":
-            if use_pipestat:
+            if is_pipestat_configured:
                 return Checker(prj)(subcommand_args)
             else:
                 raise PipestatConfigurationException("check")

From f476e68c9b3b589350dbc3929393f8b891e0fab4 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:32:10 -0400
Subject: [PATCH 210/225] remove commented code

---
 looper/looper.py | 86 ------------------------------------------------
 1 file changed, 86 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index 0827a0507..a0aea285a 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -91,12 +91,6 @@ def __call__(self, args):
 
         psms = {}
         if getattr(args, "project", None):
-            # psms = self.prj.get_pipestat_managers(project_level=True)
-            # for pipeline_name, psm in psms.items():
-            #     s = psm.get_status() or "unknown"
-            #     status.setdefault(pipeline_name, {})
-            #     status[pipeline_name][self.prj.name] = s
-            #     _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}")
 
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == "project":
@@ -117,12 +111,6 @@ def __call__(self, args):
                         _LOGGER.debug(
                             f"{sample.sample_name} ({piface.psm.pipeline_name}): {s}"
                         )
-                # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
-                # for pipeline_name, psm in psms.items():
-                #     s = psm.get_status(record_identifier=sample.sample_name)
-                #     status.setdefault(pipeline_name, {})
-                #     status[pipeline_name][sample.sample_name] = s
-                #     _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}")
 
         console = Console()
 
@@ -349,18 +337,6 @@ def __call__(self, args, **compute_kwargs):
             )
         self.counter = LooperCounter(len(project_pifaces))
         for project_piface in project_pifaces:
-            # try:
-            #     project_piface_object = PipelineInterface(
-            #         project_piface, pipeline_type="project"
-            #     )
-            # except (IOError, ValidationError) as e:
-            #     _LOGGER.warning(
-            #         "Ignoring invalid pipeline interface source: {}. "
-            #         "Caught exception: {}".format(
-            #             project_piface, getattr(e, "message", repr(e))
-            #         )
-            #     )
-            #     continue
             _LOGGER.info(
                 self.counter.show(
                     name=self.prj.name,
@@ -586,13 +562,6 @@ def __call__(self, args):
         psms = {}
 
         if project_level:
-            # psms = self.prj.get_pipestat_managers(project_level=True)
-            # print(psms)
-            # for name, psm in psms.items():
-            #     # Summarize will generate the static HTML Report Function
-            #     report_directory = psm.summarize(
-            #         looper_samples=self.prj.samples, portable=portable
-            #     )
 
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == "project":
@@ -612,24 +581,6 @@ def __call__(self, args):
                     )
                     print(f"Report directory: {report_directory}")
                     self.debug["report_directory"] = report_directory
-
-            # for piface_source_samples in self.prj._samples_by_piface(
-            #     self.prj.piface_key
-            # ).values():
-            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
-            #     # call the related pipestat manager object which will pull ALL samples when using psm.summarize
-            #     first_sample_name = list(piface_source_samples)[0]
-            #     psms = self.prj.get_pipestat_managers(
-            #         sample_name=first_sample_name, project_level=False
-            #     )
-            #     print(psms)
-            #     for name, psm in psms.items():
-            #         # Summarize will generate the static HTML Report Function
-            #         report_directory = psm.summarize(
-            #             looper_samples=self.prj.samples, portable=portable
-            #         )
-            #         print(f"Report directory: {report_directory}")
-            #         self.debug["report_directory"] = report_directory
             return self.debug
 
 
@@ -650,23 +601,7 @@ def __call__(self, args):
                     psms[piface.psm.pipeline_name] = piface.psm
                     linked_results_path = piface.psm.link(link_dir=link_dir)
                     print(f"Linked directory: {linked_results_path}")
-            # psms = self.prj.get_pipestat_managers(project_level=True)
-            # for name, psm in psms.items():
-            #     linked_results_path = psm.link(link_dir=link_dir)
-            #     print(f"Linked directory: {linked_results_path}")
         else:
-            # for piface_source_samples in self.prj._samples_by_piface(
-            #     self.prj.piface_key
-            # ).values():
-            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
-            #     # call the related pipestat manager object which will pull ALL samples when using psm.summarize
-            #     first_sample_name = list(piface_source_samples)[0]
-            #     psms = self.prj.get_pipestat_managers(
-            #         sample_name=first_sample_name, project_level=False
-            #     )
-            #     for name, psm in psms.items():
-            #         linked_results_path = psm.link(link_dir=link_dir)
-            #         print(f"Linked directory: {linked_results_path}")
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == "sample":
                     psms[piface.psm.pipeline_name] = piface.psm
@@ -690,21 +625,7 @@ def __call__(self, args):
                 if piface.psm.pipeline_type == "project":
                     psms[piface.psm.pipeline_name] = piface.psm
                     results = piface.psm.table()
-            # psms = self.prj.get_pipestat_managers(project_level=True)
-            # for name, psm in psms.items():
-            #     results = psm.table()
         else:
-            # for piface_source_samples in self.prj._samples_by_piface(
-            #     self.prj.piface_key
-            # ).values():
-            #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
-            #     # call the related pipestat manager object which will pull ALL samples when using psm.table
-            #     first_sample_name = list(piface_source_samples)[0]
-            #     psms = self.prj.get_pipestat_managers(
-            #         sample_name=first_sample_name, project_level=False
-            #     )
-            #     for name, psm in psms.items():
-            #         results = psm.table()
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == "sample":
                     psms[piface.psm.pipeline_name] = piface.psm
@@ -780,13 +701,6 @@ def destroy_summary(prj, dry_run=False, project_level=False):
         for piface in prj.pipeline_interfaces:
             if piface.psm.pipeline_type == "sample":
                 psms[piface.psm.pipeline_name] = piface.psm
-        # for piface_source_samples in prj._samples_by_piface(prj.piface_key).values():
-        #     # For each piface_key, we have a list of samples, but we only need one sample from the list to
-        #     # call the related pipestat manager object which will pull ALL samples when using psm.table
-        #     first_sample_name = list(piface_source_samples)[0]
-        #     psms = prj.get_pipestat_managers(
-        #         sample_name=first_sample_name, project_level=False
-        #     )
         for name, psm in psms.items():
             _remove_or_dry_run(
                 [

From 64c81f15a330eda8b787fd75299a6fa28acfaa8d Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:44:27 -0400
Subject: [PATCH 211/225] add "--project" argument back to Looper

---
 looper/command_models/arguments.py | 9 +++++++--
 looper/command_models/commands.py  | 1 +
 tests/test_comprehensive.py        | 9 +++++++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py
index 75855511c..8c484d33d 100644
--- a/looper/command_models/arguments.py
+++ b/looper/command_models/arguments.py
@@ -192,13 +192,13 @@ class ArgumentEnum(enum.Enum):
         name="sample_pipeline_interfaces",
         alias="-S",
         default=(List, []),
-        description="Paths to looper sample config files",
+        description="Paths to looper sample pipeline interfaces",
     )
     PROJECT_PIPELINE_INTERFACES = Argument(
         name="project_pipeline_interfaces",
         alias="-P",
         default=(List, []),
-        description="Paths to looper project config files",
+        description="Paths to looper project pipeline interfaces",
     )
     AMEND = Argument(
         name="amend", default=(List, []), description="List of amendments to activate"
@@ -276,3 +276,8 @@ class ArgumentEnum(enum.Enum):
         default=(bool, False),
         description="Makes html report portable.",
     )
+    PROJECT_LEVEL = Argument(
+        name="project",
+        default=(bool, False),
+        description="Is this command executed for project-level?",
+    )
diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py
index e9025d909..233cfd0b7 100644
--- a/looper/command_models/commands.py
+++ b/looper/command_models/commands.py
@@ -60,6 +60,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     ArgumentEnum.PIPESTAT.value,
     ArgumentEnum.SETTINGS.value,
     ArgumentEnum.AMEND.value,
+    ArgumentEnum.PROJECT_LEVEL.value,
 ]
 
 RunParser = Command(
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index eb22fb608..472df3e24 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -118,6 +118,15 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 
+    # Now use looper check to get project level statuses
+    x = ["check", "--looper-config", path_to_looper_config, "--project"]
+
+    try:
+        result = main(test_args=x)
+        assert result == {}
+    except Exception:
+        raise pytest.fail("DID RAISE {0}".format(Exception))
+
     # TEST LOOPER REPORT
 
     x = ["report", "--looper-config", path_to_looper_config]

From ba7f03ee9dbd1df7e56c253832427a339677ad8b Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 11:49:32 -0400
Subject: [PATCH 212/225] refactor obtaining results_file_path during pipestat
 configuration creation

---
 looper/project.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/looper/project.py b/looper/project.py
index 0969572f8..5c952c72e 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -584,13 +584,23 @@ def _create_pipestat_config(self, piface):
             )
 
         try:
-            # TODO if user gives non-absolute path should we force results to be in a pipeline folder?
-            # TODO otherwise pipelines could write to the same results file!
             results_file_path = expandpath(pipestat_config_dict["results_file_path"])
-            if not os.path.exists(os.path.dirname(results_file_path)):
-                results_file_path = os.path.join(
-                    os.path.dirname(output_dir), results_file_path
-                )
+
+            if not os.path.isabs(results_file_path):
+                # e.g. user configures "results.yaml" as results_file_path
+                if "{record_identifier}" in results_file_path:
+                    # this is specifically to check if the user wishes tro generate a file for EACH record
+                    if not os.path.exists(os.path.dirname(results_file_path)):
+                        results_file_path = os.path.join(output_dir, results_file_path)
+                else:
+                    if not os.path.exists(os.path.dirname(results_file_path)):
+                        results_file_path = os.path.join(
+                            output_dir, f"{pipeline_name}/", results_file_path
+                        )
+            else:
+                # Do nothing because the user has given an absolute file path
+                pass
+
             pipestat_config_dict.update({"results_file_path": results_file_path})
         except KeyError:
             results_file_path = None

From 19d3130c6480c301b2359e2c62e57be4613083d6 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 11:55:31 -0400
Subject: [PATCH 213/225] remove commented code

---
 looper/conductor.py          |  1 -
 looper/plugins.py            |  9 ---------
 tests/smoketests/test_run.py | 29 -----------------------------
 tests/test_comprehensive.py  |  2 +-
 4 files changed, 1 insertion(+), 40 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 378e2c9a1..3f04f1450 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -304,7 +304,6 @@ def add_sample(self, sample, rerun=False):
             )
         )
         if self.prj.pipestat_configured:
-            # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
             sample_statuses = self.pl_iface.psm.get_status(
                 record_identifier=sample.sample_name
             )
diff --git a/looper/plugins.py b/looper/plugins.py
index b0c4a246c..dc34283e0 100644
--- a/looper/plugins.py
+++ b/looper/plugins.py
@@ -158,12 +158,3 @@ def write_sample_yaml(namespaces):
     )
     sample.to_yaml(sample["sample_yaml_path"], add_prj_ref=False)
     return {"sample": sample}
-
-
-# def write_local_pipestat_config(namespaces):
-#
-#     config_path = ""
-#
-#     print(config_path)
-#
-#     return config_path
diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py
index a94fa9b8a..05231f594 100644
--- a/tests/smoketests/test_run.py
+++ b/tests/smoketests/test_run.py
@@ -443,35 +443,6 @@ def test_looper_command_templates_hooks(self, prep_temp_pep, cmd):
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, "test.txt", 3)
 
-    # @pytest.mark.parametrize(
-    #     "plugin,appendix",
-    #     [
-    #         ("looper.write_local_pipestat_config", "submission.yaml"),
-    #     ],
-    # )
-    # def test_looper_pipestat_plugins(self, prep_temp_pep_pipestat, plugin, appendix):
-    #     # tp = prep_temp_pep
-    #     tp = prep_temp_pep_pipestat
-    #     pep_dir = os.path.dirname(tp)
-    #     pipeline_interface1 = os.path.join(
-    #         pep_dir, "pipeline_pipestat/pipeline_interface.yaml"
-    #     )
-    #
-    #     with mod_yaml_data(pipeline_interface1) as piface_data:
-    #         piface_data.update({PRE_SUBMIT_HOOK_KEY: {}})
-    #         piface_data[PRE_SUBMIT_HOOK_KEY].update({PRE_SUBMIT_PY_FUN_KEY: {}})
-    #         piface_data[PRE_SUBMIT_HOOK_KEY][PRE_SUBMIT_PY_FUN_KEY] = [plugin]
-    #
-    #     # x = test_args_expansion(tp, "run")
-    #     x = ["run", "--looper-config", tp, "--dry-run"]
-    #     # x.pop(-1)
-    #     try:
-    #         main(test_args=x)
-    #     except Exception as err:
-    #         raise pytest.fail(f"DID RAISE {err}")
-    #     sd = os.path.join(get_outdir(tp), "submission")
-    #     verify_filecount_in_dir(sd, appendix, 3)
-
 
 class TestLooperRunSubmissionScript:
     def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index 472df3e24..c0f0f81fb 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -104,7 +104,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
     path_to_pipestat_config = os.path.join(
         pipestat_dir, f"results/pipestat_config_{pipeline_name}.yaml"
     )
-    # pipestat_config_example_pipestat_pipeline.yaml
+
     psm = PipestatManager(config_file=path_to_pipestat_config)
     psm.set_status(record_identifier="frog_1", status_identifier="completed")
     psm.set_status(record_identifier="frog_2", status_identifier="completed")

From 5c9ea759b63849377cc32736d407840dc90cae75 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 12:06:27 -0400
Subject: [PATCH 214/225] remove print statements, change info to debug

---
 looper/looper.py  | 2 +-
 looper/project.py | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/looper/looper.py b/looper/looper.py
index a0aea285a..934793aea 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -420,7 +420,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
-        _LOGGER.info(f"Pipestat compatible: {self.prj.pipestat_configured}")
+        _LOGGER.debug(f"Pipestat compatible: {self.prj.pipestat_configured}")
         self.debug["Pipestat compatible"] = self.prj.pipestat_configured
 
         for sample in select_samples(prj=self.prj, args=args):
diff --git a/looper/project.py b/looper/project.py
index 5c952c72e..9eec91067 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -469,8 +469,6 @@ def _get_pipestat_configuration(self, pipeline_type="sample"):
 
         if pipeline_type == "sample":
             for piface in self.pipeline_interfaces:
-                print(piface)
-                # first check if this piface has a psm?
 
                 pipestat_config_path = self._check_for_existing_pipestat_config(piface)
 

From c1825c16cd8c85e7fec9d642fc30364a7599cb4e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 12:14:08 -0400
Subject: [PATCH 215/225] ensure we are checking for project level pipestat
 configuration if using --project

---
 looper/__init__.py     | 1 -
 looper/cli_pydantic.py | 6 +++++-
 looper/looper.py       | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/looper/__init__.py b/looper/__init__.py
index 5db46a828..fe751d02d 100644
--- a/looper/__init__.py
+++ b/looper/__init__.py
@@ -25,7 +25,6 @@
     write_sample_yaml_cwl,
     write_sample_yaml_prj,
     write_custom_template,
-    # write_local_pipestat_config,
 )
 from .const import *
 from .pipeline_interface import PipelineInterface
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 6f60d8b8b..74d6d4e31 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -243,7 +243,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
     ) as prj:
 
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
-        is_pipestat_configured = prj._check_if_pipestat_configured()
+        is_pipestat_configured = (
+            prj._check_if_pipestat_configured(pipeline_type="project")
+            if getattr(args, "project", None)
+            else prj._check_if_pipestat_configured()
+        )
 
         if subcommand_name in ["run", "rerun"]:
             rerun = subcommand_name == "rerun"
diff --git a/looper/looper.py b/looper/looper.py
index 934793aea..45a4bf0b0 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -251,7 +251,7 @@ def __call__(self, args, preview_flag=True):
         """
 
         use_pipestat = (
-            self.prj.pipestat_configured
+            self.prj.pipestat_configured_project
             if getattr(args, "project", None)
             else self.prj.pipestat_configured
         )

From 541113949b726cf2b82c5c63a5d325b5aaf4a227 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 12:27:21 -0400
Subject: [PATCH 216/225] reduce looper verbosity by switching to logger.debug
 in some places

---
 looper/conductor.py | 1 -
 looper/looper.py    | 4 ++--
 looper/project.py   | 2 --
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/looper/conductor.py b/looper/conductor.py
index 3f04f1450..7b2930326 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -635,7 +635,6 @@ def write_script(self, pool, size):
                 argstring = jinja_render_template_strictly(
                     template=templ, namespaces=namespaces
                 )
-                print(argstring)
             except UndefinedError as jinja_exception:
                 _LOGGER.warning(NOT_SUB_MSG.format(str(jinja_exception)))
             except KeyError as e:
diff --git a/looper/looper.py b/looper/looper.py
index 45a4bf0b0..3edc254bc 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -493,7 +493,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
                 len(processed_samples), num_samples
             )
         )
-        _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
+        _LOGGER.debug("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
         self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
         if getattr(args, "dry_run", None):
             job_sub_total_if_real = job_sub_total
@@ -501,7 +501,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
             _LOGGER.info(
                 f"Dry run. No jobs were actually submitted, but {job_sub_total_if_real} would have been."
             )
-        _LOGGER.info("Jobs submitted: {}".format(job_sub_total))
+        _LOGGER.debug("Jobs submitted: {}".format(job_sub_total))
         self.debug[DEBUG_JOBS] = job_sub_total
 
         # Restructure sample/failure data for display.
diff --git a/looper/project.py b/looper/project.py
index 9eec91067..46f32ad00 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -613,7 +613,6 @@ def _create_pipestat_config(self, piface):
 
         # Pipestat_dict_ is now updated from all sources and can be written to a yaml.
         pipestat_config_path = os.path.join(
-            # os.path.dirname(output_dir), f"pipestat_config_{pipeline_name}.yaml"
             output_dir,
             f"pipestat_config_{pipeline_name}.yaml",
         )
@@ -621,7 +620,6 @@ def _create_pipestat_config(self, piface):
         # Two end goals, create a config file
         write_pipestat_config(pipestat_config_path, pipestat_config_dict)
 
-        # piface['psm'] = PipestatManager(config_file=pipestat_config_path)
         piface.psm = PipestatManager(
             config_file=pipestat_config_path, multi_pipelines=True
         )

From d4f576c4b28f50321ec4ae6539a9d292c1fc8de7 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 14:44:21 -0400
Subject: [PATCH 217/225] bumbp pipestat req to v0.9.2

---
 requirements/requirements-all.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
index ab0eff0f2..3a558c224 100644
--- a/requirements/requirements-all.txt
+++ b/requirements/requirements-all.txt
@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
 peppy>=0.40.0
-pipestat>=0.9.2a1
+pipestat>=0.9.2
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2

From affd8d4e206ab768b3732c9b789d55fafca8fc69 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Mon, 3 Jun 2024 16:33:17 -0400
Subject: [PATCH 218/225] add enum for pipeline type #360

---
 looper/cli_pydantic.py |  3 ++-
 looper/conductor.py    |  3 ++-
 looper/const.py        |  8 ++++++++
 looper/looper.py       | 24 +++++++++++++-----------
 looper/project.py      | 21 +++++++++++++--------
 5 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 74d6d4e31..bfd189fdd 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -29,6 +29,7 @@
 
 from divvy import select_divvy_config
 
+from .const import PipelineLevel
 from . import __version__
 
 from .command_models.arguments import ArgumentEnum
@@ -244,7 +245,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
 
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
         is_pipestat_configured = (
-            prj._check_if_pipestat_configured(pipeline_type="project")
+            prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
             if getattr(args, "project", None)
             else prj._check_if_pipestat_configured()
         )
diff --git a/looper/conductor.py b/looper/conductor.py
index 7b2930326..ffbb1b547 100644
--- a/looper/conductor.py
+++ b/looper/conductor.py
@@ -27,6 +27,7 @@
 from .exceptions import JobSubmissionException, SampleFailedException
 from .processed_project import populate_sample_paths
 from .utils import fetch_sample_flags, jinja_render_template_strictly
+from .const import PipelineLevel
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -276,7 +277,7 @@ def is_project_submittable(self, force=False):
         psms = {}
         if self.prj.pipestat_configured_project:
             for piface in self.prj.project_pipeline_interfaces:
-                if piface.psm.pipeline_type == "project":
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
             psm = psms[self.pl_name]
             status = psm.get_status()
diff --git a/looper/const.py b/looper/const.py
index a866f2d84..ca70851da 100644
--- a/looper/const.py
+++ b/looper/const.py
@@ -1,6 +1,7 @@
 """ Shared project constants """
 
 import os
+from enum import Enum
 
 __author__ = "Databio lab"
 __email__ = "nathan@code.databio.org"
@@ -268,3 +269,10 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
     "init-piface": "Initialize generic pipeline interface.",
     "link": "Create directory of symlinks for reported results.",
 }
+
+# Add project/sample enum
+
+
+class PipelineLevel(Enum):
+    SAMPLE = "sample"
+    PROJECT = "project"
diff --git a/looper/looper.py b/looper/looper.py
index 3edc254bc..ee3670c28 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -16,6 +16,8 @@
 # Need specific sequence of actions for colorama imports?
 from colorama import init
 
+from .const import PipelineLevel
+
 init()
 from shutil import rmtree
 
@@ -93,7 +95,7 @@ def __call__(self, args):
         if getattr(args, "project", None):
 
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "project":
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     s = piface.psm.get_status() or "unknown"
                     status.setdefault(piface.psm.pipeline_name, {})
@@ -103,7 +105,7 @@ def __call__(self, args):
         else:
             for sample in self.prj.samples:
                 for piface in sample.project.pipeline_interfaces:
-                    if piface.psm.pipeline_type == "sample":
+                    if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                         psms[piface.psm.pipeline_name] = piface.psm
                         s = piface.psm.get_status(record_identifier=sample.sample_name)
                         status.setdefault(piface.psm.pipeline_name, {})
@@ -275,7 +277,7 @@ def __call__(self, args, preview_flag=True):
             else:
                 if use_pipestat:
                     for piface in sample.project.pipeline_interfaces:
-                        if piface.psm.pipeline_type == "sample":
+                        if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                             psms[piface.psm.pipeline_name] = piface.psm
                     for pipeline_name, psm in psms.items():
                         psm.backend.remove_record(
@@ -564,7 +566,7 @@ def __call__(self, args):
         if project_level:
 
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "project":
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     report_directory = piface.psm.summarize(
                         looper_samples=self.prj.samples, portable=portable
@@ -574,7 +576,7 @@ def __call__(self, args):
             return self.debug
         else:
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "sample":
+                if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     report_directory = piface.psm.summarize(
                         looper_samples=self.prj.samples, portable=portable
@@ -597,13 +599,13 @@ def __call__(self, args):
 
         if project_level:
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "project":
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     linked_results_path = piface.psm.link(link_dir=link_dir)
                     print(f"Linked directory: {linked_results_path}")
         else:
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "sample":
+                if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     linked_results_path = piface.psm.link(link_dir=link_dir)
                     print(f"Linked directory: {linked_results_path}")
@@ -622,12 +624,12 @@ def __call__(self, args):
         psms = {}
         if project_level:
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "project":
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     results = piface.psm.table()
         else:
             for piface in self.prj.pipeline_interfaces:
-                if piface.psm.pipeline_type == "sample":
+                if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     results = piface.psm.table()
         # Results contains paths to stats and object summaries.
@@ -672,7 +674,7 @@ def destroy_summary(prj, dry_run=False, project_level=False):
     psms = {}
     if project_level:
         for piface in prj.pipeline_interfaces:
-            if piface.psm.pipeline_type == "project":
+            if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                 psms[piface.psm.pipeline_name] = piface.psm
 
         for name, psm in psms.items():
@@ -699,7 +701,7 @@ def destroy_summary(prj, dry_run=False, project_level=False):
             )
     else:
         for piface in prj.pipeline_interfaces:
-            if piface.psm.pipeline_type == "sample":
+            if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
                 psms[piface.psm.pipeline_name] = piface.psm
         for name, psm in psms.items():
             _remove_or_dry_run(
diff --git a/looper/project.py b/looper/project.py
index 46f32ad00..16684ba74 100644
--- a/looper/project.py
+++ b/looper/project.py
@@ -28,6 +28,7 @@
 from .pipeline_interface import PipelineInterface
 from .processed_project import populate_project_paths, populate_sample_paths
 from .utils import *
+from .const import PipelineLevel
 
 __all__ = ["Project"]
 
@@ -308,7 +309,7 @@ def project_pipeline_interfaces(self):
         :return list[looper.PipelineInterface]: list of pipeline interfaces
         """
         return [
-            PipelineInterface(pi, pipeline_type="project")
+            PipelineInterface(pi, pipeline_type=PipelineLevel.PROJECT.value)
             for pi in self.project_pipeline_interface_sources
         ]
 
@@ -351,7 +352,9 @@ def pipestat_configured_project(self):
 
         :return bool: whether pipestat configuration is complete
         """
-        return self._check_if_pipestat_configured(pipeline_type="project")
+        return self._check_if_pipestat_configured(
+            pipeline_type=PipelineLevel.PROJECT.value
+        )
 
     def get_sample_piface(self, sample_name):
         """
@@ -449,7 +452,7 @@ def get_schemas(pifaces, schema_key=INPUT_SCHEMA_KEY):
                 schema_set.update([schema_file])
         return list(schema_set)
 
-    def _check_if_pipestat_configured(self, pipeline_type="sample"):
+    def _check_if_pipestat_configured(self, pipeline_type=PipelineLevel.SAMPLE.value):
 
         # First check if pipestat key is in looper_config, if not return false
 
@@ -463,11 +466,11 @@ def _check_if_pipestat_configured(self, pipeline_type="sample"):
                 # This should return True OR raise an exception at this point.
                 return self._get_pipestat_configuration(pipeline_type)
 
-    def _get_pipestat_configuration(self, pipeline_type="sample"):
+    def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value):
 
         # First check if it already exists
 
-        if pipeline_type == "sample":
+        if pipeline_type == PipelineLevel.SAMPLE.value:
             for piface in self.pipeline_interfaces:
 
                 pipestat_config_path = self._check_for_existing_pipestat_config(piface)
@@ -479,7 +482,7 @@ def _get_pipestat_configuration(self, pipeline_type="sample"):
                         config_file=pipestat_config_path, multi_pipelines=True
                     )
 
-        elif pipeline_type == "project":
+        elif pipeline_type == PipelineLevel.PROJECT.value:
             for prj_piface in self.project_pipeline_interfaces:
                 pipestat_config_path = self._check_for_existing_pipestat_config(
                     prj_piface
@@ -691,7 +694,7 @@ def _piface_by_samples(self):
         pifaces_by_sample = {}
         for source, sample_names in self._samples_by_interface.items():
             try:
-                pi = PipelineInterface(source, pipeline_type="sample")
+                pi = PipelineInterface(source, pipeline_type=PipelineLevel.SAMPLE.value)
             except PipelineInterfaceConfigError as e:
                 _LOGGER.debug(f"Skipping pipeline interface creation: {e}")
             else:
@@ -742,7 +745,9 @@ def _samples_by_piface(self, piface_key):
                 for source in piface_srcs:
                     source = self._resolve_path_with_cfg(source)
                     try:
-                        PipelineInterface(source, pipeline_type="sample")
+                        PipelineInterface(
+                            source, pipeline_type=PipelineLevel.SAMPLE.value
+                        )
                     except (
                         ValidationError,
                         IOError,

From 6fb615c9a8da2a8839e474952ced5cc05098ebe0 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 4 Jun 2024 11:27:50 -0400
Subject: [PATCH 219/225] add looper/command_models to MANIFEST.in

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 5bc61acec..15473d351 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,3 +6,4 @@ include looper/default_config/*
 include looper/default_config/divvy_templates/*
 include looper/jinja_templates_old/*
 include looper/schemas/*
+include looper/command_models/*

From f451e71b04663f103c72d22a2695c08083a2cb54 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 4 Jun 2024 13:14:42 -0400
Subject: [PATCH 220/225] update changelog for alpha release

---
 docs/changelog.md  | 2 +-
 looper/_version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 9b0291898..f86e8178f 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,7 +2,7 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [1.8.0] -- 2024-03-29
+## [1.8.0] -- 2024-XX-XX
 
 ### Added
 - looper destroy now destroys individual results when pipestat is configured: https://github.com/pepkit/looper/issues/469
diff --git a/looper/_version.py b/looper/_version.py
index 29654eec0..6105bb2d2 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1 +1 @@
-__version__ = "1.8.0"
+__version__ = "1.8.0a1"

From 8b5d025b0321f11d06202290f8e95103d651f3d5 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 4 Jun 2024 14:04:22 -0400
Subject: [PATCH 221/225] Revert "update changelog for alpha release"

This reverts commit f451e71b04663f103c72d22a2695c08083a2cb54.
---
 docs/changelog.md  | 2 +-
 looper/_version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index f86e8178f..9b0291898 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,7 +2,7 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [1.8.0] -- 2024-XX-XX
+## [1.8.0] -- 2024-03-29
 
 ### Added
 - looper destroy now destroys individual results when pipestat is configured: https://github.com/pepkit/looper/issues/469
diff --git a/looper/_version.py b/looper/_version.py
index 6105bb2d2..29654eec0 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1 +1 @@
-__version__ = "1.8.0a1"
+__version__ = "1.8.0"

From 6b1cf9becb14ea674227331ccfe0ddfba687d00f Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 4 Jun 2024 14:16:47 -0400
Subject: [PATCH 222/225] add -v and --version, change version to 1.8.1a1

---
 docs/changelog.md      | 7 ++++++-
 looper/_version.py     | 3 ++-
 looper/cli_pydantic.py | 1 +
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 9b0291898..e9775235d 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,7 +2,12 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [1.8.0] -- 2024-03-29
+## [1.8.1] -- 2024-06-04
+
+### Fixed
+- added `-v` and `--version` to the CLI
+
+## [1.8.0] -- 2024-06-04
 
 ### Added
 - looper destroy now destroys individual results when pipestat is configured: https://github.com/pepkit/looper/issues/469
diff --git a/looper/_version.py b/looper/_version.py
index 29654eec0..c5aeb1653 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1 +1,2 @@
-__version__ = "1.8.0"
+__version__ = "1.8.1a1"
+# You must change the version in parser = pydantic2_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index bfd189fdd..0b7735e43 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -326,6 +326,7 @@ def main(test_args=None) -> None:
         prog="looper",
         description="Looper Pydantic Argument Parser",
         add_help=True,
+        version="1.8.1a1",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)

From 92ca137c44d8985ff696eb52525727077f928bfd Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Tue, 4 Jun 2024 15:22:20 -0400
Subject: [PATCH 223/225] update changelog.md

---
 docs/changelog.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index e9775235d..c14e0f45a 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,7 +2,7 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [1.8.1] -- 2024-06-04
+## [1.8.1] -- 2024-06-05
 
 ### Fixed
 - added `-v` and `--version` to the CLI
@@ -29,6 +29,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Load PEP from CSV: https://github.com/pepkit/looper/issues/456
 - looper now works with sample_table_index https://github.com/pepkit/looper/issues/458
 
+## [1.7.1] -- 2024-05-28
+
+### Fixed
+- pin pipestat version to be between pipestat>=0.8.0,<0.9.0  https://github.com/pepkit/looper/issues/494
 
 ## [1.7.0] -- 2024-01-26
 

From f5347d2fffbeaab9f280a29a2a1850c7edc3c788 Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Wed, 5 Jun 2024 10:57:18 -0400
Subject: [PATCH 224/225] fix --project

---
 docs/changelog.md           |   1 +
 docs/usage.md               | 976 +++++++++++++++++++++---------------
 looper/cli_pydantic.py      |   6 +-
 looper/looper.py            |   8 +-
 tests/test_comprehensive.py |   3 +-
 5 files changed, 595 insertions(+), 399 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index c14e0f45a..6b8fd2825 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ### Fixed
 - added `-v` and `--version` to the CLI
+- fixed running project level with `--project` argument
 
 ## [1.8.0] -- 2024-06-04
 
diff --git a/docs/usage.md b/docs/usage.md
index fe102ddcd..c8c58a5fe 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -26,16 +26,14 @@ Each task is controlled by one of the following commands: `run`, `rerun`, `runp`
 Here you can see the command-line usage instructions for the main looper command and for each subcommand:
 ## `looper --help`
 ```console
-version: 1.7.0
-usage: looper [-h] [--version] [--logfile LOGFILE] [--dbg] [--silent]
-              [--verbosity V] [--logdev] [--commands]
-              {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface,link}
+usage: looper [-h] [-v] [--silent] [--verbosity VERBOSITY] [--logdev]
+              {run,rerun,runp,table,report,destroy,check,clean,init,init_piface,link,inspect}
               ...
 
-looper - A project job submission engine and project manager.
+Looper Pydantic Argument Parser
 
-positional arguments:
-  {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface,link}
+commands:
+  {run,rerun,runp,table,report,destroy,check,clean,init,init_piface,link,inspect}
     run                 Run or submit sample jobs.
     rerun               Resubmit sample jobs with failed flags.
     runp                Run or submit project jobs.
@@ -44,436 +42,628 @@ positional arguments:
     destroy             Remove output files of the project.
     check               Check flag status of current runs.
     clean               Run clean scripts of already processed jobs.
-    inspect             Print information about a project.
     init                Initialize looper config file.
-    init-piface         Initialize generic pipeline interface.
+    init_piface         Initialize generic pipeline interface.
     link                Create directory of symlinks for reported results.
+    inspect             Print information about a project.
 
-options:
+optional arguments:
+  --silent              Whether to silence logging (default: False)
+  --verbosity VERBOSITY
+                        Alternate mode of expression for logging level that
+                        better accords with intuition about how to convey
+                        this. (default: None)
+  --logdev              Whether to log in development mode; possibly among
+                        other behavioral changes to logs handling, use a more
+                        information-rich message format template. (default:
+                        False)
+
+help:
   -h, --help            show this help message and exit
-  --version             show program's version number and exit
-  --logfile LOGFILE     Optional output file for looper logs (default: None)
-  --dbg                 Turn on debug mode (default: False)
-  --silent              Silence logging. Overrides verbosity.
-  --verbosity V         Set logging level (1-5 or logging module level name)
-  --logdev              Expand content of logging message format.
-  --commands            show program's version number and exit
-
-For subcommand-specific options, type: 'looper <subcommand> -h'
-https://github.com/pepkit/looper
+  -v, --version         show program's version number and exit
 ```
 
 ## `looper run --help`
 ```console
-usage: looper run [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S]
-                  [-c K [K ...]] [-u X] [-n N] [-j J] [--looper-config LOOPER_CONFIG]
-                  [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
-                  [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                  [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
-                  [config_file]
-
-Run or submit sample jobs.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  -i, --ignore-flags                 Ignore run status flags? Default=False
-  -d, --dry-run                      Don't actually submit the jobs. Default=False
-  -t S, --time-delay S               Time delay in seconds between job submissions
-  -x S, --command-extra S            String to append to every command
-  -y S, --command-extra-override S   Same as command-extra, but overrides values in PEP
-  -f, --skip-file-checks             Do not perform input file checks
-  -u X, --lump-s X                   Lump by size: total input file size (GB) to batch
-                                     into one job
-  -n N, --lump-n N                   Lump by number: number of samples to batch into one
-                                     job
-  -j J, --lump-j J                   Lump samples into number of jobs.
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-
-divvy arguments:
-  Configure divvy to change computing settings
-
-  --divvy DIVCFG                     Path to divvy configuration file. Default=$DIVCFG env
-                                     variable. Currently: not set
-  -p P, --package P                  Name of computing resource package to use
-  -s S, --settings S                 Path to a YAML settings file with compute settings
-  -c K [K ...], --compute K [K ...]  List of key-value pairs (k1=v1)
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper run [-h] [-i] [-t TIME_DELAY] [-d] [-x COMMAND_EXTRA]
+                  [-y COMMAND_EXTRA_OVERRIDE] [-u LUMP] [-n LUMP_N]
+                  [-j LUMP_J] [--divvy DIVVY] [-f] [-c COMPUTE [COMPUTE ...]]
+                  [--package PACKAGE] [--settings SETTINGS]
+                  [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                  [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                  [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                  [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                  [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                  [--looper-config LOOPER_CONFIG]
+                  [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                  [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                  [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                  [--project]
+
+optional arguments:
+  -i, --ignore-flags    Ignore run status flags (default: False)
+  -t TIME_DELAY, --time-delay TIME_DELAY
+                        Time delay in seconds between job submissions (min: 0,
+                        max: 30) (default: 0)
+  -d, --dry-run         Don't actually submit jobs (default: False)
+  -x COMMAND_EXTRA, --command-extra COMMAND_EXTRA
+                        String to append to every command (default: )
+  -y COMMAND_EXTRA_OVERRIDE, --command-extra-override COMMAND_EXTRA_OVERRIDE
+                        Same as command-extra, but overrides values in PEP
+                        (default: )
+  -u LUMP, --lump LUMP  Total input file size (GB) to batch into one job
+                        (default: None)
+  -n LUMP_N, --lump-n LUMP_N
+                        Number of commands to batch into one job (default:
+                        None)
+  -j LUMP_J, --lump-j LUMP_J
+                        Lump samples into number of jobs. (default: None)
+  --divvy DIVVY         Path to divvy configuration file. Default=$DIVCFG env
+                        variable. Currently: not set (default: None)
+  -f, --skip-file-checks
+                        Do not perform input file checks (default: False)
+  -c COMPUTE [COMPUTE ...], --compute COMPUTE [COMPUTE ...]
+                        List of key-value pairs (k1=v1) (default: [])
+  --package PACKAGE     Name of computing resource package to use (default:
+                        None)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper runp --help`
 ```console
-usage: looper runp [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S]
-                   [-c K [K ...]] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                   [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                   [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                   [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
-                   [config_file]
-
-Run or submit project jobs.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  -i, --ignore-flags                 Ignore run status flags? Default=False
-  -d, --dry-run                      Don't actually submit the jobs. Default=False
-  -t S, --time-delay S               Time delay in seconds between job submissions
-  -x S, --command-extra S            String to append to every command
-  -y S, --command-extra-override S   Same as command-extra, but overrides values in PEP
-  -f, --skip-file-checks             Do not perform input file checks
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-
-divvy arguments:
-  Configure divvy to change computing settings
-
-  --divvy DIVCFG                     Path to divvy configuration file. Default=$DIVCFG env
-                                     variable. Currently: not set
-  -p P, --package P                  Name of computing resource package to use
-  -s S, --settings S                 Path to a YAML settings file with compute settings
-  -c K [K ...], --compute K [K ...]  List of key-value pairs (k1=v1)
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper runp [-h] [-i] [-t TIME_DELAY] [-d] [-x COMMAND_EXTRA]
+                   [-y COMMAND_EXTRA_OVERRIDE] [-u LUMP] [-n LUMP_N]
+                   [--divvy DIVVY] [-f] [-c COMPUTE [COMPUTE ...]]
+                   [--package PACKAGE] [--settings SETTINGS]
+                   [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                   [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                   [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                   [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                   [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                   [--looper-config LOOPER_CONFIG]
+                   [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                   [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                   [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                   [--project]
+
+optional arguments:
+  -i, --ignore-flags    Ignore run status flags (default: False)
+  -t TIME_DELAY, --time-delay TIME_DELAY
+                        Time delay in seconds between job submissions (min: 0,
+                        max: 30) (default: 0)
+  -d, --dry-run         Don't actually submit jobs (default: False)
+  -x COMMAND_EXTRA, --command-extra COMMAND_EXTRA
+                        String to append to every command (default: )
+  -y COMMAND_EXTRA_OVERRIDE, --command-extra-override COMMAND_EXTRA_OVERRIDE
+                        Same as command-extra, but overrides values in PEP
+                        (default: )
+  -u LUMP, --lump LUMP  Total input file size (GB) to batch into one job
+                        (default: None)
+  -n LUMP_N, --lump-n LUMP_N
+                        Number of commands to batch into one job (default:
+                        None)
+  --divvy DIVVY         Path to divvy configuration file. Default=$DIVCFG env
+                        variable. Currently: not set (default: None)
+  -f, --skip-file-checks
+                        Do not perform input file checks (default: False)
+  -c COMPUTE [COMPUTE ...], --compute COMPUTE [COMPUTE ...]
+                        List of key-value pairs (k1=v1) (default: [])
+  --package PACKAGE     Name of computing resource package to use (default:
+                        None)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper rerun --help`
 ```console
-usage: looper rerun [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P]
-                    [-s S] [-c K [K ...]] [-u X] [-n N] [-j J]
-                    [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                    [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
-                    [config_file]
-
-Resubmit sample jobs with failed flags.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  -i, --ignore-flags                 Ignore run status flags? Default=False
-  -d, --dry-run                      Don't actually submit the jobs. Default=False
-  -t S, --time-delay S               Time delay in seconds between job submissions
-  -x S, --command-extra S            String to append to every command
-  -y S, --command-extra-override S   Same as command-extra, but overrides values in PEP
-  -f, --skip-file-checks             Do not perform input file checks
-  -u X, --lump-s X                   Lump by size: total input file size (GB) to batch
-                                     into one job
-  -n N, --lump-n N                   Lump by number: number of samples to batch into one
-                                     job
-  -j J, --lump-j J                   Lump samples into number of jobs.
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-
-divvy arguments:
-  Configure divvy to change computing settings
-
-  --divvy DIVCFG                     Path to divvy configuration file. Default=$DIVCFG env
-                                     variable. Currently: not set
-  -p P, --package P                  Name of computing resource package to use
-  -s S, --settings S                 Path to a YAML settings file with compute settings
-  -c K [K ...], --compute K [K ...]  List of key-value pairs (k1=v1)
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper rerun [-h] [-i] [-t TIME_DELAY] [-d] [-x COMMAND_EXTRA]
+                    [-y COMMAND_EXTRA_OVERRIDE] [-u LUMP] [-n LUMP_N]
+                    [-j LUMP_J] [--divvy DIVVY] [-f]
+                    [-c COMPUTE [COMPUTE ...]] [--package PACKAGE]
+                    [--settings SETTINGS] [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                    [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                    [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                    [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                    [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                    [--looper-config LOOPER_CONFIG]
+                    [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                    [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                    [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                    [--project]
+
+optional arguments:
+  -i, --ignore-flags    Ignore run status flags (default: False)
+  -t TIME_DELAY, --time-delay TIME_DELAY
+                        Time delay in seconds between job submissions (min: 0,
+                        max: 30) (default: 0)
+  -d, --dry-run         Don't actually submit jobs (default: False)
+  -x COMMAND_EXTRA, --command-extra COMMAND_EXTRA
+                        String to append to every command (default: )
+  -y COMMAND_EXTRA_OVERRIDE, --command-extra-override COMMAND_EXTRA_OVERRIDE
+                        Same as command-extra, but overrides values in PEP
+                        (default: )
+  -u LUMP, --lump LUMP  Total input file size (GB) to batch into one job
+                        (default: None)
+  -n LUMP_N, --lump-n LUMP_N
+                        Number of commands to batch into one job (default:
+                        None)
+  -j LUMP_J, --lump-j LUMP_J
+                        Lump samples into number of jobs. (default: None)
+  --divvy DIVVY         Path to divvy configuration file. Default=$DIVCFG env
+                        variable. Currently: not set (default: None)
+  -f, --skip-file-checks
+                        Do not perform input file checks (default: False)
+  -c COMPUTE [COMPUTE ...], --compute COMPUTE [COMPUTE ...]
+                        List of key-value pairs (k1=v1) (default: [])
+  --package PACKAGE     Name of computing resource package to use (default:
+                        None)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper report --help`
 ```console
-usage: looper report [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                     [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                     [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                     [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project] [--portable]
-                     [config_file]
-
-Create browsable HTML report of project results.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-  --project                          Process project-level pipelines
-  --portable                         Makes html report portable.
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper report [-h] [--portable] [--settings SETTINGS]
+                     [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                     [--sel-flag SEL_FLAG [SEL_FLAG ...]]
+                     [--sel-attr SEL_ATTR]
+                     [--sel-incl SEL_INCL [SEL_INCL ...]]
+                     [--sel-excl SEL_EXCL] [-l LIMIT] [-k SKIP]
+                     [--pep-config PEP_CONFIG] [-o OUTPUT_DIR]
+                     [--config-file CONFIG_FILE]
+                     [--looper-config LOOPER_CONFIG]
+                     [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                     [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                     [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                     [--project]
+
+optional arguments:
+  --portable            Makes html report portable. (default: False)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper table --help`
 ```console
-usage: looper table [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                    [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
-                    [config_file]
-
-Write summary stats table for project samples.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-  --project                          Process project-level pipelines
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper table [-h] [--settings SETTINGS]
+                    [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                    [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                    [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                    [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                    [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                    [--looper-config LOOPER_CONFIG]
+                    [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                    [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                    [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                    [--project]
+
+optional arguments:
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper inspect --help`
 ```console
-usage: looper inspect [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                      [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                      [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                      [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
-                      [--sample-names [SAMPLE_NAMES ...]] [--attr-limit ATTR_LIMIT]
-                      [config_file]
-
-Print information about a project.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-  --sample-names [SAMPLE_NAMES ...]  Names of the samples to inspect
-  --attr-limit ATTR_LIMIT            Number of attributes to display
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper inspect [-h] [--settings SETTINGS]
+                      [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                      [--sel-flag SEL_FLAG [SEL_FLAG ...]]
+                      [--sel-attr SEL_ATTR]
+                      [--sel-incl SEL_INCL [SEL_INCL ...]]
+                      [--sel-excl SEL_EXCL] [-l LIMIT] [-k SKIP]
+                      [--pep-config PEP_CONFIG] [-o OUTPUT_DIR]
+                      [--config-file CONFIG_FILE]
+                      [--looper-config LOOPER_CONFIG]
+                      [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                      [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                      [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                      [--project]
+
+optional arguments:
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper init --help`
 ```console
-usage: looper init [-h] [-f] [-o DIR] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-p]
-                   pep_config
-
-Initialize looper config file.
-
-positional arguments:
-  pep_config                         Project configuration file (PEP)
-
-options:
-  -h, --help                         show this help message and exit
-  -f, --force                        Force overwrite
-  -o DIR, --output-dir DIR
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -p, --piface                       Generates generic pipeline interface
+usage: looper init [-h] [-f] [-o OUTPUT_DIR] [--pep-config PEP_CONFIG]
+                   [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                   [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+
+optional arguments:
+  -f, --force-yes       Provide upfront confirmation of destruction intent, to
+                        skip console query. Default=False (default: False)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper destroy --help`
 ```console
-usage: looper destroy [-h] [-d] [--force-yes] [--looper-config LOOPER_CONFIG]
-                      [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
-                      [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                      [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
+usage: looper destroy [-h] [-d] [-f] [--settings SETTINGS]
+                      [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                      [--sel-flag SEL_FLAG [SEL_FLAG ...]]
+                      [--sel-attr SEL_ATTR]
+                      [--sel-incl SEL_INCL [SEL_INCL ...]]
+                      [--sel-excl SEL_EXCL] [-l LIMIT] [-k SKIP]
+                      [--pep-config PEP_CONFIG] [-o OUTPUT_DIR]
+                      [--config-file CONFIG_FILE]
+                      [--looper-config LOOPER_CONFIG]
+                      [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                      [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                      [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
                       [--project]
-                      [config_file]
-
-Remove output files of the project.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  -d, --dry-run                      Don't actually submit the jobs. Default=False
-  --force-yes                        Provide upfront confirmation of destruction intent,
-                                     to skip console query. Default=False
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-  --project                          Process project-level pipelines
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+
+optional arguments:
+  -d, --dry-run         Don't actually submit jobs (default: False)
+  -f, --force-yes       Provide upfront confirmation of destruction intent, to
+                        skip console query. Default=False (default: False)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper check --help`
 ```console
-usage: looper check [-h] [--describe-codes] [--itemized] [-f [F ...]]
-                    [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
-                    [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
-                    [config_file]
-
-Check flag status of current runs.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  --describe-codes                   Show status codes description
-  --itemized                         Show a detailed, by sample statuses
-  -f [F ...], --flags [F ...]        Check on only these flags/status values
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-  --project                          Process project-level pipelines
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper check [-h] [--describe-codes] [--itemized]
+                    [-f FLAGS [FLAGS ...]] [--settings SETTINGS]
+                    [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                    [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                    [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                    [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                    [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                    [--looper-config LOOPER_CONFIG]
+                    [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                    [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                    [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                    [--project]
+
+optional arguments:
+  --describe-codes      Show status codes description. Default=False (default:
+                        False)
+  --itemized            Show detailed overview of sample statuses.
+                        Default=False (default: False)
+  -f FLAGS [FLAGS ...], --flags FLAGS [FLAGS ...]
+                        Only check samples based on these status flags.
+                        (default: [])
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
 ## `looper clean --help`
 ```console
-usage: looper clean [-h] [-d] [--force-yes] [--looper-config LOOPER_CONFIG]
-                    [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
-                    [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                    [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
-                    [config_file]
-
-Run clean scripts of already processed jobs.
-
-positional arguments:
-  config_file                        Project configuration file (YAML) or pephub registry
-                                     path.
-
-options:
-  -h, --help                         show this help message and exit
-  -d, --dry-run                      Don't actually submit the jobs. Default=False
-  --force-yes                        Provide upfront confirmation of destruction intent,
-                                     to skip console query. Default=False
-  --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
-  -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper sample config file
-  -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...]
-                                     Path to looper project config file
-  -a A [A ...], --amend A [A ...]    List of amendments to activate
-
-sample selection arguments:
-  Specify samples to include or exclude based on sample attribute values
-
-  -l N, --limit N                    Limit to n samples
-  -k N, --skip N                     Skip samples by numerical index
-  --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
-  --sel-excl [E ...]                 Exclude samples with these values
-  --sel-incl [I ...]                 Include only samples with these values
-  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
-  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
+usage: looper clean [-h] [-d] [-f] [--settings SETTINGS]
+                    [--exc-flag EXC_FLAG [EXC_FLAG ...]]
+                    [--sel-flag SEL_FLAG [SEL_FLAG ...]] [--sel-attr SEL_ATTR]
+                    [--sel-incl SEL_INCL [SEL_INCL ...]] [--sel-excl SEL_EXCL]
+                    [-l LIMIT] [-k SKIP] [--pep-config PEP_CONFIG]
+                    [-o OUTPUT_DIR] [--config-file CONFIG_FILE]
+                    [--looper-config LOOPER_CONFIG]
+                    [-S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]]
+                    [-P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]]
+                    [--pipestat PIPESTAT] [--amend AMEND [AMEND ...]]
+                    [--project]
+
+optional arguments:
+  -d, --dry-run         Don't actually submit jobs (default: False)
+  -f, --force-yes       Provide upfront confirmation of destruction intent, to
+                        skip console query. Default=False (default: False)
+  --settings SETTINGS   Path to a YAML settings file with compute settings
+                        (default: )
+  --exc-flag EXC_FLAG [EXC_FLAG ...]
+                        Sample exclusion flag (default: [])
+  --sel-flag SEL_FLAG [SEL_FLAG ...]
+                        Sample selection flag (default: [])
+  --sel-attr SEL_ATTR   Attribute for sample exclusion OR inclusion (default:
+                        toggle)
+  --sel-incl SEL_INCL [SEL_INCL ...]
+                        Include only samples with these values (default: [])
+  --sel-excl SEL_EXCL   Exclude samples with these values (default: )
+  -l LIMIT, --limit LIMIT
+                        Limit to n samples (default: None)
+  -k SKIP, --skip SKIP  Skip samples by numerical index (default: None)
+  --pep-config PEP_CONFIG
+                        PEP configuration file (default: None)
+  -o OUTPUT_DIR, --output-dir OUTPUT_DIR
+                        Output directory (default: None)
+  --config-file CONFIG_FILE
+                        Project configuration file (default: None)
+  --looper-config LOOPER_CONFIG
+                        Looper configuration file (YAML) (default: None)
+  -S SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...], --sample-pipeline-interfaces SAMPLE_PIPELINE_INTERFACES [SAMPLE_PIPELINE_INTERFACES ...]
+                        Paths to looper sample pipeline interfaces (default:
+                        [])
+  -P PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...], --project-pipeline-interfaces PROJECT_PIPELINE_INTERFACES [PROJECT_PIPELINE_INTERFACES ...]
+                        Paths to looper project pipeline interfaces (default:
+                        [])
+  --pipestat PIPESTAT   Path to pipestat files. (default: None)
+  --amend AMEND [AMEND ...]
+                        List of amendments to activate (default: [])
+  --project             Is this command executed for project-level? (default:
+                        False)
+
+help:
+  -h, --help            show this help message and exit
 ```
 
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 0b7735e43..8a54a5928 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -246,11 +246,15 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
         # Check at the beginning if user wants to use pipestat and pipestat is configurable
         is_pipestat_configured = (
             prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
-            if getattr(args, "project", None)
+            if getattr(subcommand_args, "project", None)
             else prj._check_if_pipestat_configured()
         )
 
         if subcommand_name in ["run", "rerun"]:
+            if getattr(subcommand_args, "project", None):
+                _LOGGER.warning(
+                    "Project flag set but 'run' command was used. Please use 'runp' to run at project-level."
+                )
             rerun = subcommand_name == "rerun"
             run = Runner(prj)
             try:
diff --git a/looper/looper.py b/looper/looper.py
index ee3670c28..1eea6edd6 100755
--- a/looper/looper.py
+++ b/looper/looper.py
@@ -94,7 +94,7 @@ def __call__(self, args):
         psms = {}
         if getattr(args, "project", None):
 
-            for piface in self.prj.pipeline_interfaces:
+            for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     s = piface.psm.get_status() or "unknown"
@@ -565,7 +565,7 @@ def __call__(self, args):
 
         if project_level:
 
-            for piface in self.prj.pipeline_interfaces:
+            for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     report_directory = piface.psm.summarize(
@@ -598,7 +598,7 @@ def __call__(self, args):
         psms = {}
 
         if project_level:
-            for piface in self.prj.pipeline_interfaces:
+            for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     linked_results_path = piface.psm.link(link_dir=link_dir)
@@ -623,7 +623,7 @@ def __call__(self, args):
         results = []
         psms = {}
         if project_level:
-            for piface in self.prj.pipeline_interfaces:
+            for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
                     psms[piface.psm.pipeline_name] = piface.psm
                     results = piface.psm.table()
diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py
index c0f0f81fb..cce74ca54 100644
--- a/tests/test_comprehensive.py
+++ b/tests/test_comprehensive.py
@@ -123,7 +123,8 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     try:
         result = main(test_args=x)
-        assert result == {}
+        assert result == {"example_pipestat_project_pipeline": {"project": "unknown"}}
+
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))
 

From 78dea99fb49c8df1930bb61cf43a8de0d11cbb3e Mon Sep 17 00:00:00 2001
From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com>
Date: Thu, 6 Jun 2024 10:16:23 -0400
Subject: [PATCH 225/225] update versions for 1.8.1 release

---
 docs/changelog.md      | 2 +-
 looper/_version.py     | 2 +-
 looper/cli_pydantic.py | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 6b8fd2825..4450ed9e0 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,7 +2,7 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [1.8.1] -- 2024-06-05
+## [1.8.1] -- 2024-06-06
 
 ### Fixed
 - added `-v` and `--version` to the CLI
diff --git a/looper/_version.py b/looper/_version.py
index c5aeb1653..2ce1f6586 100644
--- a/looper/_version.py
+++ b/looper/_version.py
@@ -1,2 +1,2 @@
-__version__ = "1.8.1a1"
+__version__ = "1.8.1"
 # You must change the version in parser = pydantic2_argparse.ArgumentParser in cli_pydantic.py!!!
diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py
index 8a54a5928..035a80434 100644
--- a/looper/cli_pydantic.py
+++ b/looper/cli_pydantic.py
@@ -328,9 +328,9 @@ def main(test_args=None) -> None:
     parser = pydantic2_argparse.ArgumentParser(
         model=TopLevelParser,
         prog="looper",
-        description="Looper Pydantic Argument Parser",
+        description="Looper: A job submitter for Portable Encapsulated Projects",
         add_help=True,
-        version="1.8.1a1",
+        version="1.8.1",
     )
 
     parser = add_short_arguments(parser, ArgumentEnum)