Skip preprare and QC if output already exists (#216)

* add rich progress and checkpoionts

* refactor tests

* refactor

* fix

* fix

* tmp

* fix

* remove joinpath

* update system test

* store artefacts

.circleci/config.yml

@@ -100,9 +100,41 @@ jobs:
                           --space MNI152NLin2009cAsym \
                           --reset_database \
                           -vv
-                no_output_timeout: 6h
+        -   run:
+                name: rerun prepare - fast as output already exists
+                command: |
+                    user_name=cpplab
+                    repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
+                    docker run -ti --rm \
+                      -v /tmp/workspace/data/ds002799/derivatives/fmriprep:/bids_dataset \
+                      -v ${HOME}/outputs:/outputs \
+                        ${user_name}/${repo_name} \
+                          /bids_dataset \
+                          /outputs \
+                          participant \
+                          prepare \
+                          --participant_label 302 307 \
+                          --space MNI152NLin2009cAsym \
+                          --reset_database \
+                          -vv
+        -   run:
+                name: run group level QC
+                command: |
+                    user_name=cpplab
+                    repo_name=$(echo "${CIRCLE_PROJECT_REPONAME}" | tr '[:upper:]' '[:lower:]')
+                    docker run -ti --rm \
+                      -v ${HOME}/outputs:/outputs \
+                        ${user_name}/${repo_name} \
+                          /outputs/bidsmreye \
+                          /outputs \
+                          group \
+                          qc \
+                          --participant_label 302 307 \
+                          --space MNI152NLin2009cAsym \
+                          --reset_database \
+                          -vv
         -   store_artifacts:
-                path: ~/output
+                path: /home/circleci/outputs/bidsmreye
 
     deploy:
         machine:

.github/workflows/system_tests.yml

@@ -23,7 +23,6 @@ jobs:
         strategy:
             matrix:
                 python-version: ['3.11']
-                dataset: [demo]
 
         steps:
 
@@ -50,5 +49,10 @@ jobs:
             run: |
                 datalad wtf
 
-        -   name: Run ${{ matrix.dataset }}
-            run: make ${{ matrix.dataset }}
+        -   name: Run demo
+            run: make demo
+
+        -   name: Re run demo (should be faster)
+            run: |
+                make prepare
+                make generalize

Makefile

@@ -185,11 +185,46 @@ ds002799: clean-ds002799 get_ds002799
 				--reset_database \
 				-vv
 
-## ds002799
+## ds000114
 get_ds000114:
 	datalad install -s ///openneuro-derivatives/ds000114-fmriprep tests/data/ds000114-fmriprep
 	cd tests/data/ds000114-fmriprep && datalad get sub-0[1-2]/ses-*/func/*MNI*desc-preproc*bold.nii.gz -J 12
 
+ds000114_all: get_ds000114
+	bidsmreye 	$$PWD/tests/data/ds000114-fmriprep \
+				$$PWD/outputs/ds000114/derivatives \
+				participant \
+				all \
+				--participant_label 01 02 \
+				--space MNI152NLin2009cAsym \
+				--task linebisection overtverbgeneration -vv --force
+
+ds000114_prepare: get_ds000114
+	bidsmreye 	$$PWD/tests/data/ds000114-fmriprep \
+				$$PWD/outputs/ds000114/derivatives \
+				participant \
+				prepare \
+				--participant_label 01 02 \
+				--space MNI152NLin2009cAsym \
+				--task linebisection -vv
+
+ds000114_generalize:
+	bidsmreye 	$$PWD/tests/data/ds000114-fmriprep \
+				$$PWD/outputs/ds000114/derivatives \
+				participant \
+				generalize \
+				--participant_label 01 02 \
+				--space MNI152NLin2009cAsym
+
+ds000114_qc:
+	bidsmreye 	$$PWD/outputs/ds000114/derivatives/bidsmreye \
+				$$PWD/outputs/ds000114/derivatives \
+				group \
+				qc \
+				--participant_label 01 02 \
+				--space MNI152NLin2009cAsym \
+				-vvv
+
 ## DOCKER
 .PHONY:
 

bidsmreye/_cli.py

@@ -39,7 +39,7 @@ def cli(argv: Any = sys.argv) -> None:
 
     model_weights_file = None
     if getattr(args, "model", None) is not None:
-        model_weights_file = str(getattr(args, "model"))
+        model_weights_file = str(getattr(args, "model"))  # noqa: B009
 
     bidsmreye(
         bids_dir=args.bids_dir[0],
@@ -56,6 +56,7 @@ def cli(argv: Any = sys.argv) -> None:
         bids_filter_file=args.bids_filter_file,
         non_linear_coreg=bool(getattr(args, "non_linear_coreg", False)),
         log_level_name=log_level_name,
+        force=bool(getattr(args, "force", False)),
     )
 
 

bidsmreye/_parsers.py

@@ -127,6 +127,13 @@ def _add_common_arguments(parser: ArgumentParser) -> ArgumentParser:
 For further details, please check out TBD.
         """,
     )
+    parser.add_argument(
+        "--force",
+        help="""
+Overwrite previous output.
+        """,
+        action="store_true",
+    )
     return parser
 
 
@@ -233,7 +240,7 @@ def download_parser(
         help="""
 The directory where the model files will be stored.
         """,
-        default=Path.cwd().joinpath("models"),
+        default=Path.cwd() / "models",
     )
 
     return parser

bidsmreye/bids_utils.py

@@ -119,9 +119,8 @@ def create_bidsname(
 
     output_file = layout.build_path(entities, bids_name_config[filetype], validate=False)
 
-    output_file = Path(layout.root).joinpath(output_file)
-
-    return output_file.resolve()
+    output_file = Path(layout.root) / output_file
+    return output_file.absolute()
 
 
 def create_sidecar(
@@ -180,7 +179,7 @@ def get_dataset_layout(
         dataset_path = Path(dataset_path)
     create_dir_if_absent(dataset_path)
 
-    dataset_path = dataset_path.resolve()
+    dataset_path = dataset_path.absolute()
 
     pybids_config = config
     if config is None:
@@ -193,7 +192,7 @@ def get_dataset_layout(
             dataset_path, validate=False, derivatives=False, config=pybids_config
         )
 
-    database_path = dataset_path.joinpath("pybids_db")
+    database_path = dataset_path / "pybids_db"
     return BIDSLayout(
         dataset_path,
         validate=False,
@@ -321,7 +320,7 @@ def write_dataset_description(layout: BIDSLayout) -> None:
     :param layout: BIDSLayout of the dataset to update.
     :type layout: BIDSLayout
     """
-    output_file = Path(layout.root).joinpath("dataset_description.json")
+    output_file = Path(layout.root) / "dataset_description.json"
 
     with open(output_file, "w") as ff:
         json.dump(layout.dataset_description, ff, indent=4)

bidsmreye/bidsmreye.py

@@ -29,6 +29,7 @@ def bidsmreye(
     bids_filter_file: str | None = None,
     non_linear_coreg: bool = False,
     log_level_name: str | None = None,
+    force: bool = False,
 ) -> None:
     bids_filter = None
     if bids_filter_file is not None and Path(bids_filter_file).is_file():
@@ -50,6 +51,7 @@ def bidsmreye(
         reset_database=reset_database,
         bids_filter=bids_filter,
         non_linear_coreg=non_linear_coreg,
+        force=force,
     )  # type: ignore
 
     if log_level_name is None:

bidsmreye/configuration.py

@@ -31,14 +31,14 @@ class Config:
     def _check_input_dir(self, attribute: str, value: Path) -> None:
         if not value.is_dir:  # type: ignore
             raise ValueError(
-                f"input_dir must be an existing directory:\n{value.resolve()}."
+                f"input_dir must be an existing directory:\n{value.absolute()}."
             )
 
-        if not value.joinpath("dataset_description.json").is_file():
+        if not (value / "dataset_description.json").is_file():
             raise ValueError(
                 f"""input_dir does not seem to be a valid BIDS dataset.
 No dataset_description.json found:
-\t{value.resolve()}."""
+\t{value.absolute()}."""
             )
 
     output_dir: Path = field(default=None, converter=Path)
@@ -55,6 +55,7 @@ def _check_input_dir(self, attribute: str, value: Path) -> None:
     debug: str | bool | None = field(kw_only=True, default=None)
     reset_database: bool = field(kw_only=True, default=False)
     non_linear_coreg: bool = field(kw_only=True, default=False)
+    force: bool = field(kw_only=True, default=False)
 
     has_GPU: bool = False
 
@@ -74,11 +75,11 @@ def __attrs_post_init__(self) -> None:
         if not self.bids_filter:
             self.bids_filter = get_bids_filter_config()
 
-        self.output_dir = self.output_dir.joinpath("bidsmreye")
+        self.output_dir = self.output_dir / "bidsmreye"
         if not self.output_dir:
             self.output_dir.mkdir(parents=True, exist_ok=True)
 
-        database_path = self.input_dir.joinpath("pybids_db")
+        database_path = self.input_dir / "pybids_db"
 
         layout_in = BIDSLayout(
             self.input_dir,
@@ -225,8 +226,8 @@ def get_config(config_file: Path | None = None, default: str = "") -> dict[str,
     :rtype: dict
     """
     if config_file is None or not Path(config_file).exists():
-        my_path = Path(__file__).resolve().parent.joinpath("config")
-        config_file = my_path.joinpath(default)
+        my_path = Path(__file__).absolute().parent / "config"
+        config_file = my_path / default
 
     if config_file is None or not Path(config_file).exists():
         raise FileNotFoundError(f"Config file {config_file} not found")

bidsmreye/download.py

@@ -33,13 +33,13 @@ def download(
         model_name = default_model()
     if isinstance(model_name, Path):
         assert model_name.is_file()
-        return model_name.resolve()
+        return model_name.absolute()
     if model_name not in available_models():
         warnings.warn(f"{model_name} is not a valid model name.", stacklevel=3)
         return None
 
     if output_dir is None:
-        output_dir = Path.cwd().joinpath("models")
+        output_dir = Path.cwd() / "models"
     if isinstance(output_dir, str):
         output_dir = Path(output_dir)
 
@@ -48,11 +48,11 @@ def download(
         base_url="https://osf.io/download/",
         registry=None,
     )
-    source = resources.files(bidsmreye).joinpath("models/registry.txt")
+    source = resources.files(bidsmreye) / "models" / "registry.txt"
     with resources.as_file(source) as registry_file:
         POOCH.load_registry(registry_file)
 
-    output_file = output_dir.joinpath(f"dataset_{model_name}")
+    output_file = output_dir / f"dataset_{model_name}"
 
     if not output_file.is_file():
         file_idx = available_models().index(model_name)

bidsmreye/generalize.py

@@ -1,4 +1,4 @@
-"""TODO."""
+"""Compute eyetracking movement from preprocessed extracted data."""
 
 from __future__ import annotations
 
@@ -29,6 +29,7 @@
     check_if_file_found,
     create_dir_for_file,
     move_file,
+    progress_bar,
     set_this_filter,
 )
 
@@ -130,9 +131,7 @@ def create_confounds_tsv(layout_out: BIDSLayout, file: str, subject_label: str)
     """
     confound_numpy = create_bidsname(layout_out, file, "confounds_numpy")
 
-    source_file = Path(layout_out.root).joinpath(
-        f"sub-{subject_label}", "results_tmp.npy"
-    )
+    source_file = Path(layout_out.root) / f"sub-{subject_label}" / "results_tmp.npy"
 
     move_file(
         source_file,
@@ -210,17 +209,21 @@ def generalize(cfg: Config) -> None:
     :param cfg: Configuration object
     :type cfg: Config
     """
-    log.info("GENERALIZING")
-    log.info(f"Using model: {cfg.model_weights_file}")
-
     layout_out = get_dataset_layout(cfg.output_dir)
     check_layout(cfg, layout_out)
 
     add_sidecar_in_root(layout_out)
 
     subjects = list_subjects(cfg, layout_out)
 
-    for subject_label in subjects:
-        process_subject(cfg, layout_out, subject_label)
+    text = "GENERALIZING"
+    with progress_bar(text=text) as progress:
+        subject_loop = progress.add_task(
+            description="processing subject", total=len(subjects)
+        )
+        log.info(f"Using model: {cfg.model_weights_file}")
+        for subject_label in subjects:
+            process_subject(cfg, layout_out, subject_label)
+            progress.update(subject_loop, advance=1)
 
     quality_control_output(cfg)

bidsmreye/methods.py

@@ -33,12 +33,12 @@ def methods(
         output_dir = Path(".")
     if isinstance(output_dir, str):
         output_dir = Path(output_dir)
-    output_dir = output_dir.joinpath("logs")
+    output_dir = output_dir / "logs"
 
-    output_file = output_dir.joinpath("CITATION.md")
+    output_file = output_dir / "CITATION.md"
     create_dir_for_file(output_file)
 
-    bib_file = str(Path(__file__).parent.joinpath("templates", "CITATION.bib"))
+    bib_file = str(Path(__file__).parent / "templates" / "CITATION.bib")
     shutil.copy(bib_file, output_dir)
 
     if not model_name:
@@ -54,7 +54,7 @@ def methods(
     if not is_known_models:
         warnings.warn(f"{model_name} is not a known model name.", stacklevel=3)
 
-    template_file = str(Path(__file__).parent.joinpath("templates", "CITATION.mustache"))
+    template_file = str(Path(__file__).parent / "templates" / "CITATION.mustache")
     with open(template_file) as template:
         output = chevron.render(
             template=template,