From de035b81e015c6fe802f029862ddbc205450504e Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 1 Aug 2024 16:34:25 +0200 Subject: [PATCH] speed up CLI and pin docker dependencies --- Dockerfile | 9 +- bidsmreye/bidsmreye.py | 18 +- requirements.txt | 573 +++++++++++++++++++++++++++++++++++++++++ tox.ini | 33 +++ 4 files changed, 624 insertions(+), 9 deletions(-) create mode 100644 requirements.txt create mode 100644 tox.ini diff --git a/Dockerfile b/Dockerfile index 2982add..d1a1fc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.9-bullseye +FROM python:3.11.9-slim-bullseye@sha256:8850f5e6e8da9081a6d156252a11161aa22f04d6ed1723c57ca2d5a5d48132bc ARG DEBIAN_FRONTEND="noninteractive" @@ -12,11 +12,12 @@ WORKDIR /home/neuro/bidsMReye COPY [".", "/home/neuro/bidsMReye"] RUN pip install --upgrade pip && \ - pip3 install -e . + pip3 install -r requirements.txt && \ + pip3 install . RUN bidsmreye_model -ENTRYPOINT [ "//home/neuro/bidsMReye/entrypoint.sh" ] +ENTRYPOINT [ "/home/neuro/bidsMReye/entrypoint.sh" ] COPY ["./docker/entrypoint.sh", \ - "//home/neuro/bidsMReye/entrypoint.sh"] + "/home/neuro/bidsMReye/entrypoint.sh"] RUN chmod +x /home/neuro/bidsMReye/entrypoint.sh diff --git a/bidsmreye/bidsmreye.py b/bidsmreye/bidsmreye.py index 6610cea..c120209 100755 --- a/bidsmreye/bidsmreye.py +++ b/bidsmreye/bidsmreye.py @@ -19,12 +19,7 @@ default_model, log_levels, ) -from bidsmreye.download import download -from bidsmreye.generalize import generalize from bidsmreye.logging import bidsmreye_log -from bidsmreye.prepare_data import prepare_data -from bidsmreye.quality_control import quality_control_input -from bidsmreye.visualize import group_report log = bidsmreye_log(name="bidsmreye") @@ -122,6 +117,8 @@ def bidsmreye( log.debug(f"Configuration:\n{cfg}") if action in {"all", "generalize"} and isinstance(cfg.model_weights_file, str): + from bidsmreye.download import download + cfg.model_weights_file = download(cfg.model_weights_file) dispatch(analysis_level=analysis_level, action=action, cfg=cfg) @@ -130,6 +127,8 @@ def bidsmreye( def dispatch(analysis_level: str, action: str, cfg: Config) -> None: if analysis_level == "group": if action == "qc": + from bidsmreye.visualize import group_report + group_report(cfg=cfg) else: log.error("Unknown group level action") @@ -137,13 +136,22 @@ def dispatch(analysis_level: str, action: str, cfg: Config) -> None: elif analysis_level == "participant": if action == "all": + from bidsmreye.generalize import generalize + from bidsmreye.prepare_data import prepare_data + prepare_data(cfg) generalize(cfg) elif action == "prepare": + from bidsmreye.prepare_data import prepare_data + prepare_data(cfg) elif action == "generalize": + from bidsmreye.generalize import generalize + generalize(cfg) elif action == "qc": + from bidsmreye.quality_control import quality_control_input + quality_control_input(cfg) else: log.error("Unknown participant level action") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d656165 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,573 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements.txt --strip-extras pyproject.toml +# +absl-py==2.1.0 + # via + # tensorboard + # tensorflow +antspyx==0.4.2 + # via deepmreye +anyio==4.4.0 + # via + # httpx + # jupyter-server +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +astor==0.8.1 + # via formulaic +asttokens==2.4.1 + # via stack-data +astunparse==1.6.3 + # via tensorflow +async-lru==2.0.4 + # via jupyterlab +attrs==23.2.0 + # via + # bidsmreye (pyproject.toml) + # jsonschema + # referencing +babel==2.15.0 + # via jupyterlab-server +beautifulsoup4==4.12.3 + # via nbconvert +bids-validator==1.14.6 + # via pybids +bleach==6.1.0 + # via nbconvert +cachetools==5.4.0 + # via google-auth +certifi==2024.7.4 + # via + # httpcore + # httpx + # requests +cffi==1.16.0 + # via argon2-cffi-bindings +charset-normalizer==3.3.2 + # via requests +chart-studio==1.1.0 + # via antspyx +chevron==0.14.0 + # via bidsmreye (pyproject.toml) +click==8.1.7 + # via pybids +comm==0.2.2 + # via + # ipykernel + # ipywidgets +contourpy==1.2.1 + # via matplotlib +cycler==0.12.1 + # via matplotlib +debugpy==1.8.2 + # via ipykernel +decorator==5.1.1 + # via ipython +deepmreye==0.2.1 + # via bidsmreye (pyproject.toml) +defusedxml==0.7.1 + # via nbconvert +docopt==0.6.2 + # via num2words +executing==2.0.1 + # via stack-data +fastjsonschema==2.20.0 + # via nbformat +flatbuffers==24.3.25 + # via tensorflow +fonttools==4.53.1 + # via matplotlib +formulaic==0.5.2 + # via pybids +fqdn==1.5.1 + # via jsonschema +fsspec==2024.6.1 + # via universal-pathlib +gast==0.6.0 + # via tensorflow +google-auth==2.32.0 + # via + # google-auth-oauthlib + # tensorboard +google-auth-oauthlib==1.2.1 + # via tensorboard +google-pasta==0.2.0 + # via tensorflow +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.65.2 + # via + # tensorboard + # tensorflow +h11==0.14.0 + # via httpcore +h5py==3.11.0 + # via tensorflow +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via jupyterlab +idna==3.7 + # via + # anyio + # httpx + # jsonschema + # requests +imageio==2.34.2 + # via scikit-image +interface-meta==1.3.0 + # via formulaic +ipykernel==6.29.5 + # via + # jupyter + # jupyter-console + # jupyterlab + # qtconsole +ipython==8.26.0 + # via + # ipykernel + # ipywidgets + # jupyter-console +ipywidgets==8.1.3 + # via jupyter +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython +jinja2==3.1.4 + # via + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert +joblib==1.4.2 + # via scikit-learn +json5==0.9.25 + # via jupyterlab-server +jsonpointer==3.0.0 + # via jsonschema +jsonschema==4.23.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter==1.0.0 + # via deepmreye +jupyter-client==8.6.2 + # via + # ipykernel + # jupyter-console + # jupyter-server + # nbclient + # qtconsole +jupyter-console==6.6.3 + # via jupyter +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat + # qtconsole +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.2 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.4 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.3 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.11 + # via ipywidgets +kaleido==0.2.1 + # via bidsmreye (pyproject.toml) +keras==2.15.0 + # via + # bidsmreye (pyproject.toml) + # tensorflow +kiwisolver==1.4.5 + # via matplotlib +lazy-loader==0.4 + # via scikit-image +libclang==18.1.1 + # via tensorflow +markdown==3.6 + # via tensorboard +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # nbconvert + # werkzeug +matplotlib==3.9.1 + # via + # antspyx + # deepmreye +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.2 + # via nbconvert +ml-dtypes==0.3.2 + # via tensorflow +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via + # jupyter + # jupyter-server +nbformat==5.10.4 + # via + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +networkx==3.3 + # via scikit-image +nibabel==5.2.1 + # via + # antspyx + # pybids +notebook==7.2.1 + # via jupyter +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +num2words==0.5.13 + # via pybids +numpy==1.26.4 + # via + # antspyx + # contourpy + # deepmreye + # formulaic + # h5py + # imageio + # matplotlib + # ml-dtypes + # nibabel + # opt-einsum + # pandas + # patsy + # pybids + # scikit-image + # scikit-learn + # scipy + # statsmodels + # tensorboard + # tensorflow + # tifffile +oauthlib==3.2.2 + # via requests-oauthlib +opt-einsum==3.3.0 + # via tensorflow +overrides==7.7.0 + # via jupyter-server +packaging==24.1 + # via + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # lazy-loader + # matplotlib + # nbconvert + # nibabel + # plotly + # pooch + # qtconsole + # qtpy + # scikit-image + # statsmodels + # tensorflow +pandas==2.2.2 + # via + # antspyx + # deepmreye + # formulaic + # pybids + # statsmodels +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +patsy==0.5.6 + # via statsmodels +pexpect==4.9.0 + # via ipython +pillow==10.4.0 + # via + # antspyx + # imageio + # matplotlib + # scikit-image +platformdirs==4.2.2 + # via + # jupyter-core + # pooch +plotly==5.23.0 + # via + # chart-studio + # deepmreye +pooch==1.8.2 + # via bidsmreye (pyproject.toml) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.47 + # via + # ipython + # jupyter-console +protobuf==4.25.4 + # via + # tensorboard + # tensorflow +psutil==6.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybids==0.17.0 + # via bidsmreye (pyproject.toml) +pycparser==2.22 + # via cffi +pygments==2.18.0 + # via + # ipython + # jupyter-console + # nbconvert + # qtconsole + # rich +pyparsing==3.1.2 + # via matplotlib +python-dateutil==2.9.0.post0 + # via + # arrow + # jupyter-client + # matplotlib + # pandas +python-json-logger==2.0.7 + # via jupyter-events +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # antspyx + # jupyter-events +pyzmq==26.0.3 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # qtconsole +qtconsole==5.5.2 + # via jupyter +qtpy==2.4.1 + # via qtconsole +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +requests==2.32.3 + # via + # chart-studio + # jupyterlab-server + # pooch + # requests-oauthlib + # tensorboard +requests-oauthlib==2.0.0 + # via google-auth-oauthlib +retrying==1.3.4 + # via chart-studio +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rich==13.7.1 + # via bidsmreye (pyproject.toml) +rpds-py==0.19.1 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +scikit-image==0.24.0 + # via antspyx +scikit-learn==1.5.1 + # via + # antspyx + # deepmreye +scipy==1.14.0 + # via + # antspyx + # deepmreye + # formulaic + # pybids + # scikit-image + # scikit-learn + # statsmodels +send2trash==1.8.3 + # via jupyter-server +six==1.16.0 + # via + # asttokens + # astunparse + # bleach + # chart-studio + # google-pasta + # patsy + # python-dateutil + # retrying + # rfc3339-validator + # tensorboard + # tensorflow +sniffio==1.3.1 + # via + # anyio + # httpx +soupsieve==2.5 + # via beautifulsoup4 +sqlalchemy==2.0.31 + # via pybids +stack-data==0.6.3 + # via ipython +statsmodels==0.14.2 + # via antspyx +tenacity==9.0.0 + # via plotly +tensorboard==2.15.2 + # via tensorflow +tensorboard-data-server==0.7.2 + # via tensorboard +tensorflow==2.15.1 + # via deepmreye +tensorflow-estimator==2.15.0 + # via tensorflow +tensorflow-io-gcs-filesystem==0.37.1 + # via tensorflow +termcolor==2.4.0 + # via tensorflow +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +threadpoolctl==3.5.0 + # via scikit-learn +tifffile==2024.7.24 + # via scikit-image +tinycss2==1.3.0 + # via nbconvert +tornado==6.4.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +tqdm==4.66.4 + # via bidsmreye (pyproject.toml) +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-console + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # qtconsole +types-python-dateutil==2.9.0.20240316 + # via arrow +typing-extensions==4.12.2 + # via + # formulaic + # ipython + # sqlalchemy + # tensorflow +tzdata==2024.1 + # via pandas +universal-pathlib==0.2.2 + # via pybids +uri-template==1.3.0 + # via jsonschema +urllib3==2.2.2 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==24.6.0 + # via + # antspyx + # jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via jupyter-server +werkzeug==3.0.3 + # via tensorboard +wheel==0.43.0 + # via astunparse +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.14.1 + # via + # formulaic + # tensorflow + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..44c7f61 --- /dev/null +++ b/tox.ini @@ -0,0 +1,33 @@ +; See https://tox.wiki/en +[tox] +requires = + tox>=4 +; run lint by default when just calling "tox" +env_list = lint + +; ENVIRONMENTS +; ------------ +[style] +description = common environment for style checkers (rely on pre-commit hooks) +skip_install = true +deps = + pre-commit + +; COMMANDS +; -------- +[testenv:lint] +description = install pre-commit hooks and run all linters and formatters +skip_install = true +deps = + {[style]deps} +commands = + pre-commit install + pre-commit run --all-files --show-diff-on-failure {posargs:} + +[testenv:update_dependencies] +description = update requirements.txt +skip_install = true +deps = + pip-tools +commands = + pip-compile --strip-extras -o requirements.txt pyproject.toml{posargs:}