Merge pull request #188 from GilesStrong/refactor_poetry

Refactor poetry
GilesStrong · Feb 17, 2024 · 5ecc38f · 5ecc38f
2 parents e03bae1 + e1f7fd4
commit 5ecc38f
Show file tree

Hide file tree

Showing 109 changed files with 6,911 additions and 1,834 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -13,8 +13,8 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-      - uses: psf/black@24.1.1
+          python-version: "3.10"
+      - uses: psf/black@24.2.0
         with:
           args: ". --check"
 
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - uses: py-actions/flake8@v1
 
   isort:
@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - uses: isort/isort-action@master
 
   mypy:
@@ -42,10 +42,10 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: "3.8"
+        python-version: "3.10"
     - name: Install Dependencies
       run: |
-        pip install -r requirements.txt
-        pip install -e .
+        curl -sSL https://install.python-poetry.org | python3 -
+        poetry install
     - name: mypy
       run: ./run-mypy
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -17,8 +17,7 @@ jobs:
           python-version: "3.8"
       - name: Install Dependencies
         run: |
-          pip install -r requirements.txt
-          pip install -e .
-          # pip install 
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
       - name: pytest
-        run: pytest tests
+        run: poetry run pytest tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,18 +1,26 @@
 repos:
+  - repo: https://github.com/python-poetry/poetry
+    rev: '1.7.0'
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+      - id: poetry-export
+        args: ["-f", "requirements.txt", "-o", "requirements.txt", "--with", "docs", "--with", "dev"]
+
   - repo: https://github.com/psf/black
-    rev: 24.1.1
+    rev: 24.2.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/pycqa/flake8.git
-    rev: 3.9.1
+    rev: 7.0.0
     hooks:
     - id: flake8
       args: [--ignore=E741 W503 E203 E501 C901]
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -0,0 +1,15 @@
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.10
+  install:
+    - requirements: requirements.txt
diff --git a/README.md b/README.md
@@ -1,83 +1,111 @@
-[![CI-tests](https://github.com/GilesStrong/mode_muon_tomography/actions/workflows/tests.yml/badge.svg)](https://github.com/GilesStrong/mode_muon_tomography/actions)
-[![CI-lints](https://github.com/GilesStrong/mode_muon_tomography/actions/workflows/linting.yml/badge.svg)](https://github.com/GilesStrong/mode_muon_tomography/actions)
+[![CI-tests](https://github.com/GilesStrong/tomopt/actions/workflows/tests.yml/badge.svg)](https://github.com/GilesStrong/tomopt/actions)
+[![CI-lints](https://github.com/GilesStrong/tomopt/actions/workflows/linting.yml/badge.svg)](https://github.com/GilesStrong/tomopt/actions)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![pypi tomopt version](https://img.shields.io/pypi/v/tomopt.svg)](https://pypi.python.org/pypi/tomopt)
+[![tomopt python compatibility](https://img.shields.io/pypi/pyversions/tomopt.svg)](https://pypi.python.org/pypi/tomopt) [![tomopt license](https://img.shields.io/pypi/l/tomopt.svg)](https://pypi.python.org/pypi/tomopt)
+[![Documentation Status](https://readthedocs.org/projects/tomopt/badge/?version=stable)](https://tomopt.readthedocs.io/en/stable/?badge=stable)
+<!-- [![DOI](https://zenodo.org/badge/163840693.svg)](https://zenodo.org/badge/latestdoi/163840693) -->
 
 # TomOpt: Differential Muon Tomography Optimisation
 
-## Installation
+This repo provides a library for the differential optimisation of scattering muon tomography systems. For an overview, please read our first publication [here](https://arxiv.org/abs/2309.14027).
 
+As a disclaimer, this is a library designed to be extended by users for their specific tasks: e.g. passive volume definition, inference methods, and loss functions. Additionally, optimisation in TomOpt can be unstable, and requires careful tuning by users. This is to say that it is not a polished product for the general public, but rather fellow researchers in the field of optimisation and muon tomography.
 
-N.B. Whilst the repo is private, you will need to make sure that you have registered the public ssh key of your computer/instance with your [GitHub profile](https://github.com/settings/keys). Follow [these instructions](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh/checking-for-existing-ssh-keys) to check for existing keys or [these](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent) to generate a new key. After that follow [this](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account) to associate the key.
+If you are interested in using this library seriously, please contact us;  we would love to here if you have a specific use-case you wish to work on.
 
-Checkout package:
 
-```
-git clone git@github.com:GilesStrong/mode_muon_tomography.git
-cd mode_muon_tomography
-```
+## Overview
 
-*N.B.* For GPU usage, it is recommended to manually setup conda and install PyTorch according to system, e.g.:
-```
-conda activate root
-conda install nb_conda_kernels
-conda create -n tomopt python=3.8 pip ipykernel
-conda activate tomopt
-pip install torch==1.8.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
-pip install -r requirements.txt
-```
+The TomOpt library is designed to optimise the design of a muon tomography system. The detector system is defined by a set of parameters, which are used to define the geometry of the detectors. The optimisation is performed by minimising a loss function, which is defined by the user. The loss function is evaluated by simulating the muon scattering process through the detector system and passive volumes. The information recorded by the detectors is then passed through an inference system to arrive at a set of task-specific parameters. These are then compared to the ground truth, and the loss is calculated. The gradient of the loss with respect to the detector parameters is then used to update the detector parameters.
 
-Minimum python version is 3.8. Recommend creating a virtual environment, e.g. assuming your are using [Anaconda](https://www.anaconda.com/products/individual)/[Miniconda](https://docs.conda.io/en/latest/miniconda.html) (if installing conda for the first time, remember to restart the shell before attemting to use conda, and that by default conda writes the setup commands to `.bashrc`):
+The TomOpt library is designed to be modular, and to allow for the easy addition of new inference systems, loss functions, and passive volume definitions. The library is also designed to be easily extensible to new optimisation algorithms, and to allow for the easy addition of new constraints on the detector parameters.
 
-```
-conda activate root
-conda install nb_conda_kernels
-conda env create -f environment.yml
-conda activate tomopt
-```
+TomOpt consists of several submodules:
 
-Otherwise set up a suitable environment using your python distribution of choice using the contents of `environment.yml`. Remember to activate the correct environment each time, via e.g. `conda activate tomopt`.
+- benchmarks: and ongoing collection of concrete implementations and task-specific extensions that are used to test the library on real-world problems.
+- inference: provides classes that infer muon-trajectories from detector data, and infer properties of passive volumes from muon-trajectories.
+- muon: provides classes for handling muon batches, and generating muons from literature flux-distributions
+- optimisation: provides classes for handling the optimisation of detector parameters, and an extensive callback system to modify the optimisation process.
+- plotting: various plotting utilities for visualising the detector system, the optimisation process, and results
+- volume: contains classes for defining passive volumes and detector systems
+- core: core objects used by all parts of the code
+- utils: various utilities used throughout the codebase
 
-Install package and dependencies
-```
-pip install -r requirements.txt
-pip install -e .
-```
+## Installation
 
-Install git-hooks:
+### As a dependency
 
-```
-pre-commit install
+For dependency usage, `tomopt` can be installed via e.g. 
+
+```bash
+pip install tomopt
 ```
 
-### Windows usage
+### For development
 
-Apparently when using Windows, the environment must also be activated within ipython using:
+Check out the repo locally:
 
+```bash
+git clone git@github.com:GilesStrong/tomopt.git
+cd tomopt
 ```
-python -m ipykernel install --user --name tomopt --display-name "Python (tomopt)" 
+
+For development usage, we use [`poetry`](https://python-poetry.org/docs/#installing-with-the-official-installer) to handle dependency installation.
+Poetry can be installed via, e.g.
+
+```bash
+curl -sSL https://install.python-poetry.org | python3 -
+poetry self update
 ```
 
-## Testing
+and ensuring that `poetry` is available in your `$PATH`
 
-Testing is handled by `pytest` and is set up to run during pull requests. Tests can be manually ran locally via:
+TomOpt requires `python >= 3.10`. This can be installed via e.g. [`pyenv`](https://github.com/pyenv/pyenv):
 
-```
-pytest tests/
+```bash
+curl https://pyenv.run | bash
+pyenv update
+pyenv install 3.10
+pyenv local 3.10
 ```
 
-to run all tests, or, e.g.:
+Install the dependencies:
 
+```bash
+poetry install
+poetry self add poetry-plugin-export
+poetry config warnings.export false
+poetry run pre-commit install
 ```
-pytest tests/test_muon.py
+
+Finally, make sure everything is working as expected by running the tests:
+
+```bash
+poetry run pytest tests
 ```
 
+For those unfamiliar with `poetry`, basically just prepend commands with `poetry run` to use the stuff installed within the local environment, e.g. `poetry run jupyter notebook` to start a jupyter notebook server. This local environment is basically a python virtual environment. To correctly set up the interpreter in your IDE, use `poetry run which python` to see the path to the correct python executable.
+
+## Examples
+
+A few examples are included to introduce users and developers to the TomOpt library. These take the form of Jupyter notebooks. In `examples/getting_started` there are four ordered notebooks:
+
+- `00_Hello_World.ipynb` aims to show the user the high-level classes in TomOpt and the general workflow.
+- `01_Indepth_tutorial_single_cycle.ipynb` aims to show developers what is going on in a single update iteration.
+- `02_Indepth_tutotial_optimisation_and_callbacks.ipynb` aims to show users and developers the workings of the callback system in TomOpt
+- `03_fixed_budget_mode.ipynb` aims to show users and developers how to optimise such that the detector maintains a constant cost.
+
+In `examples/benchmarks` there is a single notebook that covers the optimisation performed in our first publication, in which we optimised a detector to estimate the fill-height of a ladle furnace at a steel plant. As a disclaimer, this notebook may not fully reproduce our result, and is designed to be used in an interactive manner by experienced users.
+
+
 ### Running notebooks in a remote cluster
 
 If you want to run notebooks on a remote cluster but access them on the browser of your local machine, you need to forward the notebook server from the cluster to your local machine.
 
 On the cluster, run:
 ```
-jupyter notebook --no-browser --port=8889
+poetry run jupyter notebook --no-browser --port=8889
 ```
 
 On your local computer, you need to set up a forwarding that picks the flux of data from the cluster via a local port, and makes it available on another port as if the server was in the local machine:
@@ -98,12 +126,12 @@ ssh -N -f -L localhost:8890:localhost:8888 username@gateway_hostname # on your l
 
 ## External repos
 
-N.B. Not currently public
+N.B. Most are not currently public
 
 - [tomo_deepinfer](https://github.com/GilesStrong/mode_muon_tomo_inference) (contact @GilesStrong for access) separately handles training and model definition of GNNs used for passive volume inference. Models are exported as JIT-traced scripts, and loaded here using the `DeepVolumeInferer` class. We still need to find a good way to host the trained models for easy download.
 - [mode_muon_tomography_scattering](https://github.com/GilesStrong/mode_muon_tomography_scattering)  (contact @GilesStrong for access) separately handles conversion of PGeant model from root to HDF5, and Geant validation data from csv to HDF5.
 - [tomopt_sphinx_theme](https://github.com/GilesStrong/tomopt_sphinx_theme) public. Controls the appearance of the docs.
 
 ## Authors
 
-The TomOpt project, and its continued development and support, is the result of the combined work of many people, whose contributions are summarised in [the author list](https://github.com/GilesStrong/mode_muon_tomography/blob/main/AUTHORS.md)
+The TomOpt project, and its continued development and support, is the result of the combined work of many people, whose contributions are summarised in [the author list](https://github.com/GilesStrong/tomopt/blob/main/AUTHORS.md)
diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle
diff --git a/docs/_build/doctrees/index.doctree b/docs/_build/doctrees/index.doctree
diff --git a/docs/_build/doctrees/installation.doctree b/docs/_build/doctrees/installation.doctree
diff --git a/docs/_build/doctrees/introduction.doctree b/docs/_build/doctrees/introduction.doctree
diff --git a/docs/_build/doctrees/modules.doctree b/docs/_build/doctrees/modules.doctree
diff --git a/docs/_build/doctrees/tomopt.benchmarks.doctree b/docs/_build/doctrees/tomopt.benchmarks.doctree
diff --git a/docs/_build/doctrees/tomopt.benchmarks.ladle_furnace.doctree b/docs/_build/doctrees/tomopt.benchmarks.ladle_furnace.doctree
diff --git a/docs/_build/doctrees/tomopt.benchmarks.small_walls.doctree b/docs/_build/doctrees/tomopt.benchmarks.small_walls.doctree
diff --git a/docs/_build/doctrees/tomopt.benchmarks.u_lorry.doctree b/docs/_build/doctrees/tomopt.benchmarks.u_lorry.doctree
diff --git a/docs/_build/doctrees/tomopt.doctree b/docs/_build/doctrees/tomopt.doctree
diff --git a/docs/_build/doctrees/tomopt.inference.doctree b/docs/_build/doctrees/tomopt.inference.doctree
diff --git a/docs/_build/doctrees/tomopt.muon.doctree b/docs/_build/doctrees/tomopt.muon.doctree
diff --git a/docs/_build/doctrees/tomopt.optimisation.callbacks.doctree b/docs/_build/doctrees/tomopt.optimisation.callbacks.doctree
diff --git a/docs/_build/doctrees/tomopt.optimisation.data.doctree b/docs/_build/doctrees/tomopt.optimisation.data.doctree
diff --git a/docs/_build/doctrees/tomopt.optimisation.doctree b/docs/_build/doctrees/tomopt.optimisation.doctree
diff --git a/docs/_build/doctrees/tomopt.optimisation.loss.doctree b/docs/_build/doctrees/tomopt.optimisation.loss.doctree
diff --git a/docs/_build/doctrees/tomopt.optimisation.wrapper.doctree b/docs/_build/doctrees/tomopt.optimisation.wrapper.doctree
diff --git a/docs/_build/doctrees/tomopt.plotting.doctree b/docs/_build/doctrees/tomopt.plotting.doctree
diff --git a/docs/_build/doctrees/tomopt.volume.doctree b/docs/_build/doctrees/tomopt.volume.doctree
diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: a1d4bf3194fba8eeb8c569809446c981
+config: 93fb6211ea9420d8aaccbcb5bb4f12fc
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/_build/html/_modules/index.html b/docs/_build/html/_modules/index.html
@@ -14,8 +14,6 @@
 
 
 
-    <link rel="shortcut icon" href="../_static/tomopt_logo.png"/>
-
 
 
 
@@ -153,7 +151,7 @@
 
               <ul>
 <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../installation.html#testing">Testing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html#examples">Examples</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../installation.html#external-repos">External repos</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../installation.html#authors">Authors</a></li>
 </ul>
@@ -314,9 +312,6 @@ <h1>All modules for which code is available</h1>
 
        <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
          <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
-         <script src="../_static/jquery.js"></script>
-         <script src="../_static/underscore.js"></script>
-         <script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
          <script src="../_static/doctools.js"></script>
          <script src="../_static/sphinx_highlight.js"></script>
 

diff --git a/docs/_build/html/_modules/tomopt/benchmarks/ladle_furnace/data.html b/docs/_build/html/_modules/tomopt/benchmarks/ladle_furnace/data.html
@@ -14,8 +14,6 @@
 
 
 
-    <link rel="shortcut icon" href="../../../../_static/tomopt_logo.png"/>
-
 
 
 
@@ -249,7 +247,7 @@ <h1>Source code for tomopt.benchmarks.ladle_furnace.data</h1><div class="highlig
 
 
 <div class="viewcode-block" id="LadleFurnacePassiveGenerator"><a class="viewcode-back" href="../../../../tomopt.benchmarks.ladle_furnace.html#tomopt.benchmarks.ladle_furnace.data.LadleFurnacePassiveGenerator">[docs]</a><span class="k">class</span> <span class="nc">LadleFurnacePassiveGenerator</span><span class="p">(</span><span class="n">AbsPassiveGenerator</span><span class="p">):</span>
-    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Research tested only: no unit tests</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
@@ -344,9 +342,6 @@ <h1>Source code for tomopt.benchmarks.ladle_furnace.data</h1><div class="highlig
 
        <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
          <script data-url_root="../../../../" id="documentation_options" src="../../../../_static/documentation_options.js"></script>
-         <script src="../../../../_static/jquery.js"></script>
-         <script src="../../../../_static/underscore.js"></script>
-         <script src="../../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
          <script src="../../../../_static/doctools.js"></script>
          <script src="../../../../_static/sphinx_highlight.js"></script>