Merge branch 'main' into jungfrau_comissioning

.coveragerc

@@ -0,0 +1,18 @@
+[run]
+omit = 
+ */test_*
+ **/conftest.py
+data_file = /tmp/hyperion.coverage
+
+[report]
+exclude_also = 
+ if TYPE_CHECKING:
+ def __repr__
+ raise NotImplementedError
+ @(abc\.)?abstractmethod
+
+[paths]
+# Tests are run from installed location, map back to the src directory
+source =
+ src
+ **/site-packages/

.devcontainer/Dockerfile

@@ -3,7 +3,7 @@
 # The devcontainer should use the build target and run as root with podman
 # or docker with user namespaces.
 #
-FROM python:3.11 as build
+FROM python:3.11 AS build
 
 ARG PIP_OPTIONS
 
@@ -24,7 +24,7 @@ WORKDIR /context
 # install python package into /venv
 RUN pip install ${PIP_OPTIONS}
 
-FROM python:3.11-slim as runtime
+FROM python:3.11-slim AS runtime
 
 # Add apt-get system dependecies for runtime here if needed
 

.github/CONTRIBUTING.md

@@ -1,10 +1,40 @@
-# Contribute to the project
+# How to contribute to mx-bluesky
 
-Contributions and issues are most welcome! All issues and pull requests are
-handled through [GitHub](https://github.com/DiamondLightSource/mx-bluesky/issues). Also, please check for any existing issues before
-filing a new one. If you have a great idea but it involves big changes, please
-file a ticket before making a pull request! We want to make sure you don't spend
-your time coding something that might not fit the scope of the project.
+We should prioritise working, agile code over strict standards, particularly for contributors that are not full time developers. The standards below are thus guidelines that developers should try to follow as much as possible (apart from when something is specified in bold, which is required).
+
+Contributions and issues are most welcome! All issues and pull requests are handled through [GitHub](https://github.com/DiamondLightSource/mx-bluesky/issues). Also, please check for any existing issues before filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project.
+
+## General Workflow
+
+1. An issue is created for the work. This issue should describe in as much detail as possible the work that needs to be done. Anyone is free to make a ticket and/or comment on one.
+2. If a developer is going to do the work they assign themselves to the issue.
+3. The developer makes a new branch with the format `issue_short_description` e.g. `122_create_a_contributing_file`. (External developers are also welcome to make forks)
+4. The developer does the work on this branch, adding their work in small commits. Commit messages should be informative and prefixed with the issue number e.g. `(#122) Added contributing file`.
+5. The developer submits a PR for the work. In the pull request should start with `Fixes #issue_num` e.g. `Fixes #122`, this will ensure the issue is automatically closed when the PR is merged. The developer should also add some background on how the reviewer might test the change.
+6. If the developer has a particular person in mind to review the work they should assign that person to the PR as a reviewer.
+7. The reviewer and developer go back and forth on the code until the reviewer approves it. (See "Reviewing Work" below)
+8. Once the work is approved the original developer merges it.
+
+**Work should not be done without an associated ticket describing what the work is**
+
+## Reviewing Work
+
+**Work must be reviewed by another developer before it can be merged**. Remember that work is reviewed for a number of reasons:
+
+- In order to maintain quality and avoid errors
+- Help people learn
+
+It is not a judgement on someone's technical abilities so be kind.
+
+It is suggested that the reviewer prefixes comments with something like the following:
+
+- **must**: A change that must be made before merging e.g. it will break something if not made
+- **should/sugg**: A change that should be made e.g. definitely improves code quality but does not neccessarily break anything
+- **nit**: A minor suggestion that the reviewer would like to see but is happy to leave as is e.g. rename a variable to something
+
+Developers are welcome to ignore **nit** comments if they wish and can choose not to do **should** comments but the must give a reason as to why they disagree with the change.
+
+For minor changes to code reviewers are welcome to make the changes themselves but in this case the original developer should then recheck what the reviewer has done.
 
 ## Issue or Discussion?
 

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -1,8 +1,13 @@
 Fixes #ISSUE
 
+Link to dodal PR (if required): #XXX
+(remember to update `setup.cfg` with the dodal commit tag if you need it for tests to pass!)
+
 ### Instructions to reviewer on how to test:
+
 1. Do thing x
 2. Confirm thing y happens
 
 ### Checks for reviewer
+
 - [ ] Would the PR title make sense to a user on a set of release notes

.github/actions/verify-nexus/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/githubgphl/imginfo:main
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/verify-nexus/action.yml

@@ -0,0 +1,18 @@
+name: "Verify nexus"
+description: "Verify nexus files against imginfo"
+inputs:
+ filename:
+ description: "nexus file to verify"
+ required: true
+outputs:
+ imginfo_stdout:
+ description: "imginfo output"
+ imginfo_stderr:
+ description: "imginfo error output"
+ imginfo_exit_code:
+ description: "imginfo exit code"
+runs:
+ using: "docker"
+ image: "Dockerfile"
+ args:
+ - ${{ inputs.filename }}

.github/actions/verify-nexus/entrypoint.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -l
+echo "Contents of /github/workspace:"
+ls /github/workspace
+echo "current dir: $(pwd)"
+echo "Running imginfo on $1"
+/imginfo/imginfo $1 >> imginfo_out_file 2>> imginfo_err_file
+{ echo "imginfo_output<<EOF"
+ cat imginfo_out_file
+ echo EOF
+} >> "$GITHUB_OUTPUT"
+{ echo "imginfo_output<<EOF"
+ cat imginfo_err_file
+ echo EOF
+} >> "$GITHUB_OUTPUT"
+echo "imginfo_exit_code=$?" >> "$GITHUB_OUTPUT"
+echo "------------- IMGINFO STDOUT -------------"
+cat imginfo_out_file
+echo "------------- IMGINFO STDERR -------------"
+cat imginfo_err_file
+echo "------------------------------------------"
+if [ -s imginfo_err_file ]; then
+ echo "ERRORS IN IMGINFO PROCESSING!"
+ exit 1
+fi
+echo "VALIDATED SUCCESSFULLY!"

.github/workflows/_test.yml

@@ -51,7 +51,7 @@ jobs:
  pip-install: ".[dev]"
 
  - name: Run tests
- run: tox -e tests
+ run: tox -e tests -- -m "not dlstbx"
 
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v4

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
  strategy:
  matrix:
  runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest
- python-version: ["3.10", "3.11"] # , "3.12"] # add 3.12 when p4p #145 is fixed
+ python-version: ["3.11"] # , "3.12"] # add 3.12 when p4p #145 is fixed
  include:
  # Include one that runs in the dev environment
  - runs-on: "ubuntu-latest"

.gitignore

@@ -1,13 +1,20 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
+*$py.class
 
 # C extensions
 *.so
 
 # Distribution / packaging
 .Python
 env/
+# Output
+*.png
+!docs/*.png
+
+# Distribution / packaging
+.Python
 build/
 develop-eggs/
 dist/
@@ -19,10 +26,14 @@ lib64/
 parts/
 sdist/
 var/
+**/_version.py
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-**/_version.py
+MANIFEST
 
 # PyInstaller
 # Usually these files are written by a python script from a template
@@ -37,31 +48,102 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
+.nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 cov.xml
+*.cover
+*.py,cover
+.hypothesis/
 .pytest_cache/
-.mypy_cache/
+cov.xml
 
 # Translations
 *.mo
 *.pot
 
 # Django stuff:
 *.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
 
 # Sphinx documentation
 docs/_build/
 
 # PyBuilder
 target/
 
-# likely venv names
-.venv*
-venv*
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.venv*/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# gedit backup files
+*~
+
+# Log files
+**/tmp
 
 # further build artifacts
 lockfiles/
@@ -87,3 +169,5 @@ edm_serial/
 **/parameters/fixed_target/cs/fiducial_*.txt
 **/parameters/fixed_target/*.json
 **/parameters/fixed_target/*/*.map
+# idea project files
+.idea/

.pre-commit-config.yaml

@@ -3,9 +3,13 @@ repos:
  rev: v4.5.0
  hooks:
  - id: check-added-large-files
+ args: ["--maxkb=500"]
  - id: check-yaml
+ args: ["--allow-multiple-documents"]
  - id: check-merge-conflict
  - id: end-of-file-fixer
+ - id: no-commit-to-branch
+ name: "Don't commit to 'main'"
 
  - repo: local
  hooks:
@@ -29,5 +33,5 @@ repos:
  hooks:
  - id: mypy
  files: 'src/.*\.py$'
- additional_dependencies: [types-requests, types-redis]
+ additional_dependencies: [types-requests, types-redis, pydantic]
  args: ["--ignore-missing-imports", "--no-strict-optional"]

.vscode/extensions.json

@@ -1,5 +1,10 @@
 {
  "recommendations": [
  "ms-vscode-remote.remote-containers",
+ "ms-python.python",
+ "tamasfe.even-better-toml",
+ "redhat.vscode-yaml",
+ "ryanluker.vscode-coverage-gutters",
+ "charliermarsh.Ruff"
  ]
 }

Dockerfile

@@ -1,7 +1,7 @@
 # The devcontainer should use the developer target and run as root with podman
 # or docker with user namespaces.
 ARG PYTHON_VERSION=3.11
-FROM python:${PYTHON_VERSION} as developer
+FROM python:${PYTHON_VERSION} AS developer
 
 # Add any system dependencies for the developer/build environment here
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -13,13 +13,13 @@ RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
 # The build stage installs the context into the venv
-FROM developer as build
+FROM developer AS build
 COPY . /context
 WORKDIR /context
 RUN touch dev-requirements.txt && pip install -c dev-requirements.txt .
 
 # The runtime stage copies the built venv into a slim runtime container
-FROM python:${PYTHON_VERSION}-slim as runtime
+FROM python:${PYTHON_VERSION}-slim AS runtime
 # Add apt-get system dependecies for runtime here if needed
 COPY --from=build /venv/ /venv/
 ENV PATH=/venv/bin:$PATH

codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+ status:
+ project:
+ default:
+ target: 85% # the required coverage value
+ threshold: 1% # the leniency in hitting the target
+
+ignore:
+ - "**/conftest.py"