diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ba7a552e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# List of folders and files to be excluded from the docker image +.devcontainer +.github +.pytest_cache +.ruff_cache +**/__pycache__/ + +# virtualenv stuff - this gets built by the docker script +.venv +activate + +tmp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00cc5a577..3f2001b79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,6 @@ jobs: matrix: runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest python-version: ["3.11"] # , "3.12"] # add 3.12 when p4p #145 is fixed - include: - # Include one that runs in the dev environment - - runs-on: "ubuntu-latest" - python-version: "dev" fail-fast: false uses: ./.github/workflows/_test.yml with: diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml new file mode 100644 index 000000000..7984c26a2 --- /dev/null +++ b/.github/workflows/publish_docker_image.yml @@ -0,0 +1,47 @@ +name: Publish Docker Image +on: + release: + types: [published] + # Allow the workflow to be triggered manually + workflow_dispatch: +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} +jobs: + build_and_push_image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + # v4.1.7 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + - name: Log in to GHCR + # v3.3.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + # v5.5.1 + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + # v6.5.0 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Generate artifact attestation + # v1.4.0 + uses: actions/attest-build-provenance@210c1913531870065f03ce1f9440dd87bc0938cd + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64fb0b4f8..2822b0802 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,7 @@ repos: args: ["--maxkb=500"] - id: check-yaml args: ["--allow-multiple-documents"] + exclude: ^helmchart/ - id: check-merge-conflict - id: end-of-file-fixer - id: no-commit-to-branch diff --git a/Dockerfile.release b/Dockerfile.release new file mode 100644 index 000000000..2d9deb56b --- /dev/null +++ b/Dockerfile.release @@ -0,0 +1,38 @@ +FROM python:3.11 AS build + +RUN pip install setuptools_scm + +# Copy the pyproject.toml and install dependencies for better caching when developing +# & rerunning deployment scripts +COPY pyproject.toml /app/hyperion/ +WORKDIR "/app/hyperion" +RUN mkdir -p src/mx_bluesky + +# This enables us to cache the pip install without needing _version.py +# see https://setuptools-scm.readthedocs.io/en/latest/usage/ +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=1.0.0 pip install \ + --no-cache-dir --no-compile -e . + +# Check out and install dodal locally with no dependencies as this may be a different version to what +# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the +# running container +RUN mkdir ../dodal && \ +git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \ +pip install --no-cache-dir --no-compile --no-deps -e ../dodal + +# +# Everything above this line should be in the image cache unless pyproject.toml changes +# +ADD .git /app/hyperion/.git +# Restore the repository at the current commit instead of copying, to exclude uncommitted changes +# This is so that if you build a developer image from this dockerfile then _version.py will not +# append the dirty workdir hash, which causes complications during deployments that mount from a clean folder. +RUN git restore . + +# Regenerate _version.py with the correct version - this should run quickly since we already have our dependencies +RUN rm src/mx_bluesky/_version.py +RUN pip install --no-cache-dir --no-compile -e . + +ENTRYPOINT /app/hyperion/utility_scripts/docker/entrypoint.sh + +EXPOSE 5005 diff --git a/docs/developer/hyperion/deploying-hyperion.rst b/docs/developer/hyperion/deploying-hyperion.rst new file mode 100644 index 000000000..e7750b111 --- /dev/null +++ b/docs/developer/hyperion/deploying-hyperion.rst @@ -0,0 +1,109 @@ +Building a deployable Docker image +================================== + +Release builds of container images should be built by the github CI on release, ad-hoc builds can be performed via +manual invocation of the Publish Docker Image workflow. + +Development builds of container images can be made by running the ``utility_scripts/build_docker_image.sh`` script. +By default it will both build and push the image unless you specify ``--no-build`` or ``--no-push``. To push an image +you will first need to create a GH personal access token and then log in with podman as described below. + +Pushing the docker image +------------------------ + +Obtaining a GitHub access token +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You will need to obtain a GitHub personal access token (classic) - not the new fine-grained token. +It will need the specific permission scopes as detailed in the `ghcr documentation `_ + +Building the image +~~~~~~~~~~~~~~~~~~ + +If building a test image, the image should be pushed to your personal GH account: + +:: + + cat | podman login ghcr.io --username --password-stdin + +where ``mysecretfile`` contains your personal access token + +Then run the ``build_docker_image.sh`` script. + +Troubleshooting +~~~~~~~~~~~~~~~ + +If you run into issues with ``podman build .`` failing with the error message +``io: read/write on closed pipe`` then you may be running out of disk space - try setting TMPDIR environment variable + +https://github.com/containers/podman/issues/22342 + +Building image on ubuntu +~~~~~~~~~~~~~~~~~~~~~~~~ + +If you run into issues such as + +:: + + potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument + +* Ensure newuidmap is installed + +:: + + sudo apt-get install uidmap + +* Add appropriate entries to ``/etc/subuid`` and ``/etc/subgid`` e.g. + +:: + + # subuid/subgid file + myuser:10000000:65536 + +* kill any existing podman processes and retry + +For further information, see https://github.com/containers/podman/issues/2542 + + +Deploying to kubernetes +----------------------- + +Once the docker image is built, the image can be deployed to kubernetes using the ``deploy_hyperion_to_k8s.sh`` script + +Production deployment +~~~~~~~~~~~~~~~~~~~~~ + +Then create and deploy the helm release + +:: + + ./utility_scripts/deploy/deploy_hyperion_to_k8s.sh --beamline= --checkout-to-prod hyperion + +This will run the ``deploy_hyperion.py`` script to deploy the latest hyperion to ``/dls_sw``. +You will be prompted to log into the beamline cluster, then it will create a helm release "hyperion". +The source folders will be mounted as bind mounts to allow the pod to pick up changes in production. +For production these are expected to be in the normal place defined in ``values.yaml``. + +Development deployment +~~~~~~~~~~~~~~~~~~~~~~ + +From a development ``hyperion`` workspace, either with a release image or using a development image built with the +script +above, you install a dev deployment to the cluster you are currently logged into with ``kubectl``: + +:: + + ./utility_scripts/deploy/deploy_hyperion_to_k8s.sh --dev --beamline= --repository= hyperion-test + + +The dev deployment bind-mounts the current ``hyperion`` workspace and ``../dodal`` into the container so that you can +run against your own development code. **Clusters do not allow bind mounts from arbitrary directories so +your workspace will have to be in a permitted directory such as your home directory.** + +By default the script will log into the ``argus`` cluster, if you want to deploy to an alternate cluster, +log in with ``kubectl set-context --current --namespace=`` and then specify ``--no-login`` when running the +script + +Please note, the deployment script is intended to be run from a checked-out matching version of the git repository. + +``helm list`` should then show details of the installed release on a successful install diff --git a/docs/developer/hyperion/index.rst b/docs/developer/hyperion/index.rst index 3f934ad59..c53178c3a 100644 --- a/docs/developer/hyperion/index.rst +++ b/docs/developer/hyperion/index.rst @@ -14,6 +14,7 @@ Documentation is split into four categories, and each is also accessible from li reference/param-hierarchy reference/readme + deploying-hyperion +++ diff --git a/docs/user/how-to/run-container.rst b/docs/user/how-to/run-container.rst index a3b0add14..f7dd72f16 100644 --- a/docs/user/how-to/run-container.rst +++ b/docs/user/how-to/run-container.rst @@ -10,6 +10,6 @@ Starting the container To pull the container from github container registry and run:: - $ docker run ghcr.io/DiamondLightSource/mx-bluesky:main --version + $ docker run ghcr.io/diamondlightsource/mx-bluesky:main --version To get a released version, use a numbered release instead of ``main``. diff --git a/helmchart/Chart.yaml b/helmchart/Chart.yaml new file mode 100644 index 000000000..cea620e9b --- /dev/null +++ b/helmchart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: hyperion +description: Hyperion server +type: application +# version of the chart +version: 0.0.1 diff --git a/helmchart/templates/deployment.yaml b/helmchart/templates/deployment.yaml new file mode 100644 index 000000000..42958a2a6 --- /dev/null +++ b/helmchart/templates/deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hyperion-deployment +spec: + selector: + matchLabels: + app: hyperion + replicas: 1 + template: + metadata: + labels: + app: hyperion + spec: + securityContext: + # gda2 + runAsUser: {{ .Values.hyperion.runAsUser }} + runAsGroup: {{ .Values.hyperion.runAsGroup }} + supplementalGroups: {{ .Values.hyperion.supplementalGroups }} + volumes: + - name: dls-sw-bl + hostPath: + path: "/dls_sw/{{ .Values.hyperion.beamline }}" + type: Directory + - name: dls-sw-apps + hostPath: + path: "/dls_sw/apps" + type: Directory + - name: dls-sw-dasc + hostPath: + path: "/dls_sw/dasc" + type: Directory + # Bind some source folders for easier debugging + - name: src + hostPath: + path: "{{ .Values.hyperion.projectDir }}/src" + type: Directory + - name: tests + hostPath: + path: "{{ .Values.hyperion.projectDir }}/tests" + type: Directory + - name: utility-scripts + hostPath: + path: "{{ .Values.hyperion.projectDir }}/utility_scripts" + type: Directory + - name: dodal + hostPath: + path: "{{ .Values.dodal.projectDir | clean }}" + type: Directory + {{- if .Values.hyperion.dev }} + - name: devlogs + hostPath: + path: "{{ .Values.hyperion.projectDir }}/tmp" + type: Directory + {{- end }} + containers: + - name: hyperion + image: {{ .Values.hyperion.imageRepository}}/hyperion:{{ .Values.hyperion.appVersion }} + resources: + limits: + cpu: "1" + memory: "1Gi" + ports: + - name: hyperion-api + containerPort: {{ .Values.hyperion.containerPort }} + protocol: TCP + env: + - name: HYPERION_LOG_DIR + value: {{ .Values.hyperion.logDir }} + - name: BEAMLINE + value: "{{ .Values.hyperion.beamline }}" + {{- if not .Values.hyperion.dev }} + - name: ZOCALO_GO_USER + value: "gda2" + - name: ZOCALO_GO_HOSTNAME + value: "{{ .Values.hyperion.beamline }}-control" + - name: ZOCALO_CONFIG + value: "/dls_sw/apps/zocalo/live/configuration.yaml" + - name: ISPYB_CONFIG_PATH + value: "/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-{{ .Values.hyperion.beamline }}.cfg" + args: [ "--external-callbacks" ] + {{- end }} + readinessProbe: + exec: + command: [ "/app/hyperion/utility_scripts/docker/healthcheck.sh" ] + periodSeconds: 5 + volumeMounts: + - mountPath: "/dls_sw/{{ .Values.hyperion.beamline }}" + name: dls-sw-bl + readOnly: true + mountPropagation: HostToContainer + - mountPath: "/dls_sw/apps" + name: dls-sw-apps + readOnly: true + mountPropagation: HostToContainer + - mountPath: "/dls_sw/dasc" + name: dls-sw-dasc + readOnly: true + mountPropagation: HostToContainer + - mountPath: "/app/hyperion/src" + name: src + - mountPath: "/app/hyperion/tests" + name: tests + - mountPath: "/app/hyperion/utility_scripts" + name: utility-scripts + - mountPath: "/app/dodal" + name: dodal + {{- if .Values.hyperion.dev }} + - mountPath: "/app/hyperion/tmp" + name: devlogs + {{ end }} diff --git a/helmchart/templates/ingress.yaml b/helmchart/templates/ingress.yaml new file mode 100644 index 000000000..8abef49d0 --- /dev/null +++ b/helmchart/templates/ingress.yaml @@ -0,0 +1,23 @@ +{{- if not .Values.hyperion.dev }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hyperion-ingress +spec: + ingressClassName: nginx + tls: + - hosts: + - {{ .Values.hyperion.externalHostname }} + rules: + - host: {{ .Values.hyperion.externalHostname }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hyperion-svc # this must match the name of the service you want to target + port: + number: {{ .Values.hyperion.containerPort }} + {{- end }} + diff --git a/helmchart/templates/service.yaml b/helmchart/templates/service.yaml new file mode 100644 index 000000000..b7c601f35 --- /dev/null +++ b/helmchart/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: hyperion-svc +spec: + {{- if .Values.hyperion.dev }} + type: LoadBalancer + {{- else }} + type: ClusterIP + {{- end }} + ports: + - name: hyperion-api + port: {{ .Values.hyperion.servicePort }} + protocol: TCP + targetPort: {{ .Values.hyperion.containerPort }} + selector: + app: hyperion diff --git a/helmchart/values.yaml b/helmchart/values.yaml new file mode 100644 index 000000000..01eff1721 --- /dev/null +++ b/helmchart/values.yaml @@ -0,0 +1,19 @@ +hyperion: + containerPort: 5005 + servicePort: 80 + imageRepository: ghcr.io/diamondlightsource + # i03-hyperion user and group + runAsUser: 36101 + runAsGroup: 36101 + supplementalGroups: [] + beamline: i03 + dev: false + logDir: "/dls_sw/i03/logs/bluesky" + # These should be overridden at install time + projectDir: SET_ON_INSTALL + appVersion: SET_ON_INSTALL + externalHostname: i03-hyperion.diamond.ac.uk +dodal: + projectDir: SET_ON_INSTALL +service: + type: NodePort diff --git a/pyproject.toml b/pyproject.toml index ed5693a70..979727952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "matplotlib", "nexgen", "numpy", - "opencv-python", + "opencv-python-headless", "opentelemetry-distro", "opentelemetry-exporter-otlp", "pydantic", @@ -114,6 +114,7 @@ markers = [ "s03: marks tests as requiring the s03 simulator running (deselect with '-m \"not s03\"')", "dlstbx: marks tests as requiring dlstbx (deselect with '-m \"not dlstbx\"')", "skip_log_setup: marks tests so that loggers are not setup before the test.", + "skip_in_pycharm: marks test as not working in pycharm testrunner", ] addopts = """ --tb=native -vv --doctest-modules --doctest-glob="*.rst" diff --git a/run_hyperion_in_podman.sh b/run_hyperion_in_podman.sh new file mode 100755 index 000000000..4adc1c8e1 --- /dev/null +++ b/run_hyperion_in_podman.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +STOP=0 +START=0 +UP=0 +IN_DEV=false + +show_help() { + echo "$(basename $0) [options...]" +cat < HyperionArgs: action="store_true", help="Run the external hyperion-callbacks service and publish events over ZMQ", ) + parser.add_argument( + "--version", + help="Print hyperion version string", + action="version", + version=version, + ) args = parser.parse_args() return HyperionArgs( verbose_event_logging=args.verbose_event_logging or False, diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index abec294b8..e56713a0a 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -1,9 +1,12 @@ import subprocess import sys +import pytest + from mx_bluesky import __version__ +@pytest.mark.skip_in_pycharm(reason="subprocess returns tty escape sequences") def test_cli_version(): cmd = [sys.executable, "-m", "mx_bluesky", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh new file mode 100755 index 000000000..aba8d1c92 --- /dev/null +++ b/utility_scripts/build_docker_image.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -e +# builds the docker image +BUILD=1 +PUSH=1 +for option in "$@"; do + case $option in + --no-build) + BUILD=0 + shift + ;; + --no-push) + PUSH=0 + shift + ;; + --help|--info|--h) + CMD=`basename $0` + echo "$CMD [options]" + echo "Builds and/or pushes the docker container image to the repository" + echo " --help This help" + echo " --no-build Do not build the image" + echo " --no-push Do not push the image" + exit 0 + ;; + -*|--*) + echo "Unknown option ${option}. Use --help for info on option usage." + exit 1 + ;; + esac +done + +PROJECTDIR=`dirname $0`/.. +PROJECT=hyperion + +if ! git diff --cached --quiet; then + echo "Cannot build image from unclean workspace" + exit 1 +fi + + +if [[ $BUILD == 1 ]]; then + echo "Building initial image" + LATEST_TAG=$PROJECT:latest + TMPDIR=/tmp podman build \ + -f $PROJECTDIR/Dockerfile.release \ + --tag $LATEST_TAG \ + $PROJECTDIR + # Now extract the version from the built image and then rebuild with the label + IMAGE_VERSION=$(podman run --rm --entrypoint=hyperion $LATEST_TAG -c "--version" | \ + sed -e 's/[^a-zA-Z0-9 ._-]/_/g') + TAG=$PROJECT:$IMAGE_VERSION + echo "Labelling image with version $IMAGE_VERSION, tagging with tags $TAG $LATEST_TAG" + TMPDIR=/tmp podman build \ + -f $PROJECTDIR/Dockerfile.release \ + --tag $TAG \ + --tag $LATEST_TAG \ + --label "version=$IMAGE_VERSION" \ + $PROJECTDIR +fi + +if [[ $PUSH == 1 ]]; then + NAMESPACE=$(podman login --get-login ghcr.io | tr '[:upper:]' '[:lower:]') + if [[ $? != 0 ]]; then + echo "Not logged in to ghcr.io" + exit 1 + fi + echo "Pushing to ghcr.io/$NAMESPACE/$PROJECT:latest ..." + podman push $PROJECT:latest docker://ghcr.io/$NAMESPACE/$PROJECT:latest + podman push $PROJECT:latest docker://ghcr.io/$NAMESPACE/$PROJECT:$IMAGE_VERSION +fi diff --git a/utility_scripts/deploy/deploy_hyperion.py b/utility_scripts/deploy/deploy_hyperion.py index ec96eca87..6f6c8e584 100644 --- a/utility_scripts/deploy/deploy_hyperion.py +++ b/utility_scripts/deploy/deploy_hyperion.py @@ -2,6 +2,7 @@ import os import re import subprocess +from typing import NamedTuple from uuid import uuid1 from create_venv import setup_venv @@ -17,9 +18,16 @@ DEV_DEPLOY_LOCATION = "/scratch/30day_tmp/hyperion_release_test/bluesky" -class repo: +class Options(NamedTuple): + release_dir: str + kubernetes: bool = False + print_release_dir: bool = False + quiet: bool = False + + +class Deployment: # Set name, setup remote origin, get the latest version""" - def __init__(self, name: str, repo_args): + def __init__(self, name: str, repo_args, options: Options): self.name = name self.repo = Repo(repo_args) @@ -31,10 +39,13 @@ def __init__(self, name: str, repo_args): t.name for t in self.repo.tags if VERSION_PATTERN_COMPILED.match(t.name) ] self.versions.sort(key=Version, reverse=True) - print(f"Found {self.name}_versions:\n{os.linesep.join(self.versions)}") + + if not options.quiet: + print(f"Found {self.name}_versions:\n{os.linesep.join(self.versions)}") + self.latest_version_str = self.versions[0] - def deploy(self, url): + def deploy(self): print(f"Cloning latest version {self.name} into {self.deploy_location}") deploy_repo = Repo.init(self.deploy_location) @@ -64,24 +75,44 @@ def set_deploy_location(self, release_area): # Get the release directory based off the beamline and the latest hyperion version -def get_hyperion_release_dir_from_args() -> str: - parser = argparse.ArgumentParser() +def _parse_options() -> Options: + parser = argparse.ArgumentParser( + prog="deploy_hyperion", description="Deploy hyperion to a beamline" + ) + parser.add_argument( + "--kubernetes", + action=argparse.BooleanOptionalAction, + help="Prepare git workspaces for deployment to kubernetes; do not install virtual environment", + ) parser.add_argument( "beamline", type=str, choices=recognised_beamlines, - help="The beamline to deploy hyperion to", + help=f"The beamline to deploy hyperion to. 'dev' installs to {DEV_DEPLOY_LOCATION}", + ) + parser.add_argument( + "--print-release-dir", + action=argparse.BooleanOptionalAction, + help="Print the path to the release folder and then exit", ) - args = parser.parse_args() if args.beamline == "dev": print("Running as dev") - return DEV_DEPLOY_LOCATION + release_dir = DEV_DEPLOY_LOCATION else: - return f"/dls_sw/{args.beamline}/software/bluesky" + release_dir = f"/dls_sw/{args.beamline}/software/bluesky" + + return Options( + release_dir=release_dir, + kubernetes=args.kubernetes, + print_release_dir=args.print_release_dir, + quiet=args.print_release_dir, + ) -def create_environment_from_control_machine(): +def _create_environment_from_control_machine( + hyperion_repo, path_to_create_venv, path_to_dls_dev_env +): try: user = os.environ["USER"] except KeyError: @@ -108,63 +139,61 @@ def create_environment_from_control_machine(): process.kill() -if __name__ == "__main__": - # Gives path to /bluesky - release_area = get_hyperion_release_dir_from_args() - +def main(options: Options): + release_area = options.release_dir this_repo_top = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) - print(f"Repo top is {this_repo_top}") + if not options.quiet: + print(f"Repo top is {this_repo_top}") - hyperion_repo = repo( - name="hyperion", - repo_args=os.path.join(this_repo_top, ".git"), + hyperion_repo = Deployment( + name="hyperion", repo_args=os.path.join(this_repo_top, ".git"), options=options ) if hyperion_repo.name != "hyperion": raise ValueError("This function should only be used with the hyperion repo") release_area_version = os.path.join( - release_area, f"hyperion_{hyperion_repo.latest_version_str}" + release_area, f"mx_bluesky_{hyperion_repo.latest_version_str}" ) + if options.print_release_dir: + print(release_area_version) + return + print(f"Putting releases into {release_area_version}") - dodal_repo = repo( + dodal_repo = Deployment( name="dodal", repo_args=os.path.join(this_repo_top, "../dodal/.git"), + options=options, ) dodal_repo.set_deploy_location(release_area_version) hyperion_repo.set_deploy_location(release_area_version) # Deploy hyperion repo - hyperion_repo.deploy(hyperion_repo.origin.url) - - # Get version of dodal that latest hyperion version uses - with open(f"{release_area_version}/hyperion/setup.cfg") as setup_file: - dodal_url = [ - line - for line in setup_file - if "https://github.com/DiamondLightSource/python-dodal" in line - ] + hyperion_repo.deploy() # Now deploy the correct version of dodal - dodal_repo.deploy(dodal_url) + dodal_repo.deploy() - if hyperion_repo.name == "hyperion": - path_to_dls_dev_env = os.path.join( - hyperion_repo.deploy_location, "utility_scripts/dls_dev_env.sh" - ) - path_to_create_venv = os.path.join( - hyperion_repo.deploy_location, "utility_scripts/deploy/create_venv.py" - ) + if not options.kubernetes: + if hyperion_repo.name == "hyperion": + path_to_dls_dev_env = os.path.join( + hyperion_repo.deploy_location, "utility_scripts/dls_dev_env.sh" + ) + path_to_create_venv = os.path.join( + hyperion_repo.deploy_location, "utility_scripts/deploy/create_venv.py" + ) - # SSH into control machine if not in dev mode - if release_area != DEV_DEPLOY_LOCATION: - create_environment_from_control_machine() - else: - setup_venv(path_to_create_venv, hyperion_repo.deploy_location) + # SSH into control machine if not in dev mode + if release_area != DEV_DEPLOY_LOCATION: + _create_environment_from_control_machine( + hyperion_repo, path_to_create_venv, path_to_dls_dev_env + ) + else: + setup_venv(path_to_create_venv, hyperion_repo.deploy_location) def create_symlink_by_tmp_and_rename(dirname, target, linkname): tmp_name = str(uuid1()) @@ -204,3 +233,9 @@ def create_symlink_by_tmp_and_rename(dirname, target, linkname): print("To start this version run hyperion_restart from the beamline's GDA") else: print("Quitting without latest version being updated") + + +if __name__ == "__main__": + # Gives path to /bluesky + options = _parse_options() + main(options) diff --git a/utility_scripts/deploy/deploy_hyperion_to_k8s.sh b/utility_scripts/deploy/deploy_hyperion_to_k8s.sh new file mode 100755 index 000000000..f5a69d0a8 --- /dev/null +++ b/utility_scripts/deploy/deploy_hyperion_to_k8s.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# Installs helm package to kubernetes +LOGIN=true + +for option in "$@"; do + case $option in + -b=*|--beamline=*) + BEAMLINE="${option#*=}" + shift + ;; + --dev) + DEV=true + shift + ;; + --checkout-to-prod) + CHECKOUT=true + shift + ;; + --repository=*) + REPOSITORY="${option#*=}" + shift + ;; + --appVersion=*) + APP_VERSION="${option#*=}" + shift + ;; + --no-login) + LOGIN=false + shift + ;; + --help|--info|--h) + CMD=`basename $0` + echo "$CMD [options] " + cat <$callback_start_log_path 2>&1 & +fi + +echo "$(date) Starting Hyperion..." +hyperion `echo $h_commands;`>$start_log_path 2>&1 diff --git a/utility_scripts/docker/healthcheck.sh b/utility_scripts/docker/healthcheck.sh new file mode 100755 index 000000000..4fe8f294a --- /dev/null +++ b/utility_scripts/docker/healthcheck.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Healthcheck script for the container image +curl -f --head -X GET http://localhost:5005/status || exit 1 diff --git a/utility_scripts/docker/i03-compose.yml b/utility_scripts/docker/i03-compose.yml new file mode 100644 index 000000000..55a0500d8 --- /dev/null +++ b/utility_scripts/docker/i03-compose.yml @@ -0,0 +1,59 @@ +name: i03-hyperion +services: + hyperion-common: + image: localhost/hyperion + pull_policy: never + expose: + - "5005" + volumes: + - type: bind + source: /dls_sw/i03 + target: /dls_sw/i03 + read-only: true + - type: bind + source: /dls_sw/apps + target: /dls_sw/apps + read-only: true + - type: bind + source: /dls_sw/dasc + target: /dls_sw/dasc + read-only: true + network_mode: "host" + annotations: + # Required in order to read config files readable by dls_dasc + - run.oci.keep_original_groups=1 + hyperion: + extends: + service: hyperion-common + volumes: + - type: bind + source: /dls/i03 + target: /dls/i03 + - type: bind + source: /dls_sw/i03/logs + target: /dls_sw/i03/logs + environment: + BEAMLINE: i03 + HYPERION_LOG_DIR: /dls_sw/i03/logs/bluesky + ZOCALO_GO_USER: gda2 + ZOCALO_GO_HOSTNAME: i03-control + ZOCALO_CONFIG: /dls_sw/apps/zocalo/live/configuration.yaml + ISPYB_CONFIG_PATH: /dls_sw/dasc/mariadb/credentials/ispyb-hyperion-i03.cfg + command: [ "--external-callbacks" ] + hyperion-dev: + extends: + service: hyperion-common + volumes: + # Bind some source folders for easier debugging + - type: bind + source: ../../src + target: /project/src + - type: bind + source: ../../utility_scripts + target: /project/utility_scripts + - type: bind + source: ../../tests + target: /project/tests + environment: + HYPERION_LOG_DIR: /tmp/dev + entrypoint: [ "/bin/bash" ]