From 5555d71943a41ea103395bbbb0ea0d6d834aba4f Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 30 May 2024 13:20:54 +0200 Subject: [PATCH] unit tests --- chart/Chart.yaml | 4 +- chart/f7t4jhub/Chart.yaml | 2 +- chart/f7t4jhub/files/jupyterhub-config.py | 2 +- chart/values.yaml | 2 +- dockerfiles/Dockerfile | 4 - dockerfiles/README.md | 19 +- tests/context.py | 13 + tests/fc_handlers.py | 411 ++++++++++++++++++++++ tests/test_spawner.py | 398 +++++++++++++++++++++ 9 files changed, 828 insertions(+), 27 deletions(-) create mode 100644 tests/context.py create mode 100644 tests/fc_handlers.py create mode 100644 tests/test_spawner.py diff --git a/chart/Chart.yaml b/chart/Chart.yaml index fea43c5..811e2b4 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -2,11 +2,11 @@ apiVersion: v2 name: f7t4jhub description: A Helm chart to Deploy JupyterHub with the FirecREST Spawner type: application -version: 0.5.0 +version: 0.5.1 appVersion: "4.1.5" dependencies: - name: f7t4jhub - version: 0.5.0 + version: 0.5.1 repository: "file://./f7t4jhub" - name: reloader version: v1.0.51 diff --git a/chart/f7t4jhub/Chart.yaml b/chart/f7t4jhub/Chart.yaml index 6383651..73eaa45 100644 --- a/chart/f7t4jhub/Chart.yaml +++ b/chart/f7t4jhub/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: f7t4jhub description: A Helm chart to Deploy JupyterHub with the FirecREST Spawner type: application -version: 0.5.0 +version: 0.5.1 appVersion: "4.1.5" diff --git a/chart/f7t4jhub/files/jupyterhub-config.py b/chart/f7t4jhub/files/jupyterhub-config.py index d671ae6..7d87ff0 100644 --- a/chart/f7t4jhub/files/jupyterhub-config.py +++ b/chart/f7t4jhub/files/jupyterhub-config.py @@ -795,7 +795,7 @@ async def refresh_user(self, user, handler=None): # Default: 'jupyterhub.spawner.LocalProcessSpawner' # c.JupyterHub.spawner_class = 'jupyterhub.spawner.LocalProcessSpawner' -c.JupyterHub.spawner_class = 'firecrest_spawner.spawner.SlurmSpawner' +c.JupyterHub.spawner_class = 'firecrestspawner.spawner.SlurmSpawner' # c.FirecRESTSpawnerBase.req_host = '{{ .Values.config.spawner.host }}' c.FirecRESTSpawnerBase.node_name_template = '{{ .Values.config.spawner.nodeNameTemplate }}' diff --git a/chart/values.yaml b/chart/values.yaml index e4c22a3..cf99852 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -27,7 +27,7 @@ f7t4jhub: hub: # Image for the JupyterHub application (replace with your own JupyterHub image) - image: 'rsarm/jhub:4.1.5-f7t-x86_64' + image: 'ghcr.io/eth-cscs/f7t4fjhub:4.1.5' vault: # Enable or disable Vault integration diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 4f5221b..64c30dd 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -60,10 +60,6 @@ RUN . /opt/conda/bin/activate && \ # git clone https://github.com/moment/moment.git -b 2.29.1 && \ # git clone https://github.com/requirejs/requirejs.git -b 2.3.6 -# COPY socket.py /opt/conda/envs/py311/lib/python3.11/socket.py -# COPY user.py /opt/conda/envs/py311/lib/python3.11/site-packages/jupyterhub/user.py -# COPY app.py /opt/conda/envs/py311/lib/python3.11/site-packages/jupyterhub/app.py - EXPOSE 8000 RUN useradd -ms /bin/bash juhu diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 3e4fc89..ae03090 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -2,21 +2,4 @@ This is a dockerfile to build the image used for the deployment of the Hub. -The image can be found in DockerHub: [rsarm/jhub:2.0.0-f7t-x86_64](https://hub.docker.com/layers/rsarm/jhub/2.0.0-f7t-x86_64/images/sha256-3474b7295728c7a61caad03711c2ae64e0615ed8c9c6ae15f8e273d72c7b7027?context=explore). - -## Notes - -### Build for linux/amd64 on an arm64 macbook - -From the based directory of the repo: - -```bash -docker build -f dockerfiles/Dockerfile --platform linux/amd64 -t rsarm/jhub:2.0.0-f7t-x86_64 . -``` -The dockerfile has a `COPY` that needs the repository to be in the build directory. - -### Push -```bash -docker login # only once per shell -docker push rsarm/jhub:2.0.0-f7t-x86_64 -``` +The available images can be found [here](https://github.com/eth-cscs/firecrestspawner/pkgs/container/f7t4fjhub). diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..0fb3584 --- /dev/null +++ b/tests/context.py @@ -0,0 +1,13 @@ +import os +import sys + + +os.environ["FIRECREST_URL"] = "https://firecrest-url-pytest.com" + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from firecrestspawner.spawner import ( + SlurmSpawner, + format_template, + FirecrestAccessTokenAuth, +) diff --git a/tests/fc_handlers.py b/tests/fc_handlers.py new file mode 100644 index 0000000..6aa7fa0 --- /dev/null +++ b/tests/fc_handlers.py @@ -0,0 +1,411 @@ +import json +import re +import pytest +from werkzeug.wrappers import Response + + +def tasks_handler(request): + taskid = request.args.get("tasks") + if taskid == "acct_352_id": + ret = { + "tasks": { + taskid: { + "data": [ + { + "jobid": "352", + "name": "firecrest_job_test", + "nodelist": "nid06227", + "nodes": "3", + "partition": "normal", + "start_time": "2021-11-29T16:31:07", + "state": "COMPLETED", + "time": "00:48:00", + "time_left": "2021-11-29T16:31:47", + "user": "username", + }, + ], + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-06T09:53:48", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "acct_353_id": + ret = { + "tasks": { + taskid: { + "data": { + "0": { + "job_data_err": "", + "job_data_out": "", + "job_file": "(null)", + "job_file_err": "stderr-file-not-found", + "job_file_out": "stdout-file-not-found", + "jobid": "353", + "name": "interactive", + "nodelist": "nid02357", + "nodes": "1", + "partition": "debug", + "start_time": "6:38", + "state": "RUNNING", + "time": "2022-03-10T10:11:34", + "time_left": "23:22", + "user": "username", + } + }, + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-06T09:53:48", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "acct_354_id": + ret = { + "tasks": { + taskid: { + "data": { + "0": { + "job_data_err": "", + "job_data_out": "", + "job_file": "(null)", + "job_file_err": "stderr-file-not-found", + "job_file_out": "stdout-file-not-found", + "jobid": "354", + "name": "interactive", + "nodelist": "", + "nodes": "1", + "partition": "debug", + "start_time": "", + "state": "PENDING", + "time": "2022-03-10T10:11:34", + "time_left": "23:22", + "user": "username", + } + }, + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-06T09:53:48", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "acct_355_id": + ret = { + "tasks": { + taskid: { + "data": [ + { + "jobid": "355", + "name": "firecrest_job_test", + "nodelist": "nid06227", + "nodes": "3", + "partition": "normal", + "start_time": "2021-11-29T16:31:07", + "state": "FAILED", + "time": "00:48:00", + "time_left": "2021-11-29T16:31:47", + "user": "username", + }, + ], + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-06T09:53:48", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "cancel_job_id": + ret = { + "tasks": { + taskid: { + "data": "", + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-06T10:42:06", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "submit_upload_job_id_good": + jobid = 353 + ret = { + "tasks": { + taskid: { + "data": { + "job_data_err": "", + "job_data_out": "", + "job_file": f"/path/to/firecrest/{taskid}/script.sh", + "job_file_err": f"/path/to/firecrest/{taskid}/slurm-35342667.out", # noqa E501 + "job_file_out": f"/path/to/firecrest/{taskid}/slurm-35342667.out", # noqa E501 + "jobid": jobid, + "result": "Job submitted", + }, + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-04T11:52:11", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "submit_upload_job_id_job_failed": + jobid = 355 + ret = { + "tasks": { + taskid: { + "data": { + "job_data_err": "", + "job_data_out": "", + "job_file": f"/path/to/firecrest/{taskid}/script.sh", + "job_file_err": f"/path/to/firecrest/{taskid}/slurm-35342667.out", # noqa E501 + "job_file_out": f"/path/to/firecrest/{taskid}/slurm-35342667.out", # noqa E501 + "jobid": jobid, + "result": "Job submitted", + }, + "description": "Finished successfully", + "hash_id": taskid, + "last_modify": "2021-12-04T11:52:11", + "service": "compute", + "status": "200", + "task_id": taskid, + "task_url": f"TASK_IP/tasks/{taskid}", + "user": "username", + } + } + } + ret_status = 200 + + if taskid == "submit_upload_job_id_no_jobid": + ret = { + "tasks": { + "a46bed48e1841ccf56f7dbd02e815bc4": { + "created_at": "2024-03-13T16:24:36", + "data": "sbatch: error: cli_filter plugin terminated with error", # noqa E501 + "description": "Finished with errors", + "hash_id": "a46bed48e1841ccf56f7dbd02e815bc4", + "last_modify": "2024-03-13T16:24:39", + "service": "compute", + "status": "400", + "system": "daint", + "task_id": "a46bed48e1841ccf56f7dbd02e815bc4", + "updated_at": "2024-03-13T16:24:39", + "user": "sarafael", + } + } + } + ret_status = 400 + + return Response(json.dumps(ret), status=ret_status, content_type="application/json") + + +def submit_upload_handler(request): + if request.headers["Authorization"] != "Bearer VALID_TOKEN": + return Response( + json.dumps({"message": "Bad token; invalid JSON"}), + status=401, + content_type="application/json", + ) + + if request.headers["X-Machine-Name"] != "cluster1": + return Response( + json.dumps( + { + "description": "Failed to submit job", + "error": "Machine does not exist", + } + ), + status=400, + headers={"X-Machine-Does-Not-Exist": "Machine does not exist"}, + content_type="application/json", + ) + + extra_headers = None + if request.files["file"].filename == "script.batch": + if "job_failed" in str(request._cached_data): + ret = { + "success": "Task created", + "task_id": "submit_upload_job_id_job_failed", + "task_url": "TASK_IP/tasks/submit_upload_job_id_job_failed", + } + status_code = 201 + elif "no_jobid" in str(request._cached_data): + ret = { + "success": "Task created", + "task_id": "submit_upload_job_id_no_jobid", + "task_url": "TASK_IP/tasks/submit_upload_job_id_no_jobid", + } + status_code = 201 + else: + ret = { + "success": "Task created", + "task_id": "submit_upload_job_id_good", + "task_url": "TASK_IP/tasks/submit_upload_job_id_good", + } + status_code = 201 + else: + extra_headers = {"X-Invalid-Path": f"path is an invalid path."} + ret = {"description": "Failed to submit job"} + status_code = 400 + + return Response( + json.dumps(ret), + status=status_code, + headers=extra_headers, + content_type="application/json", + ) + + +def systems_handler(request): + if request.headers["Authorization"] != "Bearer VALID_TOKEN": + return Response( + json.dumps({"message": "Bad token; invalid JSON"}), + status=401, + content_type="application/json", + ) + + ret = { + "description": "List of systems with status and description.", + "out": [ + { + "description": "System ready", + "status": "available", + "system": "cluster1", + }, + { + "description": "System ready", + "status": "available", + "system": "cluster2", + }, + ], + } + ret_status = 200 + return Response(json.dumps(ret), status=ret_status, content_type="application/json") + + +def sacct_handler(request): + if request.headers["Authorization"] != "Bearer VALID_TOKEN": + return Response( + json.dumps({"message": "Bad token; invalid JSON"}), + status=401, + content_type="application/json", + ) + + if request.headers["X-Machine-Name"] != "cluster1": + return Response( + json.dumps( + { + "description": "Failed to retrieve account information", + "error": "Machine does not exist", + } + ), + status=400, + headers={"X-Machine-Does-Not-Exist": "Machine does not exist"}, + content_type="application/json", + ) + + jobs = request.args.get("jobs", "").split(",") + if set(jobs) == {"352"}: + ret = { + "success": "Task created", + "task_id": "acct_352_id", + "task_url": "TASK_IP/tasks/acct_352_id", + } + status_code = 200 + elif set(jobs) == {"353"}: + ret = { + "success": "Task created", + "task_id": "acct_353_id", + "task_url": "TASK_IP/tasks/acct_353_id", + } + status_code = 200 + elif set(jobs) == {"354"}: + ret = { + "success": "Task created", + "task_id": "acct_354_id", + "task_url": "TASK_IP/tasks/acct_354_id", + } + status_code = 200 + elif set(jobs) == {"355"}: + ret = { + "success": "Task created", + "task_id": "acct_355_id", + "task_url": "TASK_IP/tasks/acct_355_id", + } + status_code = 200 + + return Response( + json.dumps(ret), status=status_code, content_type="application/json" + ) + + +def cancel_handler(request): + uri = request.url + jobid = uri.split("/")[-1] + if jobid in ("352", "353", "354"): + ret = { + "success": "Task created", + "task_id": "cancel_job_id", + "task_url": "TASK_IP/tasks/cancel_job_id", + } + status_code = 200 + + return Response( + json.dumps(ret), status=status_code, content_type="application/json" + ) + + +@pytest.fixture +def fc_server(httpserver): + httpserver.expect_request( + "/compute/jobs/upload", method="POST" + ).respond_with_handler(submit_upload_handler) + httpserver.expect_request( + re.compile("^/status/systems.*"), method="GET" + ).respond_with_handler(systems_handler) + httpserver.expect_request("/tasks", method="GET").respond_with_handler( + tasks_handler + ) + httpserver.expect_request("/compute/acct", method="GET").respond_with_handler( + sacct_handler + ) + httpserver.expect_request( + re.compile("^/compute/jobs.*"), method="DELETE" + ).respond_with_handler(cancel_handler) + return httpserver diff --git a/tests/test_spawner.py b/tests/test_spawner.py new file mode 100644 index 0000000..be6f955 --- /dev/null +++ b/tests/test_spawner.py @@ -0,0 +1,398 @@ +import json +import re +import firecrest +import getpass +import pytest +from werkzeug.wrappers import Response +from context import FirecrestAccessTokenAuth, SlurmSpawner, format_template +from fc_handlers import ( + tasks_handler, + submit_upload_handler, + systems_handler, + sacct_handler, + cancel_handler, +) +from jupyterhub.tests.conftest import db +from jupyterhub.user import User +from jupyterhub.objects import Hub +from jupyterhub.utils import random_port +from jupyterhub import orm +from oauthenticator.generic import GenericOAuthenticator + + +testport = random_port() + + +class DummyOAuthenticator(GenericOAuthenticator): + async def refresh_user(self, user, handler=None): + auth_state = {"access_token": "VALID_TOKEN"} + return {"auth_state": auth_state} + + +def new_spawner(db, spawner_class=SlurmSpawner, **kwargs): + user = db.query(orm.User).first() + hub = Hub() + user = User(user, {"authenticator": DummyOAuthenticator()}) + + _spawner = user._new_spawner( + "", + spawner_class=spawner_class, + hub=hub, + user=user, + req_srun="", + req_host="cluster1", + port=testport, + node_name_template="{}.cluster1.ch", + ) + return _spawner + + +@pytest.fixture +def fc_server(httpserver): + httpserver.expect_request( + "/compute/jobs/upload", method="POST" + ).respond_with_handler(submit_upload_handler) + + httpserver.expect_request( + re.compile("^/status/systems.*"), method="GET" + ).respond_with_handler(systems_handler) + + httpserver.expect_request( + "/tasks", method="GET" + ).respond_with_handler(tasks_handler) + + httpserver.expect_request( + "/compute/acct", method="GET" + ).respond_with_handler(sacct_handler) + + httpserver.expect_request( + re.compile("^/compute/jobs.*"), method="DELETE" + ).respond_with_handler(cancel_handler) + + return httpserver + + +def test_format_template(): + template = "{{key_1}} and {{key_2}}" + templated = format_template( + template, + key_1="value_1", + key_2="value_2", + ) + assert templated == "value_1 and value_2" + + +def test_get_access_token(): + auth = FirecrestAccessTokenAuth("access_token") + assert auth.get_access_token() == "access_token" + + +@pytest.mark.asyncio +async def test_get_req_subvars(db): + spawner = new_spawner(db=db) + expected_subvars = { + "account": "", + "cluster": "", + "constraint": "", + "epilogue": "", + "gres": "", + "homedir": "", + "host": "cluster1", + "memory": "", + "ngpus": "", + "nprocs": "", + "options": "", + "partition": "", + "prologue": "", + "qos": "", + "queue": "", + "reservation": "", + "runtime": "", + "srun": "", + "submitenv": "", + "username": getpass.getuser(), + } + assert spawner.get_req_subvars() == expected_subvars + + +@pytest.mark.asyncio +async def test_cmd_formatted_for_batch(db): + spawner = new_spawner(db=db) + assert spawner.cmd_formatted_for_batch() == "jupyterhub-singleuser" + + +@pytest.mark.asyncio +async def test_get_batch_script(db): + spawner = new_spawner(db=db) + batch_script = await spawner._get_batch_script() + ref_batch_script = """#!/bin/bash +##SBATCH --output=/jupyterhub_slurmspawner_%j.log +#SBATCH --job-name=spawner-jupyterhub +#SBATCH --chdir= +#SBATCH --get-user-env=L + + + + + + + + + + + +hostname -i + +set -euo pipefail + +trap 'echo SIGTERM received' TERM + +which jupyterhub-singleuser + +echo "jupyterhub-singleuser ended gracefully" +""" + assert batch_script == ref_batch_script + + +@pytest.mark.asyncio +async def test_get_batch_script_subvars(db): + spawner = new_spawner(db=db) + spawner.set_trait("req_partition", "partition1") + spawner.set_trait("req_account", "account1") + spawner.set_trait("req_runtime", "00:15:00") + spawner.set_trait("req_memory", "64") + spawner.set_trait("req_gres", "resource:1") + spawner.set_trait("req_nprocs", "12") + spawner.set_trait("req_reservation", "reservation1") + spawner.set_trait("req_constraint", "constraint1") + subvars = spawner.get_req_subvars() + batch_script = await spawner._get_batch_script(**subvars) + ref_batch_script = f"""#!/bin/bash +##SBATCH --output=/jupyterhub_slurmspawner_%j.log +#SBATCH --job-name=spawner-jupyterhub +#SBATCH --chdir= +#SBATCH --get-user-env=L + +#SBATCH --partition=partition1 +#SBATCH --account=account1 +#SBATCH --time=00:15:00 +#SBATCH --mem=64 +#SBATCH --gres=resource:1 +#SBATCH --cpus-per-task=12 +#SBATCH --reservation=reservation1 +#SBATCH --constraint=constraint1 + + +hostname -i + +set -euo pipefail + +trap 'echo SIGTERM received' TERM + +which jupyterhub-singleuser + +echo "jupyterhub-singleuser ended gracefully" +""" + assert batch_script == ref_batch_script + + +@pytest.mark.asyncio +async def test_get_firecrest_client(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + client = await spawner.get_firecrest_client() + systems = await client.all_systems() + ref_systems = [ + { + "description": "System ready", + "status": "available", + "system": "cluster1" + }, + { + "description": "System ready", + "status": "available", + "system": "cluster2" + }, + ] + assert systems == ref_systems + + +@pytest.mark.asyncio +async def test_query_job_status_completed(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "352" + job_status = await spawner.query_job_status() + poll = await spawner.poll() + assert job_status.name == "NOTFOUND" + assert poll == 1 + + +@pytest.mark.asyncio +async def test_query_job_status_running(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "353" + job_status = await spawner.query_job_status() + poll = await spawner.poll() + assert job_status.name == "RUNNING" + assert poll is None + + +@pytest.mark.asyncio +async def test_query_job_status_pending(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "354" + job_status = await spawner.query_job_status() + poll = await spawner.poll() + assert job_status.name == "PENDING" + assert poll is None + + +@pytest.mark.asyncio +async def _test_query_job_status_fail(db, fc_server): + # TODO: Test the case where the job failed afte start + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "355" + job_status = await spawner.query_job_status() + poll = await spawner.poll() + assert job_status.name == "UNKNOWN" + assert poll is None + + +@pytest.mark.asyncio +async def test_cancel_batch_job(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "354" + # Make sure this doesn't raise an error + await spawner.cancel_batch_job() + + +def test_load_and_clear_state(db): + spawner = new_spawner(db=db) + state = {"job_id": "354", "job_status": "RUNNING nid02000"} + + spawner.load_state(state) + assert spawner.job_id == "354" + assert spawner.job_status == "RUNNING nid02000" + + spawner.get_state() + assert spawner.job_id == "354" + assert spawner.job_status == "RUNNING nid02000" + + spawner.clear_state() + assert spawner.job_id == "" + assert spawner.job_status == "" + + +def test_load_state_nostate(db): + spawner = new_spawner(db=db) + + spawner.get_state() + assert spawner.job_id == "" + assert spawner.job_status == "" + + spawner.load_state({}) + assert spawner.job_id == "" + assert spawner.job_status == "" + + +@pytest.mark.asyncio +async def test_start_job_fail(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + spawner.set_trait("req_partition", "job_failed") + with pytest.raises(RuntimeError) as excinfo: + await spawner.start() + + assert str(excinfo.value) == ( + "The Jupyter batch job has disappeared " + "while pending in the queue or died " + "immediately after starting." + ) + assert spawner.job_status == "" # `spawner.job_status` is cleared + + +@pytest.mark.asyncio +async def test_start_no_jobid(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + spawner.set_trait("req_partition", "no_jobid") + with pytest.raises(RuntimeError) as excinfo: + await spawner.start() + + assert str(excinfo.value) == ( + "Jupyter batch job submission " "failure: (no jobid in output)" + ) + assert spawner.job_status == "" + + +@pytest.mark.asyncio +async def test_start(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + ip, port = await spawner.start() + assert spawner.job_id == "353" + assert port == testport + assert ip == "nid02357.cluster1.ch" + assert spawner.job_status == "RUNNING nid02357" + env = spawner.get_env() + assert env["JUPYTERHUB_SERVICE_URL"] == f"http://nid02357.cluster1.ch:{testport}/" # noqa 505 + + # Since the job 353 has status RUNNING, to stop the job, + # we have to trick the spawner into using a the job 352 that + # has status COMPLETED + spawner.job_id = "352" + await spawner.stop() + assert spawner.job_status == "COMPLETED nid06227" + + +@pytest.mark.asyncio +async def test_submit_batch_script(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + await spawner.submit_batch_script() + assert spawner.job_id == "353" + + +@pytest.mark.asyncio +async def test_state_gethost(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + # force setting `host` and `job_id` since they + # are set only set when calling `spawner.start()` + spawner.host = "cluster1" + spawner.job_id = "352" + host = await spawner.state_gethost() + assert host == "nid06227.cluster1.ch" + + +@pytest.mark.asyncio +async def test_stop_fail(db, fc_server): + spawner = new_spawner(db=db) + spawner.firecrest_url = fc_server.url_for("/") + spawner.host = "cluster1" + spawner.job_id = "353" # returns 'RUNNING' + # the spawner retries many poll calls, but + # since the job status keeps being RUNNING, + # it throws the warning + # Notebook server job 353 at 0.0.0.0:55171 possibly failed to terminate + await spawner.stop()