From fabfa4a8e4ed32cc8d971f590ed4067bcf9348bc Mon Sep 17 00:00:00 2001 From: David Perl Date: Wed, 25 Oct 2023 16:50:29 +0100 Subject: [PATCH 01/10] \#818 fix fgs plan system tests --- .vscode/hyperion-dodal-nexgen.code-workspace | 2 +- .vscode/settings.json | 2 +- .../experiment_plans/test_fgs_plan.py | 113 +++++++++++++----- 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/.vscode/hyperion-dodal-nexgen.code-workspace b/.vscode/hyperion-dodal-nexgen.code-workspace index 97f11cbaf..e74558da2 100644 --- a/.vscode/hyperion-dodal-nexgen.code-workspace +++ b/.vscode/hyperion-dodal-nexgen.code-workspace @@ -16,7 +16,7 @@ "esbonio.sphinx.confDir": "", "search.useIgnoreFiles": false, "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "charliermarsh.ruff" }, "python.formatting.provider": "none" } diff --git a/.vscode/settings.json b/.vscode/settings.json index 9e91dc235..4e927f6bc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,7 +13,7 @@ "source.fixAll.ruff": "explicit", "source.organizeImports.ruff": "explicit" }, - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "charliermarsh.ruff" }, "terminal.integrated.gpuAcceleration": "off", "python.analysis.typeCheckingMode": "basic", diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 218427931..d44f9a438 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -2,6 +2,7 @@ from typing import Callable from unittest.mock import MagicMock, patch +import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp import pytest from bluesky.run_engine import RunEngine @@ -17,6 +18,9 @@ read_hardware_for_ispyb_during_collection, read_hardware_for_ispyb_pre_collection, ) +from hyperion.device_setup_plans.xbpm_feedback import ( + transmission_and_xbpm_feedback_for_collection_decorator, +) from hyperion.exceptions import WarningException from hyperion.experiment_plans.flyscan_xray_centre_plan import ( FlyScanXRayCentreComposite, @@ -45,11 +49,23 @@ def params(): params = GridscanInternalParameters(**default_raw_params()) params.hyperion_params.beamline = SIM_BEAMLINE - return params + yield params + + +@pytest.fixture +def RE(): + yield RunEngine({}) + + +@pytest.fixture() +def callbacks(params): + callbacks = XrayCentreCallbackCollection.setup() + callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG + yield callbacks @pytest.fixture -def fgs_composite(): +def fxc_composite(): composite = FlyScanXRayCentreComposite( attenuator=i03.attenuator(), aperture_scatterguard=i03.aperture_scatterguard(), @@ -87,9 +103,19 @@ def fgs_composite(): composite.aperture_scatterguard.scatterguard.x.set_lim(-4.8, 5.7) + composite.xbpm_feedback.pos_ok.sim_put(1) # type: ignore + composite.xbpm_feedback.pos_stable.sim_put(1) # type: ignore + return composite +@pytest.mark.s03 +def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): + assert fxc_composite.aperture_scatterguard + assert fxc_composite.backlight + + +@pytest.mark.skip(reason="Broken due to eiger issues in s03") @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -105,9 +131,10 @@ def test_run_gridscan( wait: MagicMock, params: GridscanInternalParameters, RE: RunEngine, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, ): - RE(run_gridscan(fgs_composite, params)) + # Would be better to use flyscan_xray_centre instead but eiger doesn't work well in S03 + RE(run_gridscan(fxc_composite, params)) @pytest.mark.s03 @@ -133,14 +160,14 @@ def test_run_gridscan_and_move( @pytest.mark.s03 def test_read_hardware_for_ispyb_pre_collection( RE: RunEngine, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, ): - undulator = fgs_composite.undulator - synchrotron = fgs_composite.synchrotron - slit_gaps = fgs_composite.s4_slit_gaps - attenuator = fgs_composite.attenuator - flux = fgs_composite.flux - dcm = fgs_composite.dcm + undulator = fxc_composite.undulator + synchrotron = fxc_composite.synchrotron + slit_gaps = fxc_composite.s4_slit_gaps + attenuator = fxc_composite.attenuator + flux = fxc_composite.flux + dcm = fxc_composite.dcm @bpp.run_decorator() def read_run(u, s, g, a, f, dcm): @@ -150,6 +177,28 @@ def read_run(u, s, g, a, f, dcm): RE(read_run(undulator, synchrotron, slit_gaps, attenuator, flux, dcm)) +def test_xbpm_feedback_decorator( + RE: RunEngine, + fxc_composite: FlyScanXRayCentreComposite, + params: GridscanInternalParameters, + callbacks: XrayCentreCallbackCollection, +): + # This test is currently kind of more a unit test since we are faking XBPM feedback + # with ophyd.sim, but it should continue to pass when we replace it with something + # in S03 + + @transmission_and_xbpm_feedback_for_collection_decorator( + fxc_composite.xbpm_feedback, + fxc_composite.attenuator, + params.hyperion_params.ispyb_params.transmission_fraction, + ) + def decorated_plan(): + yield from bps.sleep(0.1) + + RE(decorated_plan()) + assert fxc_composite.xbpm_feedback.pos_stable.get() == 1 + + @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -168,11 +217,11 @@ def test_full_plan_tidies_at_end( complete: MagicMock, kickoff: MagicMock, wait: MagicMock, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, params: GridscanInternalParameters, RE: RunEngine, + callbacks: XrayCentreCallbackCollection, ): - callbacks = XrayCentreCallbackCollection.setup() callbacks.nexus_handler.nexus_writer_1 = MagicMock() callbacks.nexus_handler.nexus_writer_2 = MagicMock() callbacks.ispyb_handler.ispyb_ids = IspybIds( @@ -182,7 +231,7 @@ def test_full_plan_tidies_at_end( "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.setup", return_value=callbacks, ): - RE(flyscan_xray_centre(fgs_composite, params)) + RE(flyscan_xray_centre(fxc_composite, params)) set_shutter_to_manual.assert_called_once() @@ -204,22 +253,23 @@ def test_full_plan_tidies_at_end_when_plan_fails( complete: MagicMock, kickoff: MagicMock, wait: MagicMock, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, params: GridscanInternalParameters, RE: RunEngine, ): run_gridscan_and_move.side_effect = Exception() with pytest.raises(Exception): - RE(flyscan_xray_centre(fgs_composite, params)) + RE(flyscan_xray_centre(fxc_composite, params)) set_shutter_to_manual.assert_called_once() @pytest.mark.s03 def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_entry( RE: RunEngine, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, fetch_comment: Callable, params: GridscanInternalParameters, + callbacks: XrayCentreCallbackCollection, ): params.hyperion_params.detector_params.directory = "./tmp" params.hyperion_params.detector_params.prefix = str(uuid.uuid1()) @@ -228,17 +278,20 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 100 - callbacks = XrayCentreCallbackCollection.setup() - callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG mock_start_zocalo = MagicMock() callbacks.zocalo_handler.zocalo_interactor.run_start = mock_start_zocalo - with pytest.raises(WarningException): - RE(flyscan_xray_centre(fgs_composite, params)) + with pytest.raises(WarningException), patch( + "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.from_params", + lambda _: callbacks, + ): + RE(flyscan_xray_centre(fxc_composite, params)) dcid_used = callbacks.ispyb_handler.ispyb_ids = IspybIds( data_collection_ids=(0, 0), data_collection_group_id=0, grid_ids=(0,) ) + assert callbacks.ispyb_handler.ispyb.data_collection_ids is not None + dcid_used = callbacks.ispyb_handler.ispyb.data_collection_ids[0] comment = fetch_comment(dcid_used) @@ -253,7 +306,7 @@ def test_WHEN_plan_run_THEN_move_to_centre_returned_from_zocalo_expected_centre( complete: MagicMock, kickoff: MagicMock, RE: RunEngine, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, zocalo_env: None, params: GridscanInternalParameters, ): @@ -267,15 +320,19 @@ def test_WHEN_plan_run_THEN_move_to_centre_returned_from_zocalo_expected_centre( # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 1 - fgs_composite.eiger.stage = MagicMock() - fgs_composite.eiger.unstage = MagicMock() + fxc_composite.eiger.stage = MagicMock() + fxc_composite.eiger.unstage = MagicMock() callbacks = XrayCentreCallbackCollection.setup() callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG - RE(flyscan_xray_centre(fgs_composite, params)) + with patch( + "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.from_params", + lambda _: callbacks, + ): + RE(flyscan_xray_centre(fxc_composite, params)) # The following numbers are derived from the centre returned in fake_zocalo - assert fgs_composite.sample_motors.x.user_readback.get() == pytest.approx(-0.05) - assert fgs_composite.sample_motors.y.user_readback.get() == pytest.approx(0.05) - assert fgs_composite.sample_motors.z.user_readback.get() == pytest.approx(0.15) + assert fxc_composite.sample_motors.x.user_readback.get() == pytest.approx(0.05) + assert fxc_composite.sample_motors.y.user_readback.get() == pytest.approx(0.15) + assert fxc_composite.sample_motors.z.user_readback.get() == pytest.approx(0.25) From 6f6afa38c3bdfde5ababdc83c334c2bbc73dcb22 Mon Sep 17 00:00:00 2001 From: David Perl Date: Thu, 2 Nov 2023 12:27:01 +0000 Subject: [PATCH 02/10] correctly mark system test --- tests/system_tests/experiment_plans/test_fgs_plan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index d44f9a438..a40094cf0 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -177,6 +177,7 @@ def read_run(u, s, g, a, f, dcm): RE(read_run(undulator, synchrotron, slit_gaps, attenuator, flux, dcm)) +@pytest.mark.s03 def test_xbpm_feedback_decorator( RE: RunEngine, fxc_composite: FlyScanXRayCentreComposite, From a7a0d4a5957996a2ca701aec5583df0c45b3be1a Mon Sep 17 00:00:00 2001 From: David Perl Date: Thu, 1 Feb 2024 17:05:14 +0000 Subject: [PATCH 03/10] #818 do environment manipulation before any ophyd is loaded --- conftest.py | 11 +++++++++++ tests/conftest.py | 18 ------------------ .../experiment_plans/test_fgs_plan.py | 1 - 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/conftest.py b/conftest.py index 2a83757de..a68f50264 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,19 @@ +from os import environ, getenv from typing import Iterator from unittest.mock import patch import pytest +print("Adjusting S03 EPICS environment ...") +s03_epics_server_port = getenv("S03_EPICS_CA_SERVER_PORT") +s03_epics_repeater_port = getenv("S03_EPICS_CA_REPEATER_PORT") +if s03_epics_server_port: + environ["EPICS_CA_SERVER_PORT"] = s03_epics_server_port + print(f"[EPICS_CA_SERVER_PORT] = {s03_epics_server_port}") +if s03_epics_repeater_port: + environ["EPICS_CA_REPEATER_PORT"] = s03_epics_repeater_port + print(f"[EPICS_CA_REPEATER_PORT] = {s03_epics_repeater_port}") + def pytest_addoption(parser): parser.addoption( diff --git a/tests/conftest.py b/tests/conftest.py index 4119866be..e8aced196 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ import sys from functools import partial -from os import environ, getenv from typing import Callable, Generator, Optional, Sequence from unittest.mock import MagicMock, patch @@ -90,23 +89,6 @@ def pytest_runtest_setup(item): print("Skipping log setup for log test - deleting existing handlers") _destroy_loggers([*ALL_LOGGERS, dodal_logger]) - if "s03" in markers: - print("Running s03 test - setting EPICS server ports variables...") - s03_epics_server_port = getenv("S03_EPICS_CA_SERVER_PORT") - s03_epics_repeater_port = getenv("S03_EPICS_CA_REPEATER_PORT") - - assert ( - s03_epics_server_port is not None and s03_epics_repeater_port is not None - ), ( - "Please run the S03 launch script with the '-f' flag to run it on a port " - " which doesn't clash with the real EPICS ports." - ) - - environ["EPICS_CA_SERVER_PORT"] = s03_epics_server_port - print(f"[EPICS_CA_SERVER_PORT] = {s03_epics_server_port}") - environ["EPICS_CA_REPEATER_PORT"] = s03_epics_repeater_port - print(f"[EPICS_CA_REPEATER_PORT] = {s03_epics_repeater_port}") - def pytest_runtest_teardown(): if "dodal.beamlines.beamline_utils" in sys.modules: diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index a40094cf0..8df5721f0 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -109,7 +109,6 @@ def fxc_composite(): return composite -@pytest.mark.s03 def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.aperture_scatterguard assert fxc_composite.backlight From d422eca9068b24f2c1bfeadd3a2d6de62aeb9ab6 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Feb 2024 09:47:59 +0000 Subject: [PATCH 04/10] #818 update system test composite --- tests/system_tests/experiment_plans/test_fgs_plan.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 8df5721f0..da5ca548c 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -66,6 +66,10 @@ def callbacks(params): @pytest.fixture def fxc_composite(): + with patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"), patch( + "dodal.devices.zocalo.zocalo_results.workflows.recipe" + ), patch("dodal.devices.zocalo.zocalo_results.workflows.recipe"): + zocalo = i03.zocalo() composite = FlyScanXRayCentreComposite( attenuator=i03.attenuator(), aperture_scatterguard=i03.aperture_scatterguard(), @@ -82,7 +86,7 @@ def fxc_composite(): synchrotron=i03.synchrotron(fake_with_ophyd_sim=True), xbpm_feedback=i03.xbpm_feedback(fake_with_ophyd_sim=True), zebra=i03.zebra(), - zocalo=MagicMock(), + zocalo=zocalo, ) gda_beamline_parameters = GDABeamlineParameters.from_file( @@ -151,9 +155,9 @@ def test_run_gridscan_and_move( wait: MagicMock, params: GridscanInternalParameters, RE: RunEngine, - fgs_composite: FlyScanXRayCentreComposite, + fxc_composite: FlyScanXRayCentreComposite, ): - RE(run_gridscan_and_move(fgs_composite, params)) + RE(run_gridscan_and_move(fxc_composite, params)) @pytest.mark.s03 From 5627937cb890c428a8042b6468eeb27e0d462bd1 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Feb 2024 14:11:52 +0000 Subject: [PATCH 05/10] #818 run system test on whole plan --- .../experiment_plans/test_fgs_plan.py | 30 +++---------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index da5ca548c..3dc1af837 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -25,8 +25,6 @@ from hyperion.experiment_plans.flyscan_xray_centre_plan import ( FlyScanXRayCentreComposite, flyscan_xray_centre, - run_gridscan, - run_gridscan_and_move, ) from hyperion.external_interaction.callbacks.xray_centre.callback_collection import ( XrayCentreCallbackCollection, @@ -49,6 +47,7 @@ def params(): params = GridscanInternalParameters(**default_raw_params()) params.hyperion_params.beamline = SIM_BEAMLINE + params.hyperion_params.zocalo_environment = "dev_artemis" yield params @@ -60,7 +59,6 @@ def RE(): @pytest.fixture() def callbacks(params): callbacks = XrayCentreCallbackCollection.setup() - callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG yield callbacks @@ -118,28 +116,6 @@ def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.backlight -@pytest.mark.skip(reason="Broken due to eiger issues in s03") -@pytest.mark.s03 -@patch("bluesky.plan_stubs.wait", autospec=True) -@patch("bluesky.plan_stubs.kickoff", autospec=True) -@patch("bluesky.plan_stubs.complete", autospec=True) -@patch( - "hyperion.experiment_plans.flyscan_xray_centre_plan.wait_for_gridscan_valid", - autospec=True, -) -def test_run_gridscan( - wait_for_gridscan_valid: MagicMock, - complete: MagicMock, - kickoff: MagicMock, - wait: MagicMock, - params: GridscanInternalParameters, - RE: RunEngine, - fxc_composite: FlyScanXRayCentreComposite, -): - # Would be better to use flyscan_xray_centre instead but eiger doesn't work well in S03 - RE(run_gridscan(fxc_composite, params)) - - @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -156,8 +132,10 @@ def test_run_gridscan_and_move( params: GridscanInternalParameters, RE: RunEngine, fxc_composite: FlyScanXRayCentreComposite, + callbacks, ): - RE(run_gridscan_and_move(fxc_composite, params)) + [RE.subscribe(cb) for cb in callbacks] + RE(flyscan_xray_centre(fxc_composite, params)) @pytest.mark.s03 From 5fd0c93b0c20a7f2ef25608fe69c78a4f2f2dd6a Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Feb 2024 15:58:34 +0000 Subject: [PATCH 06/10] #818 fix xray_centre system tests --- .../experiment_plans/test_fgs_plan.py | 83 +++++++------------ 1 file changed, 30 insertions(+), 53 deletions(-) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 3dc1af837..11f6b288e 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -12,6 +12,7 @@ GDABeamlineParameters, ) from dodal.devices.aperturescatterguard import AperturePositions +from dodal.devices.smargon import Smargon from ophyd.status import Status from hyperion.device_setup_plans.read_hardware_for_setup import ( @@ -30,7 +31,6 @@ XrayCentreCallbackCollection, ) from hyperion.external_interaction.ispyb.store_datacollection_in_ispyb import IspybIds -from hyperion.parameters.constants import DEV_ISPYB_DATABASE_CFG as ISPYB_CONFIG from hyperion.parameters.constants import SIM_BEAMLINE from hyperion.parameters.external_parameters import from_file as default_raw_params from hyperion.parameters.plan_specific.gridscan_internal_params import ( @@ -47,6 +47,7 @@ def params(): params = GridscanInternalParameters(**default_raw_params()) params.hyperion_params.beamline = SIM_BEAMLINE + params.hyperion_params.ispyb_params.current_energy_ev = 10000 params.hyperion_params.zocalo_environment = "dev_artemis" yield params @@ -58,10 +59,17 @@ def RE(): @pytest.fixture() def callbacks(params): - callbacks = XrayCentreCallbackCollection.setup() + with patch( + "hyperion.external_interaction.callbacks.xray_centre.nexus_callback.NexusWriter" + ): + callbacks = XrayCentreCallbackCollection.setup() yield callbacks +def reset_positions(smargon: Smargon): + yield from bps.mv(smargon.x, -1, smargon.y, -1, smargon.z, -1) + + @pytest.fixture def fxc_composite(): with patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"), patch( @@ -87,6 +95,8 @@ def fxc_composite(): zocalo=zocalo, ) + composite.dcm.energy_in_kev.user_readback.sim_put(12.345) # type: ignore + gda_beamline_parameters = GDABeamlineParameters.from_file( BEAMLINE_PARAMETER_PATHS["i03"] ) @@ -116,28 +126,6 @@ def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.backlight -@pytest.mark.s03 -@patch("bluesky.plan_stubs.wait", autospec=True) -@patch("bluesky.plan_stubs.kickoff", autospec=True) -@patch("bluesky.plan_stubs.complete", autospec=True) -@patch( - "hyperion.experiment_plans.flyscan_xray_centre_plan.wait_for_gridscan_valid", - autospec=True, -) -def test_run_gridscan_and_move( - wait_for_gridscan_valid: MagicMock, - complete: MagicMock, - kickoff: MagicMock, - wait: MagicMock, - params: GridscanInternalParameters, - RE: RunEngine, - fxc_composite: FlyScanXRayCentreComposite, - callbacks, -): - [RE.subscribe(cb) for cb in callbacks] - RE(flyscan_xray_centre(fxc_composite, params)) - - @pytest.mark.s03 def test_read_hardware_for_ispyb_pre_collection( RE: RunEngine, @@ -204,16 +192,13 @@ def test_full_plan_tidies_at_end( RE: RunEngine, callbacks: XrayCentreCallbackCollection, ): - callbacks.nexus_handler.nexus_writer_1 = MagicMock() - callbacks.nexus_handler.nexus_writer_2 = MagicMock() + RE(reset_positions(fxc_composite.smargon)) + callbacks.ispyb_handler.ispyb_ids = IspybIds( data_collection_ids=(0, 0), data_collection_group_id=0, grid_ids=(0,) ) - with patch( - "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.setup", - return_value=callbacks, - ): - RE(flyscan_xray_centre(fxc_composite, params)) + [RE.subscribe(cb) for cb in callbacks] + RE(flyscan_xray_centre(fxc_composite, params)) set_shutter_to_manual.assert_called_once() @@ -245,8 +230,12 @@ def test_full_plan_tidies_at_end_when_plan_fails( set_shutter_to_manual.assert_called_once() +@patch( + "hyperion.external_interaction.callbacks.xray_centre.zocalo_callback.ZocaloTrigger" +) @pytest.mark.s03 def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_entry( + zocalo_trigger: MagicMock, RE: RunEngine, fxc_composite: FlyScanXRayCentreComposite, fetch_comment: Callable, @@ -259,14 +248,10 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 100 + RE(reset_positions(fxc_composite.smargon)) - mock_start_zocalo = MagicMock() - callbacks.zocalo_handler.zocalo_interactor.run_start = mock_start_zocalo - - with pytest.raises(WarningException), patch( - "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.from_params", - lambda _: callbacks, - ): + [RE.subscribe(cb) for cb in callbacks] + with pytest.raises(WarningException): RE(flyscan_xray_centre(fxc_composite, params)) dcid_used = callbacks.ispyb_handler.ispyb_ids = IspybIds( @@ -278,23 +263,22 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en comment = fetch_comment(dcid_used) assert "too long/short/bent" in comment - mock_start_zocalo.assert_not_called() + zocalo_trigger.run_start.assert_not_called() @pytest.mark.s03 +@patch("bps.plan_stubs.wait") @patch("hyperion.experiment_plans.flyscan_xray_centre_plan.bps.kickoff", autospec=True) @patch("hyperion.experiment_plans.flyscan_xray_centre_plan.bps.complete", autospec=True) -def test_WHEN_plan_run_THEN_move_to_centre_returned_from_zocalo_expected_centre( +def test_complete_xray_centre_plan_and_returned_centre( complete: MagicMock, kickoff: MagicMock, RE: RunEngine, fxc_composite: FlyScanXRayCentreComposite, zocalo_env: None, params: GridscanInternalParameters, + callbacks, ): - """This test currently avoids hardware interaction and is mostly confirming - interaction with dev_ispyb and fake_zocalo""" - params.hyperion_params.detector_params.directory = "./tmp" params.hyperion_params.detector_params.prefix = str(uuid.uuid1()) params.hyperion_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" @@ -302,17 +286,10 @@ def test_WHEN_plan_run_THEN_move_to_centre_returned_from_zocalo_expected_centre( # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 1 - fxc_composite.eiger.stage = MagicMock() - fxc_composite.eiger.unstage = MagicMock() - - callbacks = XrayCentreCallbackCollection.setup() - callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG + RE(reset_positions(fxc_composite.smargon)) - with patch( - "hyperion.experiment_plans.flyscan_xray_centre_plan.XrayCentreCallbackCollection.from_params", - lambda _: callbacks, - ): - RE(flyscan_xray_centre(fxc_composite, params)) + [RE.subscribe(cb) for cb in callbacks] + RE(flyscan_xray_centre(fxc_composite, params)) # The following numbers are derived from the centre returned in fake_zocalo assert fxc_composite.sample_motors.x.user_readback.get() == pytest.approx(0.05) From b2fc9779013629121178bf1caf8501ee8d13045c Mon Sep 17 00:00:00 2001 From: David Perl Date: Mon, 5 Feb 2024 11:17:53 +0000 Subject: [PATCH 07/10] #818 fix xray_centre system tests --- .../flyscan_xray_centre_plan.py | 2 +- .../experiment_plans/test_fgs_plan.py | 50 ++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 0e9e832a9..22f7a50d2 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -193,7 +193,7 @@ def run_gridscan( LOGGER.info("Setting fgs params") yield from set_flyscan_params(fgs_motors, parameters.experiment_params) - + LOGGER.info("Waiting for gridscan validity check") yield from wait_for_gridscan_valid(fgs_motors) @bpp.set_run_key_decorator(DO_FGS) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 11f6b288e..887e69ff5 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -267,22 +267,60 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en @pytest.mark.s03 -@patch("bps.plan_stubs.wait") -@patch("hyperion.experiment_plans.flyscan_xray_centre_plan.bps.kickoff", autospec=True) -@patch("hyperion.experiment_plans.flyscan_xray_centre_plan.bps.complete", autospec=True) -def test_complete_xray_centre_plan_and_returned_centre( - complete: MagicMock, - kickoff: MagicMock, +def test_complete_xray_centre_plan_with_no_callbacks_falls_back_to_centre( + RE: RunEngine, + fxc_composite: FlyScanXRayCentreComposite, + zocalo_env: None, + params: GridscanInternalParameters, + callbacks, + done_status, +): + fxc_composite.fast_grid_scan.kickoff = MagicMock(return_value=done_status) + fxc_composite.fast_grid_scan.complete = MagicMock(return_value=done_status) + + params.hyperion_params.detector_params.directory = "./tmp" + params.hyperion_params.detector_params.prefix = str(uuid.uuid1()) + params.hyperion_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" + + params.experiment_params.set_stub_offsets = False + + # Currently s03 calls anything with z_steps > 1 invalid + params.experiment_params.z_steps = 1 + + RE(reset_positions(fxc_composite.smargon)) + + def zocalo_trigger(): + fxc_composite.zocalo._raw_results_received.put({"results": []}) + return done_status + + # [RE.subscribe(cb) for cb in callbacks] + fxc_composite.zocalo.trigger = MagicMock(side_effect=zocalo_trigger) + RE(flyscan_xray_centre(fxc_composite, params)) + + # The following numbers are derived from the centre returned in fake_zocalo + assert fxc_composite.sample_motors.x.user_readback.get() == pytest.approx(-1) + assert fxc_composite.sample_motors.y.user_readback.get() == pytest.approx(-1) + assert fxc_composite.sample_motors.z.user_readback.get() == pytest.approx(-1) + + +@pytest.mark.s03 +def test_complete_xray_centre_plan_with_callbacks_moves_to_centre( RE: RunEngine, fxc_composite: FlyScanXRayCentreComposite, zocalo_env: None, params: GridscanInternalParameters, callbacks, + done_status, ): + fxc_composite.fast_grid_scan.kickoff = MagicMock(return_value=done_status) + fxc_composite.fast_grid_scan.complete = MagicMock(return_value=done_status) + params.hyperion_params.detector_params.directory = "./tmp" params.hyperion_params.detector_params.prefix = str(uuid.uuid1()) params.hyperion_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" + params.experiment_params.set_stub_offsets = False + # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 1 From 8298311557da0674f4a9c72a46e54c4b8f1944a8 Mon Sep 17 00:00:00 2001 From: David Perl Date: Mon, 5 Feb 2024 11:24:09 +0000 Subject: [PATCH 08/10] #818 fix s03 mark on test --- tests/system_tests/experiment_plans/test_fgs_plan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 887e69ff5..63b7d2d5f 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -121,6 +121,7 @@ def fxc_composite(): return composite +@pytest.mark.s03 def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.aperture_scatterguard assert fxc_composite.backlight From 0b892a337360942af1fc7f3b295ef5919d289f30 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 8 Mar 2024 15:15:33 +0000 Subject: [PATCH 09/10] fix typing --- .../experiment_plans/test_grid_detection_plan.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py index 5ce300208..f7db130c0 100644 --- a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py +++ b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py @@ -84,7 +84,6 @@ def test_grid_detection_plan_runs_and_triggers_snapshots( test_config_files, fake_devices, ): - params = OAVParameters("loopCentring", test_config_files["oav_config_json"]) cb = OavSnapshotCallback() RE.subscribe(cb) @@ -249,15 +248,21 @@ def test_when_grid_detection_plan_run_then_grid_detection_callback_gets_correct_ my_grid_params = cb.get_grid_parameters() test_x_grid_axis = GridAxis( - my_grid_params.x_start, my_grid_params.x_step_size, my_grid_params.x_steps + start=my_grid_params.x_start, + step_size=my_grid_params.x_step_size, + full_steps=my_grid_params.x_steps, ) test_y_grid_axis = GridAxis( - my_grid_params.y1_start, my_grid_params.y_step_size, my_grid_params.y_steps + start=my_grid_params.y1_start, + step_size=my_grid_params.y_step_size, + full_steps=my_grid_params.y_steps, ) test_z_grid_axis = GridAxis( - my_grid_params.z2_start, my_grid_params.z_step_size, my_grid_params.z_steps + start=my_grid_params.z2_start, + step_size=my_grid_params.z_step_size, + full_steps=my_grid_params.z_steps, ) assert my_grid_params.x_start == pytest.approx(-0.7942199999999999) From f76251049eee6fa7b28a202c85792e29c80247a8 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 8 Mar 2024 16:31:16 +0000 Subject: [PATCH 10/10] #818 fix system tests --- setup.cfg | 2 +- .../experiment_plans/test_fgs_plan.py | 44 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5bd19db51..dae2d0553 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = xarray doct databroker - dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@ca9d6df8f17f88ce0128af9cbdfd40d0cfc44a26 + dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@97e3cdc11b1b5092c7f12ab6bc5ea1d702401b68 pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 scipy pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 25f2e3ea3..9d6db4722 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -5,6 +5,7 @@ import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp import pytest +import pytest_asyncio from bluesky.run_engine import RunEngine from dodal.beamlines import i03 from dodal.beamlines.beamline_parameters import ( @@ -49,6 +50,8 @@ def params(): params.hyperion_params.beamline = CONST.SIM.BEAMLINE params.hyperion_params.ispyb_params.current_energy_ev = 10000 params.hyperion_params.zocalo_environment = "dev_artemis" + params.hyperion_params.ispyb_params.microns_per_pixel_x = 10 + params.hyperion_params.ispyb_params.microns_per_pixel_y = 10 yield params @@ -66,12 +69,13 @@ def reset_positions(smargon: Smargon): yield from bps.mv(smargon.x, -1, smargon.y, -1, smargon.z, -1) -@pytest.fixture -def fxc_composite(): +@pytest_asyncio.fixture +async def fxc_composite(): with patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"), patch( "dodal.devices.zocalo.zocalo_results.workflows.recipe" ), patch("dodal.devices.zocalo.zocalo_results.workflows.recipe"): zocalo = i03.zocalo() + composite = FlyScanXRayCentreComposite( attenuator=i03.attenuator(), aperture_scatterguard=i03.aperture_scatterguard(), @@ -92,6 +96,7 @@ def fxc_composite(): zocalo=zocalo, ) + await composite.robot.barcode.bare_signal._backend.put(["ABCDEFGHIJ"]) # type: ignore composite.dcm.energy_in_kev.user_readback.sim_put(12.345) # type: ignore gda_beamline_parameters = GDABeamlineParameters.from_file( @@ -102,9 +107,9 @@ def fxc_composite(): gda_beamline_parameters ) composite.aperture_scatterguard.load_aperture_positions(aperture_positions) - composite.aperture_scatterguard.aperture.z.move( - aperture_positions.LARGE.location[2], wait=True - ) + composite.aperture_scatterguard._set_raw_unsafe( + aperture_positions.LARGE.location + ).wait() composite.eiger.cam.manual_trigger.put("Yes") composite.eiger.odin.check_odin_initialised = lambda: (True, "") composite.eiger.stage = MagicMock(return_value=Status(done=True, success=True)) @@ -118,12 +123,14 @@ def fxc_composite(): return composite +@pytest.mark.asyncio @pytest.mark.s03 def test_s03_devices_connect(fxc_composite: FlyScanXRayCentreComposite): assert fxc_composite.aperture_scatterguard assert fxc_composite.backlight +@pytest.mark.asyncio @pytest.mark.s03 def test_read_hardware_for_ispyb_pre_collection( RE: RunEngine, @@ -140,7 +147,13 @@ def test_read_hardware_for_ispyb_pre_collection( @bpp.run_decorator() def read_run(u, s, g, r, a, f, dcm, ap_sg): - yield from read_hardware_for_ispyb_pre_collection(u, s, g, r, ap_sg) + yield from read_hardware_for_ispyb_pre_collection( + undulator=u, + synchrotron=s, + s4_slit_gaps=g, + aperture_scatterguard=ap_sg, + robot=r, + ) yield from read_hardware_for_ispyb_during_collection(a, f, dcm) RE( @@ -157,6 +170,7 @@ def read_run(u, s, g, r, a, f, dcm, ap_sg): ) +@pytest.mark.asyncio @pytest.mark.s03 def test_xbpm_feedback_decorator( RE: RunEngine, @@ -180,6 +194,7 @@ def decorated_plan(): assert fxc_composite.xbpm_feedback.pos_stable.get() == 1 +@pytest.mark.asyncio @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -215,6 +230,7 @@ def test_full_plan_tidies_at_end( set_shutter_to_manual.assert_called_once() +@pytest.mark.asyncio @pytest.mark.s03 @patch("bluesky.plan_stubs.wait", autospec=True) @patch("bluesky.plan_stubs.kickoff", autospec=True) @@ -243,9 +259,8 @@ def test_full_plan_tidies_at_end_when_plan_fails( set_shutter_to_manual.assert_called_once() -@patch( - "hyperion.external_interaction.callbacks.xray_centre.zocalo_callback.ZocaloTrigger" -) +@patch("hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger") +@pytest.mark.asyncio @pytest.mark.s03 def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_entry( zocalo_trigger: MagicMock, @@ -262,18 +277,13 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en # Currently s03 calls anything with z_steps > 1 invalid params.experiment_params.z_steps = 100 RE(reset_positions(fxc_composite.smargon)) - mock_start_zocalo = MagicMock() - - callbacks.ispyb_handler.emit_cb.zocalo_interactor.run_start = mock_start_zocalo # type: ignore [RE.subscribe(cb) for cb in callbacks] with pytest.raises(WarningException): RE(flyscan_xray_centre(fxc_composite, params)) - dcid_used = callbacks.ispyb_handler.ispyb_ids = IspybIds( - data_collection_ids=(0, 0), data_collection_group_id=0, grid_ids=(0,) - ) - assert callbacks.ispyb_handler.ispyb_ids.data_collection_ids is not None + ids = callbacks.ispyb_handler.ispyb_ids + assert ids.data_collection_group_id is not None dcid_used = callbacks.ispyb_handler.ispyb_ids.data_collection_ids[0] comment = fetch_comment(dcid_used) @@ -282,6 +292,7 @@ def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_en zocalo_trigger.run_start.assert_not_called() +@pytest.mark.asyncio @pytest.mark.s03 def test_complete_xray_centre_plan_with_no_callbacks_falls_back_to_centre( RE: RunEngine, @@ -319,6 +330,7 @@ def zocalo_trigger(): assert fxc_composite.sample_motors.z.user_readback.get() == pytest.approx(-1) +@pytest.mark.asyncio @pytest.mark.s03 def test_complete_xray_centre_plan_with_callbacks_moves_to_centre( RE: RunEngine,