Update pydantic and ophyd_async to latest versions (#456)

* Initial pydantic conversion using tool, also removed the dynamic switching of allowing extras and added test to ensure no regression

* Fix various other pydantic issues

* Fix issues from ophyd-async update

* Fix all the hyperion issues

* Fix all i24 SSX unit tests

* Add issues references for ignored warnings

* Remove ignoring zocalo as no longer needed

pyproject.toml

@@ -40,12 +40,12 @@ dependencies = [
     # These dependencies may be issued as pre-release versions and should have a pin constraint
     # as by default pip-install will not upgrade to a pre-release.
     # 
-    "blueapi == 0.4.5a1",
+    "blueapi >= 0.5.0",
     "daq-config-server >= 0.1.1",
     "ophyd == 1.9.0",
     "ophyd-async >= 0.3a5",
     "bluesky >= 1.13.0a4",
-    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@9e0e09ac0166899d49128e6c6a01e7c14e443f1f",
+    "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@5e8b6fc2933a2b2b2e788d432c8408317ac4a5cc",
 ]
 
 
@@ -58,7 +58,8 @@ requires-python = ">=3.11"
 dev = [
     "black",
     "build",
-    "copier",
+    # Commented out due to dependency version conflict with pydantic 1.x
+    # "copier",
     "diff-cover",
     "GitPython",
     "ipython",
@@ -120,10 +121,7 @@ addopts = """
 # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings
 filterwarnings = [
     "error",
-    # Ignore deprecation warning from zocalo (https://github.com/DiamondLightSource/python-zocalo/issues/256)
-    "ignore::DeprecationWarning:zocalo",
-    "ignore::DeprecationWarning:pkg_resources",
-    # Ignore incorrect errors  from bluesky (remove when https://github.com/bluesky/bluesky/issues/1797 is released)
+    # Ignore incorrect errors from bluesky (remove when https://github.com/bluesky/bluesky/issues/1797 is released)
     "ignore:((.|\n)*)was never iterated.*:RuntimeWarning",
     # Ignore deprecation warning from sqlalchemy
     "ignore::sqlalchemy.exc.MovedIn20Warning",
@@ -135,6 +133,8 @@ filterwarnings = [
     "ignore:(.*)unclosed file(.*)name='(.*)dodal.log'(.*):ResourceWarning",
     "ignore:(.*)unclosed <socket(.*):ResourceWarning",
     "ignore:(.*)unclosed event loop(.*):ResourceWarning",
+    # Ignore pydantic 2 issues from blueapi, see https://github.com/DiamondLightSource/blueapi/issues/622
+    "ignore::DeprecationWarning:blueapi",
 ]
 # Doctest python code in docs, python code in src docstrings, test functions in tests
 testpaths = "docs src tests/unit_tests"

src/mx_bluesky/beamlines/i24/serial/log.py

@@ -4,13 +4,12 @@
 from os import environ
 from pathlib import Path
 
-from bluesky.log import logger as bluesky_logger
 from dodal.log import (
     ERROR_LOG_BUFFER_LINES,
+    integrate_bluesky_and_ophyd_logging,
     set_up_all_logging_handlers,
 )
 from dodal.log import LOGGER as dodal_logger
-from ophyd_async.log import logger as ophyd_async_logger
 
 VISIT_PATH = Path("/dls_sw/i24/etc/ssx_current_visit.txt")
 
@@ -88,13 +87,6 @@ def _get_logging_file_path() -> Path:
     return logging_path
 
 
-def integrate_bluesky_and_ophyd_logging(parent_logger: logging.Logger):
-    """Integrate only bluesky and ophyd_async loggers."""
-    for logger in [bluesky_logger, ophyd_async_logger]:
-        logger.parent = parent_logger
-        logger.setLevel(logging.DEBUG)
-
-
 def default_logging_setup(dev_mode: bool = False):
     """ Default log setup for i24 serial.
 

src/mx_bluesky/beamlines/i24/serial/parameters/experiment_parameters.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, validator
+from pydantic import BaseModel, field_validator
 
 from mx_bluesky.beamlines.i24.serial.fixed_target.ft_utils import (
     ChipType,
@@ -21,7 +21,8 @@ class SerialExperiment(BaseModel):
     detector_distance_mm: float
     detector_name: Literal["eiger", "pilatus"]
 
-    @validator("visit", pre=True)
+    @field_validator("visit", mode="before")
+    @classmethod
     def _parse_visit(cls, visit: str | Path):
         if isinstance(visit, str):
             return Path(visit)
@@ -56,8 +57,6 @@ def from_file(cls, filename: str | Path):
 class ChipDescription(BaseModel):
     """Parameters defining the chip in use for FT collection."""
 
-    model_config = ConfigDict(use_enum_values=True)
-
     chip_type: ChipType
     x_num_steps: int
     y_num_steps: int
@@ -68,13 +67,6 @@ class ChipDescription(BaseModel):
     b2b_horz: float
     b2b_vert: float
 
-    @validator("chip_type", pre=True)
-    def _parse_chip(cls, chip_type: str | int):
-        if isinstance(chip_type, str):
-            return ChipType[chip_type]
-        else:
-            return ChipType(chip_type)
-
     @property
     def chip_format(self) -> list[int]:
         return [self.x_blocks, self.y_blocks, self.x_num_steps, self.y_num_steps]
@@ -97,26 +89,13 @@ def y_block_size(self) -> float:
 class FixedTargetParameters(SerialExperiment, LaserExperiment):
     """Fixed target parameter model."""
 
-    model_config = ConfigDict(use_enum_values=True)
-
     num_exposures: int
     chip: ChipDescription
     map_type: MappingType
     pump_repeat: PumpProbeSetting
     checker_pattern: bool = False
     total_num_images: int = 0  # Calculated in the code for now
 
-    @validator("map_type", pre=True)
-    def _parse_map(cls, map_type: str | int):
-        if isinstance(map_type, str):
-            return MappingType[map_type]
-        else:
-            return MappingType(map_type)
-
-    @validator("pump_repeat", pre=True)
-    def _parse_pump(cls, pump_repeat: int):
-        return PumpProbeSetting(pump_repeat)
-
     @classmethod
     def from_file(cls, filename: str | Path):
         with open(filename) as fh:

src/mx_bluesky/hyperion/__main__.py

@@ -71,9 +71,13 @@ def __init__(self, status: Status, message: str = "") -> None:
 class ErrorStatusAndMessage(StatusAndMessage):
     exception_type: str = ""
 
-    def __init__(self, exception: Exception) -> None:
-        super().__init__(Status.FAILED, repr(exception))
-        self.exception_type = type(exception).__name__
+
+def make_error_status_and_message(exception: Exception):
+    return ErrorStatusAndMessage(
+        status=Status.FAILED.value,
+        message=repr(exception),
+        exception_type=type(exception).__name__,
+    )
 
 
 class BlueskyRunner:
@@ -119,7 +123,7 @@ def start(
         plan_name: str,
         callbacks: CallbacksFactory | None,
     ) -> StatusAndMessage:
-        LOGGER.info(f"Started with parameters: {parameters.json(indent=2)}")
+        LOGGER.info(f"Started with parameters: {parameters.model_dump_json(indent=2)}")
 
         devices: Any = PLAN_REGISTRY[plan_name]["setup"](self.context)
 
@@ -146,7 +150,7 @@ def stopping_thread(self):
             self.RE.abort()
             self.current_status = StatusAndMessage(Status.IDLE)
         except Exception as e:
-            self.current_status = ErrorStatusAndMessage(e)
+            self.current_status = make_error_status_and_message(e)
 
     def stop(self) -> StatusAndMessage:
         if self.current_status.status == Status.IDLE.value:
@@ -197,15 +201,15 @@ def wait_on_queue(self):
                     self.last_run_aborted = False
                 except WarningException as exception:
                     LOGGER.warning("Warning Exception", exc_info=True)
-                    self.current_status = ErrorStatusAndMessage(exception)
+                    self.current_status = make_error_status_and_message(exception)
                 except Exception as exception:
                     LOGGER.error("Exception on running plan", exc_info=True)
 
                     if self.last_run_aborted:
                         # Aborting will cause an exception here that we want to swallow
                         self.last_run_aborted = False
                     else:
-                        self.current_status = ErrorStatusAndMessage(exception)
+                        self.current_status = make_error_status_and_message(exception)
                 finally:
                     [
                         self.RE.unsubscribe(cb)
@@ -231,6 +235,8 @@ def compose_start_args(context: BlueskyContext, plan_name: str, action: Actions)
         )
     try:
         parameters = experiment_internal_param_type(**json.loads(request.data))
+        if parameters.model_extra:
+            raise ValueError(f"Extra fields not allowed {parameters.model_extra}")
     except Exception as e:
         raise ValueError(
             f"Supplied parameters don't match the plan for this endpoint {request.data}"
@@ -255,7 +261,7 @@ def put(self, plan_name: str, action: Actions):
                     plan, params, plan_name, callback_type
                 )
             except Exception as e:
-                status_and_message = ErrorStatusAndMessage(e)
+                status_and_message = make_error_status_and_message(e)
                 LOGGER.error(format_exception(e))
 
         elif action == Actions.STOP.value:

src/mx_bluesky/hyperion/device_setup_plans/setup_panda.py

@@ -5,10 +5,10 @@
 
 import bluesky.plan_stubs as bps
 from blueapi.core import MsgGenerator
-from dodal.common.beamlines.beamline_utils import get_directory_provider
+from dodal.common.beamlines.beamline_utils import get_path_provider
 from dodal.devices.fast_grid_scan import PandAGridScanParams
 from ophyd_async.core import load_device
-from ophyd_async.panda import (
+from ophyd_async.fastcs.panda import (
     HDFPanda,
     SeqTable,
     SeqTableRow,
@@ -212,6 +212,6 @@ def set_panda_directory(panda_directory: Path) -> MsgGenerator:
     suffix = datetime.now().strftime("_%Y%m%d%H%M%S")
 
     async def set_panda_dir():
-        await get_directory_provider().update(directory=panda_directory, suffix=suffix)
+        await get_path_provider().update(directory=panda_directory, suffix=suffix)
 
     yield from bps.wait_for([set_panda_dir])

src/mx_bluesky/hyperion/experiment_plans/flyscan_xray_centre_plan.py

@@ -43,7 +43,7 @@
     get_processing_result,
 )
 from dodal.plans.check_topup import check_topup_and_wait_if_necessary
-from ophyd_async.panda import HDFPanda
+from ophyd_async.fastcs.panda import HDFPanda
 from scanspec.core import AxesPoints, Axis
 
 from mx_bluesky.hyperion.device_setup_plans.manipulate_sample import move_x_y_z
@@ -138,7 +138,7 @@ def flyscan_xray_centre(
             "subplan_name": CONST.PLAN.GRIDSCAN_OUTER,
             CONST.TRIGGER.ZOCALO: CONST.PLAN.DO_FGS,
             "zocalo_environment": parameters.zocalo_environment,
-            "hyperion_parameters": parameters.json(),
+            "hyperion_parameters": parameters.model_dump_json(),
             "activate_callbacks": [
                 "GridscanNexusFileCallback",
             ],

src/mx_bluesky/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py

@@ -26,7 +26,7 @@
 from dodal.devices.zebra import Zebra
 from dodal.devices.zebra_controlled_shutter import ZebraShutter
 from dodal.devices.zocalo import ZocaloResults
-from ophyd_async.panda import HDFPanda
+from ophyd_async.fastcs.panda import HDFPanda
 
 from mx_bluesky.hyperion.device_setup_plans.manipulate_sample import (
     move_aperture_if_required,
@@ -95,7 +95,7 @@ def create_parameters_for_flyscan_xray_centre(
     grid_scan_with_edge_params: GridScanWithEdgeDetect,
     grid_parameters: GridParamUpdate,
 ) -> ThreeDGridScan:
-    params_json = grid_scan_with_edge_params.dict()
+    params_json = grid_scan_with_edge_params.model_dump()
     params_json.update(grid_parameters)
     flyscan_xray_centre_parameters = ThreeDGridScan(**params_json)
     LOGGER.info(f"Parameters for FGS: {flyscan_xray_centre_parameters}")

src/mx_bluesky/hyperion/experiment_plans/pin_centre_then_xray_centre_plan.py

@@ -39,11 +39,11 @@ def create_devices(context: BlueskyContext) -> GridDetectThenXRayCentreComposite
 def create_parameters_for_grid_detection(
     pin_centre_parameters: PinTipCentreThenXrayCentre,
 ) -> GridScanWithEdgeDetect:
-    params_json = json.loads(pin_centre_parameters.json())
+    params_json = json.loads(pin_centre_parameters.model_dump_json())
     del params_json["tip_offset_um"]
     grid_detect_and_xray_centre = GridScanWithEdgeDetect(**params_json)
     LOGGER.info(
-        f"Parameters for grid detect and xray centre: {grid_detect_and_xray_centre.json(indent=2)}"
+        f"Parameters for grid detect and xray centre: {grid_detect_and_xray_centre.model_dump_json(indent=2)}"
     )
     return grid_detect_and_xray_centre
 

src/mx_bluesky/hyperion/experiment_plans/robot_load_then_centre_plan.py

@@ -32,7 +32,7 @@
 from dodal.devices.zebra import Zebra
 from dodal.devices.zocalo import ZocaloResults
 from dodal.plans.motor_util_plans import MoveTooLarge, home_and_reset_wrapper
-from ophyd_async.panda import HDFPanda
+from ophyd_async.fastcs.panda import HDFPanda
 
 from mx_bluesky.hyperion.device_setup_plans.utils import (
     start_preparing_data_collection_then_do_plan,

src/mx_bluesky/hyperion/experiment_plans/rotation_scan_plan.py

@@ -344,7 +344,7 @@ def rotation_scan(
         md={
             "subplan_name": CONST.PLAN.ROTATION_OUTER,
             CONST.TRIGGER.ZOCALO: CONST.PLAN.ROTATION_MAIN,
-            "hyperion_parameters": parameters.json(),
+            "hyperion_parameters": parameters.model_dump_json,
             "activate_callbacks": [
                 "RotationISPyBCallback",
                 "RotationNexusFileCallback",
@@ -419,7 +419,7 @@ def _multi_rotation_scan():
                 md={
                     "subplan_name": CONST.PLAN.ROTATION_OUTER,
                     CONST.TRIGGER.ZOCALO: CONST.PLAN.ROTATION_MAIN,
-                    "hyperion_parameters": single_scan.json(),
+                    "hyperion_parameters": single_scan.model_dump_json(),
                 }
             )
             def rotation_scan_core(

src/mx_bluesky/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py

@@ -53,7 +53,7 @@ def ispyb_activation_wrapper(plan_generator: MsgGenerator, parameters):
         md={
             "activate_callbacks": ["GridscanISPyBCallback"],
             "subplan_name": CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN,
-            "hyperion_parameters": parameters.json(),
+            "hyperion_parameters": parameters.model_dump_json(),
         },
     )
 
@@ -93,9 +93,7 @@ def activity_gated_start(self, doc: RunStart):
                 "ISPyB callback received start document with experiment parameters and "
                 f"uid: {self.uid_to_finalize_on}"
             )
-            self.params = GridCommon.from_json(
-                doc.get("hyperion_parameters"), allow_extras=True
-            )
+            self.params = GridCommon.from_json(doc.get("hyperion_parameters"))
             self.ispyb = StoreInIspyb(self.ispyb_config)
             data_collection_group_info = populate_data_collection_group(self.params)