Merge branch 'main' into ft-edm-updates

pyproject.toml

@@ -40,12 +40,12 @@ dependencies = [
  # These dependencies may be issued as pre-release versions and should have a pin constraint
  # as by default pip-install will not upgrade to a pre-release.
  # 
- "blueapi == 0.4.5a1",
+ "blueapi >= 0.5.0",
  "daq-config-server >= 0.1.1",
  "ophyd == 1.9.0",
  "ophyd-async >= 0.3a5",
  "bluesky >= 1.13.0a4",
- "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git",
+ "dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@5e8b6fc2933a2b2b2e788d432c8408317ac4a5cc",
 ]
 
 
@@ -58,7 +58,8 @@ requires-python = ">=3.11"
 dev = [
  "black",
  "build",
- "copier",
+ # Commented out due to dependency version conflict with pydantic 1.x
+ # "copier",
  "diff-cover",
  "GitPython",
  "ipython",
@@ -120,10 +121,7 @@ addopts = """
 # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings
 filterwarnings = [
  "error",
- # Ignore deprecation warning from zocalo (https://github.com/DiamondLightSource/python-zocalo/issues/256)
- "ignore::DeprecationWarning:zocalo",
- "ignore::DeprecationWarning:pkg_resources",
- # Ignore incorrect errors from bluesky (remove when https://github.com/bluesky/bluesky/issues/1797 is released)
+ # Ignore incorrect errors from bluesky (remove when https://github.com/bluesky/bluesky/issues/1797 is released)
  "ignore:((.|\n)*)was never iterated.*:RuntimeWarning",
  # Ignore deprecation warning from sqlalchemy
  "ignore::sqlalchemy.exc.MovedIn20Warning",
@@ -135,6 +133,8 @@ filterwarnings = [
  "ignore:(.*)unclosed file(.*)name='(.*)dodal.log'(.*):ResourceWarning",
  "ignore:(.*)unclosed <socket(.*):ResourceWarning",
  "ignore:(.*)unclosed event loop(.*):ResourceWarning",
+ # Ignore pydantic 2 issues from blueapi, see https://github.com/DiamondLightSource/blueapi/issues/622
+ "ignore::DeprecationWarning:blueapi",
 ]
 # Doctest python code in docs, python code in src docstrings, test functions in tests
 testpaths = "docs src tests/unit_tests"

src/mx_bluesky/beamlines/i04/__init__.py

@@ -1,3 +1,3 @@
-from mx_bluesky.beamlines.i04.thawing_plan import thaw, thaw_and_center
+from mx_bluesky.beamlines.i04.thawing_plan import thaw, thaw_and_stream_to_redis
 
-__all__ = ["thaw", "thaw_and_center"]
+__all__ = ["thaw", "thaw_and_stream_to_redis"]

src/mx_bluesky/beamlines/i04/thawing_plan.py

@@ -13,7 +13,7 @@
 from mx_bluesky.beamlines.i04.callbacks.murko_callback import MurkoCallback
 
 
-def thaw_and_center(
+def thaw_and_stream_to_redis(
  time_to_thaw: float,
  rotation: float = 360,
  robot: BartRobot = inject("robot"),
@@ -26,6 +26,7 @@ def thaw_and_center(
  sample_id = yield from bps.rd(robot.sample_id)
 
  yield from bps.abs_set(oav.zoom_controller.level, "1.0x", wait=True)
+ yield from bps.abs_set(oav_to_redis_forwarder.sample_id, sample_id)
 
  @subs_decorator(MurkoCallback(REDIS_HOST, REDIS_PASSWORD, MURKO_REDIS_DB))
  @run_decorator(
@@ -38,14 +39,14 @@ def thaw_and_center(
  "sample_id": sample_id,
  }
  )
- def _thaw_and_center():
+ def _thaw_and_stream_to_redis():
  yield from bps.kickoff(oav_to_redis_forwarder, wait=True)
  yield from bps.monitor(smargon.omega.user_readback, name="smargon")
  yield from bps.monitor(oav_to_redis_forwarder.uuid, name="oav")
  yield from thaw(time_to_thaw, rotation, thawer, smargon)
  yield from bps.complete(oav_to_redis_forwarder)
 
- yield from _thaw_and_center()
+ yield from _thaw_and_stream_to_redis()
 
 
 def thaw(

src/mx_bluesky/beamlines/i24/serial/fixed_target/FT-gui-edm/DiamondChipI24-py3v1.edl

@@ -3,8 +3,8 @@ beginScreenProperties
 major 4
 minor 0
 release 1
-x 4011
-y 263
+x 2870
+y 282
 w 1030
 h 950
 font "arial-medium-r-18.0"
@@ -3586,24 +3586,27 @@ topShadowColor index 1
 botShadowColor index 8
 controlPv "ME14E-MO-IOC-01:GP1"
 font "arial-medium-r-18.0"
-numItems 4
+numItems 5
 symbolTag {
  0 "Oxford"
  1 "Oxford Inner"
  2 "Custom"
  3 "Minichip"
+ 4 "MISP"
 }
 symbol0 {
  0 "OX"
  1 "B1"
  2 "Cu"
  3 "Mini"
+ 4 "MI"
 }
 value0 {
  0 "oxford"
  1 "Oxford Inner"
  2 "Custom"
  3 "Minichip"
+ 4 "MISP"
 }
 endObjectProperties
 

src/mx_bluesky/beamlines/i24/serial/fixed_target/FT-gui-edm/MappingLite-oxford_py3v1.edl

@@ -4,7 +4,7 @@ major 4
 minor 0
 release 1
 x 3168
-y 118
+y 155
 w 650
 h 400
 font "arial-medium-r-18.0"
@@ -219,7 +219,7 @@ font "arial-medium-r-18.0"
 buttonLabel "Upload Parameters"
 numCmds 1
 command {
- 0 "blueapi -c CONFIG_LOCATION controller run upload_parameters '\{\"chipid\":\"oxford\"\}'"
+ 0 "blueapi -c CONFIG_LOCATION controller run upload_parameters"
 }
 endObjectProperties
 

src/mx_bluesky/beamlines/i24/serial/fixed_target/i24ssx_Chip_Collect_py3v1.py

@@ -475,7 +475,7 @@ def start_i24(
  caput(pv.pilat_acquire, "1") # Arm pilatus
  yield from arm_zebra(zebra)
  caput(pv.pilat_filename, filename)
- time.sleep(1.5)
+ sleep(1.5)
 
  elif parameters.detector_name == "eiger":
  logger.info("Using Eiger detector")
@@ -549,7 +549,7 @@ def start_i24(
  )
  yield from arm_zebra(zebra)
 
- time.sleep(1.5)
+ sleep(1.5)
 
  else:
  msg = f"Unknown Detector Type, det_type = {parameters.detector_name}"

src/mx_bluesky/beamlines/i24/serial/fixed_target/i24ssx_Chip_Manager_py3v1.py

@@ -256,24 +256,18 @@ def save_screen_map() -> MsgGenerator:
 
 
 @log.log_on_entry
-def upload_parameters(
- chipid: str = "oxford", pmac: PMAC = inject("pmac"), width: int | None = None
-) -> MsgGenerator:
+def upload_parameters(pmac: PMAC = inject("pmac")) -> MsgGenerator:
  setup_logging()
- logger.info("Uploading Parameters to the GeoBrick")
- if chipid == "oxford":
- caput(CHIPTYPE_PV, 0)
- width = 8
- else:
- if width is None:
- raise Exception("Supply a width if chipid is not oxford")
+ logger.info("Uploading Parameters for Oxford Chip to the GeoBrick")
+ caput(CHIPTYPE_PV, 0)
+ width = 8
 
  map_file: Path = LITEMAP_PATH / "currentchip.map"
  if not map_file.exists():
  raise FileNotFoundError(f"The file {map_file} has not yet been created")
 
  with open(map_file) as f:
- logger.info(f"Chipid {chipid}")
+ logger.info(f"Chipid {ChipType.Oxford}")
  logger.info(f"width {width}")
  x = 1
  for line in f.readlines()[: width**2]:

src/mx_bluesky/beamlines/i24/serial/fixed_target/i24ssx_moveonclick.py

@@ -132,10 +132,7 @@ def update_ui(oav, frame):
  cv.imshow("OAV1view", frame)
 
 
-def start_viewer(oav1: str = OAV1_CAM):
- # Get devices out of dodal
- oav: OAV = i24.oav()
- pmac: PMAC = i24.pmac()
+def start_viewer(oav: OAV, pmac: PMAC, oav1: str = OAV1_CAM):
  # Create a video caputure from OAV1
  cap = cv.VideoCapture(oav1)
 
@@ -198,4 +195,7 @@ def start_viewer(oav1: str = OAV1_CAM):
 
 if __name__ == "__main__":
  RE = RunEngine()
- RE(start_viewer())
+ # Get devices out of dodal
+ oav: OAV = i24.oav()
+ pmac: PMAC = i24.pmac()
+ RE(start_viewer(oav, pmac))

src/mx_bluesky/beamlines/i24/serial/log.py

@@ -4,13 +4,12 @@
 from os import environ
 from pathlib import Path
 
-from bluesky.log import logger as bluesky_logger
 from dodal.log import (
  ERROR_LOG_BUFFER_LINES,
+ integrate_bluesky_and_ophyd_logging,
  set_up_all_logging_handlers,
 )
 from dodal.log import LOGGER as dodal_logger
-from ophyd_async.log import logger as ophyd_async_logger
 
 VISIT_PATH = Path("/dls_sw/i24/etc/ssx_current_visit.txt")
 
@@ -88,13 +87,6 @@ def _get_logging_file_path() -> Path:
  return logging_path
 
 
-def integrate_bluesky_and_ophyd_logging(parent_logger: logging.Logger):
- """Integrate only bluesky and ophyd_async loggers."""
- for logger in [bluesky_logger, ophyd_async_logger]:
- logger.parent = parent_logger
- logger.setLevel(logging.DEBUG)
-
-
 def default_logging_setup(dev_mode: bool = False):
  """ Default log setup for i24 serial.
 

src/mx_bluesky/beamlines/i24/serial/parameters/experiment_parameters.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, validator
+from pydantic import BaseModel, field_validator
 
 from mx_bluesky.beamlines.i24.serial.fixed_target.ft_utils import (
  ChipType,
@@ -21,7 +21,8 @@ class SerialExperiment(BaseModel):
  detector_distance_mm: float
  detector_name: Literal["eiger", "pilatus"]
 
- @validator("visit", pre=True)
+ @field_validator("visit", mode="before")
+ @classmethod
  def _parse_visit(cls, visit: str | Path):
  if isinstance(visit, str):
  return Path(visit)
@@ -56,8 +57,6 @@ def from_file(cls, filename: str | Path):
 class ChipDescription(BaseModel):
  """Parameters defining the chip in use for FT collection."""
 
- model_config = ConfigDict(use_enum_values=True)
-
  chip_type: ChipType
  x_num_steps: int
  y_num_steps: int
@@ -68,13 +67,6 @@ class ChipDescription(BaseModel):
  b2b_horz: float
  b2b_vert: float
 
- @validator("chip_type", pre=True)
- def _parse_chip(cls, chip_type: str | int):
- if isinstance(chip_type, str):
- return ChipType[chip_type]
- else:
- return ChipType(chip_type)
-
  @property
  def chip_format(self) -> list[int]:
  return [self.x_blocks, self.y_blocks, self.x_num_steps, self.y_num_steps]
@@ -97,26 +89,13 @@ def y_block_size(self) -> float:
 class FixedTargetParameters(SerialExperiment, LaserExperiment):
  """Fixed target parameter model."""
 
- model_config = ConfigDict(use_enum_values=True)
-
  num_exposures: int
  chip: ChipDescription
  map_type: MappingType
  pump_repeat: PumpProbeSetting
  checker_pattern: bool = False
  total_num_images: int = 0 # Calculated in the code for now
 
- @validator("map_type", pre=True)
- def _parse_map(cls, map_type: str | int):
- if isinstance(map_type, str):
- return MappingType[map_type]
- else:
- return MappingType(map_type)
-
- @validator("pump_repeat", pre=True)
- def _parse_pump(cls, pump_repeat: int):
- return PumpProbeSetting(pump_repeat)
-
  @classmethod
  def from_file(cls, filename: str | Path):
  with open(filename) as fh:

src/mx_bluesky/hyperion/__main__.py

@@ -71,9 +71,13 @@ def __init__(self, status: Status, message: str = "") -> None:
 class ErrorStatusAndMessage(StatusAndMessage):
  exception_type: str = ""
 
- def __init__(self, exception: Exception) -> None:
- super().__init__(Status.FAILED, repr(exception))
- self.exception_type = type(exception).__name__
+
+def make_error_status_and_message(exception: Exception):
+ return ErrorStatusAndMessage(
+ status=Status.FAILED.value,
+ message=repr(exception),
+ exception_type=type(exception).__name__,
+ )
 
 
 class BlueskyRunner:
@@ -119,7 +123,7 @@ def start(
  plan_name: str,
  callbacks: CallbacksFactory | None,
  ) -> StatusAndMessage:
- LOGGER.info(f"Started with parameters: {parameters.json(indent=2)}")
+ LOGGER.info(f"Started with parameters: {parameters.model_dump_json(indent=2)}")
 
  devices: Any = PLAN_REGISTRY[plan_name]["setup"](self.context)
 
@@ -146,7 +150,7 @@ def stopping_thread(self):
  self.RE.abort()
  self.current_status = StatusAndMessage(Status.IDLE)
  except Exception as e:
- self.current_status = ErrorStatusAndMessage(e)
+ self.current_status = make_error_status_and_message(e)
 
  def stop(self) -> StatusAndMessage:
  if self.current_status.status == Status.IDLE.value:
@@ -197,15 +201,15 @@ def wait_on_queue(self):
  self.last_run_aborted = False
  except WarningException as exception:
  LOGGER.warning("Warning Exception", exc_info=True)
- self.current_status = ErrorStatusAndMessage(exception)
+ self.current_status = make_error_status_and_message(exception)
  except Exception as exception:
  LOGGER.error("Exception on running plan", exc_info=True)
 
  if self.last_run_aborted:
  # Aborting will cause an exception here that we want to swallow
  self.last_run_aborted = False
  else:
- self.current_status = ErrorStatusAndMessage(exception)
+ self.current_status = make_error_status_and_message(exception)
  finally:
  [
  self.RE.unsubscribe(cb)
@@ -231,6 +235,8 @@ def compose_start_args(context: BlueskyContext, plan_name: str, action: Actions)
  )
  try:
  parameters = experiment_internal_param_type(**json.loads(request.data))
+ if parameters.model_extra:
+ raise ValueError(f"Extra fields not allowed {parameters.model_extra}")
  except Exception as e:
  raise ValueError(
  f"Supplied parameters don't match the plan for this endpoint {request.data}"
@@ -255,7 +261,7 @@ def put(self, plan_name: str, action: Actions):
  plan, params, plan_name, callback_type
  )
  except Exception as e:
- status_and_message = ErrorStatusAndMessage(e)
+ status_and_message = make_error_status_and_message(e)
  LOGGER.error(format_exception(e))
 
  elif action == Actions.STOP.value: