allowed backends to take pydantic models

Also finished a `PvaTable` type.

src/ophyd_async/core/__init__.py

@@ -63,7 +63,6 @@
 )
 from ._signal_backend import (
  BackendConverterFactory,
- ProtocolDatatypeAbstraction,
  RuntimeSubsetEnum,
  SignalBackend,
  SubsetEnum,
@@ -124,7 +123,6 @@
  "NameProvider",
  "PathInfo",
  "PathProvider",
- "ProtocolDatatypeAbstraction",
  "ShapeProvider",
  "StaticFilenameProvider",
  "StaticPathProvider",

src/ophyd_async/core/_device_save_loader.py

@@ -7,10 +7,10 @@
 from bluesky.plan_stubs import abs_set, wait
 from bluesky.protocols import Location
 from bluesky.utils import Msg
+from pydantic import BaseModel
 
 from ._device import Device
 from ._signal import SignalRW
-from ._signal_backend import ProtocolDatatypeAbstraction
 
 
 def ndarray_representer(dumper: yaml.Dumper, array: npt.NDArray[Any]) -> yaml.Node:
@@ -19,14 +19,12 @@ def ndarray_representer(dumper: yaml.Dumper, array: npt.NDArray[Any]) -> yaml.No
  )
 
 
-def protocol_datatype_abstraction_representer(
- dumper: yaml.Dumper, protocol_datatype_abstraction: ProtocolDatatypeAbstraction
+def pydantic_model_abstraction_representer(
+ dumper: yaml.Dumper, model: BaseModel
 ) -> yaml.Node:
  """Uses the protocol datatype since it has to be serializable."""
 
- return dumper.represent_data(
- protocol_datatype_abstraction.convert_to_protocol_datatype()
- )
+ return dumper.represent_data(model.model_dump(mode="python"))
 
 
 class OphydDumper(yaml.Dumper):
@@ -146,8 +144,8 @@ def save_to_yaml(phases: Sequence[Dict[str, Any]], save_path: str) -> None:
 
  yaml.add_representer(np.ndarray, ndarray_representer, Dumper=yaml.Dumper)
  yaml.add_multi_representer(
- ProtocolDatatypeAbstraction,
- protocol_datatype_abstraction_representer,
+ BaseModel,
+ pydantic_model_abstraction_representer,
  Dumper=yaml.Dumper,
  )
 

src/ophyd_async/core/_signal_backend.py

@@ -13,29 +13,6 @@
 from ._utils import DEFAULT_TIMEOUT, ReadingValueCallback, T
 
 
-class ProtocolDatatypeAbstraction(ABC, Generic[T]):
- @abstractmethod
- def __init__(self):
- """The abstract datatype must be able to be intialized with no arguments."""
-
- @abstractmethod
- def convert_to_protocol_datatype(self) -> T:
- """
- Convert the abstract datatype to a form which can be sent
- over whichever protocol.
-
- This output will be used when the device is serialized.
- """
-
- @classmethod
- @abstractmethod
- def convert_from_protocol_datatype(cls, value: T) -> "ProtocolDatatypeAbstraction":
- """
- Convert the datatype received from the protocol to a
- higher level abstract datatype.
- """
-
-
 class BackendConverterFactory(ABC):
  """Convert between the signal backend and the signal type"""
 

src/ophyd_async/core/_soft_signal_backend.py

@@ -8,11 +8,11 @@
 
 import numpy as np
 from bluesky.protocols import DataKey, Dtype, Reading
+from pydantic import BaseModel
 from typing_extensions import TypedDict
 
 from ._signal_backend import (
  BackendConverterFactory,
- ProtocolDatatypeAbstraction,
  RuntimeSubsetEnum,
  SignalBackend,
 )
@@ -127,21 +127,27 @@ def make_initial_value(self, datatype: Optional[Type[T]]) -> T:
  return cast(T, self.choices[0])
 
 
-class SoftProtocolDatatypeAbstractionConverter(SoftConverter):
- def __init__(self, datatype: Type[ProtocolDatatypeAbstraction]):
+class SoftPydanticModelConverter(SoftConverter):
+ """Necessary for serializing soft signals."""
+
+ def __init__(self, datatype: Type[BaseModel]):
  self.datatype = datatype
 
  def reading(self, value: T, timestamp: float, severity: int) -> Reading:
  value = self.value(value)
  return super().reading(value, timestamp, severity)
 
  def value(self, value: Any) -> Any:
- if not isinstance(value, self.datatype):
- # For the case where we 
- value = self.datatype.convert_from_protocol_datatype(value)
+ if isinstance(value, dict):
+ value = self.datatype(**value)
  return value
 
  def write_value(self, value):
+ if isinstance(value, dict):
+ # If the device is being deserialized
+ return self.datatype(**value).model_dump(mode="python")
+ if isinstance(value, self.datatype):
+ return value.model_dump(mode="python")
  return value
 
  def make_initial_value(self, datatype: Type | None) -> Any:
@@ -162,16 +168,16 @@ def make_converter(cls, datatype):
  is_enum = inspect.isclass(datatype) and (
  issubclass(datatype, Enum) or issubclass(datatype, RuntimeSubsetEnum)
  )
- is_convertable_abstract_datatype = inspect.isclass(datatype) and issubclass(
- datatype, ProtocolDatatypeAbstraction
+ is_pydantic_model = inspect.isclass(datatype) and issubclass(
+ datatype, BaseModel
  )
 
  if is_array or is_sequence:
  return SoftArrayConverter()
  if is_enum:
  return SoftEnumConverter(datatype)
- if is_convertable_abstract_datatype:
- return SoftProtocolDatatypeAbstractionConverter(datatype)
+ if is_pydantic_model:
+ return SoftPydanticModelConverter(datatype)
 
  return SoftConverter()
 

src/ophyd_async/epics/signal/__init__.py

@@ -1,5 +1,6 @@
 from ._common import LimitPair, Limits, get_supported_values
-from ._p4p import PvaSignalBackend, PvaTableAbstraction
+from ._p4p import PvaSignalBackend
+from ._p4p_table_abstraction import PvaTable
 from ._signal import (
  epics_signal_r,
  epics_signal_rw,
@@ -13,6 +14,7 @@
  "LimitPair",
  "Limits",
  "PvaSignalBackend",
+ "PvaTable",
  "PvaTableAbstraction",
  "epics_signal_r",
  "epics_signal_rw",

src/ophyd_async/epics/signal/_p4p.py

@@ -3,7 +3,6 @@
 import inspect
 import logging
 import time
-from abc import abstractmethod
 from dataclasses import dataclass
 from enum import Enum
 from math import isnan, nan
@@ -13,12 +12,12 @@
 from bluesky.protocols import DataKey, Dtype, Reading
 from p4p import Value
 from p4p.client.asyncio import Context, Subscription
+from pydantic import BaseModel
 
 from ophyd_async.core import (
  DEFAULT_TIMEOUT,
  BackendConverterFactory,
  NotConnected,
- ProtocolDatatypeAbstraction,
  ReadingValueCallback,
  RuntimeSubsetEnum,
  SignalBackend,
@@ -288,32 +287,25 @@ def __getattribute__(self, __name: str) -> Any:
  raise NotImplementedError("No PV has been set as connect() has not been called")
 
 
-class PvaTableAbstraction(ProtocolDatatypeAbstraction[Dict]):
- @abstractmethod
- def convert_to_protocol_datatype(self) -> Dict:
- """Converts the object to a pva table (dictionary)."""
-
- @classmethod
- @abstractmethod
- def convert_from_protocol_datatype(cls, value: Dict) -> "PvaTableAbstraction":
- """Converts from a pva table (dictionary) to a Python datatype."""
-
-
-class PvaTableAbtractionConverter(PvaConverter):
- def __init__(self, datatype: PvaTableAbstraction):
+class PvaPydanticModelConverter(PvaConverter):
+ def __init__(self, datatype: BaseModel):
  self.datatype = datatype
 
  def reading(self, value: Value):
  ts = time.time()
- value = self.datatype.convert_from_protocol_datatype(value.todict())
+ value = self.value(value)
  return {"value": value, "timestamp": ts, "alarm_severity": 0}
 
  def value(self, value: Value):
- return self.datatype.convert_from_protocol_datatype(value.todict())
+ return self.datatype(**value.todict())
 
- def write_value(self, value):
+ def write_value(self, value: Union[BaseModel, Dict[str, Any]]):
+ """
+ A user can put whichever form to the signal.
+ This is required for yaml deserialization.
+ """
  if isinstance(value, self.datatype):
- return value.convert_to_protocol_datatype()
+ return value.model_dump(mode="python")
  return value
 
 
@@ -327,8 +319,8 @@ class PvaConverterFactory(BackendConverterFactory):
  np.ndarray,
  Enum,
  RuntimeSubsetEnum,
+ BaseModel,
  dict,
- PvaTableAbstraction,
  )
 
  @classmethod
@@ -411,9 +403,9 @@ def make_converter(
  if (
  datatype
  and inspect.isclass(datatype)
- and issubclass(datatype, PvaTableAbstraction)
+ and issubclass(datatype, BaseModel)
  ):
- return PvaTableAbtractionConverter(datatype)
+ return PvaPydanticModelConverter(datatype)
  return PvaDictConverter()
  else:
  raise TypeError(f"{pv}: Unsupported typeid {typeid}")

src/ophyd_async/epics/signal/_p4p_table_abstraction.py

@@ -0,0 +1,70 @@
+from typing import Dict
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, model_validator
+from pydantic_numpy.typing import NpNDArray
+
+
+class PvaTable(BaseModel):
+ """An abstraction of a PVA Table of str to python array."""
+
+ model_config = ConfigDict(validate_assignment=True, strict=False)
+
+ @classmethod
+ def row(cls, sub_cls, **kwargs) -> "PvaTable":
+ arrayified_kwargs = {
+ field_name: np.concatenate(
+ (
+ (default_arr := field_value.default_factory()),
+ np.array([kwargs[field_name]], dtype=default_arr.dtype),
+ )
+ )
+ for field_name, field_value in sub_cls.model_fields.items()
+ }
+ return sub_cls(**arrayified_kwargs)
+
+ def __add__(self, right: "PvaTable") -> "PvaTable":
+ """Concatinate the arrays in field values."""
+
+ assert isinstance(right, type(self)), (
+ f"{right} is not a `PvaTable`, or is not the same "
+ f"type of `PvaTable` as {self}."
+ )
+
+ return type(self)(
+ **{
+ field_name: np.concatenate(
+ (getattr(self, field_name), getattr(right, field_name))
+ )
+ for field_name in self.model_fields
+ }
+ )
+
+ @model_validator(mode="after")
+ def validate_arrays(self) -> "PvaTable":
+ first_length = len(next(iter(self))[1])
+ assert all(
+ len(field_value) == first_length for _, field_value in self
+ ), "Rows should all be of equal size."
+
+ assert 0 <= first_length < 4096, f"Length {first_length} not in range."
+
+ if not all(
+ np.issubdtype(
+ self.model_fields[field_name].default_factory().dtype, field_value.dtype
+ )
+ for field_name, field_value in self
+ ):
+ raise ValueError(
+ f"Cannot construct a `{type(self).__name__}`, "
+ "some rows have incorrect types."
+ )
+
+ return self
+
+ def convert_to_pva_datatype(self) -> Dict[str, NpNDArray]:
+ return self.model_dump(mode="python")
+
+ @classmethod
+ def convert_from_pva_datatype(cls, pva_table: Dict[str, NpNDArray]):
+ return cls(**pva_table)

src/ophyd_async/fastcs/panda/__init__.py

@@ -15,9 +15,7 @@
  DatasetTable,
  PandaHdf5DatasetType,
  SeqTable,
- SeqTableRowType,
  SeqTrigger,
- seq_table_row,
 )
 from ._trigger import (
  PcompInfo,
@@ -44,9 +42,7 @@
  "DatasetTable",
  "PandaHdf5DatasetType",
  "SeqTable",
- "SeqTableRowType",
  "SeqTrigger",
- "seq_table_row",
  "PcompInfo",
  "SeqTableInfo",
  "StaticPcompTriggerLogic",