diff --git a/dev-requirements.txt b/dev-requirements.txt index 15e06165c..6e0905761 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,57 +1,61 @@ accessible-pygments==0.0.5 aioca==1.7 aiofiles==24.1.0 -aiohttp==3.9.5 +aiohappyeyeballs==2.3.5 +aiohttp==3.10.3 aiosignal==1.3.1 alabaster==0.7.16 +annotated-types==0.7.0 anyio==4.4.0 appdirs==1.4.4 asciitree==0.3.3 asttokens==2.4.1 -async-timeout==4.0.3 -attrs==23.2.0 -Babel==2.15.0 +attrs==24.2.0 +babel==2.16.0 beautifulsoup4==4.12.3 bidict==0.23.1 -bluesky==1.13.0a3 +black==24.8.0 +bluesky==1.13.0a4 bluesky-kafka==0.10.0 bluesky-live==0.0.8 boltons==24.0.0 -cachetools==5.3.3 +cachetools==5.4.0 caproto==1.1.1 certifi==2024.7.4 cfgv==3.4.0 +chardet==5.2.0 charset-normalizer==3.3.2 -click==8.1.3 +click==8.1.7 cloudpickle==3.0.0 colorama==0.4.6 colorlog==6.8.2 comm==0.2.2 -confluent-kafka==2.4.0 +confluent-kafka==2.5.0 contourpy==1.2.1 -copier==8.1.0 -coverage==7.5.4 +copier==9.3.1 +coverage==7.6.1 cycler==0.12.1 -dask==2024.7.0 +dask==2024.8.0 databroker==1.2.5 dataclasses-json==0.6.7 decorator==5.1.1 deepmerge==1.1.1 +diff_cover==9.1.1 distlib==0.3.8 -dls-bluesky-core==0.0.3 -dls-dodal==1.29.2 +dls-bluesky-core==0.0.4 +dls-dodal==1.28.0 dnspython==2.6.1 docopt==0.6.2 doct==1.1.0 docutils==0.21.2 -dunamai==1.21.2 +dunamai==1.22.0 email_validator==2.2.0 entrypoints==0.4 epicscorelibs==7.0.7.99.0.2 event-model==1.20.0 -exceptiongroup==1.2.1 executing==2.0.1 -fastapi==0.99.1 +fastapi==0.112.0 +fastapi-cli==0.0.5 fasteners==0.19 filelock==3.15.4 flexcache==0.3 @@ -60,6 +64,8 @@ fonttools==4.53.1 frozenlist==1.4.1 fsspec==2024.6.1 funcy==2.0 +gitdb==4.0.11 +GitPython==3.1.43 graypy==2.1.0 h11==0.14.0 h5py==3.11.0 @@ -73,7 +79,7 @@ identify==2.6.0 idna==3.7 imageio==2.34.2 imagesize==1.4.1 -importlib_metadata==8.0.0 +importlib_metadata==8.2.0 importlib_resources==6.4.0 iniconfig==2.0.0 intake==0.6.4 @@ -88,12 +94,11 @@ jsonschema-specifications==2023.12.1 jupyterlab_widgets==3.0.11 kiwisolver==1.4.5 ldap3==2.9.1 -livereload==2.7.0 locket==1.0.0 markdown-it-py==3.0.0 MarkupSafe==2.1.5 marshmallow==3.21.3 -matplotlib==3.9.1 +matplotlib==3.9.2 matplotlib-inline==0.1.7 mdit-py-plugins==0.4.1 mdurl==0.1.2 @@ -103,19 +108,19 @@ mongoquery==1.4.2 msgpack==1.0.8 msgpack-numpy==0.4.8 multidict==6.0.5 -mypy==1.10.1 +mypy==1.11.1 mypy-extensions==1.0.0 -myst-parser==3.0.1 +myst-parser==4.0.0 networkx==3.3 nodeenv==1.9.1 nose2==0.15.1 nslsii==0.10.3 -numcodecs==0.12.1 -numpy==1.26.4 +numcodecs==0.13.0 +numpy==2.0.1 opencv-python-headless==4.10.0.84 ophyd==1.9.0 -ophyd-async==0.3.4 -orjson==3.10.6 +ophyd-async==0.3.1 +orjson==3.10.7 p4p==4.1.12 packaging==24.1 pandas==2.2.2 @@ -128,85 +133,94 @@ picobox==4.0.0 pika==1.3.2 pillow==10.4.0 PIMS==0.7 -Pint==0.24.1 -pipdeptree==2.23.0 +Pint==0.24.3 +pipdeptree==2.23.1 platformdirs==4.2.2 pluggy==1.5.0 plumbum==1.8.3 ply==3.11 -pre-commit==3.7.1 -prettytable==3.10.0 +pre-commit==3.8.0 +prettytable==3.11.0 prompt-toolkit==3.0.36 psutil==6.0.0 ptyprocess==0.7.0 -pure-eval==0.2.2 +pure_eval==0.2.3 pvxslibs==1.3.1 py==1.11.0 pyasn1==0.6.0 pycryptodome==3.20.0 -pydantic==1.10.17 +pydantic==2.8.2 +pydantic-extra-types==2.9.0 +pydantic-settings==2.4.0 +pydantic_core==2.20.1 pydata-sphinx-theme==0.15.4 pyepics==3.5.6 Pygments==2.18.0 pymongo==4.8.0 pyOlog==4.5.0 pyparsing==3.1.2 -pytest==8.2.2 -pytest-asyncio==0.23.7 +pyright==1.1.375 +pytest==8.3.2 +pytest-asyncio==0.23.8 pytest-cov==5.0.0 +pytest-random-order==1.1.1 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-multipart==0.0.9 pytz==2024.1 -PyYAML==6.0.1 -pyyaml-include==2.1 +PyYAML==6.0.2 questionary==2.0.1 -redis==5.0.7 +redis==5.0.8 redis-json-dict==0.2.0 referencing==0.35.1 requests==2.32.3 responses==0.25.3 -rpds-py==0.19.0 -ruff==0.5.1 -scanspec==0.6.6 -setuptools-dso==2.10 +rich==13.7.1 +rpds-py==0.20.0 +ruff==0.5.7 +scanspec==0.7.1 +setuptools-dso==2.11a2 +shellingham==1.5.4 six==1.16.0 slicerator==1.1.0 +smmap==5.0.1 sniffio==1.3.1 snowballstemmer==2.2.0 soupsieve==2.5 -Sphinx==7.3.7 -sphinx-autobuild==2024.2.4 +Sphinx==7.4.5 +sphinx-autobuild==2024.4.16 +sphinx-autodoc-typehints==2.2.3 sphinx-click==6.0.0 sphinx-copybutton==0.5.2 -sphinx_design==0.6.0 -sphinx_mdinclude==0.6.1 -sphinxcontrib-applehelp==1.0.8 -sphinxcontrib-devhelp==1.0.6 -sphinxcontrib-htmlhelp==2.0.5 +sphinx_design==0.6.1 +sphinx_mdinclude==0.6.2 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-httpdomain==1.8.1 sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-mermaid==0.9.2 sphinxcontrib-openapi==0.8.4 -sphinxcontrib-qthelp==1.0.7 -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 stack-data==0.6.3 -starlette==0.27.0 +starlette==0.37.2 stomp-py==8.1.2 suitcase-mongo==0.6.0 suitcase-msgpack==0.3.0 suitcase-utils==0.5.4 super-state-machine==2.0.2 -tifffile==2024.7.2 -tomli==2.0.1 +tifffile==2024.8.10 toolz==0.12.1 -tornado==6.4.1 tox==3.28.0 tox-direct==0.4 -tqdm==4.66.4 +tqdm==4.66.5 traitlets==5.14.3 +typer==0.12.3 +types-aiofiles==24.1.0.20240626 types-mock==5.1.0.20240425 -types-PyYAML==6.0.12.20240311 -types-requests==2.32.0.20240622 +types-PyYAML==6.0.12.20240808 +types-requests==2.32.0.20240712 types-urllib3==1.26.25.14 typing-inspect==0.9.0 typing_extensions==4.12.2 @@ -214,18 +228,18 @@ tzdata==2024.1 tzlocal==5.2 ujson==5.10.0 urllib3==2.2.2 -uvicorn==0.30.1 +uvicorn==0.30.5 uvloop==0.19.0 virtualenv==20.26.3 -watchfiles==0.22.0 +watchfiles==0.23.0 wcwidth==0.2.13 websocket-client==1.8.0 websockets==12.0 widgetsnbextension==4.0.11 workflows==2.27 -xarray==2024.6.0 +xarray==2024.7.0 yarl==1.9.4 zarr==2.18.2 zict==2.2.0 -zipp==3.19.2 -zocalo==0.32.0 +zipp==3.20.0 +zocalo==1.0.0 diff --git a/docs/reference/openapi.yaml b/docs/reference/openapi.yaml index df19cd95e..09f4b256c 100644 --- a/docs/reference/openapi.yaml +++ b/docs/reference/openapi.yaml @@ -38,10 +38,12 @@ components: description: State of internal environment. properties: error_message: + anyOf: + - minLength: 1 + type: string + - type: 'null' description: If present - error loading context - minLength: 1 title: Error Message - type: string initialized: description: blueapi context initialized title: Initialized @@ -64,17 +66,21 @@ components: description: Representation of a plan properties: description: + anyOf: + - type: string + - type: 'null' description: Docstring of the plan title: Description - type: string name: description: Name of the plan title: Name type: string schema: + anyOf: + - type: object + - type: 'null' description: Schema of the plan's parameters title: Schema - type: object required: - name title: PlanModel @@ -98,16 +104,20 @@ components: description: Request to change the state of the worker. properties: defer: + anyOf: + - type: boolean + - type: 'null' default: false description: Should worker defer Pausing until the next checkpoint title: Defer - type: boolean new_state: $ref: '#/components/schemas/WorkerState' reason: + anyOf: + - type: string + - type: 'null' description: The reason for the current run to be aborted title: Reason - type: string required: - new_state title: StateChangeRequest @@ -178,6 +188,7 @@ components: type: string required: - task_id + - task title: TrackableTask type: object ValidationError: @@ -221,9 +232,13 @@ components: description: Worker's active task ID, can be None properties: task_id: + anyOf: + - type: string + - type: 'null' description: The ID of the current task, None if the worker is idle title: Task Id - type: string + required: + - task_id title: WorkerTask type: object info: @@ -340,8 +355,10 @@ paths: name: task_status required: false schema: + anyOf: + - type: string + - type: 'null' title: Task Status - type: string responses: '200': content: diff --git a/pyproject.toml b/pyproject.toml index 3b820255d..e9b09b86c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,9 @@ dependencies = [ "nslsii", "pyepics", "aioca", - "pydantic<2.0", # Leave pinned until can check incompatibility + "pydantic>=2.0", + "scanspec>=0.7.1", + "pydantic-settings", "stomp-py", "aiohttp", "PyYAML", diff --git a/src/blueapi/cli/format.py b/src/blueapi/cli/format.py index f9815c79c..f546f0c17 100644 --- a/src/blueapi/cli/format.py +++ b/src/blueapi/cli/format.py @@ -94,6 +94,13 @@ def _describe_type(spec: dict[Any, Any], required: bool = False): if all_of := spec.get("allOf"): items = (_describe_type(f, False) for f in all_of) disp += f'{" & ".join(items)}' + elif any_of := spec.get("anyOf"): + items = (_describe_type(f, False) for f in any_of) + + # Special case: Where the type is | null, + # we should just print + items = (item for item in items if item != "null" or len(any_of) != 2) + disp += f'{" | ".join(items)}' else: disp += "Any" case "array": diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 1bd8e482e..53e6e0d48 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -9,8 +9,10 @@ from bluesky.run_engine import RunEngine from dodal.utils import make_all_devices from ophyd_async.core import NotConnected -from pydantic import create_model -from pydantic.fields import FieldInfo, ModelField +from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, create_model +from pydantic.fields import FieldInfo +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema, core_schema from blueapi.config import EnvironmentConfig, SourceKind from blueapi.utils import BlueapiPlanModelConfig, load_module_all @@ -187,24 +189,27 @@ def _reference(self, target: type) -> type: class Reference(target): @classmethod - def __get_validators__(cls): - yield cls.valid + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + def valid(value): + val = self.find_device(value) + if not isinstance(val, target): + raise ValueError(f"Device {value} is not of type {target}") + return val + + return core_schema.no_info_after_validator_function( + valid, handler(str) + ) @classmethod - def valid(cls, value): - val = self.find_device(value) - if not isinstance(val, target): - raise ValueError(f"Device {value} is not of type {target}") - return val - - @classmethod - def __modify_schema__( - cls, field_schema: dict[str, Any], field: ModelField | None - ): - if field: - field_schema.update( - {"type": f"{target.__module__}.{target.__qualname__}"} - ) + def __get_pydantic_json_schema__( + cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + json_schema = handler(core_schema) + json_schema = handler.resolve_ref_schema(json_schema) + json_schema["type"] = f"{target.__module__}.{target.__qualname__}" + return json_schema self._reference_cache[target] = Reference diff --git a/src/blueapi/service/main.py b/src/blueapi/service/main.py index 99133d336..0a5c25070 100644 --- a/src/blueapi/service/main.py +++ b/src/blueapi/service/main.py @@ -70,7 +70,6 @@ async def lifespan(app: FastAPI): app = FastAPI( docs_url="/docs", - on_shutdown=[teardown_runner], title="BlueAPI Control", lifespan=lifespan, version=REST_API_VERSION, diff --git a/src/blueapi/service/model.py b/src/blueapi/service/model.py index 526ef2f0a..8a7ffb899 100644 --- a/src/blueapi/service/model.py +++ b/src/blueapi/service/model.py @@ -75,7 +75,7 @@ class PlanModel(BlueapiBaseModel): def from_plan(cls, plan: Plan) -> "PlanModel": return cls( name=plan.name, - schema=plan.model.schema(), + schema=plan.model.model_json_schema(), description=plan.description, ) diff --git a/src/blueapi/utils/base_model.py b/src/blueapi/utils/base_model.py index 3660271fd..142052a8c 100644 --- a/src/blueapi/utils/base_model.py +++ b/src/blueapi/utils/base_model.py @@ -1,30 +1,19 @@ -from pydantic import BaseConfig, BaseModel, Extra +from pydantic import BaseModel, ConfigDict +# Pydantic config for blueapi API models with common config. +BlueapiModelConfig = ConfigDict( + extra="forbid", + populate_by_name=True, +) -class BlueapiModelConfig(BaseConfig): - """ - Pydantic config for blueapi API models with - common config. - """ - - extra = Extra.forbid - allow_population_by_field_name = True - underscore_attrs_are_private = True - - -class BlueapiPlanModelConfig(BaseConfig): - """ - Pydantic config for plan parameters. - Includes arbitrary type config so that devices - can be parameters. - Validates default arguments, to allow default - arguments to be names of devices that are fetched - from the context. - """ - - extra = Extra.forbid - arbitrary_types_allowed = True - validate_all = True +# Pydantic config for plan parameters. Includes arbitrary type config so that +# devices can be parameters. Validates default arguments, to allow default +# arguments to be names of devices that are fetched from the context. +BlueapiPlanModelConfig = ConfigDict( + extra="forbid", + arbitrary_types_allowed=True, + validate_default=True, +) class BlueapiBaseModel(BaseModel): @@ -47,4 +36,4 @@ class BlueapiBaseModel(BaseModel): apischema also did not allow. """ - Config = BlueapiModelConfig + model_config = BlueapiModelConfig diff --git a/src/blueapi/worker/task.py b/src/blueapi/worker/task.py index 1e48cb4d8..91ecbfcda 100644 --- a/src/blueapi/worker/task.py +++ b/src/blueapi/worker/task.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from typing import Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, TypeAdapter from blueapi.core import BlueskyContext from blueapi.utils import BlueapiBaseModel @@ -20,15 +20,16 @@ class Task(BlueapiBaseModel): description="Values for parameters to plan, if any", default_factory=dict ) - def prepare_params(self, ctx: BlueskyContext) -> BaseModel: - return _lookup_params(ctx, self) + def prepare_params(self, ctx: BlueskyContext) -> Mapping[str, Any]: + model = _lookup_params(ctx, self) + return _model_to_kwargs(model) def do_task(self, ctx: BlueskyContext) -> None: LOGGER.info(f"Asked to run plan {self.name} with {self.params}") func = ctx.plan_functions[self.name] prepared_params = self.prepare_params(ctx) - ctx.run_engine(func(**prepared_params.dict())) + ctx.run_engine(func(**prepared_params)) def _lookup_params(ctx: BlueskyContext, task: Task) -> BaseModel: @@ -46,4 +47,24 @@ def _lookup_params(ctx: BlueskyContext, task: Task) -> BaseModel: plan = ctx.plans[task.name] model = plan.model - return model.parse_obj(task.params) + adapter = TypeAdapter(model) + return adapter.validate_python(task.params) + + +def _model_to_kwargs(model: BaseModel) -> Mapping[str, Any]: + """ + Converts an instance of BaseModel back to a dictionary that + can be passed as **kwargs. + Used instead of BaseModel.model_dump() because we don't want + the dumping to be nested and because it fires UserWarnings + about data types it is unfamiliar with + (such as ophyd devices). + + Args: + model: Pydantic model to convert to kwargs + + Returns: + Mapping[str, Any]: Dictionary that can be passed as **kwargs + """ + + return {name: getattr(model, name) for name in model.model_fields_set} diff --git a/tests/messaging/test_stomptemplate.py b/tests/messaging/test_stomptemplate.py index 9cf746ce1..a8c9750b3 100644 --- a/tests/messaging/test_stomptemplate.py +++ b/tests/messaging/test_stomptemplate.py @@ -7,7 +7,8 @@ import numpy as np import pytest -from pydantic import BaseModel, BaseSettings, Field +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings from stomp import Connection from stomp.exception import ConnectFailedException, NotConnectedException diff --git a/tests/service/test_rest_api.py b/tests/service/test_rest_api.py index 1b1661f90..7cf925932 100644 --- a/tests/service/test_rest_api.py +++ b/tests/service/test_rest_api.py @@ -7,7 +7,7 @@ from fastapi import status from fastapi.testclient import TestClient from pydantic import BaseModel, ValidationError -from pydantic.error_wrappers import ErrorWrapper +from pydantic_core import InitErrorDetails from super_state_machine.errors import TransitionError from blueapi.core.bluesky_types import Plan @@ -168,21 +168,24 @@ class MyModel(BaseModel): plan = Plan(name="my-plan", model=MyModel) get_plan_mock.return_value = PlanModel.from_plan(plan) - - submit_task_mock.side_effect = ValidationError( - [ErrorWrapper(ValueError("field required"), "id")], PlanModel + submit_task_mock.side_effect = ValidationError.from_exception_data( + title="ValueError", + line_errors=[ + InitErrorDetails( + type="missing", loc=("id",), msg="value is required for Identifier" + ) # type: ignore + ], ) - response = client.post("/tasks", json={"name": "my-plan"}) assert response.status_code == 422 assert response.json() == { - "detail": "\n" - " Input validation failed: id: field required,\n" - " suppplied params {},\n" - " do not match the expected params: {'title': 'MyModel', " - "'type': 'object', 'properties': {'id': {'title': 'Id', 'type': " - "'string'}}, 'required': ['id']}\n" - " " + "detail": ( + "\n Input validation failed: id: Field required,\n" + " suppplied params {},\n" + " do not match the expected params: {'properties': {'id': " + "{'title': 'Id', 'type': 'string'}}, 'required': ['id'], 'title': " + "'MyModel', 'type': 'object'}\n " + ) } diff --git a/tests/test_cli.py b/tests/test_cli.py index 5795ce68f..ab725c005 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -293,7 +293,7 @@ def test_env_reload_server_side_error(runner: CliRunner): @pytest.mark.parametrize( "exception, expected_exit_code", [ - (ValidationError("Invalid parameters", BaseModel), 1), + (ValidationError.from_exception_data(title="Base model", line_errors=[]), 1), (BlueskyRemoteControlError("Server error"), 1), (ValueError("Error parsing parameters"), 1), ], @@ -362,7 +362,7 @@ def test_device_output_formatting(): class ExtendedModel(BaseModel): name: str keys: list[int] - metadata: None | Mapping[str, str] + metadata: None | Mapping[str, str] = None def test_plan_output_formatting(): @@ -396,40 +396,48 @@ def test_plan_output_formatting(): output = StringIO() OutputFormat.JSON.display(plans, out=output) json_out = dedent("""\ - [ - { - "name": "my-plan", - "description": "Summary of description\\n\\nRest of description\\n", - "parameter_schema": { - "title": "ExtendedModel", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "keys": { - "title": "Keys", - "type": "array", - "items": { - "type": "integer" - } - }, - "metadata": { - "title": "Metadata", - "type": "object", + [ + { + "name": "my-plan", + "description": "Summary of description\\n\\nRest of description\\n", + "parameter_schema": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "keys": { + "items": { + "type": "integer" + }, + "title": "Keys", + "type": "array" + }, + "metadata": { + "anyOf": [ + { "additionalProperties": { "type": "string" - } + }, + "type": "object" + }, + { + "type": "null" } - }, - "required": [ - "name", - "keys" - ] + ], + "default": null, + "title": "Metadata" } - } - ] + }, + "required": [ + "name", + "keys" + ], + "title": "ExtendedModel", + "type": "object" + } + } + ] """) assert output.getvalue() == json_out _ = json.loads(output.getvalue()) @@ -437,39 +445,47 @@ def test_plan_output_formatting(): output = StringIO() OutputFormat.FULL.display(plans, out=output) full = dedent("""\ - my-plan - Summary of description - - Rest of description - Schema - { - "title": "ExtendedModel", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "keys": { - "title": "Keys", - "type": "array", - "items": { - "type": "integer" - } - }, - "metadata": { - "title": "Metadata", - "type": "object", + my-plan + Summary of description + + Rest of description + Schema + { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "keys": { + "items": { + "type": "integer" + }, + "title": "Keys", + "type": "array" + }, + "metadata": { + "anyOf": [ + { "additionalProperties": { "type": "string" - } + }, + "type": "object" + }, + { + "type": "null" } - }, - "required": [ - "name", - "keys" - ] + ], + "default": null, + "title": "Metadata" } + }, + "required": [ + "name", + "keys" + ], + "title": "ExtendedModel", + "type": "object" + } """) assert output.getvalue() == full