From 6fdea74dab3060eaf6b5acfac888e3dfa95e5f86 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 13:57:34 +0100 Subject: [PATCH 01/30] Move util functions from i22 branch of ophyd_async that may be generally useful --- pyproject.toml | 1 + src/dls_bluesky_core/core/__init__.py | 11 +++++- src/dls_bluesky_core/core/coordination.py | 14 +++++++ src/dls_bluesky_core/core/maths.py | 47 +++++++++++++++++++++++ src/dls_bluesky_core/core/scanspecs.py | 32 +++++++++++++++ src/dls_bluesky_core/core/types.py | 5 ++- 6 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 src/dls_bluesky_core/core/coordination.py create mode 100644 src/dls_bluesky_core/core/maths.py create mode 100644 src/dls_bluesky_core/core/scanspecs.py diff --git a/pyproject.toml b/pyproject.toml index fdbbb69..c5cf953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ description = "Common Diamond specific Bluesky plans and functions" dependencies = [ "blueapi", "ophyd", + "ophyd_async", "scanspec" ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] diff --git a/src/dls_bluesky_core/core/__init__.py b/src/dls_bluesky_core/core/__init__.py index dc48890..6ae23f0 100644 --- a/src/dls_bluesky_core/core/__init__.py +++ b/src/dls_bluesky_core/core/__init__.py @@ -1,6 +1,15 @@ -from .types import MsgGenerator, PlanGenerator +from .coordination import group_uuid +from .maths import step_to_num, in_micros +from .scanspecs import get_duration +from .types import MsgGenerator, PlanGenerator, ScanAxis + __all__ = [ + "get_duration", + "group_uuid", + "in_micros", "MsgGenerator", "PlanGenerator", + "ScanAxis", + "step_to_num" ] diff --git a/src/dls_bluesky_core/core/coordination.py b/src/dls_bluesky_core/core/coordination.py new file mode 100644 index 0000000..1a94742 --- /dev/null +++ b/src/dls_bluesky_core/core/coordination.py @@ -0,0 +1,14 @@ +import uuid + + +def group_uuid(name: str) -> str: + """ + Returns a unique but human-readable string, to assist debugging orchestrated groups. + + Args: + name (str): A human readable name + + Returns: + readable_uid (str): name appended with a unique string + """ + return f"{name}-{str(uuid.uuid4())[:6]}" diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py new file mode 100644 index 0000000..6f65d9d --- /dev/null +++ b/src/dls_bluesky_core/core/maths.py @@ -0,0 +1,47 @@ +from typing import Tuple + +import numpy as np + + +def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, int]: + """ + Standard handling for converting from start, stop, step to start, stop, num + Forces step to be same direction as length + Includes a final point if it is within 1% of the end point (consistent with GDA) + + Args: + start (float): + Start of length, will be returned unchanged + stop (float): + End of length, if length/step does not divide cleanly will be returned + extended up to 1% of step, or else truncated. + step (float): + Length of a step along the line formed from start to stop. + If stop < start, will be coerced to be backwards. + + Returns: + start, truncated_stop, num = Tuple[float, float, int] + start will be returned unchanged + truncated_stop = start + num * step + num is the maximal number of steps that could fit into the length. + + """ + # Make step be the right direction + step = abs(step) if stop > start else -abs(step) + # If stop is within 1% of a step then include it + num = int((stop - start) / step + 0.01) + return start, start + num * step, num + + +def in_micros(t: float) -> int: + """ + Converts between units of microT and units of T. + For example, from microseconds to seconds. + + Args: + t (float): A time in microseconds, or other measurement in units of microU + Returns: + t (int): A time in seconds rounded up to the nearest whole second, + or other measurement in units of U, rounded up to the nearest whole U. + """ + return np.ceil(t / 1e6) diff --git a/src/dls_bluesky_core/core/scanspecs.py b/src/dls_bluesky_core/core/scanspecs.py new file mode 100644 index 0000000..99b1c7e --- /dev/null +++ b/src/dls_bluesky_core/core/scanspecs.py @@ -0,0 +1,32 @@ +from typing import List, Optional + +import numpy as np +from scanspec.core import Frames +from scanspec.specs import DURATION + + +def get_duration(frames: List[Frames]) -> Optional[float]: + """ + Returns the duration of a number of ScanSpec frames, if known and consistent. + + Args: + frames (List[Frames]): A number of Frame objects + + Raises: + ValueError: If any frames do not have a Duration defined. + + Returns: + duration (float): if all frames have a consistent duration + None: otherwise + + """ + for fs in frames: + if DURATION in fs.axes(): + durations = fs.midpoints[DURATION] + first_duration = durations[0] + if np.all(durations == first_duration): + # Constant duration, return it + return first_duration + else: + return None + raise ValueError("Duration not specified in Spec") diff --git a/src/dls_bluesky_core/core/types.py b/src/dls_bluesky_core/core/types.py index 600310e..3f96214 100644 --- a/src/dls_bluesky_core/core/types.py +++ b/src/dls_bluesky_core/core/types.py @@ -1,8 +1,11 @@ -from typing import Any, Callable, Generator +from typing import Any, Callable, Generator, Union from bluesky import Msg +from ophyd_async.core import Device +from scanspec.specs import DURATION # 'A true "plan", usually the output of a generator function' MsgGenerator = Generator[Msg, Any, None] # 'A function that generates a plan' PlanGenerator = Callable[..., MsgGenerator] +ScanAxis = Union[Device, DURATION] From 7c2724b5176ee7d251840a9c29ce08cdb2bd032a Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 16:15:52 +0100 Subject: [PATCH 02/30] Add tests to boost test coverage back up --- tests/core/test_funcs.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/core/test_funcs.py diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py new file mode 100644 index 0000000..6105bd5 --- /dev/null +++ b/tests/core/test_funcs.py @@ -0,0 +1,30 @@ +from typing import Optional + +import pytest + +from dls_bluesky_core.core import in_micros, step_to_num + + +@pytest.mark.parametrize("us,s", [(4_000_001, 5), (4_999_999, 5), (4_000, 1), + (-4_000_001, -4), (-4_999_999, -4), (-4_000, 0), + (4_000_000.1, 5), (4_999_999.9, 5), (0.1, 1), + (-4_000_000.5, -4), (-4_999_999.9, -4), (-4.05, 0)]) +def test_in_micros(us: float, s: int): + assert in_micros(us) is s + + +@pytest.mark.parametrize("start,stop,step,expected_num,truncated_stop", + [(0, 0, 1, 1, None), # start=stop, 1 point at start + (0, 0.5, 1, 1, 0), # step>length, 1 point at start + (0, 1, 1, 2, None), # stop=start+step, 1 point at start, 1 at stop + (0, 0.99, 1, 2, 1), # stop >= start + 0.99*step, included + (0, 0.98, 1, 1, 0), # stop < start + 0.99*step, not included + (0, 1.01, 1, 2, 1), # stop >= start + 0.99*step, included + (0, 1.75, 0.25, 8, 1.75) + ]) +def test_step_to_num(start: float, stop: float, step: float, expected_num: int, truncated_stop: Optional[float]): + truncated_stop = stop if truncated_stop is None else truncated_stop + actual_start, actual_stop, num = step_to_num(start, stop, step) + assert actual_start == start + assert actual_stop == truncated_stop + assert num == expected_num From d6f48fa3a3a328fc675326238c35c875490bbb3f Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 16:16:07 +0100 Subject: [PATCH 03/30] Changes to step_to_num to match expected behaviour --- src/dls_bluesky_core/core/maths.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index 6f65d9d..3b21bc6 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -27,10 +27,10 @@ def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, i """ # Make step be the right direction - step = abs(step) if stop > start else -abs(step) + step = abs(step) if stop >= start else -abs(step) # If stop is within 1% of a step then include it - num = int((stop - start) / step + 0.01) - return start, start + num * step, num + steps = int((stop - start) / step + 0.01) + return start, start + steps * step, steps + 1 # include 1st point def in_micros(t: float) -> int: @@ -44,4 +44,4 @@ def in_micros(t: float) -> int: t (int): A time in seconds rounded up to the nearest whole second, or other measurement in units of U, rounded up to the nearest whole U. """ - return np.ceil(t / 1e6) + return int(np.ceil(t / 1e6)) From 1724ede24c0fadbd06f6d7690cebe616ff61d056 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 16:23:29 +0100 Subject: [PATCH 04/30] Update tests checking Annotations --- tests/plans/test_compliance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/plans/test_compliance.py b/tests/plans/test_compliance.py index 7c887ee..078545a 100644 --- a/tests/plans/test_compliance.py +++ b/tests/plans/test_compliance.py @@ -9,7 +9,7 @@ def is_bluesky_plan_generator(func: Any) -> bool: try: - return get_type_hints(func).get("return") is MsgGenerator + return get_type_hints(func).get("return") == MsgGenerator except TypeError: # get_type_hints fails on some objects (such as Union or Optional) return False @@ -45,7 +45,7 @@ def assert_metadata_requirements(plan: PlanGenerator, signature: inspect.Signatu ), f"'{plan.__name__}' does not allow metadata" metadata = signature.parameters["metadata"] assert ( - metadata.annotation is Optional[Mapping[str, Any]] + metadata.annotation == Optional[Mapping[str, Any]] and metadata.default is not inspect.Parameter.empty ), f"'{plan.__name__}' metadata is not optional" assert metadata.default is None, f"'{plan.__name__}' metadata default is mutable" From ec2c2dc0f72b1bb7afcdcd7a6af17265c06b94ba Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 16:27:17 +0100 Subject: [PATCH 05/30] Linting --- src/dls_bluesky_core/core/__init__.py | 5 ++- tests/core/test_funcs.py | 50 +++++++++++++++++++-------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/dls_bluesky_core/core/__init__.py b/src/dls_bluesky_core/core/__init__.py index 6ae23f0..4f7e36c 100644 --- a/src/dls_bluesky_core/core/__init__.py +++ b/src/dls_bluesky_core/core/__init__.py @@ -1,9 +1,8 @@ from .coordination import group_uuid -from .maths import step_to_num, in_micros +from .maths import in_micros, step_to_num from .scanspecs import get_duration from .types import MsgGenerator, PlanGenerator, ScanAxis - __all__ = [ "get_duration", "group_uuid", @@ -11,5 +10,5 @@ "MsgGenerator", "PlanGenerator", "ScanAxis", - "step_to_num" + "step_to_num", ] diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index 6105bd5..3aed11e 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -5,24 +5,46 @@ from dls_bluesky_core.core import in_micros, step_to_num -@pytest.mark.parametrize("us,s", [(4_000_001, 5), (4_999_999, 5), (4_000, 1), - (-4_000_001, -4), (-4_999_999, -4), (-4_000, 0), - (4_000_000.1, 5), (4_999_999.9, 5), (0.1, 1), - (-4_000_000.5, -4), (-4_999_999.9, -4), (-4.05, 0)]) +@pytest.mark.parametrize( + "us,s", + [ + (4_000_001, 5), + (4_999_999, 5), + (4_000, 1), + (-4_000_001, -4), + (-4_999_999, -4), + (-4_000, 0), + (4_000_000.1, 5), + (4_999_999.9, 5), + (0.1, 1), + (-4_000_000.5, -4), + (-4_999_999.9, -4), + (-4.05, 0), + ], +) def test_in_micros(us: float, s: int): assert in_micros(us) is s -@pytest.mark.parametrize("start,stop,step,expected_num,truncated_stop", - [(0, 0, 1, 1, None), # start=stop, 1 point at start - (0, 0.5, 1, 1, 0), # step>length, 1 point at start - (0, 1, 1, 2, None), # stop=start+step, 1 point at start, 1 at stop - (0, 0.99, 1, 2, 1), # stop >= start + 0.99*step, included - (0, 0.98, 1, 1, 0), # stop < start + 0.99*step, not included - (0, 1.01, 1, 2, 1), # stop >= start + 0.99*step, included - (0, 1.75, 0.25, 8, 1.75) - ]) -def test_step_to_num(start: float, stop: float, step: float, expected_num: int, truncated_stop: Optional[float]): +@pytest.mark.parametrize( + "start,stop,step,expected_num,truncated_stop", + [ + (0, 0, 1, 1, None), # start=stop, 1 point at start + (0, 0.5, 1, 1, 0), # step>length, 1 point at start + (0, 1, 1, 2, None), # stop=start+step, point at start & stop + (0, 0.99, 1, 2, 1), # stop >= start + 0.99*step, included + (0, 0.98, 1, 1, 0), # stop < start + 0.99*step, not included + (0, 1.01, 1, 2, 1), # stop >= start + 0.99*step, included + (0, 1.75, 0.25, 8, 1.75), + ], +) +def test_step_to_num( + start: float, + stop: float, + step: float, + expected_num: int, + truncated_stop: Optional[float], +): truncated_stop = stop if truncated_stop is None else truncated_stop actual_start, actual_stop, num = step_to_num(start, stop, step) assert actual_start == start From 7afeb2b48bbf2781e9cd8cd07638d5303d03c9dc Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Thu, 12 Oct 2023 16:37:55 +0100 Subject: [PATCH 06/30] test for group_uuid --- tests/core/test_funcs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index 3aed11e..a18a061 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -1,8 +1,9 @@ +import uuid from typing import Optional import pytest -from dls_bluesky_core.core import in_micros, step_to_num +from dls_bluesky_core.core import group_uuid, in_micros, step_to_num @pytest.mark.parametrize( @@ -36,6 +37,13 @@ def test_in_micros(us: float, s: int): (0, 0.98, 1, 1, 0), # stop < start + 0.99*step, not included (0, 1.01, 1, 2, 1), # stop >= start + 0.99*step, included (0, 1.75, 0.25, 8, 1.75), + (0, 0, -1, 1, None), # start=stop, 1 point at start + (0, 0.5, -1, 1, 0), # abs(step)>length, 1 point at start + (0, -1, 1, 2, None), # stop=start+-abs(step), point at start & stop + (0, -0.99, 1, 2, -1), # stop >= start + 0.99*-abs(step), included + (0, -0.98, 1, 1, 0), # stop < start + 0.99*-abs(step), not included + (0, -1.01, 1, 2, -1), # stop >= start + 0.99*-abs(step), included + (0, -1.75, 0.25, 8, -1.75), ], ) def test_step_to_num( @@ -50,3 +58,10 @@ def test_step_to_num( assert actual_start == start assert actual_stop == truncated_stop assert num == expected_num + + +@pytest.mark.parametrize("group", ["foo", "bar", "baz", str(uuid.uuid4())]) +def test_group_uid(group: str): + gid = group_uuid(group) + assert gid.startswith(f"{group}-") + assert not gid.endswith(f"{group}-") From ced4b8f0bd0bb6a65c6a88f7d0e2952084135df8 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Fri, 13 Oct 2023 15:19:41 +0100 Subject: [PATCH 07/30] Move i22 generic plans (but do not expose to BlueAPI) --- pyproject.toml | 2 +- src/dls_bluesky_core/plans/scanspec.py | 83 +++++++++++++++++++++++++- src/dls_bluesky_core/stubs/flyables.py | 23 +++++++ 3 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 src/dls_bluesky_core/stubs/flyables.py diff --git a/pyproject.toml b/pyproject.toml index c5cf953..263579c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ description = "Common Diamond specific Bluesky plans and functions" dependencies = [ "blueapi", "ophyd", - "ophyd_async", + "ophyd_async @ git+https://github.com/bluesky/ophyd-async.git@add-detector-logic", "scanspec" ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] diff --git a/src/dls_bluesky_core/plans/scanspec.py b/src/dls_bluesky_core/plans/scanspec.py index 50ae0a1..e337b1a 100644 --- a/src/dls_bluesky_core/plans/scanspec.py +++ b/src/dls_bluesky_core/plans/scanspec.py @@ -1,13 +1,18 @@ import operator from functools import reduce -from typing import Any, List, Mapping, Optional +from typing import Any, Dict, List, Mapping, Optional, Sequence +import bluesky.plan_stubs as bps import bluesky.plans as bp +import bluesky.preprocessors as bpp from bluesky.protocols import Movable, Readable from cycler import Cycler, cycler -from scanspec.specs import Spec +from ophyd_async.core import Device +from ophyd_async.core.flyer import ScanSpecFlyable +from scanspec.specs import Repeat, Spec -from dls_bluesky_core.core import MsgGenerator +from ..core import MsgGenerator +from ..stubs.flyables import fly_and_collect """ Plans related to the use of the `ScanSpec https://github.com/dls-controls/scanspec` @@ -57,6 +62,78 @@ def scan( yield from bp.scan_nd(detectors, cycler, md=_md) +def scanspec_fly( + flyer: ScanSpecFlyable, + hw_spec: Spec[ScanSpecFlyable], + sw_dets: Optional[Sequence[Device]] = None, + sw_spec: Optional[Spec[Device]] = None, + # TODO: How to allow in Blueapi REST call? + # TODO: Allow defining processors for @BeforeScan/@AfterScan equivalent logic? + # setup_detectors: Optional[Collection[Msg]] = None, + # flush_period: float = 0.5, # TODO: Should we allow overriding default? + metadata: Optional[Dict[str, Any]] = None, +) -> MsgGenerator: + """ + TODO + + Args: + flyer (ScanSpecFlyable): + A Flyable that traces the path of hw_spec Spec at every point of sw_spec. + hw_spec (Spec[Device]): + The 'inner scan' performed at each point of the outer scan. + sw_dets (Optional[Sequence[Device]]): + Any detectors to be triggered at every point of an outer scan. + sw_spec (Optional[Spec[Device]]): + The 'outer scan' trajectory to be followed by any non-flying motors. + Defaults to a one-shot no-op Spec. + # setup_detectors (Iterator[Msg]): + Any Msgs required to set up the detectors # TODO: Make a pre-processor? + # flush_period (float): # TODO: Allow non-default? + Timeout for calls to complete when the flyer is kicked off. + metadata (Optional[Dict[str, Any]]): + Key-value metadata to include in exported data, defaults to None. + + Returns: + MsgGenerator: _description_ + + Yields: + Iterator[MsgGenerator]: _description_ + """ + sw_dets = sw_dets or [] + sw_spec = sw_spec or Repeat() + detectors: Sequence[Device] = [flyer] + sw_dets + plan_args = { + "flyer": repr(flyer), + "hw_spec": repr(hw_spec), + "sw_dets": [repr(det) for det in sw_dets], + "sw_spec": repr(sw_spec), + } + + _md = { + "plan_args": plan_args, + "detectors": [det.name for det in detectors], + "hints": {}, # TODO: Dimension hints from ScanSpec? + } + _md.update(metadata or {}) + + @bpp.stage_decorator(detectors) + @bpp.run_decorator(md=_md) + def hw_scanspec_fly() -> MsgGenerator: + # yield from setup_detectors + yield from bps.declare_stream(*sw_dets, name="sw") + yield from bps.declare_stream(flyer, name="hw") + for point in sw_spec.midpoints(): + # Move flyer to start too + point[flyer] = hw_spec + # TODO: need to make pos_cache optional in this func + yield from bps.move_per_step(point, dict()) + yield from bps.trigger_and_read(sw_dets) + yield from bps.checkpoint() + yield from fly_and_collect(flyer, checkpoint_every_collect=True) + + return (yield from hw_scanspec_fly()) + + def _scanspec_to_cycler(spec: Spec[str], axes: Mapping[str, Movable]) -> Cycler: """ Convert a scanspec to a cycler for compatibility with legacy Bluesky plans such as diff --git a/src/dls_bluesky_core/stubs/flyables.py b/src/dls_bluesky_core/stubs/flyables.py new file mode 100644 index 0000000..d8758b2 --- /dev/null +++ b/src/dls_bluesky_core/stubs/flyables.py @@ -0,0 +1,23 @@ +import bluesky.plan_stubs as bps +from bluesky.protocols import Flyable + +from dls_bluesky_core.core import MsgGenerator, group_uuid + + +def fly_and_collect( + flyer: Flyable, flush_period: float = 0.5, checkpoint_every_collect: bool = False +) -> MsgGenerator: + yield from bps.kickoff(flyer) + complete_group = group_uuid("complete") + yield from bps.complete(flyer, group=complete_group) + done = False + while not done: + try: + yield from bps.wait(group=complete_group, timeout=flush_period) + except TimeoutError: + pass + else: + done = True + yield from bps.collect(flyer, stream=True, return_payload=False) + if checkpoint_every_collect: + yield from bps.checkpoint() From 2685a7d66865d358bd4ffc661010c3e9d2e3a389 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Fri, 13 Oct 2023 15:39:02 +0100 Subject: [PATCH 08/30] Inverse in_micros logic to actually convert to micros --- src/dls_bluesky_core/core/maths.py | 12 ++++++------ tests/core/test_funcs.py | 31 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index 3b21bc6..ccfc7d6 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -35,13 +35,13 @@ def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, i def in_micros(t: float) -> int: """ - Converts between units of microT and units of T. - For example, from microseconds to seconds. + Converts between units of T and units of microT. + For example, from seconds to microseconds. Args: - t (float): A time in microseconds, or other measurement in units of microU + t (float): A time in seconds, or other measurement in units of U Returns: - t (int): A time in seconds rounded up to the nearest whole second, - or other measurement in units of U, rounded up to the nearest whole U. + t (int): A time in microseconds, rounded up to the nearest whole microsecond, + or other measurement in units of microU, rounded up to the nearest whole microU. """ - return int(np.ceil(t / 1e6)) + return int(np.ceil(t * 1e6)) diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index a18a061..6855f6e 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -7,24 +7,25 @@ @pytest.mark.parametrize( - "us,s", + "s,us", [ - (4_000_001, 5), - (4_999_999, 5), - (4_000, 1), - (-4_000_001, -4), - (-4_999_999, -4), - (-4_000, 0), - (4_000_000.1, 5), - (4_999_999.9, 5), - (0.1, 1), - (-4_000_000.5, -4), - (-4_999_999.9, -4), - (-4.05, 0), + (4.000_001, 4_000_001), + (4.999_999, 4_999_999), + (4, 4_000_000), + (-4.000_001, -4_000_001), + (-4.999_999, -4_999_999), + (-4, -4_000_000), + (4.000_000_1, 4_000_001), + (4.999_999_9, 5_000_000), + (0.1, 100_000), + (0.000_000_1, 1), + (-4.000_000_5, -4_000_000), + (-4.999_999_9, -4_999_999), + (-4.05, -4_050_000), ], ) -def test_in_micros(us: float, s: int): - assert in_micros(us) is s +def test_in_micros(s: float, us: int): + assert in_micros(s) == us @pytest.mark.parametrize( From b9ed5a7cb2682f77aec016f6fefe4f8a265e30d7 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 10:47:39 +0100 Subject: [PATCH 09/30] Respond to review comments: - get_constant_duration renamed to be clearer intended function, always return None instead of throwing - in_micros to throw exception when working in negative time - Add tests for intended behaviour of get_constant_duration --- src/dls_bluesky_core/core/__init__.py | 4 +- src/dls_bluesky_core/core/maths.py | 4 ++ src/dls_bluesky_core/core/scanspecs.py | 29 ++++++------- tests/core/test_funcs.py | 15 ++++--- tests/core/test_scanspecs.py | 58 ++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 tests/core/test_scanspecs.py diff --git a/src/dls_bluesky_core/core/__init__.py b/src/dls_bluesky_core/core/__init__.py index 4f7e36c..dff8856 100644 --- a/src/dls_bluesky_core/core/__init__.py +++ b/src/dls_bluesky_core/core/__init__.py @@ -1,10 +1,10 @@ from .coordination import group_uuid from .maths import in_micros, step_to_num -from .scanspecs import get_duration +from .scanspecs import get_constant_duration from .types import MsgGenerator, PlanGenerator, ScanAxis __all__ = [ - "get_duration", + "get_constant_duration", "group_uuid", "in_micros", "MsgGenerator", diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index ccfc7d6..b73bc31 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -40,8 +40,12 @@ def in_micros(t: float) -> int: Args: t (float): A time in seconds, or other measurement in units of U + Raises: + ValueError: if t < 0 Returns: t (int): A time in microseconds, rounded up to the nearest whole microsecond, or other measurement in units of microU, rounded up to the nearest whole microU. """ + if t < 0: + raise ValueError return int(np.ceil(t * 1e6)) diff --git a/src/dls_bluesky_core/core/scanspecs.py b/src/dls_bluesky_core/core/scanspecs.py index 99b1c7e..1333a0b 100644 --- a/src/dls_bluesky_core/core/scanspecs.py +++ b/src/dls_bluesky_core/core/scanspecs.py @@ -5,28 +5,29 @@ from scanspec.specs import DURATION -def get_duration(frames: List[Frames]) -> Optional[float]: +def get_constant_duration(frames: List[Frames]) -> Optional[float]: """ Returns the duration of a number of ScanSpec frames, if known and consistent. Args: frames (List[Frames]): A number of Frame objects - Raises: - ValueError: If any frames do not have a Duration defined. - Returns: duration (float): if all frames have a consistent duration None: otherwise """ - for fs in frames: - if DURATION in fs.axes(): - durations = fs.midpoints[DURATION] - first_duration = durations[0] - if np.all(durations == first_duration): - # Constant duration, return it - return first_duration - else: - return None - raise ValueError("Duration not specified in Spec") + duration_frame = [ + f for f in frames if DURATION in f.axes() and len(f.midpoints[DURATION]) + ] + if len(duration_frame) != 1: + # Either no frame has DURATION axis, + # the frame with a DURATION axis has 0 points, + # or multiple frames have DURATION axis + return None + durations = duration_frame[0].midpoints[DURATION] + first_duration = durations[0] + if np.any(durations != first_duration): + # Not all durations are the same + return None + return first_duration diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index 6855f6e..83416c0 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -12,22 +12,25 @@ (4.000_001, 4_000_001), (4.999_999, 4_999_999), (4, 4_000_000), - (-4.000_001, -4_000_001), - (-4.999_999, -4_999_999), - (-4, -4_000_000), (4.000_000_1, 4_000_001), (4.999_999_9, 5_000_000), (0.1, 100_000), (0.000_000_1, 1), - (-4.000_000_5, -4_000_000), - (-4.999_999_9, -4_999_999), - (-4.05, -4_050_000), + (0, 0), ], ) def test_in_micros(s: float, us: int): assert in_micros(s) == us +@pytest.mark.parametrize( + "s", [-4.000_001, -4.999_999, -4, -4.000_000_5, -4.999_999_9, -4.05] +) +def test_in_micros_negative(s: float): + with pytest.raises(ValueError): + in_micros(s) + + @pytest.mark.parametrize( "start,stop,step,expected_num,truncated_stop", [ diff --git a/tests/core/test_scanspecs.py b/tests/core/test_scanspecs.py new file mode 100644 index 0000000..c4e8257 --- /dev/null +++ b/tests/core/test_scanspecs.py @@ -0,0 +1,58 @@ +import re + +import pytest +from scanspec.specs import DURATION, Line, Static + +from dls_bluesky_core.core.scanspecs import get_constant_duration + + +def test_single_frame_single_point(): + spec = Static.duration(0.1) + assert get_constant_duration(spec.calculate()) == 0.1 + + +def test_consistent_points(): + spec = Static.duration(0.1).concat(Static.duration(0.1)) + assert get_constant_duration(spec.calculate()) == 0.1 + + +def test_inconsistent_points(): + spec = Static.duration(0.1).concat(Static.duration(0.2)) + assert get_constant_duration(spec.calculate()) is None + + +def test_frame_with_multiple_axes(): + spec = Static.duration(0.1).zip(Line.bounded("x", 0, 0, 1)) + frames = spec.calculate() + assert len(frames) == 1 + assert get_constant_duration(frames) == 0.1 + + +def test_inconsistent_frame_with_multiple_axes(): + spec = ( + Static.duration(0.1) + .concat(Static.duration(0.2)) + .zip(Line.bounded("x", 0, 0, 2)) + ) + frames = spec.calculate() + assert len(frames) == 1 + assert get_constant_duration(frames) is None + + +def test_non_static_spec_duration(): + spec = Line.bounded(DURATION, 0, 0, 3) + frames = spec.calculate() + assert len(frames) == 1 + assert get_constant_duration(frames) == 0 + + +def test_multiple_duration_frames(): + spec = ( + Static.duration(0.1) + .concat(Static.duration(0.2)) + .zip(Line.bounded(DURATION, 0, 0, 2)) + ) + with pytest.raises( + AssertionError, match=re.escape("Zipping would overwrite axes ['DURATION']") + ): + spec.calculate() From 8ca1a02ae4fe4e986e799e0fc41697825fd6bed8 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 11:02:13 +0100 Subject: [PATCH 10/30] Handle Spec Product not raising on multiple Frames in axis --- tests/core/test_scanspecs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/core/test_scanspecs.py b/tests/core/test_scanspecs.py index c4e8257..4a8c8cc 100644 --- a/tests/core/test_scanspecs.py +++ b/tests/core/test_scanspecs.py @@ -56,3 +56,9 @@ def test_multiple_duration_frames(): AssertionError, match=re.escape("Zipping would overwrite axes ['DURATION']") ): spec.calculate() + spec = ( # TODO: refactor when https://github.com/dls-controls/scanspec/issues/90 + Static.duration(0.1) * Line.bounded(DURATION, 0, 0, 2) + ) + frames = spec.calculate() + assert len(frames) == 2 + assert get_constant_duration(frames) is None From c88725554619f1f1a35d62f505695a96f40a1f8e Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 11:09:16 +0100 Subject: [PATCH 11/30] Remove scanspec_fly until ScanSpecFlyable is recreated --- src/dls_bluesky_core/plans/scanspec.py | 81 +------------------------- 1 file changed, 2 insertions(+), 79 deletions(-) diff --git a/src/dls_bluesky_core/plans/scanspec.py b/src/dls_bluesky_core/plans/scanspec.py index e337b1a..327ea73 100644 --- a/src/dls_bluesky_core/plans/scanspec.py +++ b/src/dls_bluesky_core/plans/scanspec.py @@ -1,18 +1,13 @@ import operator from functools import reduce -from typing import Any, Dict, List, Mapping, Optional, Sequence +from typing import Any, List, Mapping, Optional -import bluesky.plan_stubs as bps import bluesky.plans as bp -import bluesky.preprocessors as bpp from bluesky.protocols import Movable, Readable from cycler import Cycler, cycler -from ophyd_async.core import Device -from ophyd_async.core.flyer import ScanSpecFlyable -from scanspec.specs import Repeat, Spec +from scanspec.specs import Spec from ..core import MsgGenerator -from ..stubs.flyables import fly_and_collect """ Plans related to the use of the `ScanSpec https://github.com/dls-controls/scanspec` @@ -62,78 +57,6 @@ def scan( yield from bp.scan_nd(detectors, cycler, md=_md) -def scanspec_fly( - flyer: ScanSpecFlyable, - hw_spec: Spec[ScanSpecFlyable], - sw_dets: Optional[Sequence[Device]] = None, - sw_spec: Optional[Spec[Device]] = None, - # TODO: How to allow in Blueapi REST call? - # TODO: Allow defining processors for @BeforeScan/@AfterScan equivalent logic? - # setup_detectors: Optional[Collection[Msg]] = None, - # flush_period: float = 0.5, # TODO: Should we allow overriding default? - metadata: Optional[Dict[str, Any]] = None, -) -> MsgGenerator: - """ - TODO - - Args: - flyer (ScanSpecFlyable): - A Flyable that traces the path of hw_spec Spec at every point of sw_spec. - hw_spec (Spec[Device]): - The 'inner scan' performed at each point of the outer scan. - sw_dets (Optional[Sequence[Device]]): - Any detectors to be triggered at every point of an outer scan. - sw_spec (Optional[Spec[Device]]): - The 'outer scan' trajectory to be followed by any non-flying motors. - Defaults to a one-shot no-op Spec. - # setup_detectors (Iterator[Msg]): - Any Msgs required to set up the detectors # TODO: Make a pre-processor? - # flush_period (float): # TODO: Allow non-default? - Timeout for calls to complete when the flyer is kicked off. - metadata (Optional[Dict[str, Any]]): - Key-value metadata to include in exported data, defaults to None. - - Returns: - MsgGenerator: _description_ - - Yields: - Iterator[MsgGenerator]: _description_ - """ - sw_dets = sw_dets or [] - sw_spec = sw_spec or Repeat() - detectors: Sequence[Device] = [flyer] + sw_dets - plan_args = { - "flyer": repr(flyer), - "hw_spec": repr(hw_spec), - "sw_dets": [repr(det) for det in sw_dets], - "sw_spec": repr(sw_spec), - } - - _md = { - "plan_args": plan_args, - "detectors": [det.name for det in detectors], - "hints": {}, # TODO: Dimension hints from ScanSpec? - } - _md.update(metadata or {}) - - @bpp.stage_decorator(detectors) - @bpp.run_decorator(md=_md) - def hw_scanspec_fly() -> MsgGenerator: - # yield from setup_detectors - yield from bps.declare_stream(*sw_dets, name="sw") - yield from bps.declare_stream(flyer, name="hw") - for point in sw_spec.midpoints(): - # Move flyer to start too - point[flyer] = hw_spec - # TODO: need to make pos_cache optional in this func - yield from bps.move_per_step(point, dict()) - yield from bps.trigger_and_read(sw_dets) - yield from bps.checkpoint() - yield from fly_and_collect(flyer, checkpoint_every_collect=True) - - return (yield from hw_scanspec_fly()) - - def _scanspec_to_cycler(spec: Spec[str], axes: Mapping[str, Movable]) -> Cycler: """ Convert a scanspec to a cycler for compatibility with legacy Bluesky plans such as From be17ffd14694c10d1a83478e30280a4251302c9f Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 11:17:45 +0100 Subject: [PATCH 12/30] Rename ScanAxis to ScannableAxis --- src/dls_bluesky_core/core/__init__.py | 4 ++-- src/dls_bluesky_core/core/types.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dls_bluesky_core/core/__init__.py b/src/dls_bluesky_core/core/__init__.py index dff8856..d945537 100644 --- a/src/dls_bluesky_core/core/__init__.py +++ b/src/dls_bluesky_core/core/__init__.py @@ -1,7 +1,7 @@ from .coordination import group_uuid from .maths import in_micros, step_to_num from .scanspecs import get_constant_duration -from .types import MsgGenerator, PlanGenerator, ScanAxis +from .types import MsgGenerator, PlanGenerator, ScannableAxis __all__ = [ "get_constant_duration", @@ -9,6 +9,6 @@ "in_micros", "MsgGenerator", "PlanGenerator", - "ScanAxis", + "ScannableAxis", "step_to_num", ] diff --git a/src/dls_bluesky_core/core/types.py b/src/dls_bluesky_core/core/types.py index 3f96214..86fda3d 100644 --- a/src/dls_bluesky_core/core/types.py +++ b/src/dls_bluesky_core/core/types.py @@ -8,4 +8,4 @@ MsgGenerator = Generator[Msg, Any, None] # 'A function that generates a plan' PlanGenerator = Callable[..., MsgGenerator] -ScanAxis = Union[Device, DURATION] +ScannableAxis = Union[Device, DURATION] From 994779f80b9d06913f9f805a0f44daa60b2f2b49 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 11:19:26 +0100 Subject: [PATCH 13/30] Revert unrelated nothing change --- src/dls_bluesky_core/plans/scanspec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dls_bluesky_core/plans/scanspec.py b/src/dls_bluesky_core/plans/scanspec.py index 327ea73..50ae0a1 100644 --- a/src/dls_bluesky_core/plans/scanspec.py +++ b/src/dls_bluesky_core/plans/scanspec.py @@ -7,7 +7,7 @@ from cycler import Cycler, cycler from scanspec.specs import Spec -from ..core import MsgGenerator +from dls_bluesky_core.core import MsgGenerator """ Plans related to the use of the `ScanSpec https://github.com/dls-controls/scanspec` From 6ba15cfe4787d8a3a430b8ee96624a067d04a707 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 11:27:03 +0100 Subject: [PATCH 14/30] Remove container build from CI as not application code --- .github/workflows/code.yml | 88 -------------------------------------- 1 file changed, 88 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 05a7952..b0b4de6 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -102,94 +102,6 @@ jobs: # If more than one module in src/ replace with module name to test run: python -m $(ls src | head -1) --version - container: - needs: [lint, dist, test] - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - env: - TEST_TAG: "testing" - - steps: - - name: Checkout - uses: actions/checkout@v3 - - # image names must be all lower case - - name: Generate image repo name - run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - - - name: Download wheel and lockfiles - uses: actions/download-artifact@v3 - with: - path: artifacts/ - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and export to Docker local cache - uses: docker/build-push-action@v4 - with: - # Note build-args, context, file, and target must all match between this - # step and the later build-push-action, otherwise the second build-push-action - # will attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - load: true - tags: ${{ env.TEST_TAG }} - # If you have a long docker build (2+ minutes), uncomment the - # following to turn on caching. For short build times this - # makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - - name: Create tags for publishing image - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.IMAGE_REPOSITORY }} - tags: | - type=ref,event=tag - type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} - # type=edge,branch=main - # Add line above to generate image for every commit to given branch, - # and uncomment the end of if clause in next step - - - name: Push cached image to container registry - if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v3 - # This does not build the image again, it will find the image in the - # Docker cache and publish it - with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - release: # upload to PyPI and make a release on every tag needs: [lint, dist, test] From d5d10f48bf4f030089858c3164ca20172a22a681 Mon Sep 17 00:00:00 2001 From: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:13:04 +0100 Subject: [PATCH 15/30] Update src/dls_bluesky_core/core/maths.py --- src/dls_bluesky_core/core/maths.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index b73bc31..aaa9ebb 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -35,16 +35,15 @@ def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, i def in_micros(t: float) -> int: """ - Converts between units of T and units of microT. - For example, from seconds to microseconds. + Converts between a positive number of seconds and an equivalent + number of microseconds. Args: - t (float): A time in seconds, or other measurement in units of U + t (float): A time in seconds Raises: ValueError: if t < 0 Returns: t (int): A time in microseconds, rounded up to the nearest whole microsecond, - or other measurement in units of microU, rounded up to the nearest whole microU. """ if t < 0: raise ValueError From 47557c6cabadc538684ecac364b5920db766bf0b Mon Sep 17 00:00:00 2001 From: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:13:11 +0100 Subject: [PATCH 16/30] Update src/dls_bluesky_core/core/maths.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/dls_bluesky_core/core/maths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index aaa9ebb..ff0779a 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -46,5 +46,5 @@ def in_micros(t: float) -> int: t (int): A time in microseconds, rounded up to the nearest whole microsecond, """ if t < 0: - raise ValueError + raise ValueError(f"Expected a positive time in seconds, got {t!r}") return int(np.ceil(t * 1e6)) From 06f9599234fda3d82d0d5ac6a5cf637be57b48bf Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 16 Oct 2023 15:24:41 +0100 Subject: [PATCH 17/30] linting --- src/dls_bluesky_core/core/maths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index ff0779a..a99009b 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -35,7 +35,7 @@ def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, i def in_micros(t: float) -> int: """ - Converts between a positive number of seconds and an equivalent + Converts between a positive number of seconds and an equivalent number of microseconds. Args: From 1bd64aeaa6b0e68d3918c2f98faa6c8e3099dcd0 Mon Sep 17 00:00:00 2001 From: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:56:16 +0100 Subject: [PATCH 18/30] Update src/dls_bluesky_core/core/maths.py --- src/dls_bluesky_core/core/maths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dls_bluesky_core/core/maths.py b/src/dls_bluesky_core/core/maths.py index a99009b..0505943 100644 --- a/src/dls_bluesky_core/core/maths.py +++ b/src/dls_bluesky_core/core/maths.py @@ -20,9 +20,9 @@ def step_to_num(start: float, stop: float, step: float) -> Tuple[float, float, i If stop < start, will be coerced to be backwards. Returns: - start, truncated_stop, num = Tuple[float, float, int] + start, adjusted_stop, num = Tuple[float, float, int] start will be returned unchanged - truncated_stop = start + num * step + adjusted_stop = start + (num - 1) * step num is the maximal number of steps that could fit into the length. """ From e843acc57b0a6e374f7cebbf4bdffd28100034ea Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Tue, 17 Oct 2023 14:09:06 +0100 Subject: [PATCH 19/30] Add test for negative step with positive span --- tests/core/test_funcs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index 83416c0..f2c1c14 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -48,6 +48,8 @@ def test_in_micros_negative(s: float): (0, -0.98, 1, 1, 0), # stop < start + 0.99*-abs(step), not included (0, -1.01, 1, 2, -1), # stop >= start + 0.99*-abs(step), included (0, -1.75, 0.25, 8, -1.75), + (1, 10, -0.901, 10, 9.109), # length overrules step for direction + (10, 1, -0.901, 10, 1.891) ], ) def test_step_to_num( From a31c5979cd6cbabb95a8ee13a0c74685c97cce17 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Tue, 17 Oct 2023 14:13:17 +0100 Subject: [PATCH 20/30] lint --- tests/core/test_funcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_funcs.py b/tests/core/test_funcs.py index f2c1c14..c981dc9 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_funcs.py @@ -49,7 +49,7 @@ def test_in_micros_negative(s: float): (0, -1.01, 1, 2, -1), # stop >= start + 0.99*-abs(step), included (0, -1.75, 0.25, 8, -1.75), (1, 10, -0.901, 10, 9.109), # length overrules step for direction - (10, 1, -0.901, 10, 1.891) + (10, 1, -0.901, 10, 1.891), ], ) def test_step_to_num( From d7674d27ece09f0518d9f4668bbb8879be6b3e2a Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Fri, 20 Oct 2023 09:44:28 +0100 Subject: [PATCH 21/30] Added stream name to bps.collect in fly_and_collect --- src/dls_bluesky_core/stubs/flyables.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/dls_bluesky_core/stubs/flyables.py b/src/dls_bluesky_core/stubs/flyables.py index d8758b2..bcb81bd 100644 --- a/src/dls_bluesky_core/stubs/flyables.py +++ b/src/dls_bluesky_core/stubs/flyables.py @@ -5,7 +5,10 @@ def fly_and_collect( - flyer: Flyable, flush_period: float = 0.5, checkpoint_every_collect: bool = False + flyer: Flyable, + flush_period: float = 0.5, + checkpoint_every_collect: bool = False, + stream_name: str = "primary", ) -> MsgGenerator: yield from bps.kickoff(flyer) complete_group = group_uuid("complete") @@ -18,6 +21,8 @@ def fly_and_collect( pass else: done = True - yield from bps.collect(flyer, stream=True, return_payload=False) + yield from bps.collect( + flyer, stream=True, return_payload=False, name=stream_name + ) if checkpoint_every_collect: yield from bps.checkpoint() From efba33ee01cac60dd13c578d9b6ca92baa8b1bbd Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Fri, 20 Oct 2023 15:41:34 +0100 Subject: [PATCH 22/30] Move inject method for type checked default arguments from blueapi --- src/dls_bluesky_core/core/coordination.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/dls_bluesky_core/core/coordination.py b/src/dls_bluesky_core/core/coordination.py index 1a94742..694c451 100644 --- a/src/dls_bluesky_core/core/coordination.py +++ b/src/dls_bluesky_core/core/coordination.py @@ -12,3 +12,21 @@ def group_uuid(name: str) -> str: readable_uid (str): name appended with a unique string """ return f"{name}-{str(uuid.uuid4())[:6]}" + + +def inject(name: str): + """ + Function to mark a default argument of a plan method as a reference to a device + that is stored in the Blueapi context. + Bypasses mypy linting, returning x as Any and therefore valid as a default + argument. + + Args: + name (str): Name of a device to be fetched from the Blueapi context + + Returns: + Any: name but without typing checking, valid as any default type + + """ + + return name From 7d478b0dc9349d01eccc23ab82f80ce2270a2368 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Fri, 20 Oct 2023 15:41:34 +0100 Subject: [PATCH 23/30] Move inject method for type checked default arguments from blueapi --- src/dls_bluesky_core/core/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dls_bluesky_core/core/__init__.py b/src/dls_bluesky_core/core/__init__.py index d945537..899a287 100644 --- a/src/dls_bluesky_core/core/__init__.py +++ b/src/dls_bluesky_core/core/__init__.py @@ -1,4 +1,4 @@ -from .coordination import group_uuid +from .coordination import group_uuid, inject from .maths import in_micros, step_to_num from .scanspecs import get_constant_duration from .types import MsgGenerator, PlanGenerator, ScannableAxis @@ -6,6 +6,7 @@ __all__ = [ "get_constant_duration", "group_uuid", + "inject", "in_micros", "MsgGenerator", "PlanGenerator", From 615f9ef69d81d2ee71b7bd790b21176d13c62051 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:04:39 +0100 Subject: [PATCH 24/30] Add test for fly_and_collect stub --- pyproject.toml | 5 ++-- tests/conftest.py | 24 +++++++++++++++++ tests/stubs/test_flyables.py | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/stubs/test_flyables.py diff --git a/pyproject.toml b/pyproject.toml index 263579c..995ece3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,8 @@ description = "Common Diamond specific Bluesky plans and functions" dependencies = [ "blueapi", "ophyd", - "ophyd_async @ git+https://github.com/bluesky/ophyd-async.git@add-detector-logic", - "scanspec" + "ophyd_async @ git+https://github.com/bluesky/ophyd-async.git", + "scanspec", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -85,6 +85,7 @@ addopts = """ filterwarnings = "error" # Doctest python code in docs, python code in src docstrings, test functions in tests testpaths = "docs src tests" +asyncio_mode = "auto" [tool.coverage.run] data_file = "/tmp/dls_bluesky_core.coverage" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a60c53b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,24 @@ +import asyncio + +import pytest +from bluesky.run_engine import RunEngine, TransitionError + + +@pytest.fixture(scope="function") +def RE(request): + loop = asyncio.new_event_loop() + loop.set_debug(True) + RE = RunEngine({}, call_returns_result=True, loop=loop) + + def clean_event_loop(): + if RE.state not in ("idle", "panicked"): + try: + RE.halt() + except TransitionError: + pass + loop.call_soon_threadsafe(loop.stop) + RE._th.join() + loop.close() + + request.addfinalizer(clean_event_loop) + return RE diff --git a/tests/stubs/test_flyables.py b/tests/stubs/test_flyables.py new file mode 100644 index 0000000..ca9df9a --- /dev/null +++ b/tests/stubs/test_flyables.py @@ -0,0 +1,52 @@ +import asyncio +from typing import Dict + +import bluesky.plan_stubs as bps +import pytest +from bluesky.protocols import Collectable, Descriptor, Flyable +from ophyd_async.core import AsyncStatus + +from dls_bluesky_core.stubs.flyables import fly_and_collect + + +class DummyFlyer(Flyable, Collectable): + def __init__(self, name: str) -> None: + self._name = name + self.has_flown = False + + @property + def name(self) -> str: + return self._name + + @AsyncStatus.wrap + async def kickoff(self) -> None: + self._fly_status = AsyncStatus(self._fly()) + + async def _fly(self) -> None: + self.has_flown = True + await asyncio.sleep(0.1) + + def complete(self) -> AsyncStatus: + return self._fly_status + + def describe_collect(self) -> Dict[str, Descriptor]: + return { + self.name: Descriptor( + source="some:source", shape=[], dtype="array", external="STREAM:" + ) + } + + +@pytest.fixture +def flyer() -> Flyable: + return DummyFlyer("test") + + +async def test_fly_and_collect(RE, flyer: DummyFlyer): + def open_and_close_run_for_fly_and_collect(): + yield from bps.open_run() + yield from fly_and_collect(flyer, flush_period=0.01, checkpoint_every_collect=True) + yield from bps.close_run() + + RE(open_and_close_run_for_fly_and_collect()) + assert flyer.has_flown is True From 3691bdd95a91958acc597102078ac99551641e16 Mon Sep 17 00:00:00 2001 From: "Ware, Joseph (DLSLtd,RAL,LSCI)" Date: Mon, 23 Oct 2023 11:06:06 +0100 Subject: [PATCH 25/30] Remove dependency on BlueAPI --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 995ece3..0adc7f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ classifiers = [ ] description = "Common Diamond specific Bluesky plans and functions" dependencies = [ - "blueapi", "ophyd", "ophyd_async @ git+https://github.com/bluesky/ophyd-async.git", "scanspec", From 030597cb4e9904679705a9ab0bf81c8ebe0e6c73 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:09:50 +0100 Subject: [PATCH 26/30] Add pytest-asyncio as dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0adc7f6..a2df8e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dev = [ "pre-commit", "pydata-sphinx-theme>=0.12", "pytest", + "pytest-asyncio", "pytest-cov", "sphinx-autobuild", "sphinx-copybutton", From 8421a0ba30273bc8f2996641d046dbdb10ef51d6 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:10:48 +0100 Subject: [PATCH 27/30] Ignoring untyped function definition for core.coordination.inject for now --- src/dls_bluesky_core/core/coordination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dls_bluesky_core/core/coordination.py b/src/dls_bluesky_core/core/coordination.py index 694c451..0e18624 100644 --- a/src/dls_bluesky_core/core/coordination.py +++ b/src/dls_bluesky_core/core/coordination.py @@ -14,7 +14,7 @@ def group_uuid(name: str) -> str: return f"{name}-{str(uuid.uuid4())[:6]}" -def inject(name: str): +def inject(name: str): # type: ignore """ Function to mark a default argument of a plan method as a reference to a device that is stored in the Blueapi context. From 02eb1d51e69e2542eadae316aba00b2bc0a79ffc Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:12:56 +0100 Subject: [PATCH 28/30] lint --- tests/stubs/test_flyables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/stubs/test_flyables.py b/tests/stubs/test_flyables.py index ca9df9a..366b2a0 100644 --- a/tests/stubs/test_flyables.py +++ b/tests/stubs/test_flyables.py @@ -45,7 +45,9 @@ def flyer() -> Flyable: async def test_fly_and_collect(RE, flyer: DummyFlyer): def open_and_close_run_for_fly_and_collect(): yield from bps.open_run() - yield from fly_and_collect(flyer, flush_period=0.01, checkpoint_every_collect=True) + yield from fly_and_collect( + flyer, flush_period=0.01, checkpoint_every_collect=True + ) yield from bps.close_run() RE(open_and_close_run_for_fly_and_collect()) From 03a65ece2334f4f176b7de79c004e9eee254ea29 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:43:51 +0100 Subject: [PATCH 29/30] Change test structure and add docstring to fly_and_collect --- src/dls_bluesky_core/stubs/flyables.py | 31 +++++++++++++++++++++ tests/core/test_coordination.py | 12 ++++++++ tests/core/{test_funcs.py => test_maths.py} | 10 +------ 3 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 tests/core/test_coordination.py rename tests/core/{test_funcs.py => test_maths.py} (87%) diff --git a/src/dls_bluesky_core/stubs/flyables.py b/src/dls_bluesky_core/stubs/flyables.py index bcb81bd..6b7e551 100644 --- a/src/dls_bluesky_core/stubs/flyables.py +++ b/src/dls_bluesky_core/stubs/flyables.py @@ -10,6 +10,37 @@ def fly_and_collect( checkpoint_every_collect: bool = False, stream_name: str = "primary", ) -> MsgGenerator: + """Repeatedly fly and collect a flyer, flushing with a period. + + flyer.kickoff and complete are called, which starts the fly scanning process. + bps.wait is called, which finishes after each flush period and then repeats, until + complete finishes. At this point, bps.collect is called to gather the documents + produced. + + For some flyers, this plan will need to be called in succession in order to, for + example, set up a flyer to send triggers and collect data without pause. For such + a use case, this plan can be setup to checkpoint for each collect. + + Note: this plan must be wrapped with calls to open and close a run, and the flyer + must implement the Collectable protocol. + + Args: + flyer (Flyable, Collectable): ophyd-async device which implements Flyable and + Collectable. + flush_period (float): How often to check if flyer.complete has finished. + Defaults to 0.5 + checkpoint_every_collect (bool): whether or not to checkpoint after + flyer.collect has been called. Defaults to + False. + stream_name (str): name of the stream to collect from. Defaults to "primary". + + + Returns: + MsgGenerator: Plan + + Yields: + Iterator[MsgGenerator]: Bluesky messages + """ yield from bps.kickoff(flyer) complete_group = group_uuid("complete") yield from bps.complete(flyer, group=complete_group) diff --git a/tests/core/test_coordination.py b/tests/core/test_coordination.py new file mode 100644 index 0000000..f3105d7 --- /dev/null +++ b/tests/core/test_coordination.py @@ -0,0 +1,12 @@ +import uuid + +import pytest + +from dls_bluesky_core.core.coordination import group_uuid + + +@pytest.mark.parametrize("group", ["foo", "bar", "baz", str(uuid.uuid4())]) +def test_group_uid(group: str): + gid = group_uuid(group) + assert gid.startswith(f"{group}-") + assert not gid.endswith(f"{group}-") diff --git a/tests/core/test_funcs.py b/tests/core/test_maths.py similarity index 87% rename from tests/core/test_funcs.py rename to tests/core/test_maths.py index c981dc9..c58ef6a 100644 --- a/tests/core/test_funcs.py +++ b/tests/core/test_maths.py @@ -1,9 +1,8 @@ -import uuid from typing import Optional import pytest -from dls_bluesky_core.core import group_uuid, in_micros, step_to_num +from dls_bluesky_core.core import in_micros, step_to_num @pytest.mark.parametrize( @@ -64,10 +63,3 @@ def test_step_to_num( assert actual_start == start assert actual_stop == truncated_stop assert num == expected_num - - -@pytest.mark.parametrize("group", ["foo", "bar", "baz", str(uuid.uuid4())]) -def test_group_uid(group: str): - gid = group_uuid(group) - assert gid.startswith(f"{group}-") - assert not gid.endswith(f"{group}-") From ed2cae557ad26fd9bd91333f58f4f49f84cad7c8 Mon Sep 17 00:00:00 2001 From: Rose Yemelyanova Date: Mon, 23 Oct 2023 11:57:37 +0100 Subject: [PATCH 30/30] Modify docstring --- src/dls_bluesky_core/stubs/flyables.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dls_bluesky_core/stubs/flyables.py b/src/dls_bluesky_core/stubs/flyables.py index 6b7e551..41acacb 100644 --- a/src/dls_bluesky_core/stubs/flyables.py +++ b/src/dls_bluesky_core/stubs/flyables.py @@ -10,7 +10,7 @@ def fly_and_collect( checkpoint_every_collect: bool = False, stream_name: str = "primary", ) -> MsgGenerator: - """Repeatedly fly and collect a flyer, flushing with a period. + """Fly and collect a flyer, waiting for collect to finish with a period. flyer.kickoff and complete are called, which starts the fly scanning process. bps.wait is called, which finishes after each flush period and then repeats, until @@ -18,11 +18,12 @@ def fly_and_collect( produced. For some flyers, this plan will need to be called in succession in order to, for - example, set up a flyer to send triggers and collect data without pause. For such + example, set up a flyer to send triggers multiple times and collect data. For such a use case, this plan can be setup to checkpoint for each collect. - Note: this plan must be wrapped with calls to open and close a run, and the flyer - must implement the Collectable protocol. + Note: this plan must be wrapped with calls to open and close run, and the flyer + must implement the Collectable protocol. See tests/stubs/test_flyables for an + example. Args: flyer (Flyable, Collectable): ophyd-async device which implements Flyable and