Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for using dtype_numpy as in ophyd-async #817

Merged
merged 1 commit into from
Aug 14, 2024
Merged

Add support for using dtype_numpy as in ophyd-async #817

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions databroker/mongo_normalized.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,15 +135,15 @@ def structure_from_descriptor(descriptor, sub_dict, max_seq_num, unicode_columns
shape = tuple((max_seq_num - 1, *field_metadata["shape"]))
# if we have a descr, then this is a
dtype = _try_descr(field_metadata)
dt_str = field_metadata.get("dtype_str")
dt_np = field_metadata.get("dtype_numpy") or field_metadata.get("dtype_str")
if dtype is not None:
if len(shape) > 2:
raise RuntimeError(
"We do not yet support general structured arrays, only 1D ones."
)
# if we have a detailed string, trust that
elif dt_str is not None:
dtype = BuiltinDtype.from_numpy_dtype(numpy.dtype(dt_str))
elif dt_np is not None:
dtype = BuiltinDtype.from_numpy_dtype(numpy.dtype(dt_np))
# otherwise guess!
else:
dtype = JSON_DTYPE_TO_MACHINE_DATA_TYPE[field_metadata["dtype"]]
Expand Down Expand Up @@ -622,10 +622,10 @@ def __init__(
unicode_keys = []
for key, field_metadata in descriptor["data_keys"].items():
if field_metadata["dtype"] == "string":
# Skip this if it has a dtype_str with an itemsize.
dtype_str = field_metadata.get("dtype_str")
if dtype_str is not None:
if numpy.dtype(dtype_str).itemsize != 0:
# Skip this if it has a dtype_numpy with an itemsize.
dt_np = field_metadata.get("dtype_numpy") or field_metadata.get("dtype_str")
if dt_np is not None:
if numpy.dtype(dt_np).itemsize != 0:
continue
unicode_keys.append(key)
# Load the all the data for unicode columns to figure out the itemsize.
Expand Down Expand Up @@ -719,7 +719,7 @@ def read(self, fields=None):
f"{key!r} actually has dtype {raw_array.dtype.str!r} "
f"but was reported as having dtype {dtype.str!r}. "
"It will be converted to the reported type, "
"but this should be fixed by setting 'dtype_str' "
"but this should be fixed by setting 'dtype_numpy' "
"in the data_key of the EventDescriptor. "
f"RunStart UID: {self._run.metadata()['start']['uid']!r}"
)
Expand Down Expand Up @@ -765,7 +765,7 @@ def read_block(self, variable, block, slice=None):
f"{variable!r} actually has dtype {raw_array.dtype.str!r} "
f"but was reported as having dtype {dtype.str!r}. "
"It will be converted to the reported type, "
"but this should be fixed by setting 'dtype_str' "
"but this should be fixed by setting 'dtype_numpy' "
"in the data_key of the EventDescriptor. "
f"RunStart UID: {self._run.metadata()['start']['uid']!r}"
)
Expand Down Expand Up @@ -1057,16 +1057,16 @@ def build_config_xarray(
for key, column in raw_columns.items():
field_metadata = data_keys[key]
dtype = _try_descr(field_metadata)
dt_str = field_metadata.get("dtype_str")
dt_np = field_metadata.get("dtype_numpy") or field_metadata.get("dtype_str")
if dtype is not None:
if len(getattr(column[0], "shape", ())) > 2:
raise RuntimeError(
"We do not yet support general structured arrays, only 1D ones."
)
numpy_dtype = dtype.to_numpy_dtype()
# if we have a detailed string, trust that
elif dt_str is not None:
numpy_dtype = numpy.dtype(dt_str)
elif dt_np is not None:
numpy_dtype = numpy.dtype(dt_np)
# otherwise guess!
else:
numpy_dtype = JSON_DTYPE_TO_MACHINE_DATA_TYPE[
Expand Down Expand Up @@ -2056,7 +2056,7 @@ def parse_transforms(transforms):


# These are fallback guesses when all we have is a general jsonschema "dtype"
# like "array" no specific "dtype_str" like "<u2".
# like "array" no specific "dtype_numpy" like "<u2".
BOOLEAN_DTYPE = BuiltinDtype.from_numpy_dtype(numpy.dtype("bool"))
FLOAT_DTYPE = BuiltinDtype.from_numpy_dtype(numpy.dtype("float64"))
INT_DTYPE = BuiltinDtype.from_numpy_dtype(numpy.dtype("int64"))
Expand All @@ -2066,7 +2066,7 @@ def parse_transforms(transforms):
"number": FLOAT_DTYPE,
"integer": INT_DTYPE,
"string": STRING_DTYPE,
"array": FLOAT_DTYPE, # If this is wrong, set 'dtype_str' in data_key to override.
"array": FLOAT_DTYPE, # If this is wrong, set 'dtype_numpy' in data_key to override.
}


Expand Down
Loading