Adding dq flag mapping CSVs, and IO methods

jdaviz/configs/default/plugins/data_quality/__init__.py

@@ -0,0 +1 @@
+from .dq_utils import *  # noqa

jdaviz/configs/default/plugins/data_quality/dq_utils.py

@@ -0,0 +1,73 @@
+from importlib import resources
+from pathlib import Path
+from astropy.table import Table
+
+dq_flag_map_paths = {
+    'jwst': Path('data', 'data_quality', 'jwst.csv'),
+    'roman': Path('data', 'data_quality', 'roman.csv'),
+}
+
+
+def load_flag_map(mission_or_instrument=None, path=None):
+    """
+    Load a flag map from disk.
+
+    Parameters
+    ----------
+    mission_or_instrument : {"jwst", "roman"}, optional
+        Load the flag map for this mission or instrument.
+    path : path-like
+        Path to a flag map file (CSV).
+
+    Returns
+    -------
+    flag_mapping : dict
+        Flag mapping definitions. Keys are powers of two,
+        values are the "name" and "description" for each flag.
+    """
+    if mission_or_instrument is not None and path is None:
+        flag_map_path = dq_flag_map_paths.get(mission_or_instrument)
+
+        if flag_map_path is None:
+            raise NotImplementedError("No flag map found for "
+                                      f"mission/instrument '{mission_or_instrument}'.")
+
+    elif mission_or_instrument is None and path is not None:
+        # load directly from a CSV file:
+        flag_map_path = path
+
+    with resources.as_file(resources.files('jdaviz').joinpath(flag_map_path)) as path:
+        flag_table = Table.read(path, format='ascii.csv')
+
+    flag_mapping = {}
+    for flag, name, desc in flag_table.iterrows():
+        flag_mapping[flag] = dict(name=name, description=desc)
+
+    return flag_mapping
+
+
+def write_flag_map(flag_mapping, csv_path, **kwargs):
+    """
+    Write a table containing bitmask flags and their descriptions
+    to a CSV file.
+
+    Parameters
+    ----------
+    flag_mapping : dict
+        Flag mapping definitions. Keys are powers of two,
+        values are the "name" and "description" for each flag.
+    csv_path : path-like
+        Where to save the CSV output.
+    **kwargs : dict
+        Other keywords passed to `~astropy.table.Table.write`
+    """
+    names = ['flag', 'name', 'description']
+    dtype = (int, str, str)
+    table = Table(names=names, dtype=dtype)
+
+    for key, value in flag_mapping.items():
+        row = {'flag': key}
+        row.update(value)
+        table.add_row(row)
+
+    table.write(csv_path, format='ascii.csv', **kwargs)

jdaviz/configs/default/plugins/data_quality/tests/test_data_quality.py

@@ -0,0 +1,36 @@
+import os
+import tempfile
+import pytest
+from jdaviz.configs.default.plugins.data_quality.dq_utils import (
+    load_flag_map, write_flag_map
+)
+
+
+@pytest.mark.parametrize(
+    "mission_or_instrument, flag, name,",
+    [['jwst', 26, 'OPEN'],
+     ['roman', 26, 'RESERVED_5'],
+     ['jwst', 15, 'TELEGRAPH'],
+     ['roman', 15, 'TELEGRAPH']]
+)
+def test_flag_map_utils(mission_or_instrument, flag, name):
+    flag_map = load_flag_map(mission_or_instrument)
+    assert flag_map[flag]['name'] == name
+
+
+def test_load_write_load():
+    # load the JWST flag map
+    flag_map = load_flag_map('jwst')
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+
+        # write out the flag map to a temporary CSV file
+        path = os.path.join(tmp_dir, 'test_flag_map.csv')
+        write_flag_map(flag_map, path)
+
+        # load that temporary CSV file back in
+        reloaded_flag_map = load_flag_map(path=path)
+
+        # confirm all values round-trip successfully:
+        for flag, orig_value in flag_map.items():
+            assert orig_value == reloaded_flag_map[flag]

jdaviz/data/data_quality/jwst.csv

@@ -0,0 +1,33 @@
+flag,name,description
+0,DO_NOT_USE,Bad pixel. Do not use.
+1,SATURATED,Pixel saturated during exposure
+2,JUMP_DET,Jump detected during exposure
+3,DROPOUT,Data lost in transmission
+4,OUTLIER,Flagged by outlier detection (was RESERVED_1)
+5,PERSISTENCE,High persistence (was RESERVED_2)
+6,AD_FLOOR,Below A/D floor (0 DN; was RESERVED_3)
+7,CHARGELOSS,Charge migration (was RESERVED_4)
+8,UNRELIABLE_ERROR,Uncertainty exceeds quoted error
+9,NON_SCIENCE,Pixel not on science portion of detector
+10,DEAD,Dead pixel
+11,HOT,Hot pixel
+12,WARM,Warm pixel
+13,LOW_QE,Low quantum efficiency
+14,RC,RC pixel
+15,TELEGRAPH,Telegraph pixel
+16,NONLINEAR,Pixel highly nonlinear
+17,BAD_REF_PIXEL,Reference pixel cannot be used
+18,NO_FLAT_FIELD,Flat field cannot be measured
+19,NO_GAIN_VALUE,Gain cannot be measured
+20,NO_LIN_CORR,Linearity correction not available
+21,NO_SAT_CHECK,Saturation check not available
+22,UNRELIABLE_BIAS,Bias variance large
+23,UNRELIABLE_DARK,Dark variance large
+24,UNRELIABLE_SLOPE,Slope variance large (i.e. noisy pixel)
+25,UNRELIABLE_FLAT,Flat variance large
+26,OPEN,Open pixel (counts move to adjacent pixels)
+27,ADJ_OPEN,Adjacent to open pixel
+28,FLUX_ESTIMATED,Pixel flux estimated due to missing/bad data
+29,MSA_FAILED_OPEN,Pixel sees light from failed-open shutter
+30,OTHER_BAD_PIXEL,A catch-all flag
+31,REFERENCE_PIXEL,Pixel is a reference pixel

jdaviz/data/data_quality/roman.csv

@@ -0,0 +1,32 @@
+flag,name,description
+0,DO_NOT_USE,Bad pixel. Do not use.
+1,SATURATED,Pixel saturated during exposure
+2,JUMP_DET,Jump detected during exposure
+3,DROPOUT,Data lost in transmission
+4,GW_AFFECTED_DATA,Data affected by the GW read window
+5,PERSISTENCE,High persistence (was RESERVED_2)
+6,AD_FLOOR,Below A/D floor (0 DN; was RESERVED_3)
+7,OUTLIER,Flagged by outlier detection (was RESERVED_4)
+8,UNRELIABLE_ERROR,Uncertainty exceeds quoted error
+9,NON_SCIENCE,Pixel not on science portion of detector
+10,DEAD,Dead pixel
+11,HOT,Hot pixel
+12,WARM,Warm pixel
+13,LOW_QE,Low quantum efficiency
+15,TELEGRAPH,Telegraph pixel
+16,NONLINEAR,Pixel highly nonlinear
+17,BAD_REF_PIXEL,Reference pixel cannot be used
+18,NO_FLAT_FIELD,Flat field cannot be measured
+19,NO_GAIN_VALUE,Gain cannot be measured
+20,NO_LIN_CORR,Linearity correction not available
+21,NO_SAT_CHECK,Saturation check not available
+22,UNRELIABLE_BIAS,Bias variance large
+23,UNRELIABLE_DARK,Dark variance large
+24,UNRELIABLE_SLOPE,Slope variance large (i.e.; noisy pixel)
+25,UNRELIABLE_FLAT,Flat variance large
+26,RESERVED_5,
+27,RESERVED_6,
+28,UNRELIABLE_RESET,Sensitive to reset anomaly
+29,RESERVED_7,
+30,OTHER_BAD_PIXEL,A catch-all flag
+31,REFERENCE_PIXEL,Pixel is a reference pixel