Merge branch 'main' into jp-2928-miri-lrs

changes/8909.scripts.rst

@@ -0,0 +1 @@
+Remove the outdated schema_editor script.

changes/8916.background.rst

@@ -0,0 +1 @@
+Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction.

changes/8935.general.rst

@@ -0,0 +1 @@
+Include xml and db test data as package data for lib tests.

jwst/background/background_sub.py

@@ -272,6 +272,20 @@ def average_background(input_model, bkg_list, sigma, maxiters):
     return avg_bkg
 
 
+def sufficient_background_pixels(dq_array, bkg_mask, min_pixels=100):
+    """Count number of good pixels for background use.
+
+    Check DQ flags of pixels selected for bkg use - XOR the DQ values with
+    the DO_NOT_USE flag to flip the DO_NOT_USE bit. Then count the number
+    of pixels that AND with the DO_NOT_USE flag, i.e. initially did not have
+    the DO_NOT_USE bit set.
+    """
+    return np.count_nonzero((dq_array[bkg_mask]
+                            ^ pixel['DO_NOT_USE'])
+                            & pixel['DO_NOT_USE']
+                            ) > min_pixels
+
+
 def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=None):
     """Scale and subtract a background reference image from WFSS/GRISM data.
 
@@ -310,18 +324,12 @@ def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=Non
     # i.e. in regions we can use as background.
     if got_catalog:
         bkg_mask = mask_from_source_cat(input_model, wl_range_name, mmag_extract)
-        # Ensure mask has 100 pixels and that those pixels correspond to valid
-        # pixels using model DQ array
-        if np.count_nonzero(input_model.dq[bkg_mask]
-                            ^ pixel['DO_NOT_USE']
-                            & pixel['DO_NOT_USE']
-                            ) < 100:
+        if not sufficient_background_pixels(input_model.dq, bkg_mask):
             log.warning("Not enough background pixels to work with.")
             log.warning("Step will be SKIPPED.")
             return None
     else:
         bkg_mask = np.ones(input_model.data.shape, dtype=bool)
-
     # Compute the mean values of science image and background reference
     # image, including only regions where there are no identified sources.
     # Exclude pixel values in the lower and upper 25% of the histogram.

jwst/background/tests/test_background.py

@@ -9,11 +9,13 @@
 from numpy.testing import assert_allclose
 
 from stdatamodels.jwst import datamodels
+from stdatamodels.jwst.datamodels.dqflags import pixel
 
 from jwst.assign_wcs import AssignWcsStep
 from jwst.background import BackgroundStep
 from jwst.stpipe import Step
-from jwst.background.background_sub import robust_mean, mask_from_source_cat, no_NaN
+from jwst.background.background_sub import (robust_mean, mask_from_source_cat,
+                                            no_NaN, sufficient_background_pixels)
 
 
 @pytest.fixture(scope="module")
@@ -403,3 +405,23 @@ def test_no_nan():
 
     # Make sure arrays are equal.
     assert np.array_equal(model.data, result.data)
+
+
+def test_sufficient_background_pixels():
+    model = datamodels.ImageModel(data=np.zeros((2048, 2048)),
+                                  dq=np.zeros((2048, 2048)))
+    refpix_flags = pixel['DO_NOT_USE'] | pixel['REFERENCE_PIXEL']
+    model.dq[:4, :] = refpix_flags
+    model.dq[-4:, :] = refpix_flags
+    model.dq[:, :4] = refpix_flags
+    model.dq[:, -4:] = refpix_flags
+
+    bkg_mask = np.ones((2048, 2048), dtype=bool)
+    # With full array minux refpix available for bkg, should be sufficient
+    assert sufficient_background_pixels(model.dq, bkg_mask)
+
+    bkg_mask[4: -4, :] = 0
+    bkg_mask[:, 4: -4] = 0
+    # Now mask out entire array, mocking full source coverage of detector -
+    # no pixels should be available for bkg
+    assert not sufficient_background_pixels(model.dq, bkg_mask)

jwst/datamodels/__init__.py

@@ -31,7 +31,7 @@
 _jwst_models = ["ModelContainer", "SourceModelContainer", "ModelLibrary"]
 
 # Deprecated modules in stdatamodels
-_deprecated_modules = ['schema']
+_deprecated_modules = ['schema', 'schema_editor']
 
 # Deprecated models in stdatamodels
 _deprecated_models: list[str] = []
@@ -52,6 +52,6 @@
         sys.modules[f"jwst.datamodels.{attr}"] = obj
 
 # Add a few submodules to sys.modules without exposing them locally
-for _submodule_name in ['schema_editor', 'validate']:
+for _submodule_name in ['validate']:
     _submodule = importlib.import_module(f"stdatamodels.jwst.datamodels.{_submodule_name}")
     sys.modules[f"jwst.datamodels.{_submodule_name}"] = _submodule

jwst/scripts/tests/test_scripts.py

@@ -11,7 +11,6 @@
     'create_data',
     'okify_regtests',
     'pointing_summary',
-    'schema_editor',
     'schemadoc',
     'set_telescope_pointing',
     'set_telescope_pointing.py',

pyproject.toml

@@ -77,7 +77,6 @@ csvconvert = "jwst.csv_tools.csvconvert:CSVConvertScript"
 exp_to_source = "jwst.exp_to_source.main:Main"
 okify_regtests = "jwst.scripts.okify_regtests:main"
 pointing_summary = "jwst.scripts.pointing_summary:main"
-schema_editor = "jwst.scripts.schema_editor:main"
 schemadoc = "jwst.scripts.schemadoc:main"
 set_telescope_pointing = "jwst.scripts.set_telescope_pointing:main"
 "set_telescope_pointing.py" = "jwst.scripts.set_telescope_pointing:deprecated_name"
@@ -157,7 +156,8 @@ namespaces = false
 ]
 "jwst.lib" = [
     "tests/data/*.asdf",
-    "tests/data/*.db",
+    "tests/data/**/*.db",
+    "tests/data/**/*.xml",
     "tests/data/*.ecsv",
     "tests/data/*.fits",
 ]