Merge branch 'main' into jp3598

spacetelescope · Dec 3, 2024 · dba5f68 · dba5f68
2 parents 53cdadd + f421b16
commit dba5f68
Show file tree

Hide file tree

Showing 11 changed files with 170 additions and 28 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -21,7 +21,7 @@ jobs:
   build:
     needs: [ check ]
     if: always() && (needs.check.result == 'success' || needs.check.result == 'skipped')
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
     with:
       upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }}
       targets: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -40,7 +40,7 @@ jobs:
         python-version: '3.12'
     - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1
   check:
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
     with:
       default_python: "3.12"
       envs: |
@@ -57,7 +57,7 @@ jobs:
     outputs:
       context: ${{ steps.context.outputs.context }}
   test:
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
     needs: [ crds_context ]
     with:
       setenv: |

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
@@ -34,7 +34,7 @@ jobs:
       context: ${{ steps.context.outputs.context }}
   test:
     if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests')))
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
     needs: [ crds_context ]
     with:
       setenv: |

diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml
@@ -43,7 +43,7 @@ jobs:
       context: ${{ steps.context.outputs.context }}
   test:
     if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
     needs: [ crds_context ]
     with:
       setenv: |

diff --git a/changes/8946.rscd.rst b/changes/8946.rscd.rst
@@ -0,0 +1 @@
+Updated RSCD step to work on segmented data
diff --git a/changes/8966.general.rst b/changes/8966.general.rst
@@ -0,0 +1 @@
+Increase asdf upper pin to 5
diff --git a/jwst/lib/tests/test_engdb_mast.py b/jwst/lib/tests/test_engdb_mast.py
@@ -67,22 +67,23 @@ def test_get_records(engdb):
     [
         ({}, [-0.7914494276, -0.7914494276, -0.7914494276, -0.791449368]),
         ({'include_obstime': True},
-         [EngDB_Value(obstime=Time('2022-02-02T22:24:58.053', scale='utc', format='isot'), value=-0.7914494276),
-          EngDB_Value(obstime=Time('2022-02-02T22:24:58.309', scale='utc', format='isot'), value=-0.7914494276),
-          EngDB_Value(obstime=Time('2022-02-02T22:24:58.565', scale='utc', format='isot'), value=-0.7914494276),
-          EngDB_Value(obstime=Time('2022-02-02T22:24:58.821', scale='utc', format='isot'), value=-0.791449368)]),
+         [EngDB_Value(obstime=Time(59612.9340052431, format='mjd'), value=-0.7914494276),
+          EngDB_Value(obstime=Time(59612.9340082060, format='mjd'), value=-0.7914494276),
+          EngDB_Value(obstime=Time(59612.9340111690, format='mjd'), value=-0.7914494276),
+          EngDB_Value(obstime=Time(59612.9340141319, format='mjd'), value=-0.791449368)]),
         ({'include_obstime': True, 'zip_results': False}, EngDB_Value(
-            obstime=[Time('2022-02-02T22:24:58.053', scale='utc', format='isot'),
-                     Time('2022-02-02T22:24:58.309', scale='utc', format='isot'),
-                     Time('2022-02-02T22:24:58.565', scale='utc', format='isot'),
-                     Time('2022-02-02T22:24:58.821', scale='utc', format='isot')],
+            obstime=[
+                Time(59612.9340052431, format='mjd'),
+                Time(59612.9340082060, format='mjd'),
+                Time(59612.9340111690, format='mjd'),
+                Time(59612.9340141319, format='mjd')],
             value=[-0.7914494276, -0.7914494276, -0.7914494276, -0.791449368])),
         ({'include_bracket_values': True},
          [-0.791449368, -0.7914494276, -0.7914494276, -0.7914494276, -0.791449368, -0.791449368])
     ])
 def test_get_values(engdb, pars, expected):
     values = engdb.get_values(*QUERY, **pars)
-    assert str(values) == str(expected)
+    assert values == expected
 
 
 def test_negative_aliveness():

diff --git a/jwst/regtest/test_miri_lrs_slitless.py b/jwst/regtest/test_miri_lrs_slitless.py
@@ -5,19 +5,19 @@
 
 from jwst.stpipe import Step
 from gwcs.wcstools import grid_from_bounding_box
-
 from stdatamodels.jwst import datamodels
 
 DATASET1_ID = "jw01536028001_03103_00001-seg001_mirimage"
 DATASET2_ID = "jw01536028001_03103_00001-seg002_mirimage"
+DATASET3_ID = "jw01281001001_04103_00001-seg002_trim_mirimage"
 ASN3_FILENAME = "jw01536-o028_20221202t215749_tso3_00001_asn.json"
 PRODUCT_NAME = "jw01536-o028_t008_miri_p750l-slitlessprism"
 ASN_ID = "o028"
 
 
 @pytest.fixture(scope="module")
 def run_tso1_pipeline(rtdata_module):
-    """Run the calwebb_tso1 pipeline on a MIRI LRS slitless exposure."""
+    """Run the calwebb_detector1 pipeline on a MIRI LRS slitless exposure."""
     rtdata = rtdata_module
     rtdata.get_data(f"miri/lrs/{DATASET1_ID}_uncal.fits")
 
@@ -34,6 +34,25 @@ def run_tso1_pipeline(rtdata_module):
     Step.from_cmdline(args)
 
 
+@pytest.fixture(scope="module")
+def run_detector1_pipeline(rtdata_module):
+    """Run calwebb_detector pipeline on a MIRI LRS slitless exposure for Segment 2 data. 
+       Focusing on the steps that depend on integration # and not covered by run_tso1_pipeline.
+       Also test running RSC step"""
+    rtdata = rtdata_module
+    rtdata.get_data(f"miri/lrs/{DATASET3_ID}_uncal.fits")
+
+    args = [
+        "calwebb_detector1",
+        rtdata.input,
+        "--steps.emicorr.save_results=True",
+        "--steps.rscd.skip=False",
+        "--steps.rscd.save_results=True",
+        "--steps.dark_current.save_results=True",
+    ]
+    Step.from_cmdline(args)
+
+
 @pytest.fixture(scope="module")
 def run_tso_spec2_pipeline(run_tso1_pipeline, rtdata_module):
     """Run the calwebb_tso-spec2 pipeline on a MIRI LRS slitless exposure."""
@@ -86,6 +105,25 @@ def test_miri_lrs_slitless_tso1(run_tso1_pipeline, rtdata_module, fitsdiff_defau
     assert diff.identical, diff.report()
 
 
+@pytest.mark.bigdata
+@pytest.mark.parametrize("step_suffix", ['rscd', 'emicorr',
+                                         'dark_current', 'ramp', 'rate', 'rateints'])
+def test_miri_lrs_slitless_detector1(run_detector1_pipeline, rtdata_module,
+                                          fitsdiff_default_kwargs, step_suffix):
+    """
+    Regression test of  detector1 pipeline performed on MIRI LRS slitless TSO data.
+    Testing segment 2 data for RSCD, emicorr and dark_current.
+    """
+    rtdata = rtdata_module
+    output_filename = f"{DATASET3_ID}_{step_suffix}.fits"
+    rtdata.output = output_filename
+
+    rtdata.get_truth(f"truth/test_miri_lrs_slitless_detector1/{output_filename}")
+
+    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
+    assert diff.identical, diff.report()
+
+
 @pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ["assign_wcs", "srctype", "flat_field",
                                           "pixel_replace", "calints", "x1dints"])

diff --git a/jwst/rscd/rscd_sub.py b/jwst/rscd/rscd_sub.py
@@ -80,12 +80,22 @@ def correction_skip_groups(output, group_skip):
         RSCD-corrected science data
     """
 
-    # Save some data params for easy use later
-    sci_nints = output.data.shape[0]       # number of integrations
-    sci_ngroups = output.data.shape[1]     # number of groups
+    # General exposure parameters
+    sci_ngroups = output.meta.exposure.ngroups
+    sci_nints = output.meta.exposure.nints
+
+    # values defined for segmented data 
+    sci_int_start = output.meta.exposure.integration_start
+
+    if sci_int_start is None:
+        sci_int_start = 1
+
 
     log.debug("RSCD correction using: nints=%d, ngroups=%d" %
               (sci_nints, sci_ngroups))
+    log.debug("The first integration in the data is integration: %d" %
+              (sci_int_start))
+    log.info("Number of groups to skip for integrations 2 and higher: %d " %group_skip)
 
     # If ngroups <= group_skip+3, skip the flagging
     # the +3 is to ensure there is a slope to be fit including the flagging for
@@ -96,9 +106,19 @@ def correction_skip_groups(output, group_skip):
         output.meta.cal_step.rscd = 'SKIPPED'
         return output
 
-    # If ngroups > group_skip+3, set all of the GROUPDQ in the first group to 'DO_NOT_USE'
-    output.groupdq[1:, 0:group_skip:, :] = \
-        np.bitwise_or(output.groupdq[1:, 0:group_skip, :, :], dqflags.group['DO_NOT_USE'])
+    # The RSCD correction is applied to integrations 2 and higher.
+    # For segmented data the first integration in the file may not be the first integration in the
+    # exposure. The value in meta.exposure.integration_start holds the value of the first integration
+    # in the file.
+    # If a correction is to be done and if  ngroups > group_skip+3, then  set all of the GROUPDQ
+    # in 0: group_skip to 'DO_NOT_USE'
+
+    int_start = 1
+    if sci_int_start !=1: # we have segmented data
+        int_start = 0
+
+    output.groupdq[int_start:, 0:group_skip, :, :] = \
+        np.bitwise_or(output.groupdq[int_start:, 0:group_skip, :, :], dqflags.group['DO_NOT_USE'])
     log.debug(f"RSCD Sub: adding DO_NOT_USE to GROUPDQ for the first {group_skip} groups")
     output.meta.cal_step.rscd = 'COMPLETE'
 

diff --git a/jwst/rscd/tests/test_rscd.py b/jwst/rscd/tests/test_rscd.py
@@ -12,18 +12,27 @@ def test_rscd_baseline_set_groupdq():
     groupdq flags in the 1st integration
     """
 
+    exposure = {
+        'integration_start' : None,
+        'integration_end':  None,
+        'ngroups' : 10,
+        'nints' : 2
+        }
     # size of integration
-    ngroups = 10
+    ngroups = exposure['ngroups']
+    nints  = exposure['nints']
+
     xsize = 10
     ysize = 10
 
     # create the data and groupdq arrays
-    csize = (2, ngroups, ysize, xsize)
+    csize = (nints, ngroups, ysize, xsize)
     data = np.full(csize, 1.0, dtype=np.float32)
     groupdq = np.zeros(csize, dtype=np.uint8)
 
     # create a JWST datamodel for MIRI data
     dm_ramp = RampModel(data=data, groupdq=groupdq)
+    dm_ramp.meta.exposure._instance.update(exposure)
 
     # get the number of groups to flag
     nflag = 3
@@ -71,19 +80,31 @@ def test_rscd_baseline_too_few_groups():
     """
 
     # size of exposure
-    nints = 2
-    ngroups = 3
     xsize = 10
     ysize = 10
 
+    exposure = {
+        'integration_start' : None,
+        'integration_end':  None,
+        'ngroups' : 3,
+        'nints' : 2
+        }
+    # size of integration
+    ngroups = exposure['ngroups']
+    nints  = exposure['nints']
+
+    xsize = 10
+    ysize = 10
+
     # create the data and groupdq arrays
     csize = (nints, ngroups, ysize, xsize)
     data = np.full(csize, 1.0, dtype=np.float32)
     groupdq = np.zeros(csize, dtype=np.uint8)
 
     # create a JWST datamodel for MIRI data on a copy (the copy is created at the step script)
     dm_ramp = RampModel(data=data, groupdq=groupdq)
-
+    dm_ramp.meta.exposure._instance.update(exposure)
+
     # get the number of groups to flag
     nflag = 3
 
@@ -99,3 +120,63 @@ def test_rscd_baseline_too_few_groups():
                                   dq_diff,
                                   err_msg='groupdq flags changed when '
                                   + 'not enough groups are present')
+
+
+def test_rscd_tso():
+    """
+    The RSCD correction is generally skipped for TSO data, but some users
+    have been running it on TSO data. So this test was added. 
+    Test for TSO segmented data that the correct groups are flagged as 'DO_NOT_USE'
+    for integration 2 and higher. Set up the segmented data so the segment
+    is for integration 25 to 50. A rscd correction should be applied to all 
+    the data.  
+ 
+    """
+    exposure = {
+        'integration_start' : 25,
+        'integration_end':  50,
+        'ngroups' : 8,
+        'nints' : 50
+        }
+
+    xsize = 10
+    ysize = 10
+    ngroups = exposure['ngroups']
+    seg_nints = exposure['integration_end'] - exposure['integration_start'] + 1
+    input_model = RampModel((seg_nints, exposure['ngroups'],
+                                        ysize, xsize))
+
+    input_model.groupdq[:,:,:,:] = 0 # initize to 0 - all good
+    input_model.meta.exposure._instance.update(exposure)
+
+    # get the number of groups to flag
+    nflag = 4
+
+    # run the RSCD baseline correction step on a copy (the copy is created at the step script)
+    ramp_rscd = correction_skip_groups(input_model.copy(), nflag)
+
+
+    # check that the difference in the groupdq flags is equal to
+    #  the 'DO_NOT_USE' flag for the 1st integration in the segment
+    # which is actually the 25th integration
+    dq_diff = (ramp_rscd.groupdq[0, 0:nflag, :, :]
+               - input_model.groupdq[0, 0:nflag, :, :])
+
+    np.testing.assert_array_equal(np.full((nflag, ysize, xsize),
+                                          dqflags.group['DO_NOT_USE'],
+                                          dtype=int),
+                                  dq_diff,
+                                  err_msg='Diff in groupdq flags is not '
+                                  + 'equal to the DO_NOT_USE flag')
+
+    # test that the groupdq flags are not changed for the rest of the groups
+    # in the 1st  integration in the segment
+    dq_diff = (ramp_rscd.groupdq[0, nflag:ngroups, :, :]
+               - input_model.groupdq[0, nflag:ngroups, :, :])
+    np.testing.assert_array_equal(np.full((ngroups - nflag, ysize, xsize),
+                                          0,
+                                          dtype=int),
+                                  dq_diff,
+                                  err_msg='groupdq flags changed after '
+                                  + 'maximum requested')
+
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,7 +18,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "asdf>=3.1.0,<4",
+    "asdf>=3.1.0,<5",
     "astropy>=5.3",
     "BayesicFitting>=3.0.1",
     "crds>=12.0.3",