diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f9f78fea0d..f5591067eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: build: needs: [ check ] if: always() && (needs.check.result == 'success' || needs.check.result == 'skipped') - uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@d68193b68216da64eafaa618f53c59f5d271c56e # v1.14.0 with: upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }} targets: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3185d02aea..083d21625a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: python-version: '3.12' - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 check: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e # v1.14.0 with: default_python: "3.12" envs: | @@ -57,7 +57,7 @@ jobs: outputs: context: ${{ steps.context.outputs.context }} test: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e # v1.14.0 needs: [ crds_context ] with: setenv: | diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 05146f9fb2..3e7ab20618 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -34,7 +34,7 @@ jobs: context: ${{ steps.context.outputs.context }} test: if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests'))) - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e # v1.14.0 needs: [ crds_context ] with: setenv: | diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml index e5ad86b187..57715057a6 100644 --- a/.github/workflows/tests_devdeps.yml +++ b/.github/workflows/tests_devdeps.yml @@ -43,7 +43,7 @@ jobs: context: ${{ steps.context.outputs.context }} test: if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e # v1.14.0 needs: [ crds_context ] with: setenv: | diff --git a/changes/8946.rscd.rst b/changes/8946.rscd.rst new file mode 100644 index 0000000000..5ef30ed606 --- /dev/null +++ b/changes/8946.rscd.rst @@ -0,0 +1 @@ +Updated RSCD step to work on segmented data diff --git a/changes/8966.general.rst b/changes/8966.general.rst new file mode 100644 index 0000000000..5b21ca6897 --- /dev/null +++ b/changes/8966.general.rst @@ -0,0 +1 @@ +Increase asdf upper pin to 5 diff --git a/jwst/lib/tests/test_engdb_mast.py b/jwst/lib/tests/test_engdb_mast.py index faa6d242b8..66a17df001 100644 --- a/jwst/lib/tests/test_engdb_mast.py +++ b/jwst/lib/tests/test_engdb_mast.py @@ -67,22 +67,23 @@ def test_get_records(engdb): [ ({}, [-0.7914494276, -0.7914494276, -0.7914494276, -0.791449368]), ({'include_obstime': True}, - [EngDB_Value(obstime=Time('2022-02-02T22:24:58.053', scale='utc', format='isot'), value=-0.7914494276), - EngDB_Value(obstime=Time('2022-02-02T22:24:58.309', scale='utc', format='isot'), value=-0.7914494276), - EngDB_Value(obstime=Time('2022-02-02T22:24:58.565', scale='utc', format='isot'), value=-0.7914494276), - EngDB_Value(obstime=Time('2022-02-02T22:24:58.821', scale='utc', format='isot'), value=-0.791449368)]), + [EngDB_Value(obstime=Time(59612.9340052431, format='mjd'), value=-0.7914494276), + EngDB_Value(obstime=Time(59612.9340082060, format='mjd'), value=-0.7914494276), + EngDB_Value(obstime=Time(59612.9340111690, format='mjd'), value=-0.7914494276), + EngDB_Value(obstime=Time(59612.9340141319, format='mjd'), value=-0.791449368)]), ({'include_obstime': True, 'zip_results': False}, EngDB_Value( - obstime=[Time('2022-02-02T22:24:58.053', scale='utc', format='isot'), - Time('2022-02-02T22:24:58.309', scale='utc', format='isot'), - Time('2022-02-02T22:24:58.565', scale='utc', format='isot'), - Time('2022-02-02T22:24:58.821', scale='utc', format='isot')], + obstime=[ + Time(59612.9340052431, format='mjd'), + Time(59612.9340082060, format='mjd'), + Time(59612.9340111690, format='mjd'), + Time(59612.9340141319, format='mjd')], value=[-0.7914494276, -0.7914494276, -0.7914494276, -0.791449368])), ({'include_bracket_values': True}, [-0.791449368, -0.7914494276, -0.7914494276, -0.7914494276, -0.791449368, -0.791449368]) ]) def test_get_values(engdb, pars, expected): values = engdb.get_values(*QUERY, **pars) - assert str(values) == str(expected) + assert values == expected def test_negative_aliveness(): diff --git a/jwst/regtest/test_miri_lrs_slitless.py b/jwst/regtest/test_miri_lrs_slitless.py index db92f58796..4645636118 100644 --- a/jwst/regtest/test_miri_lrs_slitless.py +++ b/jwst/regtest/test_miri_lrs_slitless.py @@ -5,11 +5,11 @@ from jwst.stpipe import Step from gwcs.wcstools import grid_from_bounding_box - from stdatamodels.jwst import datamodels DATASET1_ID = "jw01536028001_03103_00001-seg001_mirimage" DATASET2_ID = "jw01536028001_03103_00001-seg002_mirimage" +DATASET3_ID = "jw01281001001_04103_00001-seg002_trim_mirimage" ASN3_FILENAME = "jw01536-o028_20221202t215749_tso3_00001_asn.json" PRODUCT_NAME = "jw01536-o028_t008_miri_p750l-slitlessprism" ASN_ID = "o028" @@ -17,7 +17,7 @@ @pytest.fixture(scope="module") def run_tso1_pipeline(rtdata_module): - """Run the calwebb_tso1 pipeline on a MIRI LRS slitless exposure.""" + """Run the calwebb_detector1 pipeline on a MIRI LRS slitless exposure.""" rtdata = rtdata_module rtdata.get_data(f"miri/lrs/{DATASET1_ID}_uncal.fits") @@ -34,6 +34,25 @@ def run_tso1_pipeline(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1_pipeline(rtdata_module): + """Run calwebb_detector pipeline on a MIRI LRS slitless exposure for Segment 2 data. + Focusing on the steps that depend on integration # and not covered by run_tso1_pipeline. + Also test running RSC step""" + rtdata = rtdata_module + rtdata.get_data(f"miri/lrs/{DATASET3_ID}_uncal.fits") + + args = [ + "calwebb_detector1", + rtdata.input, + "--steps.emicorr.save_results=True", + "--steps.rscd.skip=False", + "--steps.rscd.save_results=True", + "--steps.dark_current.save_results=True", + ] + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_tso_spec2_pipeline(run_tso1_pipeline, rtdata_module): """Run the calwebb_tso-spec2 pipeline on a MIRI LRS slitless exposure.""" @@ -86,6 +105,25 @@ def test_miri_lrs_slitless_tso1(run_tso1_pipeline, rtdata_module, fitsdiff_defau assert diff.identical, diff.report() +@pytest.mark.bigdata +@pytest.mark.parametrize("step_suffix", ['rscd', 'emicorr', + 'dark_current', 'ramp', 'rate', 'rateints']) +def test_miri_lrs_slitless_detector1(run_detector1_pipeline, rtdata_module, + fitsdiff_default_kwargs, step_suffix): + """ + Regression test of detector1 pipeline performed on MIRI LRS slitless TSO data. + Testing segment 2 data for RSCD, emicorr and dark_current. + """ + rtdata = rtdata_module + output_filename = f"{DATASET3_ID}_{step_suffix}.fits" + rtdata.output = output_filename + + rtdata.get_truth(f"truth/test_miri_lrs_slitless_detector1/{output_filename}") + + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + @pytest.mark.bigdata @pytest.mark.parametrize("step_suffix", ["assign_wcs", "srctype", "flat_field", "pixel_replace", "calints", "x1dints"]) diff --git a/jwst/rscd/rscd_sub.py b/jwst/rscd/rscd_sub.py index 305c55f320..c4c1cb5d9c 100644 --- a/jwst/rscd/rscd_sub.py +++ b/jwst/rscd/rscd_sub.py @@ -80,12 +80,22 @@ def correction_skip_groups(output, group_skip): RSCD-corrected science data """ - # Save some data params for easy use later - sci_nints = output.data.shape[0] # number of integrations - sci_ngroups = output.data.shape[1] # number of groups + # General exposure parameters + sci_ngroups = output.meta.exposure.ngroups + sci_nints = output.meta.exposure.nints + + # values defined for segmented data + sci_int_start = output.meta.exposure.integration_start + + if sci_int_start is None: + sci_int_start = 1 + log.debug("RSCD correction using: nints=%d, ngroups=%d" % (sci_nints, sci_ngroups)) + log.debug("The first integration in the data is integration: %d" % + (sci_int_start)) + log.info("Number of groups to skip for integrations 2 and higher: %d " %group_skip) # If ngroups <= group_skip+3, skip the flagging # the +3 is to ensure there is a slope to be fit including the flagging for @@ -96,9 +106,19 @@ def correction_skip_groups(output, group_skip): output.meta.cal_step.rscd = 'SKIPPED' return output - # If ngroups > group_skip+3, set all of the GROUPDQ in the first group to 'DO_NOT_USE' - output.groupdq[1:, 0:group_skip:, :] = \ - np.bitwise_or(output.groupdq[1:, 0:group_skip, :, :], dqflags.group['DO_NOT_USE']) + # The RSCD correction is applied to integrations 2 and higher. + # For segmented data the first integration in the file may not be the first integration in the + # exposure. The value in meta.exposure.integration_start holds the value of the first integration + # in the file. + # If a correction is to be done and if ngroups > group_skip+3, then set all of the GROUPDQ + # in 0: group_skip to 'DO_NOT_USE' + + int_start = 1 + if sci_int_start !=1: # we have segmented data + int_start = 0 + + output.groupdq[int_start:, 0:group_skip, :, :] = \ + np.bitwise_or(output.groupdq[int_start:, 0:group_skip, :, :], dqflags.group['DO_NOT_USE']) log.debug(f"RSCD Sub: adding DO_NOT_USE to GROUPDQ for the first {group_skip} groups") output.meta.cal_step.rscd = 'COMPLETE' diff --git a/jwst/rscd/tests/test_rscd.py b/jwst/rscd/tests/test_rscd.py index eddec0c1a8..e18cd98447 100644 --- a/jwst/rscd/tests/test_rscd.py +++ b/jwst/rscd/tests/test_rscd.py @@ -12,18 +12,27 @@ def test_rscd_baseline_set_groupdq(): groupdq flags in the 1st integration """ + exposure = { + 'integration_start' : None, + 'integration_end': None, + 'ngroups' : 10, + 'nints' : 2 + } # size of integration - ngroups = 10 + ngroups = exposure['ngroups'] + nints = exposure['nints'] + xsize = 10 ysize = 10 # create the data and groupdq arrays - csize = (2, ngroups, ysize, xsize) + csize = (nints, ngroups, ysize, xsize) data = np.full(csize, 1.0, dtype=np.float32) groupdq = np.zeros(csize, dtype=np.uint8) # create a JWST datamodel for MIRI data dm_ramp = RampModel(data=data, groupdq=groupdq) + dm_ramp.meta.exposure._instance.update(exposure) # get the number of groups to flag nflag = 3 @@ -71,11 +80,22 @@ def test_rscd_baseline_too_few_groups(): """ # size of exposure - nints = 2 - ngroups = 3 xsize = 10 ysize = 10 + exposure = { + 'integration_start' : None, + 'integration_end': None, + 'ngroups' : 3, + 'nints' : 2 + } + # size of integration + ngroups = exposure['ngroups'] + nints = exposure['nints'] + + xsize = 10 + ysize = 10 + # create the data and groupdq arrays csize = (nints, ngroups, ysize, xsize) data = np.full(csize, 1.0, dtype=np.float32) @@ -83,7 +103,8 @@ def test_rscd_baseline_too_few_groups(): # create a JWST datamodel for MIRI data on a copy (the copy is created at the step script) dm_ramp = RampModel(data=data, groupdq=groupdq) - + dm_ramp.meta.exposure._instance.update(exposure) + # get the number of groups to flag nflag = 3 @@ -99,3 +120,63 @@ def test_rscd_baseline_too_few_groups(): dq_diff, err_msg='groupdq flags changed when ' + 'not enough groups are present') + + +def test_rscd_tso(): + """ + The RSCD correction is generally skipped for TSO data, but some users + have been running it on TSO data. So this test was added. + Test for TSO segmented data that the correct groups are flagged as 'DO_NOT_USE' + for integration 2 and higher. Set up the segmented data so the segment + is for integration 25 to 50. A rscd correction should be applied to all + the data. + + """ + exposure = { + 'integration_start' : 25, + 'integration_end': 50, + 'ngroups' : 8, + 'nints' : 50 + } + + xsize = 10 + ysize = 10 + ngroups = exposure['ngroups'] + seg_nints = exposure['integration_end'] - exposure['integration_start'] + 1 + input_model = RampModel((seg_nints, exposure['ngroups'], + ysize, xsize)) + + input_model.groupdq[:,:,:,:] = 0 # initize to 0 - all good + input_model.meta.exposure._instance.update(exposure) + + # get the number of groups to flag + nflag = 4 + + # run the RSCD baseline correction step on a copy (the copy is created at the step script) + ramp_rscd = correction_skip_groups(input_model.copy(), nflag) + + + # check that the difference in the groupdq flags is equal to + # the 'DO_NOT_USE' flag for the 1st integration in the segment + # which is actually the 25th integration + dq_diff = (ramp_rscd.groupdq[0, 0:nflag, :, :] + - input_model.groupdq[0, 0:nflag, :, :]) + + np.testing.assert_array_equal(np.full((nflag, ysize, xsize), + dqflags.group['DO_NOT_USE'], + dtype=int), + dq_diff, + err_msg='Diff in groupdq flags is not ' + + 'equal to the DO_NOT_USE flag') + + # test that the groupdq flags are not changed for the rest of the groups + # in the 1st integration in the segment + dq_diff = (ramp_rscd.groupdq[0, nflag:ngroups, :, :] + - input_model.groupdq[0, nflag:ngroups, :, :]) + np.testing.assert_array_equal(np.full((ngroups - nflag, ysize, xsize), + 0, + dtype=int), + dq_diff, + err_msg='groupdq flags changed after ' + + 'maximum requested') + diff --git a/pyproject.toml b/pyproject.toml index 26d333cffd..6d61c69b5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "asdf>=3.1.0,<4", + "asdf>=3.1.0,<5", "astropy>=5.3", "BayesicFitting>=3.0.1", "crds>=12.0.3",