From f9842372475d8fc4eb1634f2c0995a2489b0f327 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 15:06:10 -0400 Subject: [PATCH 1/8] Update test data for master background with dedicated background observation --- jwst/regtest/test_miri_lrs_dedicated_mbkg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py index 50ef78cb5c..1949f5646c 100644 --- a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py +++ b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py @@ -10,9 +10,9 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module - rtdata.get_asn("miri/lrs/miri_lrs_mbkg_dedicated_spec3_asn.json") + rtdata.get_asn("miri/lrs/jw01529-o003_spec3_with_bg_00001_asn.json") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub') return rtdata @@ -22,12 +22,12 @@ def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs): """Run a test for MIRI LRS data with dedicated background exposures.""" rtdata = run_pipeline - rtdata.output = "miri_lrs_seq2_exp2_master_background.fits" + rtdata.output = "jw01529003001_03103_00011_mirimage_mbsub.fits" # Get the truth file rtdata.get_truth(os.path.join( "truth/test_miri_lrs_dedicated_mbkg", - "miri_lrs_seq2_exp2_master_background.fits")) + "jw01529003001_03103_00011_mirimage_mbsub.fits")) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From c1898e4533e301badca971527ac55327e0c66083 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:05:28 -0400 Subject: [PATCH 2/8] Copy wavelengths for first spectrum so they are not sorted in place --- jwst/combine_1d/combine1d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/combine_1d/combine1d.py b/jwst/combine_1d/combine1d.py index c4a6021e9d..9d69109ceb 100644 --- a/jwst/combine_1d/combine1d.py +++ b/jwst/combine_1d/combine1d.py @@ -355,7 +355,7 @@ def count_input(input_spectra): # only include spectra that have more than 1 data point if len(input_wl) > 1: if wl is None: - wl = input_wl + wl = input_wl.copy() else: wl = np.hstack((input_wl, wl)) wl.sort() From 359a7b90c6f2195c29b27bee6a2cdbc199f01384 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:06:38 -0400 Subject: [PATCH 3/8] Small fixes for new ModelContainer --- jwst/master_background/master_background_step.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwst/master_background/master_background_step.py b/jwst/master_background/master_background_step.py index 727b351c47..ce9c21b560 100755 --- a/jwst/master_background/master_background_step.py +++ b/jwst/master_background/master_background_step.py @@ -90,7 +90,6 @@ def process(self, input): del _ result = ModelContainer() background_2d_collection = ModelContainer() - background_2d_collection.update(input_data) for model in input_data: background_2d = expand_to_2d(model, self.user_background) result.append(subtract_2d_background(model, background_2d)) @@ -102,7 +101,7 @@ def process(self, input): else: asn_id = None background_2d = expand_to_2d(input_data, self.user_background) - background_2d_collection = background_2d + background_2d_collection = ModelContainer([background_2d]) result = subtract_2d_background(input_data, background_2d) # Record name of user-supplied master background spectrum result.meta.background.master_background_file = basename(self.user_background) From 86b767ce78a392abc5cfeaef7d1985b5f11794f3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:19:53 -0400 Subject: [PATCH 4/8] Update data and test background products --- jwst/regtest/test_miri_lrs_dedicated_mbkg.py | 14 ++++++++------ jwst/regtest/test_miri_lrs_masterbg_user.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py index 1949f5646c..a2bcf42eb3 100644 --- a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py +++ b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py @@ -12,22 +12,24 @@ def run_pipeline(rtdata_module): rtdata.get_asn("miri/lrs/jw01529-o003_spec3_with_bg_00001_asn.json") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub') + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub', + save_background=True) return rtdata @pytest.mark.bigdata -def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs): +@pytest.mark.parametrize('output', ['jw01529-o004_t002_miri_p750l_o003_masterbg1d.fits', + 'jw01529003001_03103_00011_mirimage_o003_masterbg2d.fits', + 'jw01529003001_03103_00011_mirimage_mbsub.fits']) +def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, output): """Run a test for MIRI LRS data with dedicated background exposures.""" rtdata = run_pipeline - rtdata.output = "jw01529003001_03103_00011_mirimage_mbsub.fits" + rtdata.output = output # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_lrs_dedicated_mbkg", - "jw01529003001_03103_00011_mirimage_mbsub.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_dedicated_mbkg", output)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) diff --git a/jwst/regtest/test_miri_lrs_masterbg_user.py b/jwst/regtest/test_miri_lrs_masterbg_user.py index e57570bdbc..6e49984924 100644 --- a/jwst/regtest/test_miri_lrs_masterbg_user.py +++ b/jwst/regtest/test_miri_lrs_masterbg_user.py @@ -11,28 +11,30 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module # This is the user-supplied background file. - rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits") + rtdata.get_data("miri/lrs/jw01529-o004_t002_miri_p750l_x1d.fits") user_bkg = rtdata.input # This is the input file for the master_background step. - rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits") + rtdata.get_data("miri/lrs/jw01529003001_03103_00011_mirimage_cal.fits") MasterBackgroundStep.call(rtdata.input, user_background=user_bkg, - save_results=True, suffix='master_background') + save_results=True, suffix='user_mbsub', + save_background=True) return rtdata @pytest.mark.bigdata -def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs): +@pytest.mark.parametrize('output', ['jw01529003001_03103_00011_mirimage_masterbg2d.fits', + 'jw01529003001_03103_00011_mirimage_user_mbsub.fits']) +def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs, output): """Run a test for MIRI LRS data with a user-supplied background file.""" rtdata = run_pipeline - rtdata.output = "miri_lrs_sci+bkg_master_background.fits" + rtdata.output = output # Get the truth file - rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user", - "miri_lrs_sci+bkg_master_background.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user", output)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From c889fa393997b1a1202e54ecde127512b7616bed Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 15:30:01 -0400 Subject: [PATCH 5/8] Add mbsub to the list of known suffixes --- jwst/lib/suffix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/lib/suffix.py b/jwst/lib/suffix.py index a7af86beb7..b72306dce4 100644 --- a/jwst/lib/suffix.py +++ b/jwst/lib/suffix.py @@ -41,7 +41,7 @@ 'c1d', 'cal', 'calints', 'cat', 'crf', 'crfints', 'dark', 'i2d', - 'median', + 'mbsub', 'median', 'phot', 'psf-amiavg', 'psfalign', 'psfstack', 'psfsub', 'ramp', 'rate', 'rateints', 'residual_fringe', 's2d', 's3d', 'snr', From c2cd58f690deb92e187c6359b99dd1a85094b943 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 15:34:26 -0400 Subject: [PATCH 6/8] Add a test/demo for background extraction for nodded LRS data --- jwst/regtest/test_miri_lrs_nod_masterbg.py | 83 ++++++++++++++++++---- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_nod_masterbg.py b/jwst/regtest/test_miri_lrs_nod_masterbg.py index 0a3b2ba203..d17b65fad2 100644 --- a/jwst/regtest/test_miri_lrs_nod_masterbg.py +++ b/jwst/regtest/test_miri_lrs_nod_masterbg.py @@ -2,33 +2,92 @@ import pytest from astropy.io.fits.diff import FITSDiff -from jwst.master_background import MasterBackgroundStep +from jwst.stpipe import Step @pytest.fixture(scope="module") -def run_pipeline(rtdata_module): +def run_pipeline_with_master_bg(rtdata_module): rtdata = rtdata_module - rtdata.get_asn("miri/lrs/miri_lrs_mbkg_nodded_spec3_asn.json") + # Get rate files and custom extraction parameters + rate_files = ['jw01530005001_03103_00001_mirimage_rate.fits', + 'jw01530005001_03103_00002_mirimage_rate.fits'] + ref_files = ['jwst_miri_extract1d_nod1_bg.json', + 'jwst_miri_extract1d_nod2_bg.json'] + for rate, ref in zip(rate_files, ref_files): + rtdata.get_data(f"miri/lrs/{ref}") + rtdata.get_data(f"miri/lrs/{rate}") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') + Step.from_cmdline([ + "calwebb_spec2", + rate, + f"--steps.extract_1d.override_extract1d={ref}", + "--steps.extract_1d.use_source_pos=False", + ]) + + # Get the ASN, but not the data - use the spec2 input products + rtdata.get_data("miri/lrs/jw01530-o005_20221202t204827_spec3_00001_asn.json") + Step.from_cmdline([ + "calwebb_spec3", + rtdata.input, + "--steps.master_background.skip=False", + "--steps.master_background.save_background=True", + "--steps.master_background.save_results=True", + ]) return rtdata @pytest.mark.bigdata -@pytest.mark.parametrize("nod_seq", ["seq1", "seq2"]) -def test_miri_lrs_nod_masterbg(run_pipeline, fitsdiff_default_kwargs, nod_seq): - """Run a regression test for nodded MIRI LRS data.""" +@pytest.mark.parametrize("nod", [1, 2]) +@pytest.mark.parametrize("suffix", ["cal", "x1d", "s2d", "mbsub", "o005_crf", + "o005_masterbg2d"]) +def test_miri_lrs_nod_bg(run_pipeline_with_master_bg, fitsdiff_default_kwargs, + nod, suffix): + """Run a regression test for nodded MIRI LRS data with background extraction.""" + rtdata = run_pipeline_with_master_bg + + # Check output products for nod 1 or 2 + output_file = f"jw01530005001_03103_0000{nod}_mirimage_{suffix}.fits" + rtdata.output = output_file + + # Get the truth file + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + +def test_miri_lrs_nod_masterbg1d(run_pipeline_with_master_bg, fitsdiff_default_kwargs): + rtdata = run_pipeline_with_master_bg + + # Check 1D masterbg output product: created with root name from nod 1 + output_file = "jw01530005001_03103_00001_mirimage_o005_masterbg1d.fits" + rtdata.output = output_file + + # Get the truth file + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", ["s2d", "x1d"]) +def test_miri_lrs_nod_bg_spec3(run_pipeline_with_master_bg, fitsdiff_default_kwargs, + suffix): + """Run a regression test for nodded MIRI LRS with master background subtraction.""" + rtdata = run_pipeline_with_master_bg - rtdata = run_pipeline - rtdata.output = "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits" + # Check combined output products + output_file = f"jw01530-o005_t004_miri_p750l_{suffix}.fits" + rtdata.output = output_file # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_lrs_nod_masterbg", - "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From ce815ebd3c9d7bbcbd17fa5a5ea0de005ba051f8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 16:37:05 -0400 Subject: [PATCH 7/8] Add change notes --- changes/8927.combine_1d.rst | 1 + changes/8927.master_background.rst | 1 + changes/8927.pipeline.rst | 1 + 3 files changed, 3 insertions(+) create mode 100644 changes/8927.combine_1d.rst create mode 100644 changes/8927.master_background.rst create mode 100644 changes/8927.pipeline.rst diff --git a/changes/8927.combine_1d.rst b/changes/8927.combine_1d.rst new file mode 100644 index 0000000000..b05b3e95bc --- /dev/null +++ b/changes/8927.combine_1d.rst @@ -0,0 +1 @@ +Fix wavelength sort order for single input spectrum. diff --git a/changes/8927.master_background.rst b/changes/8927.master_background.rst new file mode 100644 index 0000000000..cf71da1e52 --- /dev/null +++ b/changes/8927.master_background.rst @@ -0,0 +1 @@ +Fix ModelContainer handling for user background input. diff --git a/changes/8927.pipeline.rst b/changes/8927.pipeline.rst new file mode 100644 index 0000000000..e03ac78254 --- /dev/null +++ b/changes/8927.pipeline.rst @@ -0,0 +1 @@ +Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``. From 952846af742cc1045ba4d50fc7907e909d80a66c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 16:56:26 -0400 Subject: [PATCH 8/8] Add missing 'bigdata' mark --- jwst/regtest/test_miri_lrs_nod_masterbg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwst/regtest/test_miri_lrs_nod_masterbg.py b/jwst/regtest/test_miri_lrs_nod_masterbg.py index d17b65fad2..69554b5a5a 100644 --- a/jwst/regtest/test_miri_lrs_nod_masterbg.py +++ b/jwst/regtest/test_miri_lrs_nod_masterbg.py @@ -60,6 +60,7 @@ def test_miri_lrs_nod_bg(run_pipeline_with_master_bg, fitsdiff_default_kwargs, assert diff.identical, diff.report() +@pytest.mark.bigdata def test_miri_lrs_nod_masterbg1d(run_pipeline_with_master_bg, fitsdiff_default_kwargs): rtdata = run_pipeline_with_master_bg