JP-2928, JP-3792: MIRI LRS master background (#8927)

changes/8927.combine_1d.rst

@@ -0,0 +1 @@
+Fix wavelength sort order for single input spectrum.

changes/8927.master_background.rst

@@ -0,0 +1 @@
+Fix ModelContainer handling for user background input.

changes/8927.pipeline.rst

@@ -0,0 +1 @@
+Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``.

jwst/combine_1d/combine1d.py

@@ -355,7 +355,7 @@ def count_input(input_spectra):
         # only include spectra that have more than 1 data point
         if len(input_wl) > 1:
             if wl is None:
-                wl = input_wl
+                wl = input_wl.copy()
             else:
                 wl = np.hstack((input_wl, wl))
     wl.sort()

jwst/lib/suffix.py

@@ -41,7 +41,7 @@
     'c1d', 'cal', 'calints', 'cat', 'crf', 'crfints',
     'dark',
     'i2d',
-    'median',
+    'mbsub', 'median',
     'phot', 'psf-amiavg', 'psfalign', 'psfstack', 'psfsub',
     'ramp', 'rate', 'rateints', 'residual_fringe',
     's2d', 's3d', 'snr',

jwst/master_background/master_background_step.py

@@ -90,7 +90,6 @@ def process(self, input):
                     del _
                     result = ModelContainer()
                     background_2d_collection = ModelContainer()
-                    background_2d_collection.update(input_data)
                     for model in input_data:
                         background_2d = expand_to_2d(model, self.user_background)
                         result.append(subtract_2d_background(model, background_2d))
@@ -102,7 +101,7 @@ def process(self, input):
                 else:
                     asn_id = None
                     background_2d = expand_to_2d(input_data, self.user_background)
-                    background_2d_collection = background_2d
+                    background_2d_collection = ModelContainer([background_2d])
                     result = subtract_2d_background(input_data, background_2d)
                     # Record name of user-supplied master background spectrum
                     result.meta.background.master_background_file = basename(self.user_background)

jwst/regtest/test_miri_lrs_dedicated_mbkg.py

@@ -10,24 +10,26 @@ def run_pipeline(rtdata_module):
 
     rtdata = rtdata_module
 
-    rtdata.get_asn("miri/lrs/miri_lrs_mbkg_dedicated_spec3_asn.json")
+    rtdata.get_asn("miri/lrs/jw01529-o003_spec3_with_bg_00001_asn.json")
 
-    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background')
+    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub',
+                              save_background=True)
 
     return rtdata
 
 
 @pytest.mark.bigdata
-def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs):
+@pytest.mark.parametrize('output', ['jw01529-o004_t002_miri_p750l_o003_masterbg1d.fits',
+                                    'jw01529003001_03103_00011_mirimage_o003_masterbg2d.fits',
+                                    'jw01529003001_03103_00011_mirimage_mbsub.fits'])
+def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, output):
     """Run a test for MIRI LRS data with dedicated background exposures."""
 
     rtdata = run_pipeline
-    rtdata.output = "miri_lrs_seq2_exp2_master_background.fits"
+    rtdata.output = output
 
     # Get the truth file
-    rtdata.get_truth(os.path.join(
-        "truth/test_miri_lrs_dedicated_mbkg",
-        "miri_lrs_seq2_exp2_master_background.fits"))
+    rtdata.get_truth(os.path.join("truth/test_miri_lrs_dedicated_mbkg", output))
 
     # Compare the results
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)

jwst/regtest/test_miri_lrs_masterbg_user.py

@@ -11,28 +11,30 @@ def run_pipeline(rtdata_module):
     rtdata = rtdata_module
 
     # This is the user-supplied background file.
-    rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits")
+    rtdata.get_data("miri/lrs/jw01529-o004_t002_miri_p750l_x1d.fits")
     user_bkg = rtdata.input
 
     # This is the input file for the master_background step.
-    rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits")
+    rtdata.get_data("miri/lrs/jw01529003001_03103_00011_mirimage_cal.fits")
 
     MasterBackgroundStep.call(rtdata.input, user_background=user_bkg,
-                              save_results=True, suffix='master_background')
+                              save_results=True, suffix='user_mbsub',
+                              save_background=True)
 
     return rtdata
 
 
 @pytest.mark.bigdata
-def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs):
+@pytest.mark.parametrize('output', ['jw01529003001_03103_00011_mirimage_masterbg2d.fits',
+                                    'jw01529003001_03103_00011_mirimage_user_mbsub.fits'])
+def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs, output):
     """Run a test for MIRI LRS data with a user-supplied background file."""
 
     rtdata = run_pipeline
-    rtdata.output = "miri_lrs_sci+bkg_master_background.fits"
+    rtdata.output = output
 
     # Get the truth file
-    rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user",
-                                  "miri_lrs_sci+bkg_master_background.fits"))
+    rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user", output))
 
     # Compare the results
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)

jwst/regtest/test_miri_lrs_nod_masterbg.py

@@ -2,33 +2,93 @@
 import pytest
 from astropy.io.fits.diff import FITSDiff
 
-from jwst.master_background import MasterBackgroundStep
+from jwst.stpipe import Step
 
 
 @pytest.fixture(scope="module")
-def run_pipeline(rtdata_module):
+def run_pipeline_with_master_bg(rtdata_module):
 
     rtdata = rtdata_module
 
-    rtdata.get_asn("miri/lrs/miri_lrs_mbkg_nodded_spec3_asn.json")
+    # Get rate files and custom extraction parameters
+    rate_files = ['jw01530005001_03103_00001_mirimage_rate.fits',
+                  'jw01530005001_03103_00002_mirimage_rate.fits']
+    ref_files = ['jwst_miri_extract1d_nod1_bg.json',
+                 'jwst_miri_extract1d_nod2_bg.json']
+    for rate, ref in zip(rate_files, ref_files):
+        rtdata.get_data(f"miri/lrs/{ref}")
+        rtdata.get_data(f"miri/lrs/{rate}")
 
-    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background')
+        Step.from_cmdline([
+            "calwebb_spec2",
+            rate,
+            f"--steps.extract_1d.override_extract1d={ref}",
+            "--steps.extract_1d.use_source_pos=False",
+        ])
+
+    # Get the ASN, but not the data - use the spec2 input products
+    rtdata.get_data("miri/lrs/jw01530-o005_20221202t204827_spec3_00001_asn.json")
+    Step.from_cmdline([
+        "calwebb_spec3",
+        rtdata.input,
+        "--steps.master_background.skip=False",
+        "--steps.master_background.save_background=True",
+        "--steps.master_background.save_results=True",
+    ])
 
     return rtdata
 
 
 @pytest.mark.bigdata
-@pytest.mark.parametrize("nod_seq", ["seq1", "seq2"])
-def test_miri_lrs_nod_masterbg(run_pipeline, fitsdiff_default_kwargs, nod_seq):
-    """Run a regression test for nodded MIRI LRS data."""
+@pytest.mark.parametrize("nod", [1, 2])
+@pytest.mark.parametrize("suffix", ["cal", "x1d", "s2d", "mbsub", "o005_crf",
+                                    "o005_masterbg2d"])
+def test_miri_lrs_nod_bg(run_pipeline_with_master_bg, fitsdiff_default_kwargs,
+                         nod, suffix):
+    """Run a regression test for nodded MIRI LRS data with background extraction."""
+    rtdata = run_pipeline_with_master_bg
+
+    # Check output products for nod 1 or 2
+    output_file = f"jw01530005001_03103_0000{nod}_mirimage_{suffix}.fits"
+    rtdata.output = output_file
+
+    # Get the truth file
+    rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file))
+
+    # Compare the results
+    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
+    assert diff.identical, diff.report()
+
+
+@pytest.mark.bigdata
+def test_miri_lrs_nod_masterbg1d(run_pipeline_with_master_bg, fitsdiff_default_kwargs):
+    rtdata = run_pipeline_with_master_bg
+
+    # Check 1D masterbg output product: created with root name from nod 1
+    output_file = "jw01530005001_03103_00001_mirimage_o005_masterbg1d.fits"
+    rtdata.output = output_file
+
+    # Get the truth file
+    rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file))
+
+    # Compare the results
+    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
+    assert diff.identical, diff.report()
+
+
+@pytest.mark.bigdata
+@pytest.mark.parametrize("suffix", ["s2d", "x1d"])
+def test_miri_lrs_nod_bg_spec3(run_pipeline_with_master_bg, fitsdiff_default_kwargs,
+                               suffix):
+    """Run a regression test for nodded MIRI LRS with master background subtraction."""
+    rtdata = run_pipeline_with_master_bg
 
-    rtdata = run_pipeline
-    rtdata.output = "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits"
+    # Check combined output products
+    output_file = f"jw01530-o005_t004_miri_p750l_{suffix}.fits"
+    rtdata.output = output_file
 
     # Get the truth file
-    rtdata.get_truth(os.path.join(
-        "truth/test_miri_lrs_nod_masterbg",
-        "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits"))
+    rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file))
 
     # Compare the results
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)