Failure in test_nirspec.py::test_against_test_data

[Witchblade101]

________________________________________________________ test_against_test_data ________________________________________________________

siaf = <pysiaf.Siaf object Instrument=nirspec >

def test_against_test_data(siaf=None):
    """NIRSpec test data comparison.

    Mean and RMS difference between the IDT computations and the pysiaf computations are
    computed and compared against acceptable thresholds.

    """
    if siaf is None:
        # Try to use pre-delivery-data since this should best match the source-data. If no data there, use PRD data
        try:
            pre_delivery_dir = os.path.join(JWST_DELIVERY_DATA_ROOT, instrument)
            siaf = Siaf(instrument, basepath=pre_delivery_dir)
        except OSError:
            siaf = Siaf(instrument)
    else:
        # safeguard against side-effects when running several tests on a provided siaf, e.g.
        # setting tilt to non-zero value
        siaf = copy.deepcopy(siaf)
    # directory that holds SIAF XML file
    # test_dir = os.path.join(JWST_TEMPORARY_DATA_ROOT, instrument, 'generate_test')
    # siaf_xml_file = os.path.join(test_dir, '{}_SIAF.xml'.format(instrument))
    # siaf = Siaf(instrument, filename=siaf_xml_file)

    # test_data_dir = os.path.join(TEST_DATA_ROOT, instrument)
    test_data_dir = os.path.join(JWST_SOURCE_DATA_ROOT, instrument, 'delivery', 'test_data')

    print(' ')
    for include_tilt in [False, True]:

        if include_tilt is False:
            ta_transform_data_dir = os.path.join(test_data_dir, 'TA_testDataNoTilt')
        else:
            ta_transform_data_dir = os.path.join(test_data_dir, 'TA_testDataWithGWATilt')

        filter_list = 'CLEAR F110W F140X'.split()
        sca_list = ['SCA491', 'SCA492']

        difference_metrics = {}
        index = 0
        for sca_name in sca_list:
            for filter_name in filter_list:

                test_data_file = os.path.join(ta_transform_data_dir, 'testDataTA_{}{}.fits'.format(sca_name, filter_name))
                test_data = Table(fits.getdata(test_data_file))
                if include_tilt is False:
                    tilt = None
                else:
                    test_header = fits.getheader(test_data_file)
                    tilt = (np.float(test_header['GWA_XTIL']), np.float(test_header['GWA_YTIL']))

                if sca_name == 'SCA491':
                    AperName = 'NRS1_FULL_OSS'
                elif sca_name == 'SCA492':
                    AperName = 'NRS2_FULL_OSS'

                aperture = siaf[AperName]
                aperture.filter_name = filter_name
                aperture.tilt = tilt

                if 0:
                    pl.figure(figsize=(8, 8), facecolor='w', edgecolor='k'); pl.clf()
                    aperture.plot(name_label=True)
                    siaf['NRS2_FULL_OSS'].plot(name_label=True)
                    pl.plot(test_data['XAN']*u.deg.to(u.arcsecond), test_data['YAN']*u.deg.to(u.arcsecond), 'b.')
                    pl.show()

                    1/0

                # SCI to GWA detector side (Step 1. in Sections 2.3.3, 5.5.2 of JWST-STScI-005921, see also Table 4.7.1)
                test_data['pysiaf_GWAout_X'], test_data['pysiaf_GWAout_Y'] = aperture.sci_to_gwa(test_data['SCA_X'], test_data['SCA_Y'])

                # effect of mirror, transform from GWA detector side to GWA skyward side
                test_data['pysiaf_GWAin_X'], test_data['pysiaf_GWAin_Y'] = aperture.gwaout_to_gwain(test_data['pysiaf_GWAout_X'] , test_data['pysiaf_GWAout_Y'])

                # transform to OTE frame (XAN, YAN)
                test_data['pysiaf_XAN'], test_data['pysiaf_YAN'] = aperture.gwa_to_ote(
                    test_data['pysiaf_GWAin_X'], test_data['pysiaf_GWAin_Y'])

                for axis_name in ['X', 'Y']:
                    for parameter_name in ['{}AN'.format(axis_name)]:

                        # compute differences between SIAF implementation and IDT test dataset
                        test_data['difference_{}'.format(parameter_name)] = test_data['pysiaf_{}'.format(parameter_name)] - test_data['{}'.format(parameter_name)]

                        for key_seed in ['mean', 'rms']:
                            key_name = 'diff_{}_{}'.format(parameter_name, key_seed)
                            if key_name not in difference_metrics.keys():
                                difference_metrics[key_name] = []
                            if key_seed == 'mean':
                                difference_metrics[key_name].append(np.mean(test_data['difference_{}'.format(parameter_name)]))
                            elif key_seed == 'rms':
                                difference_metrics[key_name].append(np.std(test_data['difference_{}'.format(parameter_name)]))

                        print('{} {} SCA_to_OTE transform comparison to {:>10}  tilt={} {:>10} MEAN={:+1.3e} RMS={:1.3e}'.format(sca_name, filter_name, AperName, include_tilt, parameter_name, difference_metrics['diff_{}_{}'.format(parameter_name, 'mean')][index], difference_metrics['diff_{}_{}'.format(parameter_name, 'rms')][index]))

                        assert difference_metrics['diff_{}_{}'.format(parameter_name, 'mean')][index] < 1e-9, "Failed for {}".format(AperName)

                      assert difference_metrics['diff_{}_{}'.format(parameter_name, 'rms')][index] < 5e-9, "Failed for {}".format(AperName)

E AssertionError: Failed for NRS1_FULL_OSS
E assert 1.4678534638290019e-05 < 5e-09

test_nirspec.py:123: AssertionError
--------------------------------------------------------- Captured stdout call ---------------------------------------------------------

SCA491 CLEAR SCA_to_OTE transform comparison to NRS1_FULL_OSS tilt=False XAN MEAN=-2.044e-04 RMS=1.468e-05

[Witchblade101]

This is the full test failure message described in a comment to #255

[Witchblade101]

NIRSpec failures begin starting with PRDOPSSOC-049, which was the PRD which included JWSTSIAF-180, the only NRS SIAF updates.