JP-3827: Remove unused error from ramp data

changes/334.apichange.rst

@@ -0,0 +1 @@
+Removed all references to the unused ramp error array in dark, jump, and ramp fitting steps.

docs/stcal/jump/description.rst

@@ -9,7 +9,7 @@ an input exposure. On output, the GROUPDQ array is updated with the DQ flag
 In addition, any pixels that have non-positive or NaN values in the gain
 reference file will have DQ flags "NO_GAIN_VALUE" and "DO_NOT_USE" set in the
 output PIXELDQ array.
-The SCI and ERR arrays of the input data are not modified.
+The SCI array of the input data is not modified.
 
 The current implementation uses the two-point difference method described
 in `Anderson & Gordon (2011) <https://ui.adsabs.harvard.edu/abs/2011PASP..123.1237A>`_.

src/stcal/dark_current/dark_class.py

@@ -27,8 +27,8 @@ def __init__(self, dims=None, dark_model=None):
             during the dark current step.  This argument is only used if the
             'dark_model' argument is None.  If a dark model is not available
             from which to create a DarkData class, but the dimensions of the
-            data array are known, then 'dims' is used (the arrays data, groupdq,
-            and err are assumed to have the same dimension).
+            data array are known, then 'dims' is used (the arrays data
+            and groupdq are assumed to have the same dimension).
 
 
         dark_model : data model, optional
@@ -92,11 +92,6 @@ def __init__(self, science_model=None):
             self.groupdq = science_model.groupdq
             self.pixeldq = science_model.pixeldq
 
-            if isinstance(science_model.err, u.Quantity):
-                self.err = science_model.err.value
-            else:
-                self.err = science_model.err
-
             self.exp_nframes = science_model.meta.exposure.nframes
             self.exp_groupgap = science_model.meta.exposure.groupgap
             try:  # JWST only
@@ -109,7 +104,6 @@ def __init__(self, science_model=None):
             self.data = None
             self.groupdq = None
             self.pixeldq = None
-            self.err = None
 
             self.exp_nframes = None
             self.exp_groupgap = None

src/stcal/dark_current/dark_sub.py

@@ -322,9 +322,9 @@ def average_dark_frames_4d(dark_data, nints, ngroups, nframes, groupgap):
 
 def subtract_dark(science_data, dark_data):
     """
-    Subtracts dark current data from science arrays, combines
-    error arrays in quadrature, and updates data quality array based on
-    DQ flags in the dark arrays.
+    Subtracts dark current data from science arrays.
+
+    Also updates data quality array based on DQ flags in the dark arrays.
 
     Parameters
     ----------

src/stcal/jump/jump.py

@@ -29,12 +29,12 @@ def detect_jumps_data(jump_data):
     turn.
 
     Note that the detection methods are currently set up on the assumption
-    that the input science and error data arrays will be in units of
+    that the input science data array will be in units of
     electrons, hence this routine scales those input arrays by the detector
     gain. The methods assume that the read noise values will be in units
     of DN.
 
-    The gain is applied to the science data and error arrays using the
+    The gain is applied to the science data array using the
     appropriate instrument- and detector-dependent values for each pixel of an
     image.  Also, a 2-dimensional read noise array with appropriate values for
     each pixel is passed to the detection methods.
@@ -66,11 +66,10 @@ def detect_jumps_data(jump_data):
 
     pdq = setup_pdq(jump_data)
 
-    # Apply gain to the SCI, ERR, and readnoise arrays so they're in units
+    # Apply gain to the SCI and readnoise arrays so they're in units
     # of electrons
     data = jump_data.data * jump_data.gain_2d
     gdq = jump_data.gdq
-    # err = jump_data.err * jump_data.gain_2d
     readnoise_2d = jump_data.rnoise_2d * jump_data.gain_2d
 
     # also apply to the after_jump thresholds
@@ -117,10 +116,9 @@ def detect_jumps_data(jump_data):
 
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", ".*in divide.*", RuntimeWarning)
-        # Back out the applied gain to the SCI, ERR, and readnoise arrays so they're
+        # Back out the applied gain to the SCI and readnoise arrays so they're
         #    back in units of DN
         data /= jump_data.gain_2d
-        # err /= jump_data.gain_2d
         readnoise_2d /= jump_data.gain_2d
 
     # Return the updated data quality arrays

src/stcal/jump/jump_class.py

@@ -434,7 +434,6 @@ def get_jump_data_arrays(self):
         self.data
         self.gdq
         self.pdq
-        self.err
         self.gain_2d
         self.rnoise_2d
         '''

src/stcal/ramp_fitting/gls_fit.py

@@ -465,11 +465,11 @@
 
     # Slice data by row
     data = ramp_data.data[:, :, start_row : start_row + nrows, :].copy()
-    err = ramp_data.err[:, :, start_row : start_row + nrows, :].copy()
     groupdq = ramp_data.groupdq[:, :, start_row : start_row + nrows, :].copy()
     pixeldq = ramp_data.pixeldq[start_row : start_row + nrows, :].copy()
+    average_dark_current = ramp_data.average_dark_current[start_row : start_row + nrows, :].copy()
 
-    ramp_data_slice.set_arrays(data, err, groupdq, pixeldq)
+    ramp_data_slice.set_arrays(data, groupdq, pixeldq, average_dark_current)
 
     # Carry over meta data.
     ramp_data_slice.set_meta(
@@ -564,18 +564,13 @@
     if ngroups == 1:
         med_rates = utils.compute_median_rates(ramp_data)
 
-    # We'll propagate error estimates from previous steps to the
-    # current step by using the variance.
-    input_var = ramp_data.err**2
-
     # Convert the data section from DN to electrons.
     data *= gain_2d
 
     for num_int in range(number_ints):
         ramp_data.current_integ = num_int
         gdq_cube = gdq[num_int, :, :, :]
         data_cube = data[num_int, :, :, :]
-        input_var_sect = input_var[num_int, :, :, :]
 
         if save_opt:
             first_group[:, :] = data[num_int, 0, :, :].copy()
@@ -590,7 +585,6 @@
         ) = determine_slope(
             ramp_data,
             data_cube,
-            input_var_sect,
             gdq_cube,
             readnoise_2d,
             gain_2d,
@@ -767,7 +761,6 @@
 def determine_slope(
     ramp_data,
     data_sect,
-    input_var_sect,
     gdq_sect,
     readnoise_sect,
     gain_sect,
@@ -876,9 +869,6 @@
         nx is the number of pixels in the X (more rapidly varying)
         direction.  The units should be electrons.
 
-    input_var_sect : 3-D ndarray, shape (ngroups, ny, nx)
-        The square of the input ERR array, matching data_sect.
-
     gdq_sect : 3-D ndarray, shape (ngroups, ny, nx)
         The group data quality array.  This may be a subarray, matching
         data_sect.
@@ -941,7 +931,6 @@
         return determine_slope_one_group(
             ramp_data,
             data_sect,
-            input_var_sect,
             gdq_sect,
             readnoise_sect,
             gain_sect,
@@ -971,7 +960,6 @@
     while not done:
         (intercept_sect, int_var_sect, slope_sect, slope_var_sect, cr_sect, cr_var_sect) = compute_slope(
             data_sect,
-            input_var_sect,
             gdq_sect,
             readnoise_sect,
             gain_sect,
@@ -1016,7 +1004,6 @@
 def determine_slope_one_group(
     ramp_data,
     data_sect,
-    input_var_sect,
     gdq_sect,
     readnoise_sect,
     gain_sect,
@@ -1043,9 +1030,6 @@
         nx is the number of pixels in the X (more rapidly varying)
         direction.  The units should be electrons.
 
-    input_var_sect : 3-D ndarray, shape (ngroups, ny, nx)
-        The square of the input ERR array, matching data_sect.
-
     gdq_sect : 3-D ndarray, shape (ngroups, ny, nx)
         The group data quality array.  This may be a subarray, matching
         data_sect.
@@ -1235,7 +1219,6 @@
 
 def compute_slope(
     data_sect,
-    input_var_sect,
     gdq_sect,
     readnoise_sect,
     gain_sect,
@@ -1261,9 +1244,6 @@
         The ramp data for one of the integrations in an exposure.  This
         may be a subarray in detector coordinates, but covering all groups.
 
-    input_var_sect : 3-D ndarray, shape (ngroups, ny, nx)
-        The square of the input ERR array, matching data_sect.
-
     gdq_sect : 3-D ndarray; shape (ngroups, ny, nx)
         The group data quality array.  This may be a subarray, matching
         data_sect.
@@ -1406,7 +1386,6 @@
         # ramp_data will be a ramp with a 1-D array of pixels copied out
         # of data_sect.
         ramp_data = np.empty((ngroups, nz), dtype=data_sect.dtype)
-        input_var_data = np.empty((ngroups, nz), dtype=data_sect.dtype)
         prev_fit_data = np.empty((ngroups, nz), dtype=prev_fit.dtype)
         prev_slope_data = np.empty(nz, dtype=prev_slope_sect.dtype)
         prev_slope_data[:] = prev_slope_sect[ncr_mask]
@@ -1423,7 +1402,6 @@
         saturated_data = np.empty((ngroups, nz), dtype=prev_fit.dtype)
         for k in range(ngroups):
             ramp_data[k] = data_sect[k][ncr_mask]
-            input_var_data[k] = input_var_sect[k][ncr_mask]
             prev_fit_data[k] = prev_fit[k][ncr_mask]
             cr_flagged_2d[k] = cr_flagged[k][ncr_mask]
             # This is for clobbering saturated pixels.

src/stcal/ramp_fitting/likely_algo_classes.py

@@ -278,6 +278,8 @@
             Bias of the best-fit count rate from using cvec plus the observed
             resultants to estimate the covariance matrix.
         """
+        raise NotImplementedError('Bias calculations are not implemented.')
+
         alpha = countrates[np.newaxis, :] * self.alpha_phnoise[:, np.newaxis]
         alpha += sig**2 * self.alpha_readnoise[:, np.newaxis]
         beta = countrates[np.newaxis, :] * self.beta_phnoise[:, np.newaxis]

src/stcal/ramp_fitting/ols_fit.py

@@ -545,12 +545,11 @@ def slice_ramp_data(ramp_data, start_row, nrows):
 
     # Slice data by row
     data = ramp_data.data[:, :, start_row : start_row + nrows, :].copy()
-    err = ramp_data.err[:, :, start_row : start_row + nrows, :].copy()
     groupdq = ramp_data.groupdq[:, :, start_row : start_row + nrows, :].copy()
     pixeldq = ramp_data.pixeldq[start_row : start_row + nrows, :].copy()
     average_dark_current = ramp_data.average_dark_current[start_row : start_row + nrows, :].copy()
 
-    ramp_data_slice.set_arrays(data, err, groupdq, pixeldq, average_dark_current)
+    ramp_data_slice.set_arrays(data, groupdq, pixeldq, average_dark_current)
 
     if ramp_data.zeroframe is not None:
         ramp_data_slice.zeroframe = ramp_data.zeroframe[:, start_row : start_row + nrows, :].copy()
@@ -785,7 +784,6 @@ def endianness_handler(ramp_data, gain_2d, readnoise_2d):
     readnoise_2d, rn_bswap = handle_array_endianness(readnoise_2d, sys_order)
 
     ramp_data.data, _ = handle_array_endianness(ramp_data.data, sys_order)
-    ramp_data.err, _ = handle_array_endianness(ramp_data.err, sys_order)
     ramp_data.average_dark_current , _ = handle_array_endianness(ramp_data.average_dark_current, sys_order)
     ramp_data.groupdq, _ = handle_array_endianness(ramp_data.groupdq, sys_order)
     ramp_data.pixeldq, _ = handle_array_endianness(ramp_data.pixeldq, sys_order)
@@ -927,7 +925,6 @@ def discard_miri_groups(ramp_data):
         True if usable data available for further processing.
     """
     data = ramp_data.data
-    err = ramp_data.err
     groupdq = ramp_data.groupdq
     orig_gdq = ramp_data.orig_gdq
 
@@ -958,7 +955,6 @@ def discard_miri_groups(ramp_data):
 
     if num_bad_slices > 0:
         data = data[:, num_bad_slices:, :, :]
-        err = err[:, num_bad_slices:, :, :]
         if orig_gdq is not None:
             orig_gdq = orig_gdq[:, num_bad_slices:, :, :]
 
@@ -978,7 +974,6 @@ def discard_miri_groups(ramp_data):
             return False
 
         data = data[:, :-1, :, :]
-        err = err[:, :-1, :, :]
         groupdq = groupdq[:, :-1, :, :]
         if orig_gdq is not None:
             orig_gdq = orig_gdq[:, :-1, :, :]
@@ -993,7 +988,6 @@ def discard_miri_groups(ramp_data):
         return False
 
     ramp_data.data = data
-    ramp_data.err = err
     ramp_data.groupdq = groupdq
     if orig_gdq is not None:
         ramp_data.orig_gdq = orig_gdq
@@ -1061,7 +1055,6 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting):
     """
     # Get image data information
     data = ramp_data.data
-    err = ramp_data.err
     groupdq = ramp_data.groupdq
     inpixeldq = ramp_data.pixeldq
 
@@ -1073,7 +1066,7 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting):
     imshape = (nrows, ncols)
     cubeshape = (ngroups, *imshape)
 
-    # Get GROUP DQ and ERR arrays from input file
+    # Get GROUP DQ array from input file
     gdq_cube = groupdq
     gdq_cube_shape = gdq_cube.shape
 
@@ -1198,7 +1191,6 @@ def ramp_fit_slopes(ramp_data, gain_2d, readnoise_2d, save_opt, weighting):
         del pixeldq_sect
 
     ramp_data.data = data
-    ramp_data.err = err
     ramp_data.groupdq = groupdq
     ramp_data.pixeldq = inpixeldq
 
@@ -1282,7 +1274,6 @@ def ramp_fit_compute_variances(ramp_data, gain_2d, readnoise_2d, fit_slopes_ans)
     """
     # Get image data information
     data = ramp_data.data
-    err = ramp_data.err
     groupdq = ramp_data.groupdq
     inpixeldq = ramp_data.pixeldq
 
@@ -1440,7 +1431,6 @@ def ramp_fit_compute_variances(ramp_data, gain_2d, readnoise_2d, fit_slopes_ans)
         del segs_4
 
     ramp_data.data = data
-    ramp_data.err = err
     ramp_data.groupdq = groupdq
     ramp_data.pixeldq = inpixeldq
 

src/stcal/ramp_fitting/ramp_fit.py

@@ -67,12 +67,11 @@
         del wh_chargeloss
 
     if isinstance(model.data, u.Quantity):
-        ramp_data.set_arrays(model.data.value, model.err.value, model.groupdq,
+        ramp_data.set_arrays(model.data.value, model.groupdq,
                              model.pixeldq, dark_current_array)
     else:
         ramp_data.set_arrays(
             model.data,
-            model.err,
             model.groupdq,
             model.pixeldq,
             dark_current_array,