From 92863b446a4b66195c436bbb0cf1082c492f6dc5 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Tue, 19 Nov 2024 08:03:37 -0700 Subject: [PATCH 01/27] testing cube build names --- jwst/cube_build/ifu_cube.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py index 84d56a8540..9c9b73e0fd 100644 --- a/jwst/cube_build/ifu_cube.py +++ b/jwst/cube_build/ifu_cube.py @@ -53,6 +53,8 @@ def __init__(self, self.input_models = input_models # needed when building single mode IFU cubes self.output_name_base = output_name_base + print('At the start output name base', self.output_name_base) + self.num_files = None self.instrument = instrument @@ -168,7 +170,9 @@ def define_cubename(self): """ Define the base output name """ if self.pipeline == 2: + print('in define_cubename', self.output_name_base) newname = self.output_name_base + self.suffix + '.fits' + print('***********NAME OF CUBE FILE', newname) else: if self.instrument == 'MIRI': @@ -203,6 +207,7 @@ def define_cubename(self): b_name = b_name + subchannels[i] b_name = b_name.lower() newname = self.output_name_base + ch_name + '-' + b_name + print('*******************NAME of cube when not pipeline2', newname) if self.coord_system == 'internal_cal': newname = self.output_name_base + ch_name + '-' + b_name + '_internal' if self.output_type == 'single': From c0e9d9c40e8065fb76706c3042f77e43db479e16 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Thu, 21 Nov 2024 05:52:08 -0700 Subject: [PATCH 02/27] fixed cube build --- jwst/cube_build/ifu_cube.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py index 9c9b73e0fd..26a4713bcc 100644 --- a/jwst/cube_build/ifu_cube.py +++ b/jwst/cube_build/ifu_cube.py @@ -53,7 +53,6 @@ def __init__(self, self.input_models = input_models # needed when building single mode IFU cubes self.output_name_base = output_name_base - print('At the start output name base', self.output_name_base) self.num_files = None @@ -170,9 +169,7 @@ def define_cubename(self): """ Define the base output name """ if self.pipeline == 2: - print('in define_cubename', self.output_name_base) - newname = self.output_name_base + self.suffix + '.fits' - print('***********NAME OF CUBE FILE', newname) + newname = self.output_name_base + '_' + self.suffix + '.fits' else: if self.instrument == 'MIRI': @@ -207,7 +204,6 @@ def define_cubename(self): b_name = b_name + subchannels[i] b_name = b_name.lower() newname = self.output_name_base + ch_name + '-' + b_name - print('*******************NAME of cube when not pipeline2', newname) if self.coord_system == 'internal_cal': newname = self.output_name_base + ch_name + '-' + b_name + '_internal' if self.output_type == 'single': From 4f547b3690f9c1c97e7837914caa3761b609f522 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Mon, 25 Nov 2024 13:20:24 -0700 Subject: [PATCH 03/27] fix names --- jwst/pixel_replace/pixel_replace.py | 2 +- jwst/pixel_replace/pixel_replace_step.py | 36 +++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index 99a540d932..b767c48325 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -29,7 +29,7 @@ class PixelReplacement: VERTICAL = 2 LOG_SLICE = ['column', 'row'] - default_suffix = 'pixrep' + #default_suffix = 'pixrep' def __init__(self, input_model, **pars): """ diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index 5191ac3930..d0607a251b 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -4,7 +4,8 @@ from ..stpipe import Step from jwst.stpipe import record_step_status from jwst import datamodels -from .pixel_replace import PixelReplacement +from .pixel_replace import PixelReplacement#from functools import wraps +#from jwst.pipeline.calwebb_spec3 import invariant_filename __all__ = ["PixelReplaceStep"] @@ -72,9 +73,9 @@ def process(self, input): 'n_adjacent_cols': self.n_adjacent_cols, } - # ___________________ - # calewbb_spec3 case - # ___________________ + # ___________________________________ + # calewbb_spec3 case / ModelContainer + # __________________________________ if isinstance(input_model, datamodels.ModelContainer): output_model = input_model # Setup output path naming if associations are involved. @@ -85,6 +86,12 @@ def process(self, input): pass if asn_id is None: asn_id = self.search_attr('asn_id') + if asn_id is None: # it is still None. It does not exist. A ModelContainer was passed in with + # no asn information + # to create the correct output name set the following. + self.output_use_model = True + self.save_model = invariant_filename(self.save_model) + if asn_id is not None: _make_output_path = self.search_attr( '_make_output_path', parent_first=True @@ -93,6 +100,7 @@ def process(self, input): _make_output_path, asn_id=asn_id ) + # Check models to confirm they are the correct type for i, model in enumerate(output_model): run_pixel_replace = True @@ -124,3 +132,23 @@ def process(self, input): record_step_status(replacement.output, 'pixel_replace', success=True) result = replacement.output return result + + +def invariant_filename(save_model_func): + """Restore meta.filename after save_model""" + + @wraps(save_model_func) + def save_model(model, **kwargs): + try: + filename = model.meta.filename + except AttributeError: + filename = None + + result = save_model_func(model, **kwargs) + + if filename: + model.meta.filename = filename + + return result + + return save_model From a4e9f92182cdcb474efb9989c057a15a86d84f69 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Mon, 25 Nov 2024 13:28:01 -0700 Subject: [PATCH 04/27] added import --- jwst/pixel_replace/pixel_replace_step.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index d0607a251b..bde1aaa716 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -4,7 +4,8 @@ from ..stpipe import Step from jwst.stpipe import record_step_status from jwst import datamodels -from .pixel_replace import PixelReplacement#from functools import wraps +from .pixel_replace import PixelReplacement +from functools import wraps #from jwst.pipeline.calwebb_spec3 import invariant_filename __all__ = ["PixelReplaceStep"] From 41e34bf7be02e77e7ef6403655d09d5fd12ea8b3 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Wed, 27 Nov 2024 07:50:53 -0700 Subject: [PATCH 05/27] updates --- jwst/pixel_replace/pixel_replace_step.py | 8 ++-- .../pixel_replace/tests/test_pixel_replace.py | 42 ++++++++++++++++++- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index bde1aaa716..88dd283db6 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -78,7 +78,7 @@ def process(self, input): # calewbb_spec3 case / ModelContainer # __________________________________ if isinstance(input_model, datamodels.ModelContainer): - output_model = input_model + output_model = input_model.copy() # Setup output path naming if associations are involved. asn_id = None try: @@ -103,7 +103,7 @@ def process(self, input): ) # Check models to confirm they are the correct type - for i, model in enumerate(output_model): + for model in output_model: run_pixel_replace = True if model.meta.model_type in ['MultiSlitModel', 'SlitModel', 'ImageModel', 'IFUImageModel', 'CubeModel']: @@ -119,8 +119,8 @@ def process(self, input): if run_pixel_replace: replacement = PixelReplacement(model, **pars) replacement.replace() - output_model[i] = replacement.output - record_step_status(output_model[i], 'pixel_replace', success=True) + model = replacement.output + record_step_status(model, 'pixel_replace', success=True) return output_model # ________________________________________ # calewbb_spec2 case - single input model diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 3ab91c7eec..0b3c31f5a8 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -2,6 +2,7 @@ import pytest from stdatamodels.jwst import datamodels +from jwst.datamodels import ModelContainer from stdatamodels.jwst.datamodels.dqflags import pixel as flags from jwst.assign_wcs import AssignWcsStep @@ -99,6 +100,7 @@ def nirspec_ifu(): model.var_poisson = test_data.var_poisson model.var_rnoise = test_data.var_rnoise model.var_flat = test_data.var_flat + test_data.close() return model, bad_idx @@ -212,11 +214,14 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): The test is otherwise the same as for other modes. """ input_model, bad_idx = input_model_function() + input_model.meta.filename = 'jwst_nirspec_cal.fits' # for this simple case, the results from either algorithm should # be the same - result = PixelReplaceStep.call(input_model, skip=False, algorithm=algorithm) + result = PixelReplaceStep.call(input_model, skip=False, algorithm=algorithm,save_results=True) + assert result.meta.filename == 'jwst_nirspec_pixelreplacestep.fits' + for ext in ['data', 'err', 'var_poisson', 'var_rnoise', 'var_flat']: # non-science edges are uncorrected assert np.all(np.isnan(getattr(result, ext)[..., :, 1])) @@ -238,3 +243,38 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): result.close() input_model.close() + + + + +@pytest.mark.slow +@pytest.mark.parametrize('input_model_function', + [nirspec_ifu]) +@pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) +def test_pixel_replace_nirspec_ifu_container_names(input_model_function, algorithm): + """ + Test pixel replacement for NIRSpec IFU using a container + + Larger data and more WCS operations required for testing make + this test take more than a minute, so marking this test 'slow'. + + The test is otherwise the same as for other modes. + """ + input_model, bad_idx = input_model_function() + input_model.meta.filename = 'jwst_nirspec_1_cal.fits' + input_model2, bad_idx2 = input_model_function() + input_model2.meta.filename = 'jwst_nirspec_2_cal.fits' + cfiles = [input_model, input_model2] + container = ModelContainer(cfiles) + + # for this simple case, the results from either algorithm should + # be the same + result = PixelReplaceStep.call(container, skip=False, algorithm=algorithm,save_results=True) + + print(result[0].meta.filename, result[1].meta.filename) + + assert result[0].meta.filename == 'jwst_nirspec_1_pixelreplacestep.fits' + assert result[1].meta.filename == 'jwst_nirspec_2_pixelreplacestep.fits' + + result.close() + input_model.close() From cc56feb189ea4f5764f64103808efe5d86b14e9b Mon Sep 17 00:00:00 2001 From: jemorrison Date: Fri, 6 Dec 2024 10:01:58 -0700 Subject: [PATCH 06/27] updates to fix names --- jwst/cube_build/data_types.py | 1 - jwst/cube_build/ifu_cube.py | 4 ++- jwst/pixel_replace/pixel_replace_step.py | 8 +++--- .../pixel_replace/tests/test_pixel_replace.py | 25 ++++++++++++++----- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/jwst/cube_build/data_types.py b/jwst/cube_build/data_types.py index 711f729bd1..50e28fba03 100644 --- a/jwst/cube_build/data_types.py +++ b/jwst/cube_build/data_types.py @@ -98,7 +98,6 @@ def __init__(self, input, single, output_file, output_dir): # Suffixes will be added to this name later, to designate the # channel+subchannel (MIRI MRS) or grating+filter (NRS IFU) the output cube covers. - if output_file is not None: basename, ext = os.path.splitext(os.path.basename(output_file)) self.output_name = basename diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py index 26a4713bcc..db05d49b33 100644 --- a/jwst/cube_build/ifu_cube.py +++ b/jwst/cube_build/ifu_cube.py @@ -179,9 +179,10 @@ def define_cubename(self): # that the remaining suffixes created below form the entire # list of optical elements in the final output name. suffix = self.output_name_base[self.output_name_base.rfind('_') + 1:] + if suffix in ['clear']: self.output_name_base = self.output_name_base[:self.output_name_base.rfind('_')] - + # Now compose the appropriate list of optical element suffix names # based on MRS channel and sub-channel channels = [] @@ -562,6 +563,7 @@ def build_ifucube(self): """ self.output_name = self.define_cubename() + print('ifu cube returend output_name', self.output_name) total_num = self.naxis1 * self.naxis2 * self.naxis3 self.spaxel_flux = np.zeros(total_num, dtype=np.float64) diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index 88dd283db6..32f5944e56 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -78,8 +78,10 @@ def process(self, input): # calewbb_spec3 case / ModelContainer # __________________________________ if isinstance(input_model, datamodels.ModelContainer): - output_model = input_model.copy() # Setup output path naming if associations are involved. + # if input is ModelContainer do not copy the input_model + # because assocation information is not copied. + # Instead update input_modelin place. asn_id = None try: asn_id = input_model.asn_table["asn_id"] @@ -103,7 +105,7 @@ def process(self, input): ) # Check models to confirm they are the correct type - for model in output_model: + for i, model in enumerate(input_model): run_pixel_replace = True if model.meta.model_type in ['MultiSlitModel', 'SlitModel', 'ImageModel', 'IFUImageModel', 'CubeModel']: @@ -121,7 +123,7 @@ def process(self, input): replacement.replace() model = replacement.output record_step_status(model, 'pixel_replace', success=True) - return output_model + return input_model # ________________________________________ # calewbb_spec2 case - single input model # ________________________________________ diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 0b3c31f5a8..47a6daa8d7 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -1,3 +1,4 @@ +import os import numpy as np import pytest @@ -8,7 +9,7 @@ from jwst.assign_wcs import AssignWcsStep from jwst.assign_wcs.tests.test_nirspec import create_nirspec_ifu_file from jwst.pixel_replace.pixel_replace_step import PixelReplaceStep - +from glob import glob def cal_data(shape, bad_idx, dispaxis=1, model='slit'): if model == 'image': @@ -251,7 +252,7 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): @pytest.mark.parametrize('input_model_function', [nirspec_ifu]) @pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) -def test_pixel_replace_nirspec_ifu_container_names(input_model_function, algorithm): +def test_pixel_replace_nirspec_ifu_container_names(tmp_cwd, tmp_path, input_model_function, algorithm): """ Test pixel replacement for NIRSpec IFU using a container @@ -260,6 +261,10 @@ def test_pixel_replace_nirspec_ifu_container_names(input_model_function, algorit The test is otherwise the same as for other modes. """ + output_dir = tmp_path / 'output' + output_dir.mkdir(exist_ok=True) + output_dir = str(output_dir) + input_model, bad_idx = input_model_function() input_model.meta.filename = 'jwst_nirspec_1_cal.fits' input_model2, bad_idx2 = input_model_function() @@ -267,14 +272,22 @@ def test_pixel_replace_nirspec_ifu_container_names(input_model_function, algorit cfiles = [input_model, input_model2] container = ModelContainer(cfiles) + expected_name = [] + expected_name.append('jwst_nirspec_1_pixelreplacestep.fits') + expected_name.append('jwst_nirspec_2_pixelreplacestep.fits') + + return_files = [] # for this simple case, the results from either algorithm should # be the same result = PixelReplaceStep.call(container, skip=False, algorithm=algorithm,save_results=True) - - print(result[0].meta.filename, result[1].meta.filename) + for dirname in [output_dir, tmp_cwd]: + result_files = glob(os.path.join(dirname, '*pixelreplacestep.fits')) + for file in result_files: + basename = os.path.basename(file) + return_files.append(basename) - assert result[0].meta.filename == 'jwst_nirspec_1_pixelreplacestep.fits' - assert result[1].meta.filename == 'jwst_nirspec_2_pixelreplacestep.fits' + assert expected_name[0] == basename[0][0] + assert expected_name[1] == basename[1][0] result.close() input_model.close() From 21024531d83ab2967cdd58a9a409366c76b29670 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Fri, 6 Dec 2024 10:35:13 -0700 Subject: [PATCH 07/27] update test to check filenames --- jwst/pixel_replace/tests/test_pixel_replace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 47a6daa8d7..171948f3b7 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -286,8 +286,8 @@ def test_pixel_replace_nirspec_ifu_container_names(tmp_cwd, tmp_path, input_mode basename = os.path.basename(file) return_files.append(basename) - assert expected_name[0] == basename[0][0] - assert expected_name[1] == basename[1][0] + assert expected_name[0] == return_files[0] + assert expected_name[1] == return_files[1] result.close() input_model.close() From 2488062961b80d202dfc27110ca7e682b9fb14f1 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Fri, 6 Dec 2024 13:54:36 -0700 Subject: [PATCH 08/27] add another test --- jwst/cube_build/ifu_cube.py | 1 - jwst/pipeline/calwebb_spec3.py | 24 +------------ jwst/pixel_replace/pixel_replace_step.py | 22 +----------- .../pixel_replace/tests/test_pixel_replace.py | 34 +++++++++++++++++++ jwst/stpipe/utilities.py | 21 ++++++++++++ 5 files changed, 57 insertions(+), 45 deletions(-) diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py index db05d49b33..3a75f33381 100644 --- a/jwst/cube_build/ifu_cube.py +++ b/jwst/cube_build/ifu_cube.py @@ -563,7 +563,6 @@ def build_ifucube(self): """ self.output_name = self.define_cubename() - print('ifu cube returend output_name', self.output_name) total_num = self.naxis1 * self.naxis2 * self.naxis3 self.spaxel_flux = np.zeros(total_num, dtype=np.float64) diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 707a67f5c0..6f5665aa0e 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -7,7 +7,7 @@ from jwst.datamodels import SourceModelContainer from jwst.stpipe import query_step_status - +from jwst.stpipe.utilities import invariant_filename from ..associations.lib.rules_level3_base import format_product from ..exp_to_source import multislit_to_container from ..master_background.master_background_step import split_container @@ -350,25 +350,3 @@ def _create_nrsmos_source_id(self, source_models): srcid = f's{str(source_id):>09s}' return srcid - -# ######### -# Utilities -# ######### -def invariant_filename(save_model_func): - """Restore meta.filename after save_model""" - - @wraps(save_model_func) - def save_model(model, **kwargs): - try: - filename = model.meta.filename - except AttributeError: - filename = None - - result = save_model_func(model, **kwargs) - - if filename: - model.meta.filename = filename - - return result - - return save_model diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index 32f5944e56..e55388736d 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -6,7 +6,7 @@ from jwst import datamodels from .pixel_replace import PixelReplacement from functools import wraps -#from jwst.pipeline.calwebb_spec3 import invariant_filename +from jwst.stpipe.utilities import invariant_filename __all__ = ["PixelReplaceStep"] @@ -135,23 +135,3 @@ def process(self, input): record_step_status(replacement.output, 'pixel_replace', success=True) result = replacement.output return result - - -def invariant_filename(save_model_func): - """Restore meta.filename after save_model""" - - @wraps(save_model_func) - def save_model(model, **kwargs): - try: - filename = model.meta.filename - except AttributeError: - filename = None - - result = save_model_func(model, **kwargs) - - if filename: - model.meta.filename = filename - - return result - - return save_model diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 171948f3b7..e18e420e40 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -291,3 +291,37 @@ def test_pixel_replace_nirspec_ifu_container_names(tmp_cwd, tmp_path, input_mode result.close() input_model.close() + + + +@pytest.mark.slow +@pytest.mark.parametrize('input_model_function', + [nirspec_ifu]) +@pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) +def test_pixel_replace_nirspec_ifu_name(tmp_cwd, tmp_path, input_model_function, algorithm): + """ + Test pixel replacement for NIRSpec IFU using a single file + + Larger data and more WCS operations required for testing make + this test take more than a minute, so marking this test 'slow'. + + The test is otherwise the same as for other modes. + """ + output_dir = tmp_path / 'output' + output_dir.mkdir(exist_ok=True) + output_dir = str(output_dir) + + input_model, bad_idx = input_model_function() + input_model.meta.filename = 'jwst_nirspec_cal.fits' + expected_name = 'jwst_nirspec_pixelreplacestep.fits' + + # for this simple case, the results from either algorithm should + # be the same + result = PixelReplaceStep.call(input_model, skip=False, algorithm=algorithm,save_results=True) + + assert expected_name == result.meta.filename + + + result.close() + input_model.close() + diff --git a/jwst/stpipe/utilities.py b/jwst/stpipe/utilities.py index ad62ba1468..fcca990de7 100644 --- a/jwst/stpipe/utilities.py +++ b/jwst/stpipe/utilities.py @@ -30,6 +30,7 @@ Utilities """ import importlib.util +from functools import wraps from importlib import import_module import inspect import logging @@ -211,3 +212,23 @@ def query_step_status(datamodel, cal_step): return getattr(datamodel[0].meta.cal_step, cal_step, NOT_SET) else: return getattr(datamodel.meta.cal_step, cal_step, NOT_SET) + + +def invariant_filename(save_model_func): + """Restore meta.filename after save_model""" + + @wraps(save_model_func) + def save_model(model, **kwargs): + try: + filename = model.meta.filename + except AttributeError: + filename = None + + result = save_model_func(model, **kwargs) + + if filename: + model.meta.filename = filename + + return result + + return save_model From c1c0f19cffdb3dc3ca7e033fc27e984c5dd67d46 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Mon, 9 Dec 2024 09:01:41 -0700 Subject: [PATCH 09/27] update changes --- changes/8979.pixel_replace.rst | 1 + jwst/pixel_replace/pixel_replace.py | 2 -- jwst/pixel_replace/pixel_replace_step.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 changes/8979.pixel_replace.rst diff --git a/changes/8979.pixel_replace.rst b/changes/8979.pixel_replace.rst new file mode 100644 index 0000000000..3c2df1c8c2 --- /dev/null +++ b/changes/8979.pixel_replace.rst @@ -0,0 +1 @@ +fix a bug in pixel_replace and cube_build output filenames diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index b767c48325..a28d209451 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -29,8 +29,6 @@ class PixelReplacement: VERTICAL = 2 LOG_SLICE = ['column', 'row'] - #default_suffix = 'pixrep' - def __init__(self, input_model, **pars): """ Initialize the class with input data model. diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index e55388736d..1124a1d1e7 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -5,7 +5,6 @@ from jwst.stpipe import record_step_status from jwst import datamodels from .pixel_replace import PixelReplacement -from functools import wraps from jwst.stpipe.utilities import invariant_filename __all__ = ["PixelReplaceStep"] From 55177843494728115a39ccad9f3dc388c7a51da5 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Mon, 9 Dec 2024 09:12:01 -0700 Subject: [PATCH 10/27] calwebb_spec3.py --- jwst/pipeline/calwebb_spec3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 6f5665aa0e..39b3b75f95 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -1,6 +1,5 @@ #!/usr/bin/env python from collections import defaultdict -from functools import wraps import os.path as op from stdatamodels.jwst import datamodels From 2c0c42ba21029ac0d68542abb0e254f0d89bd258 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Wed, 11 Dec 2024 11:37:47 -0700 Subject: [PATCH 11/27] fix test --- jwst/pixel_replace/tests/test_pixel_replace.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index e18e420e40..836082538a 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -219,7 +219,8 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): # for this simple case, the results from either algorithm should # be the same - result = PixelReplaceStep.call(input_model, skip=False, algorithm=algorithm,save_results=True) + result = PixelReplaceStep.call(input_model, skip=False, + algorithm=algorithm, save_results=True) assert result.meta.filename == 'jwst_nirspec_pixelreplacestep.fits' @@ -279,12 +280,12 @@ def test_pixel_replace_nirspec_ifu_container_names(tmp_cwd, tmp_path, input_mode return_files = [] # for this simple case, the results from either algorithm should # be the same - result = PixelReplaceStep.call(container, skip=False, algorithm=algorithm,save_results=True) - for dirname in [output_dir, tmp_cwd]: - result_files = glob(os.path.join(dirname, '*pixelreplacestep.fits')) - for file in result_files: - basename = os.path.basename(file) - return_files.append(basename) + result = PixelReplaceStep.call(container, skip=False, algorithm=algorithm, + save_results=True) + result_files = glob(os.path.join(tmp_cwd, '*pixelreplacestep.fits')) + for file in result_files: + basename = os.path.basename(file) + return_files.append(basename) assert expected_name[0] == return_files[0] assert expected_name[1] == return_files[1] @@ -321,7 +322,6 @@ def test_pixel_replace_nirspec_ifu_name(tmp_cwd, tmp_path, input_model_function, assert expected_name == result.meta.filename - result.close() input_model.close() From c384c2607c1767e70f00f3f21da2f825bd934946 Mon Sep 17 00:00:00 2001 From: jemorrison Date: Mon, 16 Dec 2024 11:48:59 -0700 Subject: [PATCH 12/27] added a test for invariant filename --- jwst/stpipe/tests/test_utilities.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/jwst/stpipe/tests/test_utilities.py b/jwst/stpipe/tests/test_utilities.py index 0a1d543d48..8c86beae0b 100644 --- a/jwst/stpipe/tests/test_utilities.py +++ b/jwst/stpipe/tests/test_utilities.py @@ -5,6 +5,10 @@ from jwst.stpipe.utilities import all_steps, NOT_SET import jwst.pipeline import jwst.step +import pytest +from stdatamodels.jwst import datamodels +from jwst.stpipe.utilities import invariant_filename + from jwst import datamodels as dm @@ -54,3 +58,32 @@ def test_record_query_step_status(): # test query not set model3 = dm.MultiSpecModel() assert query_step_status(model3, 'test_step') == NOT_SET + +# make up a datamodel for testing filename +@pytest.fixture(scope='function') +def miri_ifushort(): + """ Generate input model IFU image """ + + + mirifushort_short = { + 'detector': 'MIRIFUSHORT', + 'channel': '12', + 'band': 'SHORT', + 'name': 'MIRI' + + } + input_model = datamodels.IFUImageModel() + input_model.meta.exposure.type = 'MIR_MRS' + input_model.meta.instrument._instance.update(mirifushort_short) + input_model.meta.filename = 'test1.fits' + return input_model + + +def change_name_func(model): + model.meta.filename = "changed" + return model + +def test_invariant_filename(miri_ifushort): + invariant_save_func = invariant_filename(change_name_func) + output_model = invariant_save_func(miri_ifushort) + assert output_model.meta.filename == "test1.fits" From 36c4c252b91585d6dc8feb164acf92986f91e82d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 18 Dec 2024 16:25:46 -0500 Subject: [PATCH 13/27] Clarify invariant_filename test --- jwst/stpipe/tests/test_utilities.py | 25 ++++++++++++++++++++++--- jwst/stpipe/utilities.py | 5 +++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/jwst/stpipe/tests/test_utilities.py b/jwst/stpipe/tests/test_utilities.py index 8c86beae0b..3a5993b0fc 100644 --- a/jwst/stpipe/tests/test_utilities.py +++ b/jwst/stpipe/tests/test_utilities.py @@ -59,6 +59,7 @@ def test_record_query_step_status(): model3 = dm.MultiSpecModel() assert query_step_status(model3, 'test_step') == NOT_SET + # make up a datamodel for testing filename @pytest.fixture(scope='function') def miri_ifushort(): @@ -81,9 +82,27 @@ def miri_ifushort(): def change_name_func(model): model.meta.filename = "changed" + model.meta.cal_step.pixel_replace = "COMPLETE" return model -def test_invariant_filename(miri_ifushort): + +def test_invariant_filename(): + # Make sure the change_name_func changes the name and has side effects + # (here, setting a status variable, but normally, actually saving the file) + input_model = datamodels.IFUImageModel() + input_model.meta.filename = 'test1.fits' + change_name_func(input_model) + assert input_model.meta.filename == 'changed' + assert input_model.meta.cal_step.pixel_replace == 'COMPLETE' + + # When the function is wrapped with invariant_filename, + # the filename is not changed, but the side effect still happens + input_model = datamodels.IFUImageModel() + input_model.meta.filename = 'test2.fits' invariant_save_func = invariant_filename(change_name_func) - output_model = invariant_save_func(miri_ifushort) - assert output_model.meta.filename == "test1.fits" + output_model = invariant_save_func(input_model) + assert output_model.meta.filename == "test2.fits" + assert output_model.meta.cal_step.pixel_replace == 'COMPLETE' + + # The output model is not a copy - the name is reset in place + assert output_model is input_model diff --git a/jwst/stpipe/utilities.py b/jwst/stpipe/utilities.py index fcca990de7..629ac596e2 100644 --- a/jwst/stpipe/utilities.py +++ b/jwst/stpipe/utilities.py @@ -30,13 +30,14 @@ Utilities """ import importlib.util -from functools import wraps -from importlib import import_module import inspect import logging import os import re from collections.abc import Sequence +from functools import wraps +from importlib import import_module + from jwst import datamodels # Configure logging From 7bb1036d66596026e2f94de72ff0800f0aa24ec3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 18 Dec 2024 16:26:19 -0500 Subject: [PATCH 14/27] Add tests for output filename for cube build --- jwst/cube_build/ifu_cube.py | 3 +- jwst/cube_build/tests/test_cube_build_step.py | 38 +++++++++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py index 3a75f33381..26a4713bcc 100644 --- a/jwst/cube_build/ifu_cube.py +++ b/jwst/cube_build/ifu_cube.py @@ -179,10 +179,9 @@ def define_cubename(self): # that the remaining suffixes created below form the entire # list of optical elements in the final output name. suffix = self.output_name_base[self.output_name_base.rfind('_') + 1:] - if suffix in ['clear']: self.output_name_base = self.output_name_base[:self.output_name_base.rfind('_')] - + # Now compose the appropriate list of optical element suffix names # based on MRS channel and sub-channel channels = [] diff --git a/jwst/cube_build/tests/test_cube_build_step.py b/jwst/cube_build/tests/test_cube_build_step.py index d93c11e43d..f2b396c723 100644 --- a/jwst/cube_build/tests/test_cube_build_step.py +++ b/jwst/cube_build/tests/test_cube_build_step.py @@ -3,6 +3,7 @@ """ import numpy as np +import os import pytest from astropy.io import fits @@ -13,6 +14,7 @@ from jwst.cube_build import CubeBuildStep from jwst.cube_build.file_table import ErrorNoAssignWCS from jwst.cube_build.cube_build import ErrorNoChannels +from jwst.datamodels import ModelContainer @pytest.fixture(scope='module') @@ -158,7 +160,7 @@ def nirspec_data(): image.meta.instrument.name = 'NIRSPEC' image.meta.instrument.detector = 'NRS1' image.meta.exposure.type = 'NRS_IFU' - image.meta.filename = 'test_nirspec.fits' + image.meta.filename = 'test_nirspec_cal.fits' image.meta.observation.date = '2023-10-06' image.meta.observation.time = '00:00:00.000' # below values taken from regtest using file @@ -182,7 +184,7 @@ def nirspec_data(): @pytest.mark.parametrize("as_filename", [True, False]) def test_call_cube_build_nirspec(tmp_cwd, nirspec_data, tmp_path, as_filename): if as_filename: - fn = tmp_path / 'test_nirspec.fits' + fn = tmp_path / 'test_nirspec_cal.fits' nirspec_data.save(fn) step_input = fn else: @@ -190,4 +192,34 @@ def test_call_cube_build_nirspec(tmp_cwd, nirspec_data, tmp_path, as_filename): step = CubeBuildStep() step.channel = '1' step.coord_system = 'internal_cal' - step.run(step_input) + step.save_results = True + result = step.run(step_input) + + assert isinstance(result, ModelContainer) + assert len(result) == 1 + model = result[0] + assert model.meta.cal_step.cube_build == 'COMPLETE' + assert model.meta.filename == 'test_nirspec_g395h-f290lp_internal_s3d.fits' + assert os.path.isfile(model.meta.filename) + + +@pytest.mark.parametrize("as_filename", [True, False]) +def test_call_cube_build_nirspec_multi(tmp_cwd, nirspec_data, tmp_path, as_filename): + if as_filename: + fn = tmp_path / 'test_nirspec_cal.fits' + nirspec_data.save(fn) + step_input = fn + else: + step_input = nirspec_data + step = CubeBuildStep() + step.channel = '1' + step.coord_system = 'internal_cal' + step.save_results = True + step.output_type = 'multi' + result = step.run(step_input) + + assert isinstance(result, ModelContainer) + assert len(result) == 1 + model = result[0] + assert model.meta.cal_step.cube_build == 'COMPLETE' + assert model.meta.filename == 'test_nirspec_s3d.fits' From ff95e7b3788cf85839214aa6eaa4ff756eaa3852 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 18 Dec 2024 16:27:16 -0500 Subject: [PATCH 15/27] Restore pixel_replace output, set output_use_model default --- jwst/pixel_replace/pixel_replace_step.py | 33 +++---- .../pixel_replace/tests/test_pixel_replace.py | 94 +++++-------------- 2 files changed, 36 insertions(+), 91 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index 1124a1d1e7..1df5938f6c 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -1,11 +1,8 @@ -#! /usr/bin/env python from functools import partial -from ..stpipe import Step -from jwst.stpipe import record_step_status from jwst import datamodels -from .pixel_replace import PixelReplacement -from jwst.stpipe.utilities import invariant_filename +from jwst.pixel_replace.pixel_replace import PixelReplacement +from jwst.stpipe import record_step_status, Step __all__ = ["PixelReplaceStep"] @@ -34,6 +31,7 @@ class PixelReplaceStep(Step): algorithm = option("fit_profile", "mingrad", "N/A", default="fit_profile") n_adjacent_cols = integer(default=3) # Number of adjacent columns to use in creation of profile skip = boolean(default=True) # Step must be turned on by parameter reference or user + output_use_model = boolean(default=True) # Use input filenames in the output models """ def process(self, input): @@ -55,7 +53,7 @@ def process(self, input): if isinstance(input_model, (datamodels.MultiSlitModel, datamodels.SlitModel, - datamodels.ImageModel, + datamodels.ImageModel, datamodels.IFUImageModel, datamodels.CubeModel)): self.log.debug(f'Input is a {input_model.meta.model_type}.') @@ -77,10 +75,10 @@ def process(self, input): # calewbb_spec3 case / ModelContainer # __________________________________ if isinstance(input_model, datamodels.ModelContainer): - # Setup output path naming if associations are involved. - # if input is ModelContainer do not copy the input_model - # because assocation information is not copied. - # Instead update input_modelin place. + output_model = input_model + + # Setup output path naming if associations are involved, to + # include the ASN ID in the output asn_id = None try: asn_id = input_model.asn_table["asn_id"] @@ -88,12 +86,6 @@ def process(self, input): pass if asn_id is None: asn_id = self.search_attr('asn_id') - if asn_id is None: # it is still None. It does not exist. A ModelContainer was passed in with - # no asn information - # to create the correct output name set the following. - self.output_use_model = True - self.save_model = invariant_filename(self.save_model) - if asn_id is not None: _make_output_path = self.search_attr( '_make_output_path', parent_first=True @@ -104,7 +96,7 @@ def process(self, input): ) # Check models to confirm they are the correct type - for i, model in enumerate(input_model): + for i, model in enumerate(output_model): run_pixel_replace = True if model.meta.model_type in ['MultiSlitModel', 'SlitModel', 'ImageModel', 'IFUImageModel', 'CubeModel']: @@ -120,9 +112,10 @@ def process(self, input): if run_pixel_replace: replacement = PixelReplacement(model, **pars) replacement.replace() - model = replacement.output - record_step_status(model, 'pixel_replace', success=True) - return input_model + output_model[i] = replacement.output + record_step_status(output_model[i], 'pixel_replace', success=True) + + return output_model # ________________________________________ # calewbb_spec2 case - single input model # ________________________________________ diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 836082538a..92ec146967 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -11,6 +11,7 @@ from jwst.pixel_replace.pixel_replace_step import PixelReplaceStep from glob import glob + def cal_data(shape, bad_idx, dispaxis=1, model='slit'): if model == 'image': model = datamodels.ImageModel(shape) @@ -202,10 +203,9 @@ def test_pixel_replace_multislit(input_model_function, algorithm): @pytest.mark.slow -@pytest.mark.parametrize('input_model_function', - [nirspec_ifu]) +@pytest.mark.parametrize('input_model_function', [nirspec_ifu]) @pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) -def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): +def test_pixel_replace_nirspec_ifu(tmp_cwd, input_model_function, algorithm): """ Test pixel replacement for NIRSpec IFU. @@ -223,7 +223,9 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): algorithm=algorithm, save_results=True) assert result.meta.filename == 'jwst_nirspec_pixelreplacestep.fits' - + assert result.meta.cal_step.pixel_replace == 'COMPLETE' + assert os.path.isfile(result.meta.filename) + for ext in ['data', 'err', 'var_poisson', 'var_rnoise', 'var_flat']: # non-science edges are uncorrected assert np.all(np.isnan(getattr(result, ext)[..., :, 1])) @@ -247,81 +249,31 @@ def test_pixel_replace_nirspec_ifu(input_model_function, algorithm): input_model.close() - - -@pytest.mark.slow -@pytest.mark.parametrize('input_model_function', - [nirspec_ifu]) -@pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) -def test_pixel_replace_nirspec_ifu_container_names(tmp_cwd, tmp_path, input_model_function, algorithm): - """ - Test pixel replacement for NIRSpec IFU using a container - - Larger data and more WCS operations required for testing make - this test take more than a minute, so marking this test 'slow'. - - The test is otherwise the same as for other modes. - """ - output_dir = tmp_path / 'output' - output_dir.mkdir(exist_ok=True) - output_dir = str(output_dir) - - input_model, bad_idx = input_model_function() +@pytest.mark.parametrize('input_model_function', [nirspec_fs_slitmodel]) +def test_pixel_replace_container_names(tmp_cwd, input_model_function): + """Test pixel replace output names for input container.""" + input_model, _ = input_model_function() input_model.meta.filename = 'jwst_nirspec_1_cal.fits' - input_model2, bad_idx2 = input_model_function() + input_model2, _ = input_model_function() input_model2.meta.filename = 'jwst_nirspec_2_cal.fits' cfiles = [input_model, input_model2] container = ModelContainer(cfiles) - expected_name = [] - expected_name.append('jwst_nirspec_1_pixelreplacestep.fits') - expected_name.append('jwst_nirspec_2_pixelreplacestep.fits') + expected_name = ['jwst_nirspec_1_pixelreplacestep.fits', + 'jwst_nirspec_2_pixelreplacestep.fits'] + + result = PixelReplaceStep.call(container, skip=False, save_results=True) + for i, model in enumerate(result): + assert model.meta.filename == expected_name[i] + assert model.meta.cal_step.pixel_replace == 'COMPLETE' - return_files = [] - # for this simple case, the results from either algorithm should - # be the same - result = PixelReplaceStep.call(container, skip=False, algorithm=algorithm, - save_results=True) result_files = glob(os.path.join(tmp_cwd, '*pixelreplacestep.fits')) - for file in result_files: + for i, file in enumerate(result_files): basename = os.path.basename(file) - return_files.append(basename) - - assert expected_name[0] == return_files[0] - assert expected_name[1] == return_files[1] - - result.close() - input_model.close() - - - -@pytest.mark.slow -@pytest.mark.parametrize('input_model_function', - [nirspec_ifu]) -@pytest.mark.parametrize('algorithm', ['fit_profile', 'mingrad']) -def test_pixel_replace_nirspec_ifu_name(tmp_cwd, tmp_path, input_model_function, algorithm): - """ - Test pixel replacement for NIRSpec IFU using a single file - - Larger data and more WCS operations required for testing make - this test take more than a minute, so marking this test 'slow'. - - The test is otherwise the same as for other modes. - """ - output_dir = tmp_path / 'output' - output_dir.mkdir(exist_ok=True) - output_dir = str(output_dir) - - input_model, bad_idx = input_model_function() - input_model.meta.filename = 'jwst_nirspec_cal.fits' - expected_name = 'jwst_nirspec_pixelreplacestep.fits' - - # for this simple case, the results from either algorithm should - # be the same - result = PixelReplaceStep.call(input_model, skip=False, algorithm=algorithm,save_results=True) - - assert expected_name == result.meta.filename + assert expected_name[i] == basename + with datamodels.open(file) as model: + assert model.meta.cal_step.pixel_replace == 'COMPLETE' + assert model.meta.filename == expected_name[i] result.close() input_model.close() - From 112c6fbca4ede7095ddb4e4513cc6ab40d174475 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 18 Dec 2024 16:40:48 -0500 Subject: [PATCH 16/27] Update change note --- changes/8979.pixel_replace.rst | 1 - changes/9019.pixel_replace.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changes/8979.pixel_replace.rst create mode 100644 changes/9019.pixel_replace.rst diff --git a/changes/8979.pixel_replace.rst b/changes/8979.pixel_replace.rst deleted file mode 100644 index 3c2df1c8c2..0000000000 --- a/changes/8979.pixel_replace.rst +++ /dev/null @@ -1 +0,0 @@ -fix a bug in pixel_replace and cube_build output filenames diff --git a/changes/9019.pixel_replace.rst b/changes/9019.pixel_replace.rst new file mode 100644 index 0000000000..9c961ad048 --- /dev/null +++ b/changes/9019.pixel_replace.rst @@ -0,0 +1 @@ +Base output filenames on input filenames when the step is called outside the pipeline, in order to create sensible names when the input is a list of models. From 8f2b88d01dea04e873970131c0440cbd12f66dc6 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 19 Dec 2024 09:15:00 -0500 Subject: [PATCH 17/27] Fix filename order in test --- jwst/pixel_replace/tests/test_pixel_replace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/pixel_replace/tests/test_pixel_replace.py b/jwst/pixel_replace/tests/test_pixel_replace.py index 92ec146967..dcf8c96fa5 100644 --- a/jwst/pixel_replace/tests/test_pixel_replace.py +++ b/jwst/pixel_replace/tests/test_pixel_replace.py @@ -268,7 +268,7 @@ def test_pixel_replace_container_names(tmp_cwd, input_model_function): assert model.meta.cal_step.pixel_replace == 'COMPLETE' result_files = glob(os.path.join(tmp_cwd, '*pixelreplacestep.fits')) - for i, file in enumerate(result_files): + for i, file in enumerate(sorted(result_files)): basename = os.path.basename(file) assert expected_name[i] == basename with datamodels.open(file) as model: From 1834ec09f287fef20e283e5c464bc5231f380e88 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 19 Dec 2024 09:35:23 -0500 Subject: [PATCH 18/27] Fix typos --- jwst/pixel_replace/pixel_replace_step.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index 1df5938f6c..2bfe166c0e 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -71,14 +71,12 @@ def process(self, input): 'n_adjacent_cols': self.n_adjacent_cols, } - # ___________________________________ - # calewbb_spec3 case / ModelContainer - # __________________________________ + # calwebb_spec3 case / ModelContainer if isinstance(input_model, datamodels.ModelContainer): output_model = input_model - # Setup output path naming if associations are involved, to - # include the ASN ID in the output + # Set up output path name to include the ASN ID + # if associations are involved asn_id = None try: asn_id = input_model.asn_table["asn_id"] @@ -116,9 +114,8 @@ def process(self, input): record_step_status(output_model[i], 'pixel_replace', success=True) return output_model - # ________________________________________ - # calewbb_spec2 case - single input model - # ________________________________________ + + # calwebb_spec2 case / single input model else: # Make copy of input to prevent overwriting result = input_model.copy() From 095d364335268710228e5ad8bb39a2a8512190d8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 19 Dec 2024 09:39:40 -0500 Subject: [PATCH 19/27] Remove unused test fixture --- jwst/stpipe/tests/test_utilities.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/jwst/stpipe/tests/test_utilities.py b/jwst/stpipe/tests/test_utilities.py index 3a5993b0fc..145440c6f9 100644 --- a/jwst/stpipe/tests/test_utilities.py +++ b/jwst/stpipe/tests/test_utilities.py @@ -5,7 +5,6 @@ from jwst.stpipe.utilities import all_steps, NOT_SET import jwst.pipeline import jwst.step -import pytest from stdatamodels.jwst import datamodels from jwst.stpipe.utilities import invariant_filename @@ -60,26 +59,6 @@ def test_record_query_step_status(): assert query_step_status(model3, 'test_step') == NOT_SET -# make up a datamodel for testing filename -@pytest.fixture(scope='function') -def miri_ifushort(): - """ Generate input model IFU image """ - - - mirifushort_short = { - 'detector': 'MIRIFUSHORT', - 'channel': '12', - 'band': 'SHORT', - 'name': 'MIRI' - - } - input_model = datamodels.IFUImageModel() - input_model.meta.exposure.type = 'MIR_MRS' - input_model.meta.instrument._instance.update(mirifushort_short) - input_model.meta.filename = 'test1.fits' - return input_model - - def change_name_func(model): model.meta.filename = "changed" model.meta.cal_step.pixel_replace = "COMPLETE" From dfd1fc7f2f1a61028e623e0f29e6894e530df2cd Mon Sep 17 00:00:00 2001 From: Maria Date: Fri, 20 Dec 2024 15:51:31 -0500 Subject: [PATCH 20/27] JP-3622 Update refpix step for NIR detectors to use convolution kernel (#8726) Co-authored-by: David Law Co-authored-by: Melanie Clarke Co-authored-by: Tyler Pauly --- changes/8726.refpix.rst | 1 + docs/jwst/refpix/arguments.rst | 22 +++ docs/jwst/refpix/description.rst | 12 ++ jwst/refpix/optimized_convolution.py | 180 ++++++++++++++++++ jwst/refpix/reference_pixels.py | 136 +++++++++---- jwst/refpix/refpix_step.py | 35 +++- .../tests/test_optimized_convolution.py | 65 +++++++ jwst/refpix/tests/test_refpix.py | 76 +++++--- jwst/regtest/test_nircam_image.py | 32 ++++ 9 files changed, 493 insertions(+), 66 deletions(-) create mode 100644 changes/8726.refpix.rst create mode 100644 jwst/refpix/optimized_convolution.py create mode 100644 jwst/refpix/tests/test_optimized_convolution.py diff --git a/changes/8726.refpix.rst b/changes/8726.refpix.rst new file mode 100644 index 0000000000..0639bf5208 --- /dev/null +++ b/changes/8726.refpix.rst @@ -0,0 +1 @@ +Implemented SIRS algorithm instead of running median for side pixels of NIR full-frame data. Running median is still default. diff --git a/docs/jwst/refpix/arguments.rst b/docs/jwst/refpix/arguments.rst index 59126c2db4..a5c92404e9 100644 --- a/docs/jwst/refpix/arguments.rst +++ b/docs/jwst/refpix/arguments.rst @@ -52,3 +52,25 @@ in IRS2 mode will be processed along with the normal pixels and preserved in the output. This option is intended for calibration or diagnostic reductions only. For normal science operation, this argument should always be False, so that interleaved pixels are stripped before continuing processing. + +* ``--refpix_algorithm`` + +The ``refpix_algorithm`` argument is only relevant for all NIR full-frame +data, and can be set to 'median' (default) to use the running median or +'sirs' to use the Simple Improved Reference Subtraction (SIRS). + +* ``--sigreject`` + +The ``sigreject`` argument is the number of sigmas to reject as outliers in the +SIRS algorithm. The value is expected to be a float. + +* ``--gaussmooth`` + +The ``gaussmooth`` argument is the width of Gaussian smoothing kernel to use as +a low-pass filter. The numerical value is expected to be a float. + +* ``--halfwidth`` + +The ``halfwidth`` argument is the half-width of convolution kernel to build. The +numerical value is expected to be an integer. + diff --git a/docs/jwst/refpix/description.rst b/docs/jwst/refpix/description.rst index f5fb2d8701..0dd86b0177 100644 --- a/docs/jwst/refpix/description.rst +++ b/docs/jwst/refpix/description.rst @@ -76,6 +76,18 @@ NIR Detector Data (set by the step parameter ``side_gain``, which defaults to 1.0) is subtracted from the full group on a row-by-row basis. Note that the ``odd_even_rows`` parameter is ignored for NIR data when the side reference pixels are processed. + If the ``--refpix_algorithm`` option is set to 'sirs', the Simple Improved + Reference Subtraction (SIRS) method will be used instead of the running median. + The SIRS revision uses the left and right side reference pixels as described + in https://doi.org/10.1117/1.JATIS.8.2.028002. This implementation uses a + mathematically equivalent formulation using convolution kernels rather than + Fourier transforms, with the convolution kernel truncated where the weights + approach zero. There are two convolution kernels for each readout channel, + one for each of the left and right reference pixels. These kernels are the + Fourier transforms of the weight coefficients (further description in the paper). + The approach implemented here makes nearly optimal use of the reference pixels. + It reduces read noise by 5-10% relative to a running median filter of the + reference pixels, and reduces 1/f "striping" noise by a larger factor than this. #. Transform the data back to the JWST focal plane, or DMS, frame. MIR Detector Data diff --git a/jwst/refpix/optimized_convolution.py b/jwst/refpix/optimized_convolution.py new file mode 100644 index 0000000000..02872114b7 --- /dev/null +++ b/jwst/refpix/optimized_convolution.py @@ -0,0 +1,180 @@ +# +# Module for using the Simple Improved Reference Subtraction (SIRS) algorithm +# to improve the 1/f noise, only for full frame non-IRS2 NIR data +# + +import logging +import numpy as np + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +def make_kernels(sirs_kernel_model, detector, gaussmooth, halfwidth): + """ + Make convolution kernels from Fourier coefficients in the reference file. + + Parameters: + ----------- + + sirs_kernel_model : `~jwst.datamodels.SIRSKernelModel` + Data model containing the Fourier coefficients from the reference files for + Simple Improved Reference Subtraction (SIRS) + + detector : str + Name of the detector of the input data + + gaussmooth : float + Width of Gaussian smoothing kernel to use as a low-pass filter on reference file's coefficients + + halfwidth : int + Half-width of convolution kernel to build from reference file's coefficients + + Returns: + -------- + kernels: list + List of kernels appropriate for convolution with the left and right reference pixels. + + """ + + gamma, zeta = get_conv_kernel_coeffs(sirs_kernel_model, detector) + if gamma is None or zeta is None: + log.info(f'Optimized convolution kernel coefficients NOT found for detector {detector}') + return None + + kernels_left = [] + kernels_right = [] + for chan in range(gamma.shape[0]): + n = len(gamma[chan]) - 1 + kernel_left = np.fft.fftshift(np.fft.irfft(gamma[chan]))[n - halfwidth:n + halfwidth + 1] + kernel_right = np.fft.fftshift(np.fft.irfft(zeta[chan]))[n - halfwidth:n + halfwidth + 1] + + x = np.arange(-halfwidth, halfwidth + 1) + window = np.exp(-x ** 2 / (2 * gaussmooth ** 2)) + window /= np.sum(window) + + kernel_right = np.convolve(kernel_right, window, mode='same') + kernel_left = np.convolve(kernel_left, window, mode='same') + + kernels_right += [kernel_right] + kernels_left += [kernel_left] + + return [kernels_left, kernels_right] + + +def get_conv_kernel_coeffs(sirs_kernel_model, detector): + """ + Get the convolution kernels coefficients from the reference file + + Parameters: + ----------- + + sirs_kernel_model : `~jwst.datamodels.SIRSKernelModel` + Data model containing the Fourier coefficients from the reference files for + Simple Improved Reference Subtraction (SIRS) + + detector : str + Name of the detector of the input data + + Returns: + -------- + + gamma: numpy array + Fourier coefficients + + zeta: numpy array + Fourier coefficients + """ + mdl_dict = sirs_kernel_model.to_flat_dict() + gamma, zeta = None, None + for item in mdl_dict: + det = item.split(sep='.')[0] + if detector.lower() == det.lower(): + arr_name = item.split(sep='.')[1] + if arr_name == 'gamma': + gamma = np.array(mdl_dict[item]) + elif arr_name == 'zeta': + zeta = np.array(mdl_dict[item]) + if gamma is not None and zeta is not None: + break + return gamma, zeta + + +def apply_conv_kernel(data, kernels, sigreject=4.0): + """ + Apply the convolution kernel. + + Parameters: + ----------- + + data : 2-D numpy array + Data to be corrected + + kernels : list + List containing the left and right kernels + + sigreject: float + Number of sigmas to reject as outliers + + Returns: + -------- + + data : 2-D numpy array + Data model with convolution + """ + data = data.astype(float) + npix = data.shape[-1] + + kernels_l, kernels_r = kernels + nchan = len(kernels_l) + + L = data[:, :4] + R = data[:, -4:] + + # Find the approximate standard deviations of the reference pixels + # using an outlier-robust median approach. Mask pixels that differ + # by more than sigreject sigma from this level. + # NOTE: The Median Absolute Deviation (MAD) is calculated as the + # median of the absolute differences between data values and their + # median. For normal distribution MAD is equal to 1.48 times the + # standard deviation but is a more robust estimate of the dispersion + # of data values.The calculation of MAD is straightforward but + # time-consuming, especially if MAD estimates are needed for the + # local environment around every pixel of a large image. The + # calculation is MAD = np.median(np.abs(x-np.median(x))). + # Reference: https://www.interstellarmedium.org/numerical_tools/mad/ + MAD = 1.48 + medL = np.median(L) + sigL = MAD * np.median(np.abs(L - medL)) + medR = np.median(R) + sigR = MAD * np.median(np.abs(R - medR)) + + # nL and nR are the number of good reference pixels in the left and right + # channel in each row. These will be used in lieu of replacing the values + # of those pixels directly. + goodL = 1 * (np.abs(L - medL) <= sigreject * sigL) + nL = np.sum(goodL, axis=1) + goodR = 1 * (np.abs(R - medR) <= sigreject * sigR) + nR = np.sum(goodR, axis=1) + + # Average of the left and right channels, replacing masked pixels with zeros. + # Appropriate normalization factors will be computed later. + L = np.sum(L * goodL, axis=1) / 4 + R = np.sum(R * goodR, axis=1) / 4 + for chan in range(nchan): + kernel_l = kernels_l[chan] + kernel_r = kernels_r[chan] + + # Compute normalizations so that we don't have to directly + # replace the values of flagged/masked reference pixels. + normL = np.convolve(np.ones(nL.shape), kernel_l, mode='same') + normL /= np.convolve(nL / 4, kernel_l, mode='same') + normR = np.convolve(np.ones(nR.shape), kernel_r, mode='same') + normR /= np.convolve(nR / 4, kernel_r, mode='same') + template = np.convolve(L, kernel_l, mode='same') * normL + template += np.convolve(R, kernel_r, mode='same') * normR + data[:, chan * npix * 3 // 4:(chan + 1) * npix * 3 // 4] -= template[:, np.newaxis] + + log.debug('Optimized convolution kernel applied') + return data + diff --git a/jwst/refpix/reference_pixels.py b/jwst/refpix/reference_pixels.py index 734a1e33bd..7353af451a 100644 --- a/jwst/refpix/reference_pixels.py +++ b/jwst/refpix/reference_pixels.py @@ -51,6 +51,7 @@ from ..lib import pipe_utils, reffile_utils from .irs2_subtract_reference import make_irs2_mask +from .optimized_convolution import make_kernels, apply_conv_kernel log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -134,7 +135,7 @@ SUBARRAY_SKIPPED = 3 -class Dataset(): +class Dataset: """Base Class to handle passing stuff from routine to routine Parameters: @@ -167,6 +168,10 @@ class Dataset(): flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + for Simple Improved Reference Subtraction (SIRS) + """ def __init__(self, input_model, @@ -174,8 +179,15 @@ def __init__(self, input_model, use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows): + self.refpix_algorithm = conv_kernel_params['refpix_algorithm'] + self.sirs_kernel_model = conv_kernel_params['sirs_kernel_model'] + self.sigreject = conv_kernel_params['sigreject'] + self.gaussmooth = conv_kernel_params['gaussmooth'] + self.halfwidth = conv_kernel_params['halfwidth'] + if (input_model.meta.subarray.xstart is None or input_model.meta.subarray.ystart is None or input_model.meta.subarray.xsize is None or @@ -373,12 +385,17 @@ def log_parameters(self): if not self.is_subarray: log.info('NIR full frame data') log.info('The following parameters are valid for this mode:') - log.info(f'use_side_ref_pixels = {self.use_side_ref_pixels}') - log.info(f'odd_even_columns = {self.odd_even_columns}') - log.info(f'side_smoothing_length = {self.side_smoothing_length}') - log.info(f'side_gain = {self.side_gain}') - log.info('The following parameter is not applicable and is ignored:') - log.info(f'odd_even_rows = {self.odd_even_rows}') + if self.refpix_algorithm == 'median': + log.info(f'use_side_ref_pixels = {self.use_side_ref_pixels}') + log.info(f'odd_even_columns = {self.odd_even_columns}') + log.info(f'side_smoothing_length = {self.side_smoothing_length}') + log.info(f'side_gain = {self.side_gain}') + log.info('The following parameter is not applicable and is ignored:') + log.info(f'odd_even_rows = {self.odd_even_rows}') + elif self.refpix_algorithm == 'sirs': + log.info(f'sigreject = {self.sigreject}') + log.info(f'gaussmooth = {self.gaussmooth}') + log.info(f'halfwidth = {self.halfwidth}') else: log.info('NIR subarray data') # Transform the pixeldq array from DMS to detector coords @@ -491,19 +508,24 @@ class NIRDataset(Dataset): side_gain: float gain to use in applying the side reference pixel correction + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + """ def __init__(self, input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain): + side_gain, + conv_kernel_params): super(NIRDataset, self).__init__(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows=False) # Set appropriate NIR sections @@ -1149,10 +1171,28 @@ def do_side_correction(self, group): """ - left = self.calculate_side_ref_signal(group, 0, 3) - right = self.calculate_side_ref_signal(group, 2044, 2047) - sidegroup = self.combine_ref_signals(left, right) - corrected_group = self.apply_side_correction(group, sidegroup) + continue_apply_conv_kernel = False + # Check if convolution kernels for this detector are in the reference file + # and if not, proceed with side-pixel correction as usual + if self.refpix_algorithm == 'sirs' and self.sirs_kernel_model is not None: + kernels = make_kernels(self.sirs_kernel_model, + self.input_model.meta.instrument.detector, + self.gaussmooth, + self.halfwidth) + if kernels is None: + log.info('The REFPIX step will use the running median') + else: + continue_apply_conv_kernel = True + # + # Apply optimized convolution kernel + if continue_apply_conv_kernel: + corrected_group = apply_conv_kernel(group, kernels, sigreject=self.sigreject) + else: + # use running median + left = self.calculate_side_ref_signal(group, 0, 3) + right = self.calculate_side_ref_signal(group, 2044, 2047) + sidegroup = self.combine_ref_signals(left, right) + corrected_group = self.apply_side_correction(group, sidegroup) return corrected_group def do_corrections(self): @@ -1168,6 +1208,7 @@ def do_fullframe_corrections(self): """Do Reference Pixels Corrections for all amplifiers, NIR detectors First read of each integration is NOT subtracted, as the signal is removed in the superbias subtraction step""" + # # First transform pixeldq array to detector coordinates self.DMS_to_detector_dq() @@ -1578,13 +1619,15 @@ class MIRIDataset(Dataset): """ def __init__(self, input_model, - odd_even_rows): + odd_even_rows, + conv_kernel_params): super(MIRIDataset, self).__init__(input_model, odd_even_columns=False, use_side_ref_pixels=False, side_smoothing_length=False, side_gain=False, + conv_kernel_params=conv_kernel_params, odd_even_rows=odd_even_rows) self.reference_sections = MIR_reference_sections @@ -1923,7 +1966,8 @@ def create_dataset(input_model, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows): + odd_even_rows, + conv_kernel_params): """Create a dataset object from an input model. Parameters: @@ -1952,6 +1996,8 @@ def create_dataset(input_model, flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel """ detector = input_model.meta.instrument.detector @@ -1966,110 +2012,128 @@ def create_dataset(input_model, if detector[:3] == 'MIR': return MIRIDataset(input_model, - odd_even_rows) + odd_even_rows, + conv_kernel_params) elif detector == 'NRS1': return NRS1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRS2': return NRS2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA1': return NRCA1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA2': return NRCA2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA3': return NRCA3Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCA4': return NRCA4Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCALONG': return NRCALONGDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB1': return NRCB1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB2': return NRCB2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB3': return NRCB3Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCB4': return NRCB4Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NRCBLONG': return NRCBLONGDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'NIS': return NIRISSDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'GUIDER1': return GUIDER1Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) elif detector == 'GUIDER2': return GUIDER2Dataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) else: log.error('Unrecognized detector') return NIRDataset(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) def correct_model(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows): + odd_even_rows, + conv_kernel_params): """Wrapper to do Reference Pixel Correction on a JWST Model. Performs the correction on the datamodel @@ -2099,6 +2163,9 @@ def correct_model(input_model, odd_even_columns, flag that controls whether odd and even-numbered rows are handled separately (MIR only) + conv_kernel_params : dict + Dictionary containing the parameters needed for the optimized convolution kernel + """ if input_model.meta.instrument.name == 'MIRI': if reffile_utils.is_subarray(input_model): @@ -2110,7 +2177,8 @@ def correct_model(input_model, odd_even_columns, use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) if input_dataset is None: status = SUBARRAY_DOESNTFIT @@ -2138,10 +2206,10 @@ def reference_pixel_correction(input_dataset): Corrected dataset """ - input_dataset.do_corrections() if input_dataset.input_model.meta.exposure.zero_frame: + log.info('Processing the zero frame') process_zeroframe_correction(input_dataset) return diff --git a/jwst/refpix/refpix_step.py b/jwst/refpix/refpix_step.py index e7000ba610..bdcccfde53 100644 --- a/jwst/refpix/refpix_step.py +++ b/jwst/refpix/refpix_step.py @@ -25,12 +25,24 @@ class RefPixStep(Step): ovr_corr_mitigation_ftr = float(default=3.0) # Factor to avoid overcorrection of bad reference pixels for IRS2 preserve_irs2_refpix = boolean(default=False) # Preserve reference pixels in output irs2_mean_subtraction = boolean(default=False) # Apply a mean offset subtraction before IRS2 correction + refpix_algorithm = option("median", "sirs", default="median") # NIR full-frame side pixel algorithm + sigreject = float(default=4.0) # Number of sigmas to reject as outliers + gaussmooth = float(default=1.0) # Width of Gaussian smoothing kernel to use as a low-pass filter + halfwidth = integer(default=30) # Half-width of convolution kernel to build """ reference_file_types = ['refpix'] def process(self, step_input): + conv_kernel_params = { + 'refpix_algorithm': self.refpix_algorithm, + 'sirs_kernel_model': None, + 'sigreject': self.sigreject, + 'gaussmooth': self.gaussmooth, + 'halfwidth': self.halfwidth + } + # Open the input data model with datamodels.RampModel(step_input) as input_model: @@ -51,7 +63,8 @@ def process(self, step_input): self.use_side_ref_pixels = False reference_pixels.correct_model( result, self.odd_even_columns, self.use_side_ref_pixels, - self.side_smoothing_length, self.side_gain, self.odd_even_rows) + self.side_smoothing_length, self.side_gain, self.odd_even_rows, + conv_kernel_params) # Now that values are updated, replace bad reference pixels irs2_subtract_reference.flag_bad_refpix(result, replace_only=True) @@ -81,12 +94,30 @@ def process(self, step_input): else: # Not an NRS IRS2 exposure. Do the normal refpix correction. + + # Get the reference file from CRDS or use user-supplied one only for NIR full-frame data + if self.refpix_algorithm == 'sirs': + if input_model.meta.instrument.name != 'MIRI' and 'FULL' in input_model.meta.subarray.name: + sirs_ref_filename = self.get_reference_file(result, 'sirskernel') + if sirs_ref_filename == 'N/A': + self.log.warning('No reference file found for the optimized convolution kernel.') + self.log.warning('REFPIX step will use the running median algorithm for side pixels.') + else: + self.log.info('Using SIRS reference file: {}'.format(sirs_ref_filename)) + sirs_kernel_model = datamodels.SIRSKernelModel(sirs_ref_filename) + conv_kernel_params['refpix_algorithm'] = sirs_kernel_model + elif input_model.meta.instrument.name == 'MIRI': + self.log.info('Simple Improved Reference Subtraction (SIRS) not applied for MIRI data.') + elif 'FULL' not in input_model.meta.subarray.name: + self.log.info('Simple Improved Reference Subtraction (SIRS) not applied for subarray data.') + status = reference_pixels.correct_model(result, self.odd_even_columns, self.use_side_ref_pixels, self.side_smoothing_length, self.side_gain, - self.odd_even_rows) + self.odd_even_rows, + conv_kernel_params) if status == reference_pixels.REFPIX_OK: result.meta.cal_step.refpix = 'COMPLETE' diff --git a/jwst/refpix/tests/test_optimized_convolution.py b/jwst/refpix/tests/test_optimized_convolution.py new file mode 100644 index 0000000000..9129f62e7e --- /dev/null +++ b/jwst/refpix/tests/test_optimized_convolution.py @@ -0,0 +1,65 @@ +import numpy as np +from stdatamodels.jwst.datamodels import RampModel, SIRSKernelModel +from jwst.refpix.optimized_convolution import make_kernels, get_conv_kernel_coeffs, apply_conv_kernel + + +# create the ConvKernelModel +ckm = {'nrcb1': { + 'gamma': np.array([[0.8737859+0.j, 0.72877103-0.01848215j, 0.7474708+0.00441926j, + 0.7596158-0.01682704j, 0.7710808-0.00618939j], + [0.37835783+0.j, 0.27234325-0.03058944j, 0.38302818+0.03056235j, + 0.36819065-0.02578794j, 0.3908449+0.02115744j], + [0.36443716+0.j, 0.335223+0.02436169j, 0.32699308-0.02325623j, + 0.3830375-0.01340938j, 0.39612782+0.00736016j], + [0.00335188+0.j, 0.01759672-0.01073076j, 0.04302938+0.00353758j, + 0.08149841-0.00643084j, 0.07274915-0.002046j]]), + 'zeta': np.array([[0.14007446+0.0000000e+00j, 0.2371146+1.6455967e-02j, + 0.22727127-5.9413449e-03j, 0.2090475+7.0676603e-03j, + 0.20298977+2.2992526e-05j], + [0.6206608+0.j, 0.680701+0.02468053j, 0.57776874-0.03374288j, + 0.5873975+0.01647749j, 0.5693782-0.02531039j], + [0.6543285+0.j, 0.6167225-0.02665404j, 0.6405862+0.01494319j, + 0.57719606+0.00970044j, 0.57160926-0.01088286j], + [1.0137521+0.j, 0.9492664+0.0071805j, 0.92866725-0.00784425j, + 0.8868761-0.00237024j, 0.89918566-0.00323711j]]) + } + } +conv_kernel_model = SIRSKernelModel(ckm) + + +def mk_data_mdl(data, instrument, detector): + # create input_model + input_model = RampModel(data=data) + input_model.meta.instrument.name = instrument + input_model.meta.instrument.detector = detector + input_model.meta.subarray.name = 'FULL' + return input_model + + +def test_get_conv_kernel_coeffs(): + detector = 'NRCB1' + gamma, zeta = get_conv_kernel_coeffs(conv_kernel_model, detector) + assert gamma is not None + assert zeta is not None + + +def test_mk_kernels(): + detector = 'nothing' + gaussmooth = 1 + halfwidth = 30 + kernels = make_kernels(conv_kernel_model, detector, gaussmooth, halfwidth) + assert kernels is None + + +def test_apply_conv_kernel(): + data = np.zeros((3, 3, 2048, 2048)) + 1.999 + instrument, detector = 'NIRCAM', 'NRCB1' + input_model = mk_data_mdl(data, instrument, detector) + gaussmooth = 1 + halfwidth = 30 + kernels = make_kernels(conv_kernel_model, detector, gaussmooth, halfwidth) + sigreject = 4 + result = apply_conv_kernel(input_model.data[1, 1, ...], kernels, sigreject=sigreject) + compare = np.ones((1, 1, 2048, 2048)) + assert compare.all() == result.all() + diff --git a/jwst/refpix/tests/test_refpix.py b/jwst/refpix/tests/test_refpix.py index f7d5fccaeb..246e8bd488 100644 --- a/jwst/refpix/tests/test_refpix.py +++ b/jwst/refpix/tests/test_refpix.py @@ -8,8 +8,16 @@ Dataset, NIRDataset, correct_model, create_dataset, NRS_edgeless_subarrays) +conv_kernel_params = { + 'refpix_algorithm': 'median', + 'sirs_kernel_model': None, + 'sigreject': 4, + 'gaussmooth': 1, + 'halfwidth': 30} + + def test_refpix_subarray_miri(): - '''Check that the correction is skipped for MIR subarray data ''' + """Check that the correction is skipped for MIR subarray data """ # For MIRI, no reference pixel correction is performed on subarray data # No changes should be seen in the data arrays before and after correction @@ -73,8 +81,8 @@ def test_refpix_subarray_nirspec(subarray, ysize, xsize): def test_each_amp(): - '''Test that each amp is calculated separately using the average of left - and right pixels''' + """Test that each amp is calculated separately using the average of left + and right pixels""" # create input data # create model of data with 0 value array @@ -110,7 +118,7 @@ def test_each_amp(): def test_firstframe_sub(): - '''For MIR data, check that the first group is subtracted from each group in an integration + """For MIR data, check that the first group is subtracted from each group in an integration and added back in after the correction. This was found in testing the amp step. Make sure that the first frame is @@ -118,7 +126,7 @@ def test_firstframe_sub(): in the first group match the reference pixels in all other groups, then the subtraction will result in zeros, leaving zeros to be calculated as the reference pixel values, and the output data will match the input data after the frame is - added back in. So there should be no change to the data.''' + added back in. So there should be no change to the data.""" # create input data # create model of data with 0 value array @@ -150,7 +158,7 @@ def test_firstframe_sub(): np.testing.assert_array_equal(im.data, outim.data) def test_odd_even(): - '''Check that odd/even rows are applied when flag is set''' + """Check that odd/even rows are applied when flag is set""" # Test that odd and even rows are calculated separately @@ -280,7 +288,7 @@ def test_odd_even_amp_nirspec(detector, ysize, odd_even): def test_no_odd_even(): - '''Check that odd/even rows are not applied if flag is set to False''' + """Check that odd/even rows are not applied if flag is set to False""" # Test that odd and even rows are calculated together # create input data @@ -333,8 +341,8 @@ def test_no_odd_even(): def test_side_averaging(): - '''For MIRI data, check that the mean value in the reference pixels is calculated for each amplifier - using the average of the left and right side reference pixels.''' + """For MIRI data, check that the mean value in the reference pixels is calculated for each amplifier + using the average of the left and right side reference pixels.""" # Test that the left and right side pixels are averaged. # create input data @@ -364,8 +372,8 @@ def test_side_averaging(): def test_above_sigma(): - '''Test that a value greater than 3 sigma above mean of reference pixels is rejected - in the averaging of the reference pixels to be subtracted.''' + """Test that a value greater than 3 sigma above mean of reference pixels is rejected + in the averaging of the reference pixels to be subtracted.""" # create input data # create model of data with 0 value array @@ -395,11 +403,11 @@ def test_above_sigma(): def test_nan_refpix(): - '''Verify that the reference pixels flagged DO_NOT_USE are not used in the calculation + """Verify that the reference pixels flagged DO_NOT_USE are not used in the calculation Test that flagging a reference pixel with DO_NOT_USE does not use the pixel in the average. Set the pixel to NaN, which results in a NaN average value if used. If the test - passes, then the NaN was correctly flagged and rejected from the average.''' + passes, then the NaN was correctly flagged and rejected from the average.""" # create input data # create model of data with 0 value array @@ -430,7 +438,7 @@ def test_nan_refpix(): def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): - '''Test all corrections for subarray data with no even/odd.''' + """Test all corrections for subarray data with no even/odd.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -462,7 +470,8 @@ def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -472,7 +481,7 @@ def test_do_corrections_subarray_no_oddEven(setup_subarray_cube): def test_do_corrections_subarray(setup_subarray_cube): - '''Test all corrections for subarray data.''' + """Test all corrections for subarray data.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -504,7 +513,8 @@ def test_do_corrections_subarray(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -514,7 +524,7 @@ def test_do_corrections_subarray(setup_subarray_cube): def test_do_corrections_subarray_4amp(setup_subarray_cube): - '''Test all corrections for subarray data.''' + """Test all corrections for subarray data.""" # Create inputs and subarray SUBGRISM64 data, and set correction parameters ngroups = 3 @@ -575,7 +585,8 @@ def test_do_corrections_subarray_4amp(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) init_dataset.do_corrections() @@ -583,7 +594,7 @@ def test_do_corrections_subarray_4amp(setup_subarray_cube): def test_get_restore_group_subarray(setup_subarray_cube): - '''Test subarray input model data is replaced with group data.''' + """Test subarray input model data is replaced with group data.""" # Create inputs and subarray SUB320A335R data, and set correction parameters ngroups = 3 @@ -606,6 +617,7 @@ def test_get_restore_group_subarray(setup_subarray_cube): use_side_ref_pixels, side_smoothing_length, side_gain, + conv_kernel_params, odd_even_rows) # Make sure get_group properly copied the subarray @@ -622,7 +634,7 @@ def test_get_restore_group_subarray(setup_subarray_cube): def test_do_top_bottom_correction(setup_cube): - '''Test top/bottom correction for NIRCam data.''' + """Test top/bottom correction for NIRCam data.""" ngroups = 3 nrows = 2048 @@ -639,7 +651,8 @@ def test_do_top_bottom_correction(setup_cube): odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) abounds = [0, 512, 1024, 1536, 2048] top_even_amps = [12, 13, 14, 15] @@ -695,7 +708,7 @@ def test_do_top_bottom_correction(setup_cube): def test_do_top_bottom_correction_no_even_odd(setup_cube): - '''Test top/bottom correction with no even/odd.''' + """Test top/bottom correction with no even/odd.""" ngroups = 3 nrows = 2048 @@ -712,7 +725,8 @@ def test_do_top_bottom_correction_no_even_odd(setup_cube): odd_even_columns, use_side_ref_pixels, side_smoothing_length, - side_gain) + side_gain, + conv_kernel_params) abounds = [0, 512, 1024, 1536, 2048] top_amps = [12, 13, 14, 15] @@ -750,7 +764,7 @@ def test_do_top_bottom_correction_no_even_odd(setup_cube): def make_rampmodel(ngroups, ysize, xsize, instrument='MIRI', fill_value=None): - '''Make MIRI or NIRSpec ramp model for testing''' + """Make MIRI or NIRSpec ramp model for testing.""" # create the data and groupdq arrays csize = (1, ngroups, ysize, xsize) @@ -796,7 +810,7 @@ def make_rampmodel(ngroups, ysize, xsize, instrument='MIRI', fill_value=None): @pytest.fixture(scope='function') def setup_cube(): - ''' Set up fake data to test.''' + """ Set up fake data to test.""" def _cube(instr, detector, ngroups, nrows, ncols): @@ -822,7 +836,7 @@ def _cube(instr, detector, ngroups, nrows, ncols): @pytest.fixture(scope='function') def setup_subarray_cube(): - ''' Set up fake NIRCam subarray data to test.''' + """ Set up fake NIRCam subarray data to test.""" def _cube(name, detector, xstart, ystart, ngroups, nrows, ncols): @@ -861,7 +875,7 @@ def _cube(name, detector, xstart, ystart, ngroups, nrows, ncols): ('FGS', "GUIDER2") ]) def test_correct_model(setup_cube, instr, det): - '''Test all corrections for full frame data for all detectors.''' + """Test all corrections for full frame data for all detectors.""" ngroups = 2 nrows = 2048 @@ -885,7 +899,8 @@ def test_correct_model(setup_cube, instr, det): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, :4, 4:-4]), 0, decimal=0) np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, 4:-4, 4:-4]), dataval - rpix, decimal=0) @@ -929,7 +944,8 @@ def test_zero_frame(setup_cube): use_side_ref_pixels, side_smoothing_length, side_gain, - odd_even_rows) + odd_even_rows, + conv_kernel_params) # Make sure the SCI data is as expected. data = np.zeros(input_model.data.shape, dtype=input_model.data.dtype) diff --git a/jwst/regtest/test_nircam_image.py b/jwst/regtest/test_nircam_image.py index 89b08f7de8..40c896367d 100644 --- a/jwst/regtest/test_nircam_image.py +++ b/jwst/regtest/test_nircam_image.py @@ -22,6 +22,7 @@ def run_detector1pipeline(rtdata_module): "--steps.saturation.save_results=True", "--steps.superbias.save_results=True", "--steps.refpix.save_results=True", + "--steps.refpix.refpix_algorithm=median", "--steps.linearity.save_results=True", "--steps.dark_current.save_results=True", "--steps.jump.save_results=True", @@ -30,6 +31,21 @@ def run_detector1pipeline(rtdata_module): Step.from_cmdline(args) +@pytest.fixture(scope="module") +def run_detector1pipeline_with_sirs(rtdata_module): + """Run calwebb_detector1 on NIRCam imaging long data using the convolution kernel algorithm + Simple Improved Reference Subtraction (SIRS)""" + rtdata = rtdata_module + rtdata.get_data("nircam/image/jw01538046001_03105_00001_nrcalong_uncal.fits") + + # Run detector1 pipeline only on one of the _uncal files + args = ["calwebb_detector1", rtdata.input, + "--steps.refpix.refpix_algorithm=sirs", + "--steps.refpix.save_results=True" + ] + Step.from_cmdline(args) + + @pytest.fixture(scope="module") def run_detector1_with_clean_flicker_noise(rtdata_module): """Run detector1 pipeline on NIRCam imaging data with noise cleaning.""" @@ -89,6 +105,22 @@ def run_image3pipeline(run_image2pipeline, rtdata_module): Step.from_cmdline(args) +@pytest.mark.bigdata +def test_nircam_image_sirs(run_detector1pipeline_with_sirs, rtdata_module, fitsdiff_default_kwargs): + """Regression test of detector1 and image2 pipelines performed on NIRCam data.""" + rtdata = rtdata_module + rtdata.input = "jw01538046001_03105_00001_nrcalong_uncal.fits" + output = "jw01538046001_03105_00001_nrcalong_refpix.fits" + rtdata.output = output + rtdata.get_truth("truth/test_nircam_image_stages/jw01538046001_03105_00001_nrcalong_refpix_SIRS.fits") + + fitsdiff_default_kwargs["rtol"] = 5e-5 + fitsdiff_default_kwargs["atol"] = 1e-4 + + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["dq_init", "saturation", "superbias", "refpix", "linearity", From a4323a05bde4dab2819fbcc32c3a69bfd5afca69 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Fri, 20 Dec 2024 16:42:10 -0500 Subject: [PATCH 21/27] preparation for 1.17.0 release (#9024) --- CHANGES.rst | 270 +++++++++++++++++++++++++++ CITATION.cff | 7 +- README.md | 19 +- changes/8210.general.rst | 1 - changes/8554.assign_wcs.rst | 2 - changes/8554.resample.rst | 1 - changes/8554.skymatch.rst | 1 - changes/8671.docs.rst | 1 - changes/8726.refpix.rst | 1 - changes/8761.stpipe.rst | 1 - changes/8782.outlier_detection.0.rst | 1 - changes/8782.outlier_detection.1.rst | 1 - changes/8782.resample.rst | 1 - changes/8787.background.rst | 1 - changes/8828.outlier_detection.rst | 1 - changes/8831.datamodels.rst | 1 - changes/8840.outlier_detection.rst | 1 - changes/8843.associations.rst | 1 - changes/8846.ami.rst | 1 - changes/8847.master_background.rst | 1 - changes/8849.emicorr.rst | 1 - changes/8851.outlier_detection.rst | 1 - changes/8852.general.rst | 1 - changes/8853.outlier_detection.rst | 1 - changes/8866.resample.rst | 1 - changes/8870.outlier_detection.rst | 1 - changes/8874.assign_wcs.rst | 1 - changes/8880.outlier_detection.rst | 1 - changes/8885.general.rst | 1 - changes/8890.jump.rst | 2 - changes/8892.skymatch.rst | 1 - changes/8893.resample.rst | 1 - changes/8897.assign_wcs.rst | 1 - changes/8907.pipeline.rst | 1 - changes/8908.resample_spec.rst | 1 - changes/8909.scripts.rst | 1 - changes/8911.cube_build.rst | 1 - changes/8913.cube_build.rst | 1 - changes/8916.background.rst | 1 - changes/8918.datamodels.rst | 1 - changes/8926.docs.rst | 1 - changes/8927.combine_1d.rst | 1 - changes/8927.master_background.rst | 1 - changes/8927.pipeline.rst | 1 - changes/8932.master_background.rst | 1 - changes/8935.general.rst | 1 - changes/8945.stpipe.rst | 1 - changes/8946.rscd.rst | 1 - changes/8947.pipeline.rst | 1 - changes/8950.general.rst | 1 - changes/8952.firstframe.rst | 1 - changes/8957.general.rst | 1 - changes/8958.general.rst | 1 - changes/8961.extract_1d.rst | 1 - changes/8963.assign_wcs.rst | 3 - changes/8966.general.rst | 1 - changes/8974.ami.rst | 1 - changes/8975.resample.rst | 1 - changes/8978.assign_wcs.rst | 1 - changes/8983.extract_1d.rst | 1 - changes/8990.background.rst | 1 - pyproject.toml | 6 +- requirements-sdp.txt | 78 ++++++++ 63 files changed, 366 insertions(+), 76 deletions(-) delete mode 100644 changes/8210.general.rst delete mode 100644 changes/8554.assign_wcs.rst delete mode 100644 changes/8554.resample.rst delete mode 100644 changes/8554.skymatch.rst delete mode 100644 changes/8671.docs.rst delete mode 100644 changes/8726.refpix.rst delete mode 100644 changes/8761.stpipe.rst delete mode 100644 changes/8782.outlier_detection.0.rst delete mode 100644 changes/8782.outlier_detection.1.rst delete mode 100644 changes/8782.resample.rst delete mode 100644 changes/8787.background.rst delete mode 100644 changes/8828.outlier_detection.rst delete mode 100644 changes/8831.datamodels.rst delete mode 100644 changes/8840.outlier_detection.rst delete mode 100644 changes/8843.associations.rst delete mode 100644 changes/8846.ami.rst delete mode 100644 changes/8847.master_background.rst delete mode 100644 changes/8849.emicorr.rst delete mode 100644 changes/8851.outlier_detection.rst delete mode 100644 changes/8852.general.rst delete mode 100644 changes/8853.outlier_detection.rst delete mode 100644 changes/8866.resample.rst delete mode 100644 changes/8870.outlier_detection.rst delete mode 100644 changes/8874.assign_wcs.rst delete mode 100644 changes/8880.outlier_detection.rst delete mode 100644 changes/8885.general.rst delete mode 100644 changes/8890.jump.rst delete mode 100644 changes/8892.skymatch.rst delete mode 100644 changes/8893.resample.rst delete mode 100644 changes/8897.assign_wcs.rst delete mode 100644 changes/8907.pipeline.rst delete mode 100644 changes/8908.resample_spec.rst delete mode 100644 changes/8909.scripts.rst delete mode 100644 changes/8911.cube_build.rst delete mode 100644 changes/8913.cube_build.rst delete mode 100644 changes/8916.background.rst delete mode 100644 changes/8918.datamodels.rst delete mode 100644 changes/8926.docs.rst delete mode 100644 changes/8927.combine_1d.rst delete mode 100644 changes/8927.master_background.rst delete mode 100644 changes/8927.pipeline.rst delete mode 100644 changes/8932.master_background.rst delete mode 100644 changes/8935.general.rst delete mode 100644 changes/8945.stpipe.rst delete mode 100644 changes/8946.rscd.rst delete mode 100644 changes/8947.pipeline.rst delete mode 100644 changes/8950.general.rst delete mode 100644 changes/8952.firstframe.rst delete mode 100644 changes/8957.general.rst delete mode 100644 changes/8958.general.rst delete mode 100644 changes/8961.extract_1d.rst delete mode 100644 changes/8963.assign_wcs.rst delete mode 100644 changes/8966.general.rst delete mode 100644 changes/8974.ami.rst delete mode 100644 changes/8975.resample.rst delete mode 100644 changes/8978.assign_wcs.rst delete mode 100644 changes/8983.extract_1d.rst delete mode 100644 changes/8990.background.rst diff --git a/CHANGES.rst b/CHANGES.rst index 1041652782..55f5f2c371 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,273 @@ +1.17.0 (2024-12-20) +=================== + +General +------- + +- Rename default branch to main. (`#8210 + `_) +- Added mypy type checking to CI checks (`#8852 + `_) +- Give regtest okify results unique subdirectories. (`#8885 + `_) +- Include xml and db test data as package data for lib tests. (`#8935 + `_) +- Update minimum required version of crds to allow for "data release style" + contexts. (`#8950 `_) +- remove ``okify_regtests`` script (move to ``ci_watson``) (`#8957 + `_) +- When blending metadata don't store columns containing all missing value + (nans). (`#8958 `_) +- Increase asdf upper pin to 5 (`#8966 + `_) + + +Documentation +------------- + +- use ``towncrier`` to handle change log entries (`#8671 + `_) +- Mention possible need to provide package name to strun when using aliases. + (`#8926 `_) + + +``stpipe`` +---------- + +- Add warning that ``Step.__call__`` is deprecated. (`#8761 + `_) +- Remove all uses of Step.__call__ to allow it's deprecation. (`#8945 + `_) + + +Data Models +----------- + +- Remove memory-saving options from ModelContainer (`#8831 + `_) +- Update ModelLibrary to use meta.asn.exptype instead of meta.exptype. (`#8918 + `_) + + +Associations +------------ + +- Add mosaic association candidates to list of level three candidate types + requiring members from more than one observation (`#8843 + `_) + + +Scripts +------- + +- Remove the outdated schema_editor script. (`#8909 + `_) + + +Pipeline +-------- + +- Fixed a bug leading to incorrect area extensions, and sometimes crashes, in + the coron3 pipeline (`#8907 + `_) +- Add 'mbsub' to the list of known suffixes, for ``master_background`` + correction in ``calwebb_spec3``. (`#8927 + `_) +- Transfer wcsinfo metadata to new MultiSlitModel created during + ``calwebb_spec2`` processing of NIRSpec MSA data. (`#8947 + `_) + + +ami_analyze / ami_normalize / ami_average (ami3) +------------------------------------------------ + +- Change how AMI observables are averaged: average fringe quantities before + calculating additional observables. Update their error calculation: use + covariance of amplitudes/phases (and derived quantities) and standard error + of the mean. Code now expects an ASDF filename string for user-supplied + affine2d and bandpass input arguments. Example file creation in + documentation. (`#8846 + `_) +- Use mask and pupil geometry constants from NRM reference file, and apply + affine distortion from commissioning to LG model as default. (`#8974 + `_) + + +assign_wcs (image2, spec2) +-------------------------- + +- Use the range of points in the TabularModel to adjust the bounding_box in a + MIRI LRS FIXEDSLIT observation. + Ignore the bounding_box in running the inverse WCS transform when computing + crpix. (`#8554 `_) +- Catch NaN values in msa tables for source positions in slit and replace with + slit center. (`#8874 `_) +- Use pixel vertices to define s_region instead of pixel centers." (`#8897 + `_) +- Fix all bounding box assignments to wcs objects so that they are correctly + and + specifically assigned as order ``F`` boxes. This ensures consistency with the + assumptions made by GWCS for bounding boxes. (`#8963 + `_) +- Fix negative SHUTTRID values under numpy 2 (`#8978 + `_) + + +background (image2, spec2) +-------------------------- + +- Fixed crash when combining full and subarray observations for background + subtraction. (`#8787 `_) +- Apply bitwise operations in correct order when counting good pixels in the + background mask during WFSS background subtraction. (`#8916 + `_) +- Compute scaling for WFSS background subtraction using error-weighted mean + (`#8990 `_) + + +combine_1d (spec3) +------------------ + +- Fix wavelength sort order for single input spectrum. (`#8927 + `_) + + +cube_build (spec2 IFU, spec3) +----------------------------- + +- For moving-target IFU data, set RA, Dec header information of s3d products + according to the mean of input models instead of the first input model. + (`#8911 `_) +- Tweak the cube_build spaxel debugging option to provide filename info and + match outputs to the stated ordering. (`#8913 + `_) + + +emicorr (detector1 MIR) +----------------------- + +- Removed the interleaved noise array and do interleaving of noise and + subtraction in-place to avoid creating 2 arrays of equal dimensions to data. + (`#8849 `_) + + +extract_1d (spec2, spec3) +------------------------- + +- Refactor the core extraction algorithm and aperture definition modules for + slit and slitless extractions, for greater efficiency and maintainability. + Extraction reference files in FITS format are no longer supported. Current + behavior for extractions proceeding from extract1d reference files in JSON + format is preserved, with minor improvements: DQ arrays are populated and + error propagation is improved for some aperture types. (`#8961 + `_) +- Set order 2 weights to 1.0 for NIRISS SOSS SUBSTRIP96 data to avoid a + spurious flux drop in spectra around 1.5um. (`#8983 + `_) + + +firstframe (detector1 MIR) +-------------------------- + +- Update the firstframe step to optionally not flag when a ramp saturates in + group 3 (`#8952 `_) + + +jump (detector1) +---------------- + +- Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine + to fix accidental flagging of bright science pixels. (`#8890 + `_) + + +master_background (spec2 MOS, spec3) +------------------------------------ + +- Include resample, pixel_replace and extract_1d into MOS background pipeline + (`#8847 `_) +- Fix ModelContainer handling for user background input. (`#8927 + `_) +- Added pixel-to-pixel sigma clipping on input backgrounds in the NIRSpec MOS + master_background_mos step. Added option to median filter master background + spectrum in both the master_background and master_background_mos steps. + (`#8932 `_) + + +outlier_detection (image3, tso3, spec3, coron3) +----------------------------------------------- + +- Fix a bug that caused intermediate files to conflict for different slits when + a MultiSlitModel was processed. (`#8782 + `_) +- Decrease the amount of file I/O required to compute the median when in_memory + is set to False. (`#8782 + `_) +- For slit spectra, threshold outliers with a median error across exposures + instead of input error from the exposure itself. (`#8828 + `_) +- Moved median computers out of the jwst repository and into stcal. (`#8840 + `_) +- Update documentation to clarify the interaction between pipeline-level and + step-level `--in_memory` flags. (`#8851 + `_) +- Avoid modifying input and saving duplicate files when resample_data=False. + (`#8853 `_) +- Remove deprecated nlow and nhigh parameters from outlier detection step. + (`#8870 `_) +- reorganize outlier detection documentation (`#8880 + `_) + + +refpix (detector1) +------------------ + +- Implemented SIRS algorithm instead of running median for side pixels of NIR + full-frame data. Running median is still default. (`#8726 + `_) + + +resample (image2, image3, coron3) +--------------------------------- + +- Ignore the bounding_box in the inverse WCS transform in reproject. (`#8554 + `_) +- Permit creating drizzled models one at a time in many-to-many mode. (`#8782 + `_) +- Updated resample code to support the new ``drizzle`` API, see + https://github.com/spacetelescope/drizzle/pull/134 for more details. (`#8866 + `_) +- Use s_region list to calculate output footprint instead of re-computing via + WCS transforms (`#8893 + `_) +- Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment + variable (`#8975 `_) + + +resample_spec (spec2 MOS, spec3) +-------------------------------- + +- Update NIRSpec spectral resampling to add a missing correction factor in + resampled WCS tangent plane transformation. (`#8908 + `_) + + +rscd (detector1 MIR) +-------------------- + +- Updated RSCD step to work on segmented data (`#8946 + `_) + + +skymatch (image3) +----------------- + +- Ignore the bounding_box of an observation when computing sky statistics. + (`#8554 `_) +- Resolve warnings emitted by NumPy 2 when running skymatch. (`#8892 + `_) + + 1.16.1 (2024-10-30) =================== diff --git a/CITATION.cff b/CITATION.cff index 6d933fee0e..c660ccdba4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -92,8 +92,11 @@ authors: - family-names: "Clarke" given-names: "Melanie" orcid: "https://orcid.org/0009-0002-3561-4347" +- family-names: "Filippazzo" + given-names: "Joseph" + orcid: "https://orcid.org/0000-0002-0201-8306" title: "JWST Calibration Pipeline" -version: 1.16.0 +version: 1.17.0 doi: 10.5281/zenodo.7038885 -date-released: 2024-09-20 +date-released: 2024-12-20 url: "https://github.com/spacetelescope/jwst" diff --git a/README.md b/README.md index 84d50d76c8..91be40ef59 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ You can also install a specific version: conda create -n python=3.11 conda activate - pip install jwst==1.9.4 + pip install jwst==1.16.1 ### Installing the development version from Github @@ -89,15 +89,15 @@ used for Linux and Mac OS systems. Linux: - conda create -n jwstdp-1.12.5 --file https://ssb.stsci.edu/releases/jwstdp/1.12.5/conda_python_stable-deps.txt - conda activate jwstdp-1.12.5 - pip install -r https://ssb.stsci.edu/releases/jwstdp/1.12.5/reqs_stable-deps.txt + conda create -n jwstdp-1.16.1 --file https://ssb.stsci.edu/releases/jwstdp/1.16.1/conda_python_stable-deps.txt + conda activate jwstdp-1.16.1 + pip install -r https://ssb.stsci.edu/releases/jwstdp/1.16.1/reqs_stable-deps.txt MacOS: - conda create -n jwstdp-1.12.5 --file https://ssb.stsci.edu/releases/jwstdp/1.12.5/conda_python_macos-stable-deps.txt - conda activate jwstdp-1.12.5 - pip install -r https://ssb.stsci.edu/releases/jwstdp/1.12.5/reqs_macos-stable-deps.txt + conda create -n jwstdp-1.16.1 --file https://ssb.stsci.edu/releases/jwstdp/1.16.1/conda_python_macos-stable-deps.txt + conda activate jwstdp-1.16.1 + pip install -r https://ssb.stsci.edu/releases/jwstdp/1.16.1/reqs_macos-stable-deps.txt Each DMS delivery has its own installation instructions, which may be found in the corresponding release documentation linked from this page: @@ -216,8 +216,9 @@ the specified context and less than the context for the next release. | jwst tag | DMS build | SDP_VER | CRDS_CONTEXT | Released | Ops Install | Notes | |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------| -| 1.16.1 | B11.1.1 | 2024.3.1 | 1298 | 2024-11-13 | TBD | Final release candidate for B11.1 | -| 1.16.0 | B11.1 | 2024.3.0 | 1298 | 2024-09-20 | TBD | First release candidate for B11.1 | +| 1.17.0rc1 | B11.2 | TBD | 1321 | 2024-12-20 | TBD | First release candidate for B11.2 | +| 1.16.1 | B11.1.1 | 2024.3.1 | 1303 | 2024-11-13 | 2024-12-06 | Final release candidate for B11.1 | +| 1.16.0 | B11.1 | 2024.3.0 | 1298 | 2024-09-20 | | First release candidate for B11.1 | | 1.15.1 | B11.0 | 2024.2.2 | 1293 | 2024-07-08 | 2024-09-12 | Final release candidate for B11.0 | | 1.15.0 | B11.0rc1 | | 1274 | 2024-06-26 | | First release candidate for B11.0 | | 1.14.1 | | | 1240 | 2024-06-27 | | PyPI-only release for external users | diff --git a/changes/8210.general.rst b/changes/8210.general.rst deleted file mode 100644 index b923267b54..0000000000 --- a/changes/8210.general.rst +++ /dev/null @@ -1 +0,0 @@ -Rename default branch to main. diff --git a/changes/8554.assign_wcs.rst b/changes/8554.assign_wcs.rst deleted file mode 100644 index c36dd9d167..0000000000 --- a/changes/8554.assign_wcs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use the range of points in the TabularModel to adjust the bounding_box in a MIRI LRS FIXEDSLIT observation. -Ignore the bounding_box in running the inverse WCS transform when computing crpix. diff --git a/changes/8554.resample.rst b/changes/8554.resample.rst deleted file mode 100644 index bda0397ae8..0000000000 --- a/changes/8554.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore the bounding_box in the inverse WCS transform in reproject. diff --git a/changes/8554.skymatch.rst b/changes/8554.skymatch.rst deleted file mode 100644 index 1561af36b6..0000000000 --- a/changes/8554.skymatch.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore the bounding_box of an observation when computing sky statistics. diff --git a/changes/8671.docs.rst b/changes/8671.docs.rst deleted file mode 100644 index 724817599d..0000000000 --- a/changes/8671.docs.rst +++ /dev/null @@ -1 +0,0 @@ -use ``towncrier`` to handle change log entries diff --git a/changes/8726.refpix.rst b/changes/8726.refpix.rst deleted file mode 100644 index 0639bf5208..0000000000 --- a/changes/8726.refpix.rst +++ /dev/null @@ -1 +0,0 @@ -Implemented SIRS algorithm instead of running median for side pixels of NIR full-frame data. Running median is still default. diff --git a/changes/8761.stpipe.rst b/changes/8761.stpipe.rst deleted file mode 100644 index 3d2985c04f..0000000000 --- a/changes/8761.stpipe.rst +++ /dev/null @@ -1 +0,0 @@ -Add warning that ``Step.__call__`` is deprecated. diff --git a/changes/8782.outlier_detection.0.rst b/changes/8782.outlier_detection.0.rst deleted file mode 100644 index 469e927e44..0000000000 --- a/changes/8782.outlier_detection.0.rst +++ /dev/null @@ -1 +0,0 @@ -Decrease the amount of file I/O required to compute the median when in_memory is set to False. \ No newline at end of file diff --git a/changes/8782.outlier_detection.1.rst b/changes/8782.outlier_detection.1.rst deleted file mode 100644 index a1f9bd8e55..0000000000 --- a/changes/8782.outlier_detection.1.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug that caused intermediate files to conflict for different slits when a MultiSlitModel was processed. \ No newline at end of file diff --git a/changes/8782.resample.rst b/changes/8782.resample.rst deleted file mode 100644 index 05b3e87052..0000000000 --- a/changes/8782.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Permit creating drizzled models one at a time in many-to-many mode. diff --git a/changes/8787.background.rst b/changes/8787.background.rst deleted file mode 100644 index 9f04f7e0bd..0000000000 --- a/changes/8787.background.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed crash when combining full and subarray observations for background subtraction. diff --git a/changes/8828.outlier_detection.rst b/changes/8828.outlier_detection.rst deleted file mode 100644 index e0edcecb80..0000000000 --- a/changes/8828.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -For slit spectra, threshold outliers with a median error across exposures instead of input error from the exposure itself. diff --git a/changes/8831.datamodels.rst b/changes/8831.datamodels.rst deleted file mode 100644 index 22289006df..0000000000 --- a/changes/8831.datamodels.rst +++ /dev/null @@ -1 +0,0 @@ -Remove memory-saving options from ModelContainer \ No newline at end of file diff --git a/changes/8840.outlier_detection.rst b/changes/8840.outlier_detection.rst deleted file mode 100644 index 0d1fe91bb7..0000000000 --- a/changes/8840.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Moved median computers out of the jwst repository and into stcal. \ No newline at end of file diff --git a/changes/8843.associations.rst b/changes/8843.associations.rst deleted file mode 100644 index e6d2334960..0000000000 --- a/changes/8843.associations.rst +++ /dev/null @@ -1 +0,0 @@ -Add mosaic association candidates to list of level three candidate types requiring members from more than one observation \ No newline at end of file diff --git a/changes/8846.ami.rst b/changes/8846.ami.rst deleted file mode 100644 index a965c823ff..0000000000 --- a/changes/8846.ami.rst +++ /dev/null @@ -1 +0,0 @@ -Change how AMI observables are averaged: average fringe quantities before calculating additional observables. Update their error calculation: use covariance of amplitudes/phases (and derived quantities) and standard error of the mean. Code now expects an ASDF filename string for user-supplied affine2d and bandpass input arguments. Example file creation in documentation. \ No newline at end of file diff --git a/changes/8847.master_background.rst b/changes/8847.master_background.rst deleted file mode 100644 index f1b516a1f2..0000000000 --- a/changes/8847.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Include resample, pixel_replace and extract_1d into MOS background pipeline diff --git a/changes/8849.emicorr.rst b/changes/8849.emicorr.rst deleted file mode 100644 index aee1210d87..0000000000 --- a/changes/8849.emicorr.rst +++ /dev/null @@ -1 +0,0 @@ -Removed the interleaved noise array and do interleaving of noise and subtraction in-place to avoid creating 2 arrays of equal dimensions to data. diff --git a/changes/8851.outlier_detection.rst b/changes/8851.outlier_detection.rst deleted file mode 100644 index da2e49986d..0000000000 --- a/changes/8851.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Update documentation to clarify the interaction between pipeline-level and step-level `--in_memory` flags. \ No newline at end of file diff --git a/changes/8852.general.rst b/changes/8852.general.rst deleted file mode 100644 index 0a584e8d63..0000000000 --- a/changes/8852.general.rst +++ /dev/null @@ -1 +0,0 @@ -Added mypy type checking to CI checks diff --git a/changes/8853.outlier_detection.rst b/changes/8853.outlier_detection.rst deleted file mode 100644 index abb775f6c8..0000000000 --- a/changes/8853.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid modifying input and saving duplicate files when resample_data=False. diff --git a/changes/8866.resample.rst b/changes/8866.resample.rst deleted file mode 100644 index f31024d0ee..0000000000 --- a/changes/8866.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Updated resample code to support the new ``drizzle`` API, see https://github.com/spacetelescope/drizzle/pull/134 for more details. diff --git a/changes/8870.outlier_detection.rst b/changes/8870.outlier_detection.rst deleted file mode 100644 index 53ce01221d..0000000000 --- a/changes/8870.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated nlow and nhigh parameters from outlier detection step. diff --git a/changes/8874.assign_wcs.rst b/changes/8874.assign_wcs.rst deleted file mode 100644 index d27218f8d4..0000000000 --- a/changes/8874.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Catch NaN values in msa tables for source positions in slit and replace with slit center. \ No newline at end of file diff --git a/changes/8880.outlier_detection.rst b/changes/8880.outlier_detection.rst deleted file mode 100644 index f5642b4e30..0000000000 --- a/changes/8880.outlier_detection.rst +++ /dev/null @@ -1 +0,0 @@ -reorganize outlier detection documentation diff --git a/changes/8885.general.rst b/changes/8885.general.rst deleted file mode 100644 index 435de02426..0000000000 --- a/changes/8885.general.rst +++ /dev/null @@ -1 +0,0 @@ -Give regtest okify results unique subdirectories. diff --git a/changes/8890.jump.rst b/changes/8890.jump.rst deleted file mode 100644 index d04413a547..0000000000 --- a/changes/8890.jump.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine -to fix accidental flagging of bright science pixels. diff --git a/changes/8892.skymatch.rst b/changes/8892.skymatch.rst deleted file mode 100644 index 7662782814..0000000000 --- a/changes/8892.skymatch.rst +++ /dev/null @@ -1 +0,0 @@ -Resolve warnings emitted by NumPy 2 when running skymatch. diff --git a/changes/8893.resample.rst b/changes/8893.resample.rst deleted file mode 100644 index 9e48ecea89..0000000000 --- a/changes/8893.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Use s_region list to calculate output footprint instead of re-computing via WCS transforms diff --git a/changes/8897.assign_wcs.rst b/changes/8897.assign_wcs.rst deleted file mode 100644 index fb209a0fdc..0000000000 --- a/changes/8897.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Use pixel vertices to define s_region instead of pixel centers." diff --git a/changes/8907.pipeline.rst b/changes/8907.pipeline.rst deleted file mode 100644 index c6d1759f93..0000000000 --- a/changes/8907.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug leading to incorrect area extensions, and sometimes crashes, in the coron3 pipeline diff --git a/changes/8908.resample_spec.rst b/changes/8908.resample_spec.rst deleted file mode 100644 index 2d69d9feee..0000000000 --- a/changes/8908.resample_spec.rst +++ /dev/null @@ -1 +0,0 @@ -Update NIRSpec spectral resampling to add a missing correction factor in resampled WCS tangent plane transformation. \ No newline at end of file diff --git a/changes/8909.scripts.rst b/changes/8909.scripts.rst deleted file mode 100644 index 0addc426d1..0000000000 --- a/changes/8909.scripts.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the outdated schema_editor script. diff --git a/changes/8911.cube_build.rst b/changes/8911.cube_build.rst deleted file mode 100644 index e6de4d2827..0000000000 --- a/changes/8911.cube_build.rst +++ /dev/null @@ -1 +0,0 @@ -For moving-target IFU data, set RA, Dec header information of s3d products according to the mean of input models instead of the first input model. diff --git a/changes/8913.cube_build.rst b/changes/8913.cube_build.rst deleted file mode 100644 index 2fd2ff006f..0000000000 --- a/changes/8913.cube_build.rst +++ /dev/null @@ -1 +0,0 @@ -Tweak the cube_build spaxel debugging option to provide filename info and match outputs to the stated ordering. diff --git a/changes/8916.background.rst b/changes/8916.background.rst deleted file mode 100644 index e2e4414b14..0000000000 --- a/changes/8916.background.rst +++ /dev/null @@ -1 +0,0 @@ -Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction. \ No newline at end of file diff --git a/changes/8918.datamodels.rst b/changes/8918.datamodels.rst deleted file mode 100644 index 2d442399f7..0000000000 --- a/changes/8918.datamodels.rst +++ /dev/null @@ -1 +0,0 @@ -Update ModelLibrary to use meta.asn.exptype instead of meta.exptype. diff --git a/changes/8926.docs.rst b/changes/8926.docs.rst deleted file mode 100644 index 621c1a9254..0000000000 --- a/changes/8926.docs.rst +++ /dev/null @@ -1 +0,0 @@ -Mention possible need to provide package name to strun when using aliases. diff --git a/changes/8927.combine_1d.rst b/changes/8927.combine_1d.rst deleted file mode 100644 index b05b3e95bc..0000000000 --- a/changes/8927.combine_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Fix wavelength sort order for single input spectrum. diff --git a/changes/8927.master_background.rst b/changes/8927.master_background.rst deleted file mode 100644 index cf71da1e52..0000000000 --- a/changes/8927.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ModelContainer handling for user background input. diff --git a/changes/8927.pipeline.rst b/changes/8927.pipeline.rst deleted file mode 100644 index e03ac78254..0000000000 --- a/changes/8927.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``. diff --git a/changes/8932.master_background.rst b/changes/8932.master_background.rst deleted file mode 100644 index 4ad13be5d5..0000000000 --- a/changes/8932.master_background.rst +++ /dev/null @@ -1 +0,0 @@ -Added pixel-to-pixel sigma clipping on input backgrounds in the NIRSpec MOS master_background_mos step. Added option to median filter master background spectrum in both the master_background and master_background_mos steps. diff --git a/changes/8935.general.rst b/changes/8935.general.rst deleted file mode 100644 index fe0b6a58e2..0000000000 --- a/changes/8935.general.rst +++ /dev/null @@ -1 +0,0 @@ -Include xml and db test data as package data for lib tests. diff --git a/changes/8945.stpipe.rst b/changes/8945.stpipe.rst deleted file mode 100644 index 73937c9f0f..0000000000 --- a/changes/8945.stpipe.rst +++ /dev/null @@ -1 +0,0 @@ -Remove all uses of Step.__call__ to allow it's deprecation. diff --git a/changes/8946.rscd.rst b/changes/8946.rscd.rst deleted file mode 100644 index 5ef30ed606..0000000000 --- a/changes/8946.rscd.rst +++ /dev/null @@ -1 +0,0 @@ -Updated RSCD step to work on segmented data diff --git a/changes/8947.pipeline.rst b/changes/8947.pipeline.rst deleted file mode 100644 index 2c24cc6f0b..0000000000 --- a/changes/8947.pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Transfer wcsinfo metadata to new MultiSlitModel created during ``calwebb_spec2`` processing of NIRSpec MSA data. \ No newline at end of file diff --git a/changes/8950.general.rst b/changes/8950.general.rst deleted file mode 100644 index 466db73740..0000000000 --- a/changes/8950.general.rst +++ /dev/null @@ -1 +0,0 @@ -Update minimum required version of crds to allow for "data release style" contexts. diff --git a/changes/8952.firstframe.rst b/changes/8952.firstframe.rst deleted file mode 100644 index ff41581d2f..0000000000 --- a/changes/8952.firstframe.rst +++ /dev/null @@ -1 +0,0 @@ -Update the firstframe step to optionally not flag when a ramp saturates in group 3 \ No newline at end of file diff --git a/changes/8957.general.rst b/changes/8957.general.rst deleted file mode 100644 index 91a475dc0f..0000000000 --- a/changes/8957.general.rst +++ /dev/null @@ -1 +0,0 @@ -remove ``okify_regtests`` script (move to ``ci_watson``) diff --git a/changes/8958.general.rst b/changes/8958.general.rst deleted file mode 100644 index a84befe1ff..0000000000 --- a/changes/8958.general.rst +++ /dev/null @@ -1 +0,0 @@ -When blending metadata don't store columns containing all missing value (nans). diff --git a/changes/8961.extract_1d.rst b/changes/8961.extract_1d.rst deleted file mode 100644 index ce89d4c2b9..0000000000 --- a/changes/8961.extract_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor the core extraction algorithm and aperture definition modules for slit and slitless extractions, for greater efficiency and maintainability. Extraction reference files in FITS format are no longer supported. Current behavior for extractions proceeding from extract1d reference files in JSON format is preserved, with minor improvements: DQ arrays are populated and error propagation is improved for some aperture types. diff --git a/changes/8963.assign_wcs.rst b/changes/8963.assign_wcs.rst deleted file mode 100644 index b9f007448a..0000000000 --- a/changes/8963.assign_wcs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix all bounding box assignments to wcs objects so that they are correctly and -specifically assigned as order ``F`` boxes. This ensures consistency with the -assumptions made by GWCS for bounding boxes. diff --git a/changes/8966.general.rst b/changes/8966.general.rst deleted file mode 100644 index 5b21ca6897..0000000000 --- a/changes/8966.general.rst +++ /dev/null @@ -1 +0,0 @@ -Increase asdf upper pin to 5 diff --git a/changes/8974.ami.rst b/changes/8974.ami.rst deleted file mode 100644 index ea39be8806..0000000000 --- a/changes/8974.ami.rst +++ /dev/null @@ -1 +0,0 @@ -Use mask and pupil geometry constants from NRM reference file, and apply affine distortion from commissioning to LG model as default. \ No newline at end of file diff --git a/changes/8975.resample.rst b/changes/8975.resample.rst deleted file mode 100644 index 95d48e855e..0000000000 --- a/changes/8975.resample.rst +++ /dev/null @@ -1 +0,0 @@ -Removed allowed_memory parameter and DMODEL_ALLOWED_MEMORY environment variable diff --git a/changes/8978.assign_wcs.rst b/changes/8978.assign_wcs.rst deleted file mode 100644 index 2771c1d3c6..0000000000 --- a/changes/8978.assign_wcs.rst +++ /dev/null @@ -1 +0,0 @@ -Fix negative SHUTTRID values under numpy 2 diff --git a/changes/8983.extract_1d.rst b/changes/8983.extract_1d.rst deleted file mode 100644 index 5bf93662f2..0000000000 --- a/changes/8983.extract_1d.rst +++ /dev/null @@ -1 +0,0 @@ -Set order 2 weights to 1.0 for NIRISS SOSS SUBSTRIP96 data to avoid a spurious flux drop in spectra around 1.5um. diff --git a/changes/8990.background.rst b/changes/8990.background.rst deleted file mode 100644 index ed656009a4..0000000000 --- a/changes/8990.background.rst +++ /dev/null @@ -1 +0,0 @@ -Compute scaling for WFSS background subtraction using error-weighted mean diff --git a/pyproject.toml b/pyproject.toml index 12c5996d16..3342f6b90e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,9 +33,9 @@ dependencies = [ "scikit-image>=0.20.0", "scipy>=1.14.1", "spherical-geometry>=1.2.22", - "stcal @ git+https://github.com/spacetelescope/stcal.git@main", - "stdatamodels @ git+https://github.com/spacetelescope/stdatamodels.git@main", - "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main", + "stcal>=1.11.0,<1.12.0", + "stdatamodels>=2.2.0,<2.3.0", + "stpipe>=0.8.0,<0.9.0", "stsci.imagestats>=1.6.3", "synphot>=1.2", "tweakwcs>=0.8.8", diff --git a/requirements-sdp.txt b/requirements-sdp.txt index a110a4b011..f6f1a9c478 100644 --- a/requirements-sdp.txt +++ b/requirements-sdp.txt @@ -14,3 +14,81 @@ # conda activate sdp # pip install -e .[test,sdp] # pip freeze | grep -v jwst >> requirements-sdp.txt +asdf==4.0.0 +asdf-astropy==0.7.0 +asdf_coordinates_schemas==0.3.0 +asdf_standard==1.1.1 +asdf_transform_schemas==0.5.0 +asdf_wcs_schemas==0.4.0 +astropy==7.0.0 +astropy-iers-data==0.2024.12.16.0.35.48 +attrs==24.3.0 +BayesicFitting==3.2.3 +certifi==2024.12.14 +charset-normalizer==3.4.0 +ci_watson==0.8.0 +colorama==0.4.6 +contourpy==1.3.1 +coverage==7.6.9 +crds==12.0.8 +cycler==0.12.1 +drizzle==2.0.0 +et_xmlfile==2.0.0 +filelock==3.16.1 +fonttools==4.55.3 +future==1.0.0 +gwcs==0.22.1 +idna==3.10 +imageio==2.36.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +jmespath==1.0.1 +jplephem==2.22 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +kiwisolver==1.4.7 +lazy_loader==0.4 +lxml==5.3.0 +matplotlib==3.10.0 +networkx==3.4.2 +numpy==1.26.4 +opencv-python-headless==4.10.0.84 +openpyxl==3.1.5 +packaging==24.2 +Parsley==1.3 +photutils==2.0.2 +pillow==11.0.0 +pluggy==1.5.0 +poppy==1.1.1 +pyerfa==2.0.1.5 +pyparsing==3.2.0 +pysiaf==0.24.1 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-doctestplus==1.3.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +readchar==4.2.1 +referencing==0.35.1 +requests==2.32.3 +requests-mock==1.12.1 +rpds-py==0.22.3 +ruff==0.8.4 +scikit-image==0.25.0 +scipy==1.14.1 +semantic-version==2.10.0 +setuptools==75.1.0 +six==1.17.0 +spherical_geometry==1.3.2 +stcal==1.11.0 +stdatamodels==2.2.0 +stpipe==0.8.0 +stsci.imagestats==1.8.3 +stsci.stimage==0.2.9 +synphot==1.5.0 +tifffile==2024.12.12 +tweakwcs==0.8.9 +urllib3==2.2.3 +wheel==0.44.0 +wiimatch==0.3.2 +zipp==3.21.0 From ec3304909d54f7c72ad36ed74366267fe8839ed4 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Sat, 21 Dec 2024 09:31:22 -0500 Subject: [PATCH 22/27] Prepare for post-1.17.0 development (#9025) --- requirements-sdp.txt | 78 -------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/requirements-sdp.txt b/requirements-sdp.txt index f6f1a9c478..a110a4b011 100644 --- a/requirements-sdp.txt +++ b/requirements-sdp.txt @@ -14,81 +14,3 @@ # conda activate sdp # pip install -e .[test,sdp] # pip freeze | grep -v jwst >> requirements-sdp.txt -asdf==4.0.0 -asdf-astropy==0.7.0 -asdf_coordinates_schemas==0.3.0 -asdf_standard==1.1.1 -asdf_transform_schemas==0.5.0 -asdf_wcs_schemas==0.4.0 -astropy==7.0.0 -astropy-iers-data==0.2024.12.16.0.35.48 -attrs==24.3.0 -BayesicFitting==3.2.3 -certifi==2024.12.14 -charset-normalizer==3.4.0 -ci_watson==0.8.0 -colorama==0.4.6 -contourpy==1.3.1 -coverage==7.6.9 -crds==12.0.8 -cycler==0.12.1 -drizzle==2.0.0 -et_xmlfile==2.0.0 -filelock==3.16.1 -fonttools==4.55.3 -future==1.0.0 -gwcs==0.22.1 -idna==3.10 -imageio==2.36.1 -importlib_metadata==8.5.0 -iniconfig==2.0.0 -jmespath==1.0.1 -jplephem==2.22 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -kiwisolver==1.4.7 -lazy_loader==0.4 -lxml==5.3.0 -matplotlib==3.10.0 -networkx==3.4.2 -numpy==1.26.4 -opencv-python-headless==4.10.0.84 -openpyxl==3.1.5 -packaging==24.2 -Parsley==1.3 -photutils==2.0.2 -pillow==11.0.0 -pluggy==1.5.0 -poppy==1.1.1 -pyerfa==2.0.1.5 -pyparsing==3.2.0 -pysiaf==0.24.1 -pytest==8.3.4 -pytest-cov==6.0.0 -pytest-doctestplus==1.3.0 -python-dateutil==2.9.0.post0 -PyYAML==6.0.2 -readchar==4.2.1 -referencing==0.35.1 -requests==2.32.3 -requests-mock==1.12.1 -rpds-py==0.22.3 -ruff==0.8.4 -scikit-image==0.25.0 -scipy==1.14.1 -semantic-version==2.10.0 -setuptools==75.1.0 -six==1.17.0 -spherical_geometry==1.3.2 -stcal==1.11.0 -stdatamodels==2.2.0 -stpipe==0.8.0 -stsci.imagestats==1.8.3 -stsci.stimage==0.2.9 -synphot==1.5.0 -tifffile==2024.12.12 -tweakwcs==0.8.9 -urllib3==2.2.3 -wheel==0.44.0 -wiimatch==0.3.2 -zipp==3.21.0 From 78f06477af7dfc9a7862631a936fe9374cd740a3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 23 Dec 2024 10:55:26 -0500 Subject: [PATCH 23/27] Add suffix for SIRS refpix output so it can be okified --- jwst/regtest/test_nircam_image.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jwst/regtest/test_nircam_image.py b/jwst/regtest/test_nircam_image.py index 40c896367d..9a0b3bd596 100644 --- a/jwst/regtest/test_nircam_image.py +++ b/jwst/regtest/test_nircam_image.py @@ -33,15 +33,18 @@ def run_detector1pipeline(rtdata_module): @pytest.fixture(scope="module") def run_detector1pipeline_with_sirs(rtdata_module): - """Run calwebb_detector1 on NIRCam imaging long data using the convolution kernel algorithm - Simple Improved Reference Subtraction (SIRS)""" + """Run calwebb_detector1 on NIRCam imaging long data using SIRS. + + SIRS is the convolution kernel algorithm - Simple Improved Reference Subtraction. + """ rtdata = rtdata_module rtdata.get_data("nircam/image/jw01538046001_03105_00001_nrcalong_uncal.fits") # Run detector1 pipeline only on one of the _uncal files args = ["calwebb_detector1", rtdata.input, + "--output_file=jw01538046001_03105_00001_nrcalong_sirs", "--steps.refpix.refpix_algorithm=sirs", - "--steps.refpix.save_results=True" + "--steps.refpix.save_results=True", ] Step.from_cmdline(args) @@ -110,9 +113,9 @@ def test_nircam_image_sirs(run_detector1pipeline_with_sirs, rtdata_module, fitsd """Regression test of detector1 and image2 pipelines performed on NIRCam data.""" rtdata = rtdata_module rtdata.input = "jw01538046001_03105_00001_nrcalong_uncal.fits" - output = "jw01538046001_03105_00001_nrcalong_refpix.fits" + output = "jw01538046001_03105_00001_nrcalong_sirs_refpix.fits" rtdata.output = output - rtdata.get_truth("truth/test_nircam_image_stages/jw01538046001_03105_00001_nrcalong_refpix_SIRS.fits") + rtdata.get_truth("truth/test_nircam_image_stages/jw01538046001_03105_00001_nrcalong_sirs_refpix.fits") fitsdiff_default_kwargs["rtol"] = 5e-5 fitsdiff_default_kwargs["atol"] = 1e-4 From 071f1cd44555f730365421f472fb84d6a8426bb3 Mon Sep 17 00:00:00 2001 From: David Law Date: Mon, 30 Dec 2024 15:23:47 -0500 Subject: [PATCH 24/27] Bugfix for SIRS reference pixel subtraction --- jwst/refpix/refpix_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/refpix/refpix_step.py b/jwst/refpix/refpix_step.py index bdcccfde53..fee8d54ac2 100644 --- a/jwst/refpix/refpix_step.py +++ b/jwst/refpix/refpix_step.py @@ -105,7 +105,7 @@ def process(self, step_input): else: self.log.info('Using SIRS reference file: {}'.format(sirs_ref_filename)) sirs_kernel_model = datamodels.SIRSKernelModel(sirs_ref_filename) - conv_kernel_params['refpix_algorithm'] = sirs_kernel_model + conv_kernel_params['sirs_kernel_model'] = sirs_kernel_model elif input_model.meta.instrument.name == 'MIRI': self.log.info('Simple Improved Reference Subtraction (SIRS) not applied for MIRI data.') elif 'FULL' not in input_model.meta.subarray.name: From 82c6a0f0ba5c6986ab91e356dc8493737a09d28a Mon Sep 17 00:00:00 2001 From: David Law Date: Mon, 30 Dec 2024 15:27:34 -0500 Subject: [PATCH 25/27] Add change log entry --- changes/9037.refpix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/9037.refpix.rst diff --git a/changes/9037.refpix.rst b/changes/9037.refpix.rst new file mode 100644 index 0000000000..d38bb056ac --- /dev/null +++ b/changes/9037.refpix.rst @@ -0,0 +1 @@ +Bugfix for the new SIRS reference pixel subtraction. From bb62f0c9f0ae6ef239b0cc774a4ebebc1c9f133a Mon Sep 17 00:00:00 2001 From: David Law Date: Mon, 30 Dec 2024 17:50:32 -0500 Subject: [PATCH 26/27] Update SIRS regression test to use nrca3 --- jwst/regtest/test_nircam_image.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwst/regtest/test_nircam_image.py b/jwst/regtest/test_nircam_image.py index 9a0b3bd596..33c7bb797b 100644 --- a/jwst/regtest/test_nircam_image.py +++ b/jwst/regtest/test_nircam_image.py @@ -38,11 +38,11 @@ def run_detector1pipeline_with_sirs(rtdata_module): SIRS is the convolution kernel algorithm - Simple Improved Reference Subtraction. """ rtdata = rtdata_module - rtdata.get_data("nircam/image/jw01538046001_03105_00001_nrcalong_uncal.fits") + rtdata.get_data("nircam/image/jw01345001001_10201_00001_nrca3_uncal.fits") # Run detector1 pipeline only on one of the _uncal files args = ["calwebb_detector1", rtdata.input, - "--output_file=jw01538046001_03105_00001_nrcalong_sirs", + "--output_file=jw01345001001_10201_00001_nrca3_sirs", "--steps.refpix.refpix_algorithm=sirs", "--steps.refpix.save_results=True", ] @@ -112,10 +112,10 @@ def run_image3pipeline(run_image2pipeline, rtdata_module): def test_nircam_image_sirs(run_detector1pipeline_with_sirs, rtdata_module, fitsdiff_default_kwargs): """Regression test of detector1 and image2 pipelines performed on NIRCam data.""" rtdata = rtdata_module - rtdata.input = "jw01538046001_03105_00001_nrcalong_uncal.fits" - output = "jw01538046001_03105_00001_nrcalong_sirs_refpix.fits" + rtdata.input = "jw01345001001_10201_00001_nrca3_uncal.fits" + output = "jw01345001001_10201_00001_nrca3_sirs_refpix.fits" rtdata.output = output - rtdata.get_truth("truth/test_nircam_image_stages/jw01538046001_03105_00001_nrcalong_sirs_refpix.fits") + rtdata.get_truth("truth/test_nircam_image_stages/jw01345001001_10201_00001_nrca3_sirs_refpix.fits") fitsdiff_default_kwargs["rtol"] = 5e-5 fitsdiff_default_kwargs["atol"] = 1e-4 From 5244263c2085948f6121219069afd8cd563962c7 Mon Sep 17 00:00:00 2001 From: David Law Date: Thu, 2 Jan 2025 12:27:12 -0500 Subject: [PATCH 27/27] JP-3664: Fix pixel replace numpy 2.0 issues (#9004) Co-authored-by: Melanie Clarke --- changes/9004.pixel_replace.rst | 4 ++++ jwst/pixel_replace/pixel_replace.py | 26 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 changes/9004.pixel_replace.rst diff --git a/changes/9004.pixel_replace.rst b/changes/9004.pixel_replace.rst new file mode 100644 index 0000000000..f0599ecd4a --- /dev/null +++ b/changes/9004.pixel_replace.rst @@ -0,0 +1,4 @@ +Change from the default BFGS algorithm to Nelder-Mead when calling scipy.minimize +within the fit_profile approach to pixel replacement in order to fix numpy 2.0 +compatibility issues. Additionally, add safety catch to ensure that pixel replacement +profile fitting doesn't attempt to scale based on noise. \ No newline at end of file diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index a28d209451..098fdeed9c 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -317,20 +317,27 @@ def fit_profile(self, model): # Cut out valid neighboring profiles adjacent_condition = self.custom_slice(dispaxis, valid_adjacent_inds) profile_data = model.data[adjacent_condition] + profile_err = model.err[adjacent_condition] # Mask out bad pixels invalid_condition = (model.dq[adjacent_condition] & self.DO_NOT_USE).astype(bool) profile_data[invalid_condition] = np.nan + profile_err[invalid_condition] = np.nan # Add additional cut to pull only from region with valid data # for convenience (may not be necessary) region_condition = self.custom_slice(3 - dispaxis, range(*profile_cut)) profile_data = profile_data[region_condition] + profile_snr = np.abs(profile_data / profile_err[region_condition]) # Normalize profile data # TODO: check on signs here - absolute max sometimes picks up # large negative outliers profile_norm_scale = np.nanmax(np.abs(profile_data), axis=(dispaxis - 1), keepdims=True) + # If profile data has SNR < 5 everywhere just use unity scaling + # (so we don't normalize to noise) + if (np.nanmax(profile_snr) < 5): + profile_norm_scale[:] = 1.0 normalized = profile_data / profile_norm_scale # Get corresponding error and variance data and scale and mask to match @@ -375,12 +382,19 @@ def fit_profile(self, model): norm_current = min_current / np.max(min_current) # Scale median profile to current profile with bad pixel - minimize mse? - # TODO: check on signs here - absolute max sometimes picks up - # large negative outliers - norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), - args=(min_median, norm_current)).x - - scale = np.max(min_current) + # Only do this scaling if we didn't default to all-unity scaling above, + # and require input values below 1e20 so that we don't overflow the + # minimization routine with extremely bad noise. + if ((np.nanmedian(profile_norm_scale) != 1.0) & (np.nanmax(np.abs(min_median)) < 1e20) + & (np.nanmax(np.abs(norm_current)) < 1e20)): + # TODO: check on signs here - absolute max sometimes picks up + # large negative outliers + norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), + args=(np.abs(min_median), np.abs(norm_current)), method='Nelder-Mead').x + scale = np.max(min_current) + else: + norm_scale = 1.0 + scale = 1.0 # Replace pixels that are do-not-use but not non-science current_dq = model.dq[current_condition][range(*profile_cut)]