From fd1d608b15a49ad18dc47f895ce034c45f9bc15a Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:12:26 -0700 Subject: [PATCH 01/23] Move legacy LSSTCam calibration pipelines to LSSTCam/legacy_202409. --- pipelines/LSSTCam/{ => legacy_202409}/cpBfk.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpBias.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpCrosstalk.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpCti.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpDark.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpDarkForDefects.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpDefects.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpDefectsIndividual.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpFlat.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpFlatSingleChip.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpFringe.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpLinearizer.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpPlotPtc.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpPtc.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpPtcNoPhotodiode.yaml | 0 pipelines/LSSTCam/{ => legacy_202409}/cpSky.yaml | 0 16 files changed, 0 insertions(+), 0 deletions(-) rename pipelines/LSSTCam/{ => legacy_202409}/cpBfk.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpBias.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpCrosstalk.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpCti.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpDark.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpDarkForDefects.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpDefects.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpDefectsIndividual.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpFlat.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpFlatSingleChip.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpFringe.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpLinearizer.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpPlotPtc.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpPtc.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpPtcNoPhotodiode.yaml (100%) rename pipelines/LSSTCam/{ => legacy_202409}/cpSky.yaml (100%) diff --git a/pipelines/LSSTCam/cpBfk.yaml b/pipelines/LSSTCam/legacy_202409/cpBfk.yaml similarity index 100% rename from pipelines/LSSTCam/cpBfk.yaml rename to pipelines/LSSTCam/legacy_202409/cpBfk.yaml diff --git a/pipelines/LSSTCam/cpBias.yaml b/pipelines/LSSTCam/legacy_202409/cpBias.yaml similarity index 100% rename from pipelines/LSSTCam/cpBias.yaml rename to pipelines/LSSTCam/legacy_202409/cpBias.yaml diff --git a/pipelines/LSSTCam/cpCrosstalk.yaml b/pipelines/LSSTCam/legacy_202409/cpCrosstalk.yaml similarity index 100% rename from pipelines/LSSTCam/cpCrosstalk.yaml rename to pipelines/LSSTCam/legacy_202409/cpCrosstalk.yaml diff --git a/pipelines/LSSTCam/cpCti.yaml b/pipelines/LSSTCam/legacy_202409/cpCti.yaml similarity index 100% rename from pipelines/LSSTCam/cpCti.yaml rename to pipelines/LSSTCam/legacy_202409/cpCti.yaml diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/legacy_202409/cpDark.yaml similarity index 100% rename from pipelines/LSSTCam/cpDark.yaml rename to pipelines/LSSTCam/legacy_202409/cpDark.yaml diff --git a/pipelines/LSSTCam/cpDarkForDefects.yaml b/pipelines/LSSTCam/legacy_202409/cpDarkForDefects.yaml similarity index 100% rename from pipelines/LSSTCam/cpDarkForDefects.yaml rename to pipelines/LSSTCam/legacy_202409/cpDarkForDefects.yaml diff --git a/pipelines/LSSTCam/cpDefects.yaml b/pipelines/LSSTCam/legacy_202409/cpDefects.yaml similarity index 100% rename from pipelines/LSSTCam/cpDefects.yaml rename to pipelines/LSSTCam/legacy_202409/cpDefects.yaml diff --git a/pipelines/LSSTCam/cpDefectsIndividual.yaml b/pipelines/LSSTCam/legacy_202409/cpDefectsIndividual.yaml similarity index 100% rename from pipelines/LSSTCam/cpDefectsIndividual.yaml rename to pipelines/LSSTCam/legacy_202409/cpDefectsIndividual.yaml diff --git a/pipelines/LSSTCam/cpFlat.yaml b/pipelines/LSSTCam/legacy_202409/cpFlat.yaml similarity index 100% rename from pipelines/LSSTCam/cpFlat.yaml rename to pipelines/LSSTCam/legacy_202409/cpFlat.yaml diff --git a/pipelines/LSSTCam/cpFlatSingleChip.yaml b/pipelines/LSSTCam/legacy_202409/cpFlatSingleChip.yaml similarity index 100% rename from pipelines/LSSTCam/cpFlatSingleChip.yaml rename to pipelines/LSSTCam/legacy_202409/cpFlatSingleChip.yaml diff --git a/pipelines/LSSTCam/cpFringe.yaml b/pipelines/LSSTCam/legacy_202409/cpFringe.yaml similarity index 100% rename from pipelines/LSSTCam/cpFringe.yaml rename to pipelines/LSSTCam/legacy_202409/cpFringe.yaml diff --git a/pipelines/LSSTCam/cpLinearizer.yaml b/pipelines/LSSTCam/legacy_202409/cpLinearizer.yaml similarity index 100% rename from pipelines/LSSTCam/cpLinearizer.yaml rename to pipelines/LSSTCam/legacy_202409/cpLinearizer.yaml diff --git a/pipelines/LSSTCam/cpPlotPtc.yaml b/pipelines/LSSTCam/legacy_202409/cpPlotPtc.yaml similarity index 100% rename from pipelines/LSSTCam/cpPlotPtc.yaml rename to pipelines/LSSTCam/legacy_202409/cpPlotPtc.yaml diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/legacy_202409/cpPtc.yaml similarity index 100% rename from pipelines/LSSTCam/cpPtc.yaml rename to pipelines/LSSTCam/legacy_202409/cpPtc.yaml diff --git a/pipelines/LSSTCam/cpPtcNoPhotodiode.yaml b/pipelines/LSSTCam/legacy_202409/cpPtcNoPhotodiode.yaml similarity index 100% rename from pipelines/LSSTCam/cpPtcNoPhotodiode.yaml rename to pipelines/LSSTCam/legacy_202409/cpPtcNoPhotodiode.yaml diff --git a/pipelines/LSSTCam/cpSky.yaml b/pipelines/LSSTCam/legacy_202409/cpSky.yaml similarity index 100% rename from pipelines/LSSTCam/cpSky.yaml rename to pipelines/LSSTCam/legacy_202409/cpSky.yaml From 11f8cc3c4e223bc6cf5377ceae71684b3a325c93 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:12:58 -0700 Subject: [PATCH 02/23] Add new IsrTaskLSST pipeline for LSSTCam bfk. --- pipelines/LSSTCam/cpBfk.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 pipelines/LSSTCam/cpBfk.yaml diff --git a/pipelines/LSSTCam/cpBfk.yaml b/pipelines/LSSTCam/cpBfk.yaml new file mode 100644 index 00000000..f9fb55f4 --- /dev/null +++ b/pipelines/LSSTCam/cpBfk.yaml @@ -0,0 +1,12 @@ +description: cp_pipe LSSTCam brighter-fatter kernel calibration construction. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBfkLSST.yaml +tasks: + cpBfkPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + useEfdPhotodiodeData: false + doExtractPhotodiodeData: true + auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] + matchExposuresType: FLUX From 338cac683ad7a5fcffc18e94f3641d11808bcd16 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:13:18 -0700 Subject: [PATCH 03/23] Add new IsrTaskLSST pipeline for LSSTCam bias. --- pipelines/LSSTCam/cpBias.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpBias.yaml diff --git a/pipelines/LSSTCam/cpBias.yaml b/pipelines/LSSTCam/cpBias.yaml new file mode 100644 index 00000000..29435e36 --- /dev/null +++ b/pipelines/LSSTCam/cpBias.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTCam BIAS calibration construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBiasLSST.yaml From 2272f6e85a5a576c7f76dd76b64d3ac75072e04b Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:13:38 -0700 Subject: [PATCH 04/23] Add new IsrTaskLSST pipeline for LSSTCam biasBootstrap. --- pipelines/LSSTCam/cpBiasBootstrap.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpBiasBootstrap.yaml diff --git a/pipelines/LSSTCam/cpBiasBootstrap.yaml b/pipelines/LSSTCam/cpBiasBootstrap.yaml new file mode 100644 index 00000000..15bcad2b --- /dev/null +++ b/pipelines/LSSTCam/cpBiasBootstrap.yaml @@ -0,0 +1,4 @@ +description: LSSTCam bootstrap bias calibration construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBiasBootstrapLSST.yaml From e678a00d764448df3eb366ca20cd5d605a9b361c Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:14:04 -0700 Subject: [PATCH 05/23] Add new IsrTaskLSST pipeline for LSSTCam dark. --- pipelines/LSSTCam/cpDark.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpDark.yaml diff --git a/pipelines/LSSTCam/cpDark.yaml b/pipelines/LSSTCam/cpDark.yaml new file mode 100644 index 00000000..eb23d22c --- /dev/null +++ b/pipelines/LSSTCam/cpDark.yaml @@ -0,0 +1,4 @@ +description: cp_pipe DARK calibration construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkLSST.yaml From fc46aa3efe6799370355ff5187ce8e617f1eb18a Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:14:16 -0700 Subject: [PATCH 06/23] Add new IsrTaskLSST pipeline for LSSTCam darkBootstrap. --- pipelines/LSSTCam/cpDarkBootstrap.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpDarkBootstrap.yaml diff --git a/pipelines/LSSTCam/cpDarkBootstrap.yaml b/pipelines/LSSTCam/cpDarkBootstrap.yaml new file mode 100644 index 00000000..c5be5b21 --- /dev/null +++ b/pipelines/LSSTCam/cpDarkBootstrap.yaml @@ -0,0 +1,4 @@ +description: LSSTCam bootstrap dark calibration construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkBootstrapLSST.yaml From 1e7854d2354b55e7b1eccef955aec5d19bcb67d9 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:14:31 -0700 Subject: [PATCH 07/23] Add new IsrTaskLSST pipeline for LSSTCam defects. --- pipelines/LSSTCam/cpDefects.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 pipelines/LSSTCam/cpDefects.yaml diff --git a/pipelines/LSSTCam/cpDefects.yaml b/pipelines/LSSTCam/cpDefects.yaml new file mode 100644 index 00000000..015380d3 --- /dev/null +++ b/pipelines/LSSTCam/cpDefects.yaml @@ -0,0 +1,29 @@ +description: cp_pipe LSSTCam DEFECT calibration construction from combined images. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDefectsLSST.yaml +tasks: + cpMeasureBiasDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + badPixelsToFillColumnThreshold: 1000 + saturatedPixelsToFillColumnThreshold: 20 + saturatedColumnDilationRadius: 2 + cpMeasureDarkDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + thresholdType: "VALUE" + darkCurrentThreshold: 3.0 + badPixelsToFillColumnThreshold: 1000 + saturatedPixelsToFillColumnThreshold: 20 + saturatedColumnDilationRadius: 2 + cpMeasureFlatDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedWithFilterTask + config: + thresholdType: "VALUE" + fracThresholdFlat: 0.9 + badPixelsToFillColumnThreshold: 1000 + cpMergeDefects: + class: lsst.cp.pipe.MergeDefectsCombinedTask + config: + edgesAsDefects: true From 1aa399d98caf1524dfed3faf059d5e2e99b5d813 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:14:49 -0700 Subject: [PATCH 08/23] Add new IsrTaskLSST pipeline for LSSTCam flat. --- pipelines/LSSTCam/cpFlat.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpFlat.yaml diff --git a/pipelines/LSSTCam/cpFlat.yaml b/pipelines/LSSTCam/cpFlat.yaml new file mode 100644 index 00000000..39c63c10 --- /dev/null +++ b/pipelines/LSSTCam/cpFlat.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTCam flat calibration construction. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatLSST.yaml From 486a886458b6ec7f1eb3d4f845eff9ccae4d259f Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:15:04 -0700 Subject: [PATCH 09/23] Add new IsrTaskLSST pipeline for LSSTCam flatBootstrap. --- pipelines/LSSTCam/cpFlatBootstrap.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpFlatBootstrap.yaml diff --git a/pipelines/LSSTCam/cpFlatBootstrap.yaml b/pipelines/LSSTCam/cpFlatBootstrap.yaml new file mode 100644 index 00000000..5908f052 --- /dev/null +++ b/pipelines/LSSTCam/cpFlatBootstrap.yaml @@ -0,0 +1,4 @@ +description: LSSTCam bootstrap flat construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatBootstrapLSST.yaml From 09970cce414d9969cd1ed80f618a151bc3559e85 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:15:19 -0700 Subject: [PATCH 10/23] Add new IsrTaskLSST pipeline for LSSTCam flat (single chip). --- pipelines/LSSTCam/cpFlatSingleChip.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpFlatSingleChip.yaml diff --git a/pipelines/LSSTCam/cpFlatSingleChip.yaml b/pipelines/LSSTCam/cpFlatSingleChip.yaml new file mode 100644 index 00000000..30f21a89 --- /dev/null +++ b/pipelines/LSSTCam/cpFlatSingleChip.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTCam FLAT calibration construction, optimized for a single chip. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChipLSST.yaml From f64e7f3da0c3ddd66089e9c285ce89d08fb8fda1 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:15:36 -0700 Subject: [PATCH 11/23] Add new IsrTaskLSST pipeline for LSSTCam linearizer. --- pipelines/LSSTCam/cpLinearizer.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 pipelines/LSSTCam/cpLinearizer.yaml diff --git a/pipelines/LSSTCam/cpLinearizer.yaml b/pipelines/LSSTCam/cpLinearizer.yaml new file mode 100644 index 00000000..f6af9207 --- /dev/null +++ b/pipelines/LSSTCam/cpLinearizer.yaml @@ -0,0 +1,26 @@ +description: cp_pipe LSSTCam linearity calibration construction. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpLinearizerLSST.yaml +tasks: + cpLinearizerPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + useEfdPhotodiodeData: false + doExtractPhotodiodeData: true + auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] + matchExposuresType: FLUX + cpLinearizerSolve: + class: lsst.cp.pipe.LinearitySolveTask + config: + linearityType: Spline + usePhotodiode: true + splineGroupingColumn: CCOBCURR + splineFitTemperatureColumn: TEMP6 + splineFitWeightParsStart: [7.2e-5, 1e-4] + doSplineFitWeights: false + doSplineFitTemperature: false + minPhotodiodeCurrent: 3e-10 + doSplineFitOffset: true + splineFitMaxIter: 40 + splineKnots: 15 From 44c4dde1d7d8b0a9fd4124edba32d4db13964cdf Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:15:52 -0700 Subject: [PATCH 12/23] Add new IsrTaskLSST pipeline for LSSTCam ptc plotting. --- pipelines/LSSTCam/cpPlotPtc.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpPlotPtc.yaml diff --git a/pipelines/LSSTCam/cpPlotPtc.yaml b/pipelines/LSSTCam/cpPlotPtc.yaml new file mode 100644 index 00000000..461fd16b --- /dev/null +++ b/pipelines/LSSTCam/cpPlotPtc.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTCam task to plot Photon Transfer Curve dataset +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPlotPtc.yaml From 500805584dfecfbf92c99819570f6727e85cecd3 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:16:06 -0700 Subject: [PATCH 13/23] Add new IsrTaskLSST pipeline for LSSTCam ptc. --- pipelines/LSSTCam/cpPtc.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 pipelines/LSSTCam/cpPtc.yaml diff --git a/pipelines/LSSTCam/cpPtc.yaml b/pipelines/LSSTCam/cpPtc.yaml new file mode 100644 index 00000000..5cf767f2 --- /dev/null +++ b/pipelines/LSSTCam/cpPtc.yaml @@ -0,0 +1,12 @@ +description: LSSTCam Photon-Transfer Curve calibration construction. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcLSST.yaml +tasks: + cpPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + useEfdPhotodiodeData: false + doExtractPhotodiodeData: true + auxiliaryHeaderKeys: ["CCOBCURR", "CCOBFLUX", "TEMP6"] + matchExposuresType: FLUX From b870de2a0296ac6d336b10b5a1895178f79b4352 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:16:22 -0700 Subject: [PATCH 14/23] Add new IsrTaskLSST pipeline for LSSTCam cpSky. --- pipelines/LSSTCam/cpSky.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 pipelines/LSSTCam/cpSky.yaml diff --git a/pipelines/LSSTCam/cpSky.yaml b/pipelines/LSSTCam/cpSky.yaml new file mode 100644 index 00000000..afc954c9 --- /dev/null +++ b/pipelines/LSSTCam/cpSky.yaml @@ -0,0 +1,15 @@ +description: LSSTCam Sky frame generation pipeline definition. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpSkyLSST.yaml +tasks: + cpSkyImage: + class: lsst.cp.pipe.CpSkyImageTask + config: + # These values roughly equate to the dimensions of a single detector. + # As noted below, the sizes are in millimeters, and correspond to a + # background image of 4096*4096 pixels (4096*0.01=40.96). + # TODO: Revise these values on DM-43321. + largeScaleBackground.xSize: 40.96 # in mm + largeScaleBackground.ySize: 40.96 # in mm + largeScaleBackground.pixelSize: 0.01 # in mm per pixel From 7f5afc1531b28da32eb6c72d9d25e15344708283 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 1 Oct 2024 11:19:32 -0700 Subject: [PATCH 15/23] Update pipeline testing for LSSTCam with IsrTaskLSST. --- tests/test_pipelines.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 56fe2a4e..ccc93302 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -155,12 +155,11 @@ def test_lsstcam_pipelines(self): "cpFilterScan.yaml", "cpMonochromatorScan.yaml", "cpSpectroFlat.yaml", - # TODO DM-46358: Remove following from exclusion list. - "cpBiasBootstrap.yaml", - "cpDarkBootstrap.yaml", - "cpFlatBootstrap.yaml", - "cpPtcFixupGainRatios.yaml", - "cpPtcRename.yaml", + "cpDarkForDefects.yaml", + "cpDefectsIndividual.yaml", + # Unsupported pipelines. + "cpCrosstalk.yaml", + "cpFringe.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam", pipeline)) From 166c95be22c18a61c90fb6303229d37a2c8902ec Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Mon, 7 Oct 2024 07:58:01 -0700 Subject: [PATCH 16/23] Add cpLinearizerPtc subset for generating inputs to linearizer without fitting. --- pipelines/_ingredients/cpLinearizerLSST.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pipelines/_ingredients/cpLinearizerLSST.yaml b/pipelines/_ingredients/cpLinearizerLSST.yaml index e231a5ea..38c09733 100644 --- a/pipelines/_ingredients/cpLinearizerLSST.yaml +++ b/pipelines/_ingredients/cpLinearizerLSST.yaml @@ -43,6 +43,12 @@ tasks: splineKnots: 10 trimmedState: false usePhotodiode: true +subsets: + cpLinearizerPtc: + subset: + - cpLinearizerIsr + - cpLinearizerPtcExtract + - cpLinearizerPtcSolve contracts: - cpLinearizerIsr.doBootstrap == True - cpLinearizerPtcExtract.connections.inputExp == cpLinearizerIsr.connections.outputExposure From d0e28172c0a39daf9442e6547e18344be32cc41d Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 18 Oct 2024 10:47:21 -0700 Subject: [PATCH 17/23] Add new IsrTaskLSST pipeline for LSSTCam cti. --- pipelines/LSSTCam/cpCti.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pipelines/LSSTCam/cpCti.yaml diff --git a/pipelines/LSSTCam/cpCti.yaml b/pipelines/LSSTCam/cpCti.yaml new file mode 100644 index 00000000..057b436f --- /dev/null +++ b/pipelines/LSSTCam/cpCti.yaml @@ -0,0 +1,4 @@ +description: LSSTCam cti calibration construction +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpCtiLSST.yaml From 0b1e911c5987c5bc5ce0d98f321c4870460c16bc Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 18 Oct 2024 16:06:00 -0700 Subject: [PATCH 18/23] Ensure that top node is above the top value to avoid possible rounding errors. --- python/lsst/cp/pipe/cpLinearitySolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/cp/pipe/cpLinearitySolve.py b/python/lsst/cp/pipe/cpLinearitySolve.py index edf0b1ee..83f266ba 100644 --- a/python/lsst/cp/pipe/cpLinearitySolve.py +++ b/python/lsst/cp/pipe/cpLinearitySolve.py @@ -601,7 +601,7 @@ def run(self, inputPtc, dummy, camera, inputDims, # fits deviations from linearity, rather than the linear # function itself which is degenerate with the gain. - nodes = np.linspace(0.0, np.max(inputOrdinate[mask]), self.config.splineKnots) + nodes = np.linspace(0.0, np.max(inputOrdinate[mask]) + 1.0, self.config.splineKnots) if temperatureValues is not None: temperatureValuesScaled = temperatureValues - np.median(temperatureValues[mask]) From 2fbe5f291c9136d70f35f99aa7e7ee98b3328296 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 18 Oct 2024 16:06:31 -0700 Subject: [PATCH 19/23] Make linearity turnoff computation more robust against initial outliers. --- python/lsst/cp/pipe/cpLinearitySolve.py | 31 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/python/lsst/cp/pipe/cpLinearitySolve.py b/python/lsst/cp/pipe/cpLinearitySolve.py index 83f266ba..15eabe8e 100644 --- a/python/lsst/cp/pipe/cpLinearitySolve.py +++ b/python/lsst/cp/pipe/cpLinearitySolve.py @@ -842,11 +842,34 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW fitMask = initialMask.copy() fitMask[ordinate < self.config.minSignalFitLinearityTurnoff] = False - aa = sum(abscissa[fitMask])/sum(abscissa[fitMask]**2/ordinate[fitMask]) + found = False + while (fitMask.sum() >= 4) and not found: + aa = sum(abscissa[fitMask])/sum(abscissa[fitMask]**2/ordinate[fitMask]) + + # Use the residuals to compute the turnoff. + residuals = (ordinate - aa*abscissa)/ordinate + + goodPoints = np.abs(residuals) < self.config.maxFracLinearityDeviation + + if goodPoints.sum() > 4: + # This was an adequate fit. + found = True + turnoffIndex = np.argmax(ordinate[goodPoints]) + else: + # This was a bad fit; remove the largest outlier. + badIndex = np.argmax(np.abs(residuals)[fitMask]) + fitIndices, = np.nonzero(fitMask) + fitMask[fitIndices[badIndex]] = False + + if not found: + # Could not find any reasonable value. + self.log.warning( + "Could not find a reasonable initial linear fit to compute linearity turnoff for " + "amplifier %s; may need finer sampling of input data?", + ampName, + ) + turnoffIndex = np.nonzero(fitMask)[0][-1] - # Use the residuals to compute the turnoff. - residuals = (ordinate - aa*abscissa)/ordinate - turnoffIndex = np.argmax(ordinate[np.abs(residuals) < self.config.maxFracLinearityDeviation]) turnoff = ordinate[turnoffIndex] # Fit the maximum signal. From 830444b026383fa6caccde949f14b7bd69a327ce Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Mon, 21 Oct 2024 09:03:17 -0700 Subject: [PATCH 20/23] Fix checking/logging when all points are bad in initial linearity check. --- python/lsst/cp/pipe/cpLinearitySolve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/lsst/cp/pipe/cpLinearitySolve.py b/python/lsst/cp/pipe/cpLinearitySolve.py index 15eabe8e..e051a971 100644 --- a/python/lsst/cp/pipe/cpLinearitySolve.py +++ b/python/lsst/cp/pipe/cpLinearitySolve.py @@ -869,6 +869,7 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW ampName, ) turnoffIndex = np.nonzero(fitMask)[0][-1] + residuals = np.zeros(len(ordinate)) turnoff = ordinate[turnoffIndex] From 28b5309ac6cf3e1bc67d62d0c5360ae27aaa8e91 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 22 Oct 2024 13:43:46 -0400 Subject: [PATCH 21/23] Update linearity turnoff code to be more robust to odd amps. This also fixes an index error in the selection of the max in the case of intermediate bad points. --- python/lsst/cp/pipe/cpLinearitySolve.py | 33 ++++++++++++++++--------- tests/test_linearity.py | 6 +++-- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/python/lsst/cp/pipe/cpLinearitySolve.py b/python/lsst/cp/pipe/cpLinearitySolve.py index e051a971..e77fb9a3 100644 --- a/python/lsst/cp/pipe/cpLinearitySolve.py +++ b/python/lsst/cp/pipe/cpLinearitySolve.py @@ -487,10 +487,12 @@ def run(self, inputPtc, dummy, camera, inputDims, inputAbscissa = inputPtc.rawExpTimes[ampName].copy() # Compute linearityTurnoff and linearitySignalMax. - turnoff, maxSignal = self._computeTurnoffAndMax( - inputAbscissa[mask], - inputPtc.rawMeans[ampName][mask], - inputPtc.expIdMask[ampName][mask], + turnoffMask = inputPtc.expIdMask[ampName].copy() + turnoffMask &= mask + turnoffIndex, turnoff, maxSignal = self._computeTurnoffAndMax( + inputAbscissa, + inputPtc.rawMeans[ampName], + turnoffMask, ampName=ampName, ) linearizer.linearityTurnoff[ampName] = turnoff @@ -503,7 +505,9 @@ def run(self, inputPtc, dummy, camera, inputDims, else: # For spline fits, cut above the turnoff. self.log.info("Using linearityTurnoff of %.4f adu for amplifier %s", turnoff, ampName) - mask &= (inputOrdinate <= turnoff) + extraMask = np.ones(len(inputOrdinate), dtype=bool) + extraMask[turnoffIndex + 1:] = False + mask &= extraMask mask &= (inputOrdinate > self.config.minLinearAdu) @@ -824,6 +828,8 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW Returns ------- + turnoffIndex : `int` + Fit turnoff index (keyed to raw input). turnoff : `float` Fit turnoff value. maxSignal : `float` @@ -848,13 +854,15 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW # Use the residuals to compute the turnoff. residuals = (ordinate - aa*abscissa)/ordinate + residuals -= np.nanmedian(residuals) goodPoints = np.abs(residuals) < self.config.maxFracLinearityDeviation if goodPoints.sum() > 4: # This was an adequate fit. found = True - turnoffIndex = np.argmax(ordinate[goodPoints]) + turnoff = np.max(ordinate[goodPoints]) + turnoffIndex = np.where(np.isclose(ordinate, turnoff))[0][0] else: # This was a bad fit; remove the largest outlier. badIndex = np.argmax(np.abs(residuals)[fitMask]) @@ -868,10 +876,10 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW "amplifier %s; may need finer sampling of input data?", ampName, ) - turnoffIndex = np.nonzero(fitMask)[0][-1] - residuals = np.zeros(len(ordinate)) + turnoff = np.max(ordinate[fitMask]) + turnoffIndex = np.where(np.isclose(ordinate, turnoff))[0][0] - turnoff = ordinate[turnoffIndex] + residuals = np.zeros(len(ordinate)) # Fit the maximum signal. if turnoffIndex == (len(residuals) - 1): @@ -882,12 +890,13 @@ def _computeTurnoffAndMax(self, abscissa, ordinate, initialMask, ampName="UNKNOW ) maxSignal = ordinate[turnoffIndex] else: - maxSignalInitial = np.max(ordinate) + maxSignalInitial = np.nanmax(ordinate) - highFluxPoints = (ordinate > (1.0 - self.config.maxFracLinearityDeviation)*maxSignalInitial) + highFluxPoints = (np.nan_to_num(ordinate) + > (1.0 - self.config.maxFracLinearityDeviation)*maxSignalInitial) maxSignal = np.median(ordinate[highFluxPoints]) - return turnoff, maxSignal + return turnoffIndex, turnoff, maxSignal def debugFit(self, stepname, xVector, yVector, yModel, mask, ampName): """Debug method for linearity fitting. diff --git a/tests/test_linearity.py b/tests/test_linearity.py index b1468dbe..604c1f95 100644 --- a/tests/test_linearity.py +++ b/tests/test_linearity.py @@ -590,9 +590,10 @@ def test_linearity_turnoff(self): task = LinearitySolveTask(config=config) with self.assertNoLogs(level=logging.WARNING): - turnoff, max_signal = task._computeTurnoffAndMax(abscissa, ordinate, ptc_mask) + turnoff_index, turnoff, max_signal = task._computeTurnoffAndMax(abscissa, ordinate, ptc_mask) # This was visually inspected such that these are reasonable. + self.assertEqual(turnoff_index, 90) np.testing.assert_almost_equal(turnoff, 99756.30512572) np.testing.assert_almost_equal(max_signal, 108730.32842316) @@ -649,12 +650,13 @@ def test_linearity_turnoff(self): cutoff = (ordinate < turnoff) with self.assertLogs(level=logging.WARNING) as cm: - turnoff2, max_signal2 = task._computeTurnoffAndMax( + turnoff_index2, turnoff2, max_signal2 = task._computeTurnoffAndMax( abscissa[cutoff], ordinate[cutoff], ptc_mask[cutoff], ) self.assertIn("No linearity turnoff", cm.output[0]) + self.assertEqual(turnoff_index2, len(ptc_mask[cutoff]) - 1) def _comcam_raw_linearity_data(self): # These are LSSTComCam measurements taken from a calibration From c73b5471472f8629c33eb094a6733a435c1f5f74 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 23 Oct 2024 13:26:37 -0700 Subject: [PATCH 22/23] Add high flux linearity constraint to avoid craziness. --- python/lsst/cp/pipe/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/lsst/cp/pipe/utils.py b/python/lsst/cp/pipe/utils.py index ce4bf4cb..b541f7d8 100644 --- a/python/lsst/cp/pipe/utils.py +++ b/python/lsst/cp/pipe/utils.py @@ -1339,7 +1339,14 @@ def __call__(self, pars): log_w = np.sqrt(-2.*np.log(fact*self._w[self._w > 0])) constraint = np.hstack([constraint, log_w]) - return np.hstack([resid, constraint]) + # Don't let it get to >5% correction. + values = pars[self.par_indices["values"]] + if np.abs(values[-1])/self._nodes[-1] > 0.25: + extra_constraint = 1e10 + else: + extra_constraint = 0 + + return np.hstack([resid, constraint, extra_constraint]) def getReadNoise(exposure, ampName, taskMetadata=None, log=None): From ab5e4992c180e058f5cd429431536ecbbb162577 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 8 Nov 2024 10:50:21 -0800 Subject: [PATCH 23/23] Add new IsrTaskLSST pipelines for LSSTCam cpPtcFixupGainRatios. --- pipelines/LSSTCam/cpPtcFixupGainRatios.yaml | 4 ++++ pipelines/LSSTCam/cpPtcRename.yaml | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 pipelines/LSSTCam/cpPtcFixupGainRatios.yaml create mode 100644 pipelines/LSSTCam/cpPtcRename.yaml diff --git a/pipelines/LSSTCam/cpPtcFixupGainRatios.yaml b/pipelines/LSSTCam/cpPtcFixupGainRatios.yaml new file mode 100644 index 00000000..3a692b15 --- /dev/null +++ b/pipelines/LSSTCam/cpPtcFixupGainRatios.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTCam ptc fixup. +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcFixupGainRatios.yaml diff --git a/pipelines/LSSTCam/cpPtcRename.yaml b/pipelines/LSSTCam/cpPtcRename.yaml new file mode 100644 index 00000000..3b974fcf --- /dev/null +++ b/pipelines/LSSTCam/cpPtcRename.yaml @@ -0,0 +1,4 @@ +description: cp_pipe ptc renaming (LSSTCam). +instrument: lsst.obs.lsst.LsstCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcRename.yaml