From dce7c6034d97c3656993a4f8d2db90ffe1576f13 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 09:44:03 -0800 Subject: [PATCH 1/6] Add PhotonTransferCurveFixupGainRatiosTask. --- python/lsst/cp/pipe/ptc/__init__.py | 1 + .../lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py | 173 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py diff --git a/python/lsst/cp/pipe/ptc/__init__.py b/python/lsst/cp/pipe/ptc/__init__.py index 2db6d1cc..624aeef9 100755 --- a/python/lsst/cp/pipe/ptc/__init__.py +++ b/python/lsst/cp/pipe/ptc/__init__.py @@ -24,3 +24,4 @@ from .cpPlotPtc import * from .cpPtcExtract import * from .cpPtcSolve import * +from .cpPtcFixupGainRatios import * diff --git a/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py b/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py new file mode 100644 index 00000000..645216ab --- /dev/null +++ b/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py @@ -0,0 +1,173 @@ +# This file is part of cp_pipe. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import copy +import numpy as np + +from lsst.ip.isr import PhotonTransferCurveDataset +import lsst.pex.config as pexConfig +import lsst.pipe.base as pipeBase +import lsst.pipe.base.connectionTypes as cT + +from ..utils import ampOffsetGainRatioFixup + + +__all__ = ["PhotonTransferCurveFixupGainRatiosConfig", "PhotonTransferCurveFixupGainRatiosTask"] + + +class PhotonTransferCurveFixupGainRatiosConnections( + pipeBase.PipelineTaskConnections, + dimensions=("instrument", "detector") +): + exposureMetadata = cT.Input( + name="cpPtcFixupGainRatiosIsrExp.metadata", + doc="Input exposures for gain ratio fixup.", + storageClass="PropertyList", + dimensions=("instrument", "exposure", "detector"), + multiple=True, + ) + inputPtc = cT.PrerequisiteInput( + name="ptc", + doc="Input PTC to modify.", + storageClass="PhotonTransferCurveDataset", + dimensions=("instrument", "detector"), + isCalibration=True, + ) + outputPtc = cT.Output( + name="ptcFixed", + doc="Output modified PTC.", + storageClass="PhotonTransferCurveDataset", + dimensions=("instrument", "detector"), + multiple=False, + isCalibration=True, + ) + + +class PhotonTransferCurveFixupGainRatiosConfig( + pipeBase.PipelineTaskConfig, + pipelineConnections=PhotonTransferCurveFixupGainRatiosConnections, +): + ampOffsetGainRatioMinAdu = pexConfig.Field( + dtype=float, + doc="Minimum number of adu to use for amp offset gain ratio fixup.", + default=1000.0, + ) + ampOffsetGainRatioMaxAdu = pexConfig.Field( + dtype=float, + doc="Maximum number of adu to use for amp offset gain ratio fixup.", + default=40000.0, + ) + + +class PhotonTransferCurveFixupGainRatiosTask(pipeBase.PipelineTask): + """Task to use on-sky amp ratios to fix up gain ratios in a PTC. + + This uses the ampOffsetGainRatioFixup with on-sky data (preferably + twilight flats or similar) to update gain ratios. + """ + ConfigClass = PhotonTransferCurveFixupGainRatiosConfig + _DefaultName = "cpPhotonTransferCurveFixupGainRatios" + + def runQuantum(self, butlerQC, inputRefs, outputRefs): + # docstring inherited. + inputs = butlerQC.get(inputRefs) + outputs = self.run(inputPtc=inputs["inputPtc"], exposureMetadata=inputs["exposureMetadata"]) + butlerQC.put(outputs, outputRefs) + + def run(self, *, inputPtc, exposureMetadata): + """Run the gain ratio fixup task. + + Parameters + ---------- + inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset` + Input PTC to modify. + exposureMetadata: `list` [`lsst.daf.base.PropertyList`] + Input exposure metadata. + + Returns + ------- + results : `lsst.pipe.base.Struct` + The output struct contains: + + ``outputPtc`` + The output modified ptc. + """ + ampNames = inputPtc.ampNames + + # Create a set of fake partial PTC datasets. + fakePtc = PhotonTransferCurveDataset( + ampNames=ampNames, + ptcFitType="FAKEPTC", + covMatrixSide=1, + covMatrixSideFullCovFit=1, + ) + + for i, metadata in enumerate(exposureMetadata): + fakePartialPtc = PhotonTransferCurveDataset(ampNames=ampNames, ptcFitType="PARTIAL") + + for ampName in ampNames: + fakePartialPtc.setAmpValuesPartialDataset( + ampName, + inputExpIdPair=(2*i, 2*i + 1), + rawExpTime=float(i), + rawMean=metadata[f"LSST ISR FINAL MEDIAN {ampName}"], + rawVar=metadata[f"LSST ISR FINAL STDEV {ampName}"]**2., + ampOffset=metadata[f"LSST ISR AMPOFFSET PEDESTAL {ampName}"], + expIdMask=True, + gain=inputPtc.gainUnadjusted[ampName], + noise=metadata[f"LSST ISR READNOISE {ampName}"]*inputPtc.gain[ampName], + covariance=np.zeros((1, 1)), + covSqrtWeights=np.zeros((1, 1)), + ) + + fakePtc.appendPartialPtc(fakePartialPtc) + + detectorMeans = np.zeros(len(exposureMetadata)) + + for i in range(len(detectorMeans)): + arr = np.asarray([fakePtc.rawMeans[ampName][i] for ampName in ampNames]) + detectorMeans[i] = np.nanmean(arr) + + index = np.argsort(detectorMeans) + fakePtc.sort(index) + + for ampName in ampNames: + fakePtc.finalMeans[ampName][:] = fakePtc.rawMeans[ampName].copy() + fakePtc.finalVars[ampName][:] = fakePtc.rawVars[ampName].copy() + fakePtc.gainUnadjusted[ampName] = inputPtc.gainUnadjusted[ampName] + fakePtc.gain[ampName] = inputPtc.gainUnadjusted[ampName] + + ampOffsetGainRatioFixup( + fakePtc, + self.config.ampOffsetGainRatioMinAdu, + self.config.ampOffsetGainRatioMaxAdu, + log=self.log, + ) + + outputPtc = copy.copy(inputPtc) + + # Replace the gain (leave gainUnadjusted alone). + for ampName in ampNames: + outputPtc.gain[ampName] = fakePtc.gain[ampName] + + return pipeBase.Struct( + outputPtc=outputPtc, + ) From 1d0d8b722b837181953b93356a2183be78717aef Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 10:13:56 -0800 Subject: [PATCH 2/6] Add pipelines for cpPtcFixupGainRatiosTask. --- pipelines/LATISS/cpPtcFixupGainRatios.yaml | 4 +++ .../LSSTComCam/cpPtcFixupGainRatios.yaml | 4 +++ .../_ingredients/cpPtcFixupGainRatios.yaml | 28 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 pipelines/LATISS/cpPtcFixupGainRatios.yaml create mode 100644 pipelines/LSSTComCam/cpPtcFixupGainRatios.yaml create mode 100644 pipelines/_ingredients/cpPtcFixupGainRatios.yaml diff --git a/pipelines/LATISS/cpPtcFixupGainRatios.yaml b/pipelines/LATISS/cpPtcFixupGainRatios.yaml new file mode 100644 index 00000000..fe58ed2d --- /dev/null +++ b/pipelines/LATISS/cpPtcFixupGainRatios.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LATISS ptc fixup. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcFixupGainRatios.yaml diff --git a/pipelines/LSSTComCam/cpPtcFixupGainRatios.yaml b/pipelines/LSSTComCam/cpPtcFixupGainRatios.yaml new file mode 100644 index 00000000..3737b110 --- /dev/null +++ b/pipelines/LSSTComCam/cpPtcFixupGainRatios.yaml @@ -0,0 +1,4 @@ +description: cp_pipe LSSTComCam ptc fixup. +instrument: lsst.obs.lsst.LsstComCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcFixupGainRatios.yaml diff --git a/pipelines/_ingredients/cpPtcFixupGainRatios.yaml b/pipelines/_ingredients/cpPtcFixupGainRatios.yaml new file mode 100644 index 00000000..a9478eab --- /dev/null +++ b/pipelines/_ingredients/cpPtcFixupGainRatios.yaml @@ -0,0 +1,28 @@ +description: cp_pipe ptc fixup. +tasks: + cpPtcFixupGainRatiosIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpPtcFixupGainRatiosIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doLinearize = True + config.doDefect = True + config.doAmpOffset = True + config.ampOffset.ampEdgeMaxOffset = 100000.0 + config.ampOffset.ampEdgeInset = 10 + config.ampOffset.doBackground = False + config.ampOffset.doDetection = False + config.ampOffset.doApplyAmpOffset = False + cpPtcFixupGainRatios: + class: lsst.cp.pipe.ptc.PhotonTransferCurveFixupGainRatiosTask + +contracts: + - cpPtcFixupGainRatiosIsr.doBootstrap == True From a9ee24e0f00c393f6a29c34ab4bd093253a4ac2d Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 10:14:28 -0800 Subject: [PATCH 3/6] Update pipelines tests for cpPtcFixupGainRatiosTask. --- tests/test_pipelines.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 9f9ef5fd..6f859574 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -77,6 +77,7 @@ def _get_pipelines(self, exclude=[]): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpSpectroFlat.yaml", + "cpPtcFixupGainRatios.yaml", } for ex in exclude: @@ -157,6 +158,7 @@ def test_lsstcam_pipelines(self): "cpBiasBootstrap.yaml", "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam", pipeline)) @@ -170,6 +172,7 @@ def test_lsstcam_imsim_pipelines(self): "cpBiasBootstrap.yaml", "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam-imSim", pipeline)) @@ -204,6 +207,7 @@ def test_lsstcomcamsim_pipelines(self): "cpLinearizer.yaml", "cpCrosstalk.yaml", "cpCti.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTComCamSim", pipeline)) @@ -216,6 +220,7 @@ def test_lsst_ts8_pipelines(self): "cpBiasBootstrap.yaml", "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSST-TS8", pipeline)) @@ -229,6 +234,7 @@ def test_decam_pipelines(self): "cpBiasBootstrap.yaml", "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "DECam", pipeline)) @@ -242,6 +248,7 @@ def test_hsc_pipelines(self): "cpBiasBootstrap.yaml", "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", + "cpPtcFixupGainRatios.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "HSC", pipeline)) From 4f75b6e952e835bafc24af94cd2383862d630aee Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 14:06:25 -0800 Subject: [PATCH 4/6] Add PhotonTransferCurveRenameTask to rename a PTC dataset. --- .../lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py b/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py index 645216ab..4a903e8d 100644 --- a/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py +++ b/python/lsst/cp/pipe/ptc/cpPtcFixupGainRatios.py @@ -30,7 +30,12 @@ from ..utils import ampOffsetGainRatioFixup -__all__ = ["PhotonTransferCurveFixupGainRatiosConfig", "PhotonTransferCurveFixupGainRatiosTask"] +__all__ = [ + "PhotonTransferCurveFixupGainRatiosConfig", + "PhotonTransferCurveFixupGainRatiosTask", + "PhotonTransferCurveRenameConfig", + "PhotonTransferCurveRenameTask", +] class PhotonTransferCurveFixupGainRatiosConnections( @@ -171,3 +176,47 @@ def run(self, *, inputPtc, exposureMetadata): return pipeBase.Struct( outputPtc=outputPtc, ) + + +class PhotonTransferCurveRenameConnections( + pipeBase.PipelineTaskConnections, + dimensions=("instrument", "detector") +): + inputPtc = cT.PrerequisiteInput( + name="ptcFixed", + doc="Input PTC to rename.", + storageClass="PhotonTransferCurveDataset", + dimensions=("instrument", "detector"), + isCalibration=True, + ) + outputPtc = cT.Output( + name="ptc", + doc="Output PTC that has been renamed.", + storageClass="PhotonTransferCurveDataset", + dimensions=("instrument", "detector"), + multiple=False, + isCalibration=True, + ) + + +class PhotonTransferCurveRenameConfig( + pipeBase.PipelineTaskConfig, + pipelineConnections=PhotonTransferCurveRenameConnections, +): + pass + + +class PhotonTransferCurveRenameTask(pipeBase.PipelineTask): + """Task to rename a ptcFixed into a ptc.""" + ConfigClass = PhotonTransferCurveRenameConfig + _DefaultName = "cpPhotonTransferCurveRename" + + def runQuantum(self, butlerQC, inputRefs, outputRefs): + # docstring inherited. + inputs = butlerQC.get(inputRefs) + + outputs = pipeBase.Struct(ptc=inputs["inputPtc"]) + butlerQC.put(outputs, outputRefs) + + def run(self): + pass From 1e008a1fc55cdc9b7b9a173521d637e3bfc5cb7c Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 14:30:08 -0800 Subject: [PATCH 5/6] Add cpPtcRename pipelines. --- pipelines/LATISS/cpPtcRename.yaml | 4 ++++ pipelines/LSSTComCam/cpPtcRename.yaml | 4 ++++ pipelines/_ingredients/cpPtcRename.yaml | 4 ++++ tests/test_pipelines.py | 7 +++++++ 4 files changed, 19 insertions(+) create mode 100644 pipelines/LATISS/cpPtcRename.yaml create mode 100644 pipelines/LSSTComCam/cpPtcRename.yaml create mode 100644 pipelines/_ingredients/cpPtcRename.yaml diff --git a/pipelines/LATISS/cpPtcRename.yaml b/pipelines/LATISS/cpPtcRename.yaml new file mode 100644 index 00000000..074ff46f --- /dev/null +++ b/pipelines/LATISS/cpPtcRename.yaml @@ -0,0 +1,4 @@ +description: cp_pipe ptc renaming (LATISS). +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcRename.yaml diff --git a/pipelines/LSSTComCam/cpPtcRename.yaml b/pipelines/LSSTComCam/cpPtcRename.yaml new file mode 100644 index 00000000..01a37cd7 --- /dev/null +++ b/pipelines/LSSTComCam/cpPtcRename.yaml @@ -0,0 +1,4 @@ +description: cp_pipe ptc renaming (ComCam). +instrument: lsst.obs.lsst.LsstComCam +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcRename.yaml diff --git a/pipelines/_ingredients/cpPtcRename.yaml b/pipelines/_ingredients/cpPtcRename.yaml new file mode 100644 index 00000000..6e53e87a --- /dev/null +++ b/pipelines/_ingredients/cpPtcRename.yaml @@ -0,0 +1,4 @@ +description: cp_pipe ptc renaming. +tasks: + cpPtcRename: + class: lsst.cp.pipe.PhotonTransferCurveRenameTask diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 6f859574..56fe2a4e 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -78,6 +78,7 @@ def _get_pipelines(self, exclude=[]): "cpFlatBootstrap.yaml", "cpSpectroFlat.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", } for ex in exclude: @@ -159,6 +160,7 @@ def test_lsstcam_pipelines(self): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam", pipeline)) @@ -173,6 +175,7 @@ def test_lsstcam_imsim_pipelines(self): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam-imSim", pipeline)) @@ -208,6 +211,7 @@ def test_lsstcomcamsim_pipelines(self): "cpCrosstalk.yaml", "cpCti.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTComCamSim", pipeline)) @@ -221,6 +225,7 @@ def test_lsst_ts8_pipelines(self): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSST-TS8", pipeline)) @@ -235,6 +240,7 @@ def test_decam_pipelines(self): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "DECam", pipeline)) @@ -249,6 +255,7 @@ def test_hsc_pipelines(self): "cpDarkBootstrap.yaml", "cpFlatBootstrap.yaml", "cpPtcFixupGainRatios.yaml", + "cpPtcRename.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "HSC", pipeline)) From b246f8cd139b0de11e149d41d493a29e980fff11 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Wed, 6 Nov 2024 17:05:36 -0800 Subject: [PATCH 6/6] Change amp fixup tests to use PhotonTransferCurveFixupGainRatiosTask. --- tests/test_ptc.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/test_ptc.py b/tests/test_ptc.py index ca1627f2..5c2ddaa2 100644 --- a/tests/test_ptc.py +++ b/tests/test_ptc.py @@ -877,7 +877,8 @@ def test_ptcFitBootstrap(self): self.ptcFitAndCheckPtc(fitType=fitType, order=order, doFitBootstrap=True) def test_ampOffsetGainRatioFixup(self): - """Test the ampOffsetGainRatioFixup utility code.""" + """Test the ampOffsetGainRatioFixup code via + PhotonTransferCurveFixupGainRatiosTask.""" rng = np.random.RandomState(12345) gainsTruth = rng.normal(loc=1.7, scale=0.05, size=len(self.ampNames)) @@ -894,6 +895,12 @@ def test_ampOffsetGainRatioFixup(self): nFlat = 20 meansElectron = gainsMedian * np.linspace(500.0, 30000.0, nFlat) + # Set up the fixup task. + fixupConfig = cpPipe.ptc.PhotonTransferCurveFixupGainRatiosConfig() + fixupConfig.ampOffsetGainRatioMinAdu = 1000.0 + fixupConfig.ampOffsetGainRatioMaxAdu = 20000.0 + fixupTask = cpPipe.ptc.PhotonTransferCurveFixupGainRatiosTask(config=fixupConfig) + for testMode in ["full", "badamp"]: badAmp = None @@ -928,17 +935,23 @@ def test_ampOffsetGainRatioFixup(self): ampOffset = AmpOffsetTask(config=config) detector = self.flatExp1.getDetector() + metadatas = [] for i in range(nFlat): exp = self.flatExp1.clone() for amp in detector: - exp[amp.getBBox()].image.array[:, :] = ptc.finalMeans[amp.getName()][i] + amp_name = amp.getName() + exp[amp.getBBox()].image.array[:, :] = ptc.finalMeans[amp_name][i] + md = exp.metadata - res = ampOffset.run(exp) + md[f"LSST ISR FINAL MEDIAN {amp_name}"] = ptc.finalMeans[amp_name][i] + md[f"LSST ISR FINAL STDEV {amp_name}"] = np.sqrt(ptc.finalMeans[amp_name][i]) + md[f"LSST ISR READNOISE {amp_name}"] = 0.0 - for j, amp in enumerate(detector): - ptc.ampOffsets[amp.getName()][i] = res.pedestals[j] + ampOffset.run(exp) + metadatas.append(exp.metadata) - ampOffsetGainRatioFixup(ptc, 1000.0, 20000.0) + result = fixupTask.run(inputPtc=ptc, exposureMetadata=metadatas) + ptc = result.outputPtc # Check that the flats are flat after adjustment. for i in range(nFlat):