Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-45300: Begin deprecation of task metadata in cp_pipe/cp_verify #253

Merged
merged 2 commits into from
Jul 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 4 additions & 42 deletions python/lsst/cp/pipe/ptc/cpPtcExtract.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from lsst.geom import (Box2I, Point2I, Extent2I)
from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
arrangeFlatsByExpFlux, sigmaClipCorrection,
CovFastFourierTransform)
CovFastFourierTransform, getReadNoise)

import lsst.pipe.base.connectionTypes as cT

Expand Down Expand Up @@ -66,6 +66,7 @@ class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
storageClass="TaskMetadata",
dimensions=("instrument", "exposure", "detector"),
multiple=True,
deprecated="This connection is deprecated and will be removed after v28.",
)
outputCovariances = cT.Output(
name="ptcCovariances",
Expand Down Expand Up @@ -515,16 +516,14 @@ def run(self, inputExp, inputDims, taskMetadata, inputPhotodiodeData=None):
readNoise2 = dict()
meanReadNoise = dict()

expMetadata1 = expRef1.get(component="metadata")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I now see that we already had exp1 and exp2 loaded so this was just extra i/o.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I definitely want to deprecate all taskMetadata inputs, so I've added the notice to that connection as well.

metadataIndex1 = inputDims.index(expId1)
thisTaskMetadata1 = taskMetadata[metadataIndex1]

expMetadata2 = expRef2.get(component="metadata")
metadataIndex2 = inputDims.index(expId2)
thisTaskMetadata2 = taskMetadata[metadataIndex2]

readNoise1[ampName] = self.getReadNoise(expMetadata1, thisTaskMetadata1, ampName)
readNoise2[ampName] = self.getReadNoise(expMetadata2, thisTaskMetadata2, ampName)
readNoise1[ampName] = getReadNoise(exp1, ampName, taskMetadata=thisTaskMetadata1)
readNoise2[ampName] = getReadNoise(exp2, ampName, taskMetadata=thisTaskMetadata2)

meanReadNoise[ampName] = np.nanmean([readNoise1[ampName], readNoise2[ampName]])

Expand Down Expand Up @@ -1042,43 +1041,6 @@ def getGainFromFlatPair(self, im1Area, im2Area, imStatsCtrl, mu1, mu2,

return gain

def getReadNoise(self, exposureMetadata, taskMetadata, ampName):
"""Gets readout noise for an amp from ISR metadata.

If possible, this attempts to get the now-standard headers
added to the exposure itself. If not found there, the ISR
TaskMetadata is searched. If neither of these has the value,
warn and set the read noise to NaN.

Parameters
----------
exposureMetadata : `lsst.daf.base.PropertySet`
Metadata to check for read noise first.
taskMetadata : `lsst.pipe.base.TaskMetadata`
List of exposures metadata from ISR for this exposure.
ampName : `str`
Amplifier name.

Returns
-------
readNoise : `float`
The read noise for this set of exposure/amplifier.
"""
# Try from the exposure first.
expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}"
if expectedKey in exposureMetadata:
return exposureMetadata[expectedKey]

# If not, try getting it from the task metadata.
expectedKey = f"RESIDUAL STDEV {ampName}"
if "isr" in taskMetadata:
if expectedKey in taskMetadata["isr"]:
return taskMetadata["isr"][expectedKey]

self.log.warning("Median readout noise from ISR metadata for amp %s "
"could not be calculated." % ampName)
return np.nan

def computeGaussianHistogramParameters(self, im1Area, im2Area, imStatsCtrl, mu1, mu2):
"""Compute KS test for a Gaussian model fit to a histogram of the
difference image.
Expand Down
46 changes: 45 additions & 1 deletion python/lsst/cp/pipe/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#

__all__ = ['ddict2dict', 'CovFastFourierTransform']
__all__ = ['ddict2dict', 'CovFastFourierTransform', 'getReadNoise']


import galsim
Expand Down Expand Up @@ -1311,3 +1311,47 @@ def __call__(self, pars):
constraint = np.hstack([constraint, log_w])

return np.hstack([resid, constraint])


def getReadNoise(exposure, ampName, taskMetadata=None, log=None):
"""Gets readout noise for an amp from ISR metadata.

If possible, this attempts to get the now-standard headers
added to the exposure itself. If not found there, the ISR
TaskMetadata is searched. If neither of these has the value,
warn and set the read noise to NaN.

Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure to check for read noise first.
ampName : `str`
Amplifier name.
taskMetadata : `lsst.pipe.base.TaskMetadata`, optional
List of exposures metadata from ISR for this exposure.
log : `logging.logger`, optional
Log for messages.

Returns
-------
readNoise : `float`
The read noise for this set of exposure/amplifier.
"""
exposureMetadata = exposure.getMetadata()

# Try from the exposure first.
expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}"
if expectedKey in exposureMetadata:
return exposureMetadata[expectedKey]

# If not, try getting it from the task metadata.
if taskMetadata:
expectedKey = f"RESIDUAL STDEV {ampName}"
if "isr" in taskMetadata:
if expectedKey in taskMetadata["isr"]:
return taskMetadata["isr"][expectedKey]

log = log if log else logging.getLogger(__name__)
log.warning("Median readout noise from ISR metadata for amp %s "
"could not be found." % ampName)
return np.nan
Loading