diff --git a/pipelines/visitQualityExtended.yaml b/pipelines/visitQualityExtended.yaml
index 9a396b09d..0fdaca864 100644
--- a/pipelines/visitQualityExtended.yaml
+++ b/pipelines/visitQualityExtended.yaml
@@ -5,3 +5,11 @@ tasks:
class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
refCatSourceVisit:
class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
+ deltaSkyCorrHist:
+ class: lsst.analysis.tools.tasks.deltaSkyCorrAnalysis.DeltaSkyCorrHistTask
+ deltaSkyCorrAnalysis:
+ class: lsst.analysis.tools.tasks.deltaSkyCorrAnalysis.DeltaSkyCorrAnalysisTask
+ config:
+ atools.deltaSkyCorr: DeltaSkyCorrXYPlot
+ python: |
+ from lsst.analysis.tools.atools import *
diff --git a/python/lsst/analysis/tools/actions/scalar/scalarActions.py b/python/lsst/analysis/tools/actions/scalar/scalarActions.py
index 4756e45d4..56ad716fb 100644
--- a/python/lsst/analysis/tools/actions/scalar/scalarActions.py
+++ b/python/lsst/analysis/tools/actions/scalar/scalarActions.py
@@ -36,6 +36,8 @@
"FracInRange",
"FracNan",
"SumAction",
+ "MedianHistAction",
+ "IqrHistAction",
)
import operator
@@ -246,3 +248,90 @@ def __call__(self, data: KeyedData, **kwargs) -> Scalar:
mask = self.getMask(**kwargs)
arr = cast(Vector, data[self.vectorKey.format(**kwargs)])[mask]
return cast(Scalar, np.nansum(arr))
+
+
+class MedianHistAction(ScalarAction):
+ """Calculates the median of the given histogram data."""
+
+ histKey = Field[str]("Key of frequency Vector")
+ midKey = Field[str]("Key of bin midpoints Vector")
+
+ def getInputSchema(self) -> KeyedDataSchema:
+ return (
+ (self.histKey, Vector),
+ (self.midKey, Vector),
+ )
+
+ def histMedian(self, hist, bin_mid):
+ """Calculates the median of a histogram with binned values
+
+ Parameters
+ ----------
+ hist : `numpy.ndarray`
+ Frequency array
+ bin_mid : `numpy.ndarray`
+ Bin midpoints array
+
+ Returns
+ -------
+ median : `float`
+ Median of histogram with binned values
+ """
+ cumulative_sum = np.cumsum(hist)
+ median_index = np.searchsorted(cumulative_sum, cumulative_sum[-1] / 2)
+ median = bin_mid[median_index]
+ return median
+
+ def __call__(self, data: KeyedData, **kwargs):
+ if len(data[self.histKey.format(**kwargs)]) != 0:
+ hist = cast(Vector, data[self.histKey.format(**kwargs)])
+ bin_mid = cast(Vector, data[self.midKey.format(**kwargs)])
+ med = cast(Scalar, float(self.histMedian(hist, bin_mid)))
+ else:
+ med = np.NaN
+ return med
+
+
+class IqrHistAction(ScalarAction):
+ """Calculates the interquartile range of the given histogram data."""
+
+ histKey = Field[str]("Key of frequency Vector")
+ midKey = Field[str]("Key of bin midpoints Vector")
+
+ def getInputSchema(self) -> KeyedDataSchema:
+ return (
+ (self.histKey, Vector),
+ (self.midKey, Vector),
+ )
+
+ def histIqr(self, hist, bin_mid):
+ """Calculates the interquartile range of a histogram with binned values
+
+ Parameters
+ ----------
+ hist : `numpy.ndarray`
+ Frequency array
+ bin_mid : `numpy.ndarray`
+ Bin midpoints array
+
+ Returns
+ -------
+ iqr : `float`
+ Inter-quartile range of histogram with binned values
+ """
+ cumulative_sum = np.cumsum(hist)
+ liqr_index = np.searchsorted(cumulative_sum, cumulative_sum[-1] / 4)
+ uiqr_index = np.searchsorted(cumulative_sum, (3 / 4) * cumulative_sum[-1])
+ liqr = bin_mid[liqr_index]
+ uiqr = bin_mid[uiqr_index]
+ iqr = uiqr - liqr
+ return iqr
+
+ def __call__(self, data: KeyedData, **kwargs):
+ if len(data[self.histKey.format(**kwargs)]) != 0:
+ hist = cast(Vector, data[self.histKey.format(**kwargs)])
+ bin_mid = cast(Vector, data[self.midKey.format(**kwargs)])
+ iqr = cast(Scalar, float(self.histIqr(hist, bin_mid)))
+ else:
+ iqr = np.NaN
+ return iqr
diff --git a/python/lsst/analysis/tools/atools/__init__.py b/python/lsst/analysis/tools/atools/__init__.py
index 10c7668b1..048b3a7d6 100644
--- a/python/lsst/analysis/tools/atools/__init__.py
+++ b/python/lsst/analysis/tools/atools/__init__.py
@@ -1,6 +1,7 @@
from .astrometricRepeatability import *
from .coveragePlots import *
from .deblenderMetric import *
+from .deltaSkyCorr import *
from .diaSolarSystemObjectMetrics import *
from .diaSourceMetrics import *
from .diffimMetadataMetrics import *
diff --git a/python/lsst/analysis/tools/atools/deltaSkyCorr.py b/python/lsst/analysis/tools/atools/deltaSkyCorr.py
new file mode 100644
index 000000000..27f21e957
--- /dev/null
+++ b/python/lsst/analysis/tools/atools/deltaSkyCorr.py
@@ -0,0 +1,67 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+__all__ = ("DeltaSkyCorrXYPlot",)
+
+from ..actions.plot.xyPlot import XYPlot
+from ..actions.scalar.scalarActions import IqrHistAction, MedianHistAction
+from ..actions.vector import ConstantValue, LoadVector
+from ..interfaces import AnalysisTool
+
+
+class DeltaSkyCorrXYPlot(AnalysisTool):
+ parameterizedBand: bool = False
+
+ def setDefaults(self):
+ super().setDefaults()
+ self.process.buildActions.x = LoadVector()
+ self.process.buildActions.y = LoadVector()
+ self.process.buildActions.x.vectorKey = "bin_mid"
+ self.process.buildActions.y.vectorKey = "hist"
+ self.process.buildActions.xerr = ConstantValue(value=0)
+ self.process.buildActions.yerr = ConstantValue(value=0)
+
+ self.process.calculateActions.median = MedianHistAction()
+ self.process.calculateActions.median.histKey = "hist"
+ self.process.calculateActions.median.midKey = "bin_mid"
+ self.process.calculateActions.iqr = IqrHistAction()
+ self.process.calculateActions.iqr.histKey = "hist"
+ self.process.calculateActions.iqr.midKey = "bin_mid"
+
+ self.produce.plot = XYPlot()
+ self.produce.plot.xAxisLabel = r"$\Delta$skyCorr Flux (nJy)"
+ self.produce.plot.yAxisLabel = "Frequency"
+ self.produce.plot.yScale = "log"
+ self.produce.plot.xLine = 0
+ self.produce.plot.strKwargs = {
+ "fmt": "-",
+ "color": "b",
+ }
+
+ self.produce.metric.units = {
+ "median": "nJy",
+ "iqr": "nJy",
+ }
+
+ self.produce.metric.newNames = {
+ "median": "delta_skyCorr_median",
+ "iqr": "delta_skyCorr_iqr",
+ }
diff --git a/python/lsst/analysis/tools/tasks/deltaSkyCorrAnalysis.py b/python/lsst/analysis/tools/tasks/deltaSkyCorrAnalysis.py
new file mode 100644
index 000000000..31f5541ae
--- /dev/null
+++ b/python/lsst/analysis/tools/tasks/deltaSkyCorrAnalysis.py
@@ -0,0 +1,222 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+import logging
+
+import numpy as np
+from lsst.pex.config import Field, ListField
+from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
+from lsst.pipe.base.connectionTypes import Input, Output
+
+from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
+
+log = logging.getLogger(__name__)
+
+
+class DeltaSkyCorrHistConnections(PipelineTaskConnections, dimensions=("instrument", "visit")):
+ """Connections class for DeltaSkyCorrHistTask."""
+
+ skyCorrs = Input(
+ name="skyCorr",
+ storageClass="Background",
+ doc="Sky correction background models from a run without any synthetic source injection.",
+ multiple=True,
+ dimensions=("instrument", "visit", "detector"),
+ deferLoad=True,
+ )
+ injected_skyCorrs = Input(
+ name="injected_skyCorr",
+ storageClass="Background",
+ doc="Sky correction background models from a run with synthetic sources injected into the data.",
+ multiple=True,
+ dimensions=("instrument", "visit", "detector"),
+ deferLoad=True,
+ )
+ calexpBackgrounds = Input(
+ name="calexpBackground",
+ storageClass="Background",
+ doc="Initial per-detector background models associated with the calibrated exposure.",
+ multiple=True,
+ dimensions=("instrument", "visit", "detector"),
+ deferLoad=True,
+ )
+ photoCalib = Input(
+ name="calexp.photoCalib",
+ storageClass="PhotoCalib",
+ doc="Photometric calibration associated with the calibrated exposure.",
+ multiple=True,
+ dimensions=("instrument", "visit", "detector"),
+ deferLoad=True,
+ )
+ delta_skyCorr_hist = Output(
+ name="delta_skyCorr_hist",
+ storageClass="ArrowNumpyDict",
+ doc="A dictionary containing the histogram values, bin mid points, and bin lower/upper edges for the "
+ "aggregated skyCorr difference dataset, i.e., the difference between the injected and non-injected "
+ "sky correction background models.",
+ dimensions=("instrument", "visit"),
+ )
+
+
+class DeltaSkyCorrHistConfig(PipelineTaskConfig, pipelineConnections=DeltaSkyCorrHistConnections):
+ """Config class for DeltaSkyCorrHistTask."""
+
+ bin_range = ListField[float](
+ doc="The lower and upper range for the histogram bins, in nJy.",
+ default=[-1, 1],
+ )
+ bin_width = Field[float](
+ doc="The width of each histogram bin, in nJy.",
+ default=0.0001,
+ )
+
+
+class DeltaSkyCorrHistTask(PipelineTask):
+ """A task for generating a histogram of counts in the difference image
+ between an injected sky correction frame and a non-injected sky correction
+ frame (i.e., injected_skyCorr - skyCorr).
+ """
+
+ ConfigClass = DeltaSkyCorrHistConfig
+ _DefaultName = "deltaSkyCorrHist"
+
+ def __init__(self, initInputs=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def runQuantum(self, butlerQC, inputRefs, outputRefs):
+ inputs = butlerQC.get(inputRefs)
+ inputs["num_initial_bgs"] = len(inputs["calexpBackgrounds"][0].get())
+ delta_skyCorr_hist = self.run(**{k: v for k, v in inputs.items() if k != "calexpBackgrounds"})
+ butlerQC.put(delta_skyCorr_hist, outputRefs.delta_skyCorr_hist)
+
+ def run(self, skyCorrs, injected_skyCorrs, num_initial_bgs, photoCalib):
+ """Generate a histogram of counts in the difference image between an
+ injected sky correction frame and a non-injected sky correction frame
+ (i.e., injected_skyCorr - skyCorr).
+
+ Parameters
+ ----------
+ skyCorrs : `list`[`~lsst.daf.butler.DeferredDatasetHandle`]
+ Sky correction background models from a run without any synthetic
+ source injection.
+ These deferred dataset handles should normally resolve to
+ `~lsst.afw.math.BackgroundList` objects.
+ injected_skyCorrs : `list`[`~lsst.daf.butler.DeferredDatasetHandle`]
+ Sky correction background models from a run with synthetic sources
+ injected into the data.
+ These deferred dataset handles should normally resolve to
+ `~lsst.afw.math.BackgroundList` objects.
+ num_initial_bgs : `int`
+ The length of the initial per-detector background model list.
+ This number of background models will be skipped from the start of
+ each skyCorr/injected_skyCorr background model list.
+ See the Notes section for more details.
+ photoCalib : `list`[`~lsst.daf.butler.DeferredDatasetHandle`]
+ Photometric calibration, for conversion from counts to nJy.
+
+ Returns
+ -------
+ delta_skyCorr_hist : `dict`[`str`, `~numpy.ndarray`]
+ A dictionary containing the histogram values and bin lower/upper
+ edges for the skyCorr difference dataset.
+
+ Notes
+ -----
+ The first N background elements in the skyCorr/injected_skyCorr
+ background list are the inverse of the initial per-detector background
+ solution.
+ The effect of this is that adding a sky correction frame to a
+ background-subtracted calibrated exposure will undo the per-detector
+ background solution and apply the full focal plane sky correction in
+ its place.
+
+ For this task, we only want to compare the extra (subtractive) sky
+ correction components, so we skip the first N background models from
+ the sky frame.
+ """
+ # Generate lookup tables for the skyCorr/injected_skyCorr data.
+ lookup_skyCorrs = {x.dataId: x for x in skyCorrs}
+ lookup_injected_skyCorrs = {x.dataId: x for x in injected_skyCorrs}
+ lookup_photoCalib = {x.dataId: x for x in photoCalib}
+
+ # Set up the global histogram.
+ bin_edges = np.arange(
+ self.config.bin_range[0],
+ self.config.bin_range[1] + self.config.bin_width,
+ self.config.bin_width,
+ )
+ hist = np.zeros(len(bin_edges) - 1)
+ log.info("Generating a histogram containing %d bins.", len(hist))
+
+ # Loop over the skyCorr/injected_skyCorr data.
+ for dataId in lookup_injected_skyCorrs.keys():
+ # Get the skyCorr/injected_skyCorr data.
+ skyCorr = lookup_skyCorrs[dataId].get()
+ injected_skyCorr = lookup_injected_skyCorrs[dataId].get()
+ # And the photometric calibration
+ instFluxToNanojansky = lookup_photoCalib[dataId].get().instFluxToNanojansky(1)
+
+ # Isolate the extra (subtractive) sky correction components.
+ skyCorr_extras = skyCorr.clone()
+ skyCorr_extras._backgrounds = skyCorr_extras._backgrounds[num_initial_bgs:]
+ injected_skyCorr_extras = injected_skyCorr.clone()
+ injected_skyCorr_extras._backgrounds = injected_skyCorr_extras._backgrounds[num_initial_bgs:]
+
+ # Create the delta_skyCorr array.
+ delta_skyCorr_det = injected_skyCorr_extras.getImage().array - skyCorr_extras.getImage().array
+ delta_skyCorr_det *= instFluxToNanojansky # Convert image to nJy
+
+ # Compute the per-detector histogram; update the global histogram.
+ hist_det, _ = np.histogram(delta_skyCorr_det, bins=bin_edges)
+ hist += hist_det
+
+ # Return results.
+ num_populated_bins = len([x for x in hist if x == 0])
+ log.info("Populated %d of %d histogram bins.", len(hist) - num_populated_bins, len(hist))
+ bin_mid = bin_edges[:-1] + (self.config.bin_width / 2)
+ delta_skyCorr_hist = dict(
+ hist=hist, bin_lower=bin_edges[:-1], bin_upper=bin_edges[1:], bin_mid=bin_mid
+ )
+ return delta_skyCorr_hist
+
+
+class DeltaSkyCorrAnalysisConnections(
+ AnalysisBaseConnections,
+ dimensions=("instrument", "visit"),
+ defaultTemplates={"outputName": "deltaSkyCorr"},
+):
+ data = Input(
+ name="delta_skyCorr_hist",
+ storageClass="ArrowNumpyDict",
+ doc="A dictionary containing the histogram values, bin mid points, and bin lower/upper edges for the "
+ "aggregated skyCorr difference dataset, i.e., the difference between the injected and non-injected "
+ "sky correction background models.",
+ deferLoad=True,
+ dimensions=("instrument", "visit"),
+ )
+
+
+class DeltaSkyCorrAnalysisConfig(AnalysisBaseConfig, pipelineConnections=DeltaSkyCorrAnalysisConnections):
+ pass
+
+
+class DeltaSkyCorrAnalysisTask(AnalysisPipelineTask):
+ ConfigClass = DeltaSkyCorrAnalysisConfig
+ _DefaultName = "deltaSkyCorrAnalysis"