From 6613b0f35587f76f9a53dc42702ec565a2cbae14 Mon Sep 17 00:00:00 2001
From: Alex Broughton <abrought@slac.stanford.edu>
Date: Thu, 14 Nov 2024 09:41:29 -0800
Subject: [PATCH] Add flat metrics for mean, noise vs mjd and FP plots

---
 pipelines/cpCore.yaml                         | 42 ++++++++++++++++++-
 .../tools/atools/calibQuantityProfile.py      | 19 ++++++---
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/pipelines/cpCore.yaml b/pipelines/cpCore.yaml
index 7d8ca571b..974afee98 100644
--- a/pipelines/cpCore.yaml
+++ b/pipelines/cpCore.yaml
@@ -139,11 +139,30 @@ tasks:
       python: |
         from lsst.analysis.tools.atools import *
 
+  analyzeFlatCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
+    config:
+      connections.outputName: cpFlatCore
+      connections.data: verifyFlatResults
+
+      atools.flatMeanPerAmp: CalibStatisticFocalPlanePlot
+      atools.flatMeanPerAmp.produce.plot.addHistogram: true
+      atools.flatMeanPerAmp.quantityKey: FLAT_MEAN
+      atools.flatMeanPerAmp.unit: electron
+
+      atools.flatNoisePerAmp: CalibStatisticFocalPlanePlot
+      atools.flatNoisePerAmp.produce.plot.addHistogram: true
+      atools.flatNoisePerAmp.quantityKey: FLAT_NOISE
+      atools.flatNoisePerAmp.unit: electron
+
+      python: |
+        from lsst.analysis.tools.atools import *
+
   analyzeFlatDetCore:
     class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
     config:
       connections.outputName: cpFlatDetCore
-      connections.data: verifyFlatResults
+      connections.data: verifyFlatDetResults
 
       atools.flatTestsByDate: CalibAmpScatterTool
       atools.flatTestsByDate.prep.panelKey: amplifier
@@ -154,6 +173,27 @@ tasks:
       atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD"
       atools.flatTestsByDate.produce.plot.yAxisLabel: "Test Pass Results"
 
+      atools.flatMeansByDate: CalibAmpScatterTool
+      atools.flatMeansByDate.prep.panelKey: amplifier
+      atools.flatMeansByDate.prep.dataKey: mjd
+      atools.flatMeansByDate.prep.quantityKey: FLAT_MEAN
+      atools.flatMeansByDate.produce.plot.xAxisLabel: "MJD"
+      atools.flatMeansByDate.produce.plot.yAxisLabel: "Flat Mean (electrons)"
+
+      atools.flatNoiseByDate: CalibAmpScatterTool
+      atools.flatNoiseByDate.prep.panelKey: amplifier
+      atools.flatNoiseByDate.prep.dataKey: mjd
+      atools.flatNoiseByDate.prep.quantityKey: FLAT_NOISE
+      atools.flatNoiseByDate.produce.plot.xAxisLabel: "MJD"
+      atools.flatNoiseByDate.produce.plot.yAxisLabel: "Flat Noise (electrons)"
+
+      atools.flatNoiseByMean: CalibAmpScatterTool
+      atools.flatNoiseByMean.prep.panelKey: amplifier
+      atools.flatNoiseByMean.prep.dataKey: FLAT_MEAN
+      atools.flatNoiseByMean.prep.quantityKey: FLAT_NOISE
+      atools.flatNoiseByMean.produce.plot.xAxisLabel: "Flat Mean (electrons)"
+      atools.flatNoiseByMean.produce.plot.yAxisLabel: "Flat Noise (electrons)"
+
       python: |
         from lsst.analysis.tools.atools import *
 
diff --git a/python/lsst/analysis/tools/atools/calibQuantityProfile.py b/python/lsst/analysis/tools/atools/calibQuantityProfile.py
index 80be9a892..5b2228b6e 100644
--- a/python/lsst/analysis/tools/atools/calibQuantityProfile.py
+++ b/python/lsst/analysis/tools/atools/calibQuantityProfile.py
@@ -33,6 +33,7 @@
 
 from typing import cast
 
+import numpy
 from lsst.pex.config import Field
 from lsst.pex.config.configurableActions import ConfigurableActionField
 
@@ -90,16 +91,24 @@ class SingleValueRepacker(KeyedDataAction):
 
     def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
         repackedData = {}
-        uniquePanelKeys = list(set(data[self.panelKey]))
 
-        # Loop over data vector to repack information as it is expected.
+        if isinstance(data[self.panelKey], numpy.ma.MaskedArray):
+            good = ~data[self.panelKey].mask
+            uniquePanelKeys = numpy.unique(data[self.panelKey])
+            uniquePanelKeys = uniquePanelKeys.data[~uniquePanelKeys.mask].data
+        else:
+            good = numpy.ones(len(data[self.panelKey]), dtype=bool)
+            uniquePanelKeys = list(set(data[self.panelKey]))
+
+        # Loop over data vector to repack information as
+        # it is expected.
         for i in range(len(uniquePanelKeys)):
             repackedData[f"{uniquePanelKeys[i]}_x"] = []
             repackedData[f"{uniquePanelKeys[i]}"] = []
 
-        panelVec = cast(Vector, data[self.panelKey])
-        dataVec = cast(Vector, data[self.dataKey])
-        quantityVec = cast(Vector, data[self.quantityKey])
+        panelVec = cast(Vector, data[self.panelKey][good])
+        dataVec = cast(Vector, data[self.dataKey][good])
+        quantityVec = cast(Vector, data[self.quantityKey][good])
 
         for i in range(len(panelVec)):
             repackedData[f"{panelVec[i]}_x"].append(dataVec[i])