Merge branch 'main' into tickets/DM-38500

Merge latest changes from main

.gitignore

@@ -15,6 +15,7 @@ version.py
 
 # Pytest
 tests/.tests
+tests/tmp*
 pytest_session.txt
 .cache/
 .pytest_cache

bin.src/build-gather-resource-usage-qg

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A custom script to generate a QuantumGraph for GatherResourceUsageTask and
+ConsolidateResourceUsageTask.
+
+See ``GatherResourceUsageTask.build_quantum_graph`` for more information
+(including why this script exists), and use ``--help`` for documentation on
+arguments.
+"""
+
+from lsst.analysis.tools.tasks.gatherResourceUsage import ResourceUsageQuantumGraphBuilder
+
+if __name__ == "__main__":
+    ResourceUsageQuantumGraphBuilder.main()

doc/lsst.analysis.tools/index.rst

@@ -57,6 +57,7 @@ Script reference
    :maxdepth: 1
 
    scripts/verify_to_sasquatch.py
+   scripts/build-gather-resource-usage-qg
 
 .. _lsst.analysis_tools-pyapi:
 
@@ -69,3 +70,4 @@ Python API Reference
 .. automodapi:: lsst.analysis.tools.actions.scalar
 .. automodapi:: lsst.analysis.tools.actions.vector
 .. automodapi:: lsst.analysis.tools.actions.keyedData
+.. automodapi:: lsst.analysis.tools.tasks

doc/lsst.analysis.tools/scripts/build-gather-resource-usage-qg.rst

@@ -0,0 +1,3 @@
+.. autoprogram:: lsst.analysis.tools.tasks.gatherResourceUsage:ResourceUsageQuantumGraphBuilder.make_argument_parser()
+   :prog: build-gather-resource-usage-qg
+   :groups:

pipelines/apDetectorVisitQualityCore.yaml

@@ -30,3 +30,34 @@ tasks:
       atools.simpleSky: SimpleDiaPlot
       python: |
         from lsst.analysis.tools.atools import *
+  analyzeTrailedDiaSrcCore:
+    class: lsst.analysis.tools.tasks.TrailedDiaSrcDetectorVisitAnalysisTask
+    config:
+      # Counts up the number of dia sources in the trailed table.
+      atools.numDiaSourcesAll: NumDiaSourcesAllMetric
+      connections.outputName: trailedDiaSrcCore
+      python: |
+        from lsst.analysis.tools.atools import *
+  diffimTaskCore:
+    class: lsst.analysis.tools.tasks.DiffimDetectorVisitAnalysisTask
+    config:
+      connections.outputName: diffimMetadata
+
+      atools.diffimMetadataMetric: DiffimMetadataMetricTool
+      atools.diffimMetadataMetric.metrics:
+        # Format is "metric name in the metadata": units
+        nUnmergedDiaSources: ct
+        nMergedDiaSources: ct
+        nGoodPixels: ct
+        nBadPixels: ct
+        nPixelsDetectedPositive: ct
+        nPixelsDetectedNegative: ct
+        nBadPixelsDetectedPositive: ct
+        nBadPixelsDetectedNegative: ct
+        sciencePsfSize: pixel
+        templatePsfSize: pixel
+        scaleScienceVarianceFactor: ct
+        scaleTemplateVarianceFactor: ct
+        templateCoveragePercent: percent
+      python: |
+        from lsst.analysis.tools.atools import DiffimMetadataMetricTool

pipelines/coaddDiffMatchedQualityExtended.yaml

@@ -1,56 +1,68 @@
 description: |
   Matched difference (measured vs reference) plots/metrics
+parameters:
+  mag_x_min: 17
+  mag_x_max: 29
+  mag_diff_min: -1000
+  mag_diff_max: 1000
+  mag_chi_min: -5
+  mag_chi_max: 5
+  pos_diff_min: -100
+  pos_diff_max: 100
+  pos_chi_min: -5
+  pos_chi_max: 5
 tasks:
   diff_matched_analysis:
     class: lsst.analysis.tools.tasks.DiffMatchedAnalysisTask
     config:
       connections.outputName: diff_matched_truth_summary_objectTable_tract
 
       # plots
-      atools.matchedRefCModelMagDiff: MatchedRefCoaddDiffMagPlot
+      atools.matchedRefCModelMagDiff: MatchedRefCoaddDiffMagTool
+      # Doesn't work; interpreted as float
+      # atools.matchedRefCModelMagDiff.produce.plot.xLims: [parameters.mag_x_min, parameters.mag_x_max]
+      atools.matchedRefCModelMagDiff.produce.plot.xLims: lims_mag_x
+      atools.matchedRefCModelMagDiff.produce.plot.yLims: lims_mag_diff
 
-      atools.matchedRefMagChi: MatchedRefCoaddDiffMagPlot
-      atools.matchedRefMagChi.compute_chi: true
+      atools.matchedRefCModelMagChi: MatchedRefCoaddDiffMagTool
+      atools.matchedRefCModelMagChi.compute_chi: true
+      atools.matchedRefCModelMagChi.produce.plot.xLims: lims_mag_x
+      atools.matchedRefCModelMagChi.produce.plot.yLims: lims_mag_chi
 
       # TODO: Can this be a one liner?
-      atools.matchedRefPositionXDiff: MatchedRefCoaddDiffPositionPlot
-      atools.matchedRefPositionXDiff.variable: x
+      atools.matchedRefPositionXDiff: MatchedRefCoaddDiffPositionTool
+      atools.matchedRefPositionXDiff.coord_meas: x
+      atools.matchedRefPositionXDiff.coord_ref: refcat_x
+      atools.matchedRefPositionXDiff.produce.plot.xLims: lims_mag_x
+      atools.matchedRefPositionXDiff.produce.plot.yLims: lims_pos_diff
 
-      atools.matchedRefPositionXChi: MatchedRefCoaddDiffPositionPlot
-      atools.matchedRefPositionXChi.variable: x
+      atools.matchedRefPositionXChi: MatchedRefCoaddDiffPositionTool
+      atools.matchedRefPositionXChi.coord_meas: x
+      atools.matchedRefPositionXChi.coord_ref: refcat_x
+      atools.matchedRefPositionXChi.produce.plot.xLims: lims_mag_x
+      atools.matchedRefPositionXChi.produce.plot.yLims: lims_pos_chi
       atools.matchedRefPositionXChi.compute_chi: true
 
-      atools.matchedRefPositionYDiff: MatchedRefCoaddDiffPositionPlot
-      atools.matchedRefPositionYDiff.variable: y
+      atools.matchedRefPositionYDiff: MatchedRefCoaddDiffPositionTool
+      atools.matchedRefPositionYDiff.coord_meas: y
+      atools.matchedRefPositionYDiff.coord_ref: refcat_y
+      atools.matchedRefPositionYDiff.produce.plot.xLims: lims_mag_x
+      atools.matchedRefPositionYDiff.produce.plot.yLims: lims_pos_diff
 
-      atools.matchedRefPositionYChi: MatchedRefCoaddDiffPositionPlot
-      atools.matchedRefPositionYChi.variable: y
+      atools.matchedRefPositionYChi: MatchedRefCoaddDiffPositionTool
+      atools.matchedRefPositionYChi.coord_meas: y
+      atools.matchedRefPositionYChi.coord_ref: refcat_y
+      atools.matchedRefPositionYChi.produce.plot.xLims: lims_mag_x
+      atools.matchedRefPositionYChi.produce.plot.yLims: lims_pos_chi
       atools.matchedRefPositionYChi.compute_chi: true
 
-      # metrics
-      atools.matchedRefCModelMagDiffMetric: MatchedRefCoaddDiffMagMetric
-
-      atools.matchedRefMagChiMetric: MatchedRefCoaddDiffMagMetric
-      atools.matchedRefMagChiMetric.compute_chi: true
-
-      atools.matchedRefPositionXDiffMetric: MatchedRefCoaddDiffPositionMetric
-      atools.matchedRefPositionXDiffMetric.variable: x
-
-      atools.matchedRefPositionXChiMetric: MatchedRefCoaddDiffPositionMetric
-      atools.matchedRefPositionXChiMetric.variable: x
-      atools.matchedRefPositionXChiMetric.compute_chi: true
-
-      atools.matchedRefPositionYDiffMetric: MatchedRefCoaddDiffPositionMetric
-      atools.matchedRefPositionYDiffMetric.variable: y
-
-      atools.matchedRefPositionYChiMetric: MatchedRefCoaddDiffPositionMetric
-      atools.matchedRefPositionYChiMetric.variable: y
-      atools.matchedRefPositionYChiMetric.compute_chi: true
-
       python: |
         from lsst.analysis.tools.atools.diffMatched import (
-          MatchedRefCoaddDiffMagMetric,
-          MatchedRefCoaddDiffPositionMetric,
-          MatchedRefCoaddDiffMagPlot,
-          MatchedRefCoaddDiffPositionPlot,
+          MatchedRefCoaddDiffMagTool,
+          MatchedRefCoaddDiffPositionTool,
         )
+        lims_mag_x = (parameters.mag_x_min, parameters.mag_x_max)
+        lims_mag_chi = (parameters.mag_chi_min, parameters.mag_chi_max)
+        lims_mag_diff = (parameters.mag_diff_min, parameters.mag_diff_max)
+        lims_pos_chi = (parameters.pos_chi_min, parameters.pos_chi_max)
+        lims_pos_diff = (parameters.pos_diff_min, parameters.pos_diff_max)

pipelines/coaddQualityCore.yaml

@@ -18,6 +18,7 @@ tasks:
       atools.yPerpCModel: YPerpCModel
       atools.skyObjectSky: SkyObjectSkyPlot
       atools.skyObjectFlux: SkyObjectHistPlot
+      atools.psfCModelSky: PsfCModelSkyPlot
       python: |
         from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *
@@ -35,3 +36,37 @@ tasks:
     class: lsst.analysis.tools.tasks.catalogMatch.CatalogMatchTask
   refCatObjectTract:
     class: lsst.analysis.tools.tasks.refCatObjectAnalysis.RefCatObjectAnalysisTask
+  plotPropertyMapTract:
+    class: lsst.analysis.tools.tasks.PropertyMapTractAnalysisTask
+    config:
+      connections.outputName: propertyMapTract
+      atools.HealSparseMap: PropertyMapTool
+      zoomFactors: [2, 8]
+      python: |
+        from lsst.analysis.tools.atools import *
+        config.properties["dcr_ddec"] = PropertyMapConfig
+        config.properties["dcr_ddec"].operations = ["weighted_mean"]
+        config.properties["dcr_dra"] = PropertyMapConfig
+        config.properties["dcr_dra"].operations = ["weighted_mean"]
+        config.properties["dcr_e1"] = PropertyMapConfig
+        config.properties["dcr_e1"].operations = ["weighted_mean"]
+        config.properties["dcr_e2"] = PropertyMapConfig
+        config.properties["dcr_e2"].operations = ["weighted_mean"]
+        config.properties["epoch"] = PropertyMapConfig
+        config.properties["epoch"].operations = ["min", "max", "mean"]
+        config.properties["exposure_time"] = PropertyMapConfig
+        config.properties["exposure_time"].coaddName = "deep"
+        config.properties["exposure_time"].operations = ["sum"]
+        config.properties["exposure_time"].nBinsHist = 35
+        config.properties["psf_e1"] = PropertyMapConfig
+        config.properties["psf_e1"].operations = ["weighted_mean"]
+        config.properties["psf_e2"] = PropertyMapConfig
+        config.properties["psf_e2"].operations = ["weighted_mean"]
+        config.properties["psf_maglim"] = PropertyMapConfig
+        config.properties["psf_maglim"].operations = ["weighted_mean"]
+        config.properties["psf_size"] = PropertyMapConfig
+        config.properties["psf_size"].operations = ["weighted_mean"]
+        config.properties["sky_background"] = PropertyMapConfig
+        config.properties["sky_background"].operations = ["weighted_mean"]
+        config.properties["sky_noise"] = PropertyMapConfig
+        config.properties["sky_noise"].operations = ["weighted_mean"]

python/lsst/analysis/tools/actions/keyedData/calcDistances.py

@@ -64,14 +64,16 @@ def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
 
         Parameters
         ----------
-        data: KeyedData
+        data : KeyedData
             Catalog of data including coordinate, visit, and object group
             information.
+
         Returns
         -------
-        distanceParams: `dict`
+        distanceParams : `dict`
             Dictionary of the calculated arrays and metrics with the following
             keys:
+
             - ``rmsDistances`` : Per-object rms of separations (`np.array`).
             - ``separationResiduals`` : All separations minus per-object median
                 (`np.array`)
@@ -219,6 +221,7 @@ def sphDist(ra_mean, dec_mean, ra, dec):
         Array of RA in radians.
     dec : `numpy.array` [`float`]
         Array of Dec in radians.
+
     Notes
     -----
     Uses the Haversine formula to preserve accuracy at small angles.

python/lsst/analysis/tools/actions/plot/__init__.py

@@ -1,9 +1,11 @@
 from .barPlots import *
+from .calculateRange import *
 from .colorColorFitPlot import *
 from .diaSkyPlot import *
 from .focalPlanePlot import *
 from .histPlot import *
 from .multiVisitCoveragePlot import *
+from .propertyMapPlot import *
 from .rhoStatisticsPlot import *
 from .scatterplotWithTwoHists import *
 from .skyPlot import *

python/lsst/analysis/tools/actions/plot/calculateRange.py

@@ -0,0 +1,74 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("MinMax", "Med2Mad")
+
+from typing import cast
+
+import numpy as np
+
+from ...interfaces import Vector, VectorAction
+from ...statistics import nansigmaMad
+
+
+class MinMax(VectorAction):
+    """Return the maximum and minimum values of an input vector to use as the
+    minimum and maximum values of a colorbar range.
+
+    Parameters
+    ----------
+    data : `Vector`
+        A vector containing the data whose minimum and maximum are to be
+        returned.
+
+    Returns
+    -------
+    A two-element vector containing the minimum and maximum values of `data`.
+    """
+
+    def __call__(self, data: Vector, **kwargs) -> Vector:
+        return cast(Vector, [np.min(data), np.max(data)])
+
+
+class Med2Mad(VectorAction):
+    """Return the median +/- 2*nansigmamad values of an input vector to use
+    as the minimum and maximum values of a colorbar range.
+
+    Parameters
+    ----------
+    data : `Vector`
+        A vector containing the data whose median +/- 2*nansigmamad are to
+        be returned.
+
+    Returns
+    -------
+    A two-element vector containing the median +/- 2*nansigmamad values of
+    `data`.
+    """
+
+    def __call__(self, data: Vector, **kwargs) -> Vector:
+        med = np.nanmedian(data)
+        mad = nansigmaMad(data)
+        cmin = med - 2 * mad
+        cmax = med + 2 * mad
+        return cast(Vector, [cmin, cmax])