DM-48582: factor out configs that are repeated for each kind of visit/source analysis

config/matchedVisitCore.py

@@ -0,0 +1,31 @@
+# Configuration for standard instances of
+# lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
+
+from lsst.analysis.tools.atools import *
+
+config.atools.stellarAstrometricSelfRepeatabilityRA = AstrometricRepeatability
+config.atools.stellarAstrometricSelfRepeatabilityRA.coordinate = "RA"
+config.atools.stellarAstrometricSelfRepeatabilityRA.level = 2
+config.atools.stellarAstrometricSelfRepeatabilityDec = AstrometricRepeatability
+config.atools.stellarAstrometricSelfRepeatabilityDec.coordinate = "Dec"
+config.atools.stellarAstrometricSelfRepeatabilityDec.level = 2
+config.atools.stellarPhotometricRepeatability = StellarPhotometricRepeatability
+config.atools.stellarPhotometricResiduals = StellarPhotometricResidualsFocalPlane
+config.atools.stellarPhotometricRepeatabilityCalib = StellarPhotometricRepeatability
+config.atools.stellarPhotometricRepeatabilityCalib.fluxType = "calibFlux"
+config.atools.stellarPhotometricResidualsCalib = StellarPhotometricResidualsFocalPlane
+config.atools.stellarPhotometricResidualsCalib.fluxType = "calibFlux"
+config.atools.stellarAstrometricResidualsRA = StellarAstrometricResidualsRAFocalPlanePlot
+config.atools.stellarAstrometricResidualsDec = StellarAstrometricResidualsDecFocalPlanePlot
+config.atools.stellarAstrometricResidualStdDevRA = StellarAstrometricResidualStdDevRAFocalPlanePlot
+config.atools.stellarAstrometricResidualStdDevDec = StellarAstrometricResidualStdDevDecFocalPlanePlot
+config.atools.stellarAstrometricRepeatability1 = AstrometricRelativeRepeatability
+config.atools.stellarAstrometricRepeatability1.xValue = 1
+config.atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus = 5
+config.atools.stellarAstrometricRepeatability2 = AstrometricRelativeRepeatability
+config.atools.stellarAstrometricRepeatability2.xValue = 2
+config.atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus = 20
+config.atools.stellarAstrometricRepeatability3 = AstrometricRelativeRepeatability
+config.atools.stellarAstrometricRepeatability3.xValue = 3
+config.atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus = 200
+config.atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD = 30

config/matchedVisitCoreWholeSkyPlot.py

@@ -0,0 +1,26 @@
+# Configuration for instances of lsst.analysis.tools.tasks.WholeSkyAnalysisTask
+# that aggregate metrics from the standard configuration of
+# lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask.
+
+from lsst.analysis.tools.atools import *
+
+config.atools.wholeSkyMetric = WholeSkyPlotTool
+config.atools.wholeSkyMetric.plotKeys = []
+config.atools.wholeSkyMetric.keysWithBand = [
+    "stellarAstrometricRepeatability1_{band}_AM1",
+    "stellarAstrometricRepeatability1_{band}_AF1",
+    "stellarAstrometricRepeatability1_{band}_AD1",
+    "stellarAstrometricRepeatability2_{band}_AM2",
+    "stellarAstrometricRepeatability2_{band}_AF2",
+    "stellarAstrometricRepeatability2_{band}_AD2",
+    "stellarAstrometricRepeatability3_{band}_AM3",
+    "stellarAstrometricRepeatability3_{band}_AF3",
+    "stellarAstrometricRepeatability3_{band}_AD3",
+    "stellarAstrometricSelfRepeatabilityDec_{band}_dmL2AstroErr_Dec",
+    "stellarAstrometricSelfRepeatabilityRA_{band}_dmL2AstroErr_RA",
+    "stellarPhotometricRepeatability_{band}_stellarPhotRepeatStdev",
+    "stellarPhotometricRepeatability_{band}_stellarPhotRepeatOutlierFraction",
+    "stellarPhotometricResiduals_{band}_photResidTractSigmaMad",
+    "stellarPhotometricResiduals_{band}_photResidTractStdev",
+    "stellarPhotometricResiduals_{band}_photResidTractMedian",
+]

config/matchedVisitExtended.py

@@ -0,0 +1,53 @@
+# Configuration for extended instances of
+# lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
+#
+# This does not include the Core configuration, as it may be used by instances
+# that are configured to optionally run alongside instances with that
+# configurations.
+
+from lsst.analysis.tools.atools import *
+from lsst.analysis.tools.interfaces import *
+
+config.atools.modelPhotRepStarSn5to10 = StellarPhotometricRepeatability
+config.atools.modelPhotRepStarSn5to10.fluxType = "gaussianFlux"
+config.atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 5
+config.atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 10
+config.atools.modelPhotRepStarSn5to10.produce.plot = NoPlot
+
+config.atools.modelPhotRepStarSn10to20 = StellarPhotometricRepeatability
+config.atools.modelPhotRepStarSn10to20.fluxType = "gaussianFlux"
+config.atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 10
+config.atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 20
+config.atools.modelPhotRepStarSn10to20.produce.plot = NoPlot
+
+config.atools.modelPhotRepStarSn20to40 = StellarPhotometricRepeatability
+config.atools.modelPhotRepStarSn20to40.fluxType = "gaussianFlux"
+config.atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 20
+config.atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 40
+config.atools.modelPhotRepStarSn20to40.produce.plot = NoPlot
+
+config.atools.modelPhotRepStarSn40to80 = StellarPhotometricRepeatability
+config.atools.modelPhotRepStarSn40to80.fluxType = "gaussianFlux"
+config.atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 40
+config.atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 80
+config.atools.modelPhotRepStarSn40to80.produce.plot = NoPlot
+
+config.atools.psfPhotRepStarSn5to10 = StellarPhotometricRepeatability
+config.atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 5
+config.atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 10
+config.atools.psfPhotRepStarSn5to10.produce.plot = NoPlot
+
+config.atools.psfPhotRepStarSn10to20 = StellarPhotometricRepeatability
+config.atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 10
+config.atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 20
+config.atools.psfPhotRepStarSn10to20.produce.plot = NoPlot
+
+config.atools.psfPhotRepStarSn20to40 = StellarPhotometricRepeatability
+config.atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 20
+config.atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 40
+config.atools.psfPhotRepStarSn20to40.produce.plot = NoPlot
+
+config.atools.psfPhotRepStarSn40to80 = StellarPhotometricRepeatability
+config.atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum = 40
+config.atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum = 80
+config.atools.psfPhotRepStarSn40to80.produce.plot = NoPlot

config/refCatSourceAstrometricCore.py

@@ -0,0 +1,16 @@
+# Configuration for standard instances of
+# lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
+#
+# These configurations are not in the class default configs because there may
+# be instances of the class that are intended to optionally run alongside one
+# that includes this standard set.
+
+from lsst.analysis.tools.atools import *
+from lsst.analysis.tools.contexts import VisitContext
+
+config.atools.astromDiffRASkyVisitPlot = TargetRefCatDeltaRASkyVisitPlot
+config.atools.astromDiffDecSkyVisitPlot = TargetRefCatDeltaDecSkyVisitPlot
+config.atools.astromDiffRAScatterVisitPlot = TargetRefCatDeltaRAScatterVisitPlot
+config.atools.astromDiffDecScatterVisitPlot = TargetRefCatDeltaDecScatterVisitPlot
+config.atools.astromDiffMetrics = TargetRefCatDeltaMetrics
+config.atools.astromDiffMetrics.applyContext = VisitContext

config/refCatSourcePhotometricCore.py

@@ -0,0 +1,13 @@
+# Configuration for standard instances of
+# lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
+#
+# These configurations are not in the class default configs because there may
+# be instances of the class that are intended to optionally run alongside one
+# that includes this standard set.
+
+from lsst.analysis.tools.atools import *
+
+config.atools.photomDiffPsfSkyVisitPlot = TargetRefCatDeltaPsfSkyVisitPlot
+config.atools.photomDiffAp09SkyVisitPlot = TargetRefCatDeltaAp09SkyVisitPlot
+config.atools.photoDiffPsfScatterVisitPlot = TargetRefCatDeltaPsfScatterVisitPlot
+config.atools.photoDiffCModelScatterVisitPlot = TargetRefCatDeltaAp09ScatterVisitPlot

config/sourceTableCore.py

@@ -0,0 +1,15 @@
+# Configuration for standard instances of
+# lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
+#
+# These configurations are not in the class default configs because there may
+# be instances of the class that are intended to optionally run alongside one
+# that includes this standard set.
+
+from lsst.analysis.tools.atools import *
+from lsst.analysis.tools.contexts import VisitContext
+
+config.atools.skyFluxVisitStatisticMetric = SkyFluxStatisticMetric
+config.atools.skyFluxVisitStatisticMetric.applyContext = VisitContext
+config.atools.skySourceSky = SkySourceSkyPlot
+config.atools.skySourceFlux = SkySourceHistPlot
+config.atools.relativeSizeResidualPlot = RelativeSizeResidualPlot

doc/lsst.analysis.tools/faqs.rst

@@ -72,3 +72,10 @@ Do atools only work on a per detector/patch/tract/visit basis?
 --------------------------------------------------------------
 
 You can write an atool that rolls up the data from a bunch of smaller parts.
+
+Some tasks are configured directly in pipeline YAML, but others reference files in the `config` directory.  Why?
+----------------------------------------------------------------------------------------------------------------
+
+If multiple very similar configurations of an analysis task are run multiple times in a pipeline (for example, we want to run visit-level analysis both before and after our final calibration steps, with only the input connections changing), we put the common configuration for those tasks in the ``config`` directory and use the ``config: file:`` directive to apply those in the pipeline YAML file.
+
+When this sort of duplication is not a concern, the configuration for an analysis task should go directly in a pipeline YAML file to make it easier to read.

pipelines/matchedVisitQualityCore.yaml

@@ -4,95 +4,18 @@ tasks:
   analyzeMatchedVisitCore:
     class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
     config:
+      file: $ANALYSIS_TOOLS_DIR/config/matchedVisitCore.py
       connections.outputName: matchedVisitCore
-      atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability
-      atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane
-      atools.stellarPhotometricRepeatabilityCalib: StellarPhotometricRepeatability
-      atools.stellarPhotometricRepeatabilityCalib.fluxType: 'calibFlux'
-      atools.stellarPhotometricResidualsCalib: StellarPhotometricResidualsFocalPlane
-      atools.stellarPhotometricResidualsCalib.fluxType: 'calibFlux'
-      atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot
-      atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot
-      atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot
-      atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot
-      atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability
-      atools.stellarAstrometricRepeatability1.xValue: 1
-      atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5
-      atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability
-      atools.stellarAstrometricRepeatability2.xValue: 2
-      atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20
-      atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability
-      atools.stellarAstrometricRepeatability3.xValue: 3
-      atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200
-      atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30
-      atools.stellarAstrometricSelfRepeatabilityRA: AstrometricRepeatability
-      atools.stellarAstrometricSelfRepeatabilityRA.level: 2
-      atools.stellarAstrometricSelfRepeatabilityRA.coordinate: 'RA'
-      atools.stellarAstrometricSelfRepeatabilityDec: AstrometricRepeatability
-      atools.stellarAstrometricSelfRepeatabilityDec.level: 2
-      atools.stellarAstrometricSelfRepeatabilityDec.coordinate: 'Dec'
-      python: |
-        from lsst.analysis.tools.atools import *
   analyzeMatchedVisitExtended:
     class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
     config:
+      file: $ANALYSIS_TOOLS_DIR/config/matchedVisitExtended.py
       connections.outputName: matchedVisitExtended
-      atools.modelPhotRepStarSn5to10: StellarPhotometricRepeatability
-      atools.modelPhotRepStarSn5to10.fluxType: gaussianFlux
-      atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 5
-      atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 10
-      atools.modelPhotRepStarSn5to10.produce.plot: NoPlot
-
-      atools.modelPhotRepStarSn10to20: StellarPhotometricRepeatability
-      atools.modelPhotRepStarSn10to20.fluxType: gaussianFlux
-      atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 10
-      atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 20
-      atools.modelPhotRepStarSn10to20.produce.plot: NoPlot
-
-      atools.modelPhotRepStarSn20to40: StellarPhotometricRepeatability
-      atools.modelPhotRepStarSn20to40.fluxType: gaussianFlux
-      atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 20
-      atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 40
-      atools.modelPhotRepStarSn20to40.produce.plot: NoPlot
-
-      atools.modelPhotRepStarSn40to80: StellarPhotometricRepeatability
-      atools.modelPhotRepStarSn40to80.fluxType: gaussianFlux
-      atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 40
-      atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 80
-      atools.modelPhotRepStarSn40to80.produce.plot: NoPlot
-
-      atools.psfPhotRepStarSn5to10: StellarPhotometricRepeatability
-      atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 5
-      atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 10
-      atools.psfPhotRepStarSn5to10.produce.plot: NoPlot
-
-      atools.psfPhotRepStarSn10to20: StellarPhotometricRepeatability
-      atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 10
-      atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 20
-      atools.psfPhotRepStarSn10to20.produce.plot: NoPlot
-
-      atools.psfPhotRepStarSn20to40: StellarPhotometricRepeatability
-      atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 20
-      atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 40
-      atools.psfPhotRepStarSn20to40.produce.plot: NoPlot
-
-      atools.psfPhotRepStarSn40to80: StellarPhotometricRepeatability
-      atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 40
-      atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 80
-      atools.psfPhotRepStarSn40to80.produce.plot: NoPlot
-      python: |
-        from lsst.analysis.tools.atools import *
-        from lsst.analysis.tools.interfaces import *
   analyzeMatchedPreVisitCore:
     class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
     config:
+      file: $ANALYSIS_TOOLS_DIR/config/matchedVisitCore.py
       connections.sourceCatalogs: preSourceTable_visit
       connections.outputName: matchedPreVisitCore
       # Proper motion catalogs are not available for `preSourceTables`
       applyAstrometricCorrections: false
-      atools.stellarAstrometricSelfRepeatabilityRA: AstrometricRepeatability
-      atools.stellarAstrometricSelfRepeatabilityRA.coordinate: 'RA'
-      atools.stellarAstrometricSelfRepeatabilityDec: AstrometricRepeatability
-      atools.stellarAstrometricSelfRepeatabilityDec.coordinate: 'Dec'
-      python: |
-        from lsst.analysis.tools.atools import *

pipelines/visitQualityCore.yaml

@@ -4,49 +4,37 @@ tasks:
   analyzeSourceTableCore:
     class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
     config:
+      file: $ANALYSIS_TOOLS_DIR/config/sourceTableCore.py
       connections.inputName: sourceTable_visit
       connections.outputName: sourceTableCore
-      atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric
-      atools.skyFluxVisitStatisticMetric.applyContext: VisitContext
-      atools.skySourceSky: SkySourceSkyPlot
-      atools.skySourceFlux: SkySourceHistPlot
-      atools.relativeSizeResidualPlot: RelativeSizeResidualPlot
-      python: |
-        from lsst.analysis.tools.atools import *
-        from lsst.analysis.tools.contexts import *
+  analyzePreSourceTableCore:
+    class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
+    config:
+      file: $ANALYSIS_TOOLS_DIR/config/sourceTableCore.py
+      connections.inputName: preSourceTable_visit
+      connections.outputName: preSourceTableCore
   catalogMatchVisit:
     class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
+  catalogMatchPreVisit:
+    class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
+    config:
+      connections.catalog: "preSourceTable_visit"
+      connections.targetCatalog: "preSourceTable_visit"
+      connections.visitSummaryTable: "visitSummary"
   astrometricRefCatSourceVisit:
     class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
     config:
-      atools.astromDiffRASkyVisitPlot: TargetRefCatDeltaRASkyVisitPlot
-      atools.astromDiffDecSkyVisitPlot: TargetRefCatDeltaDecSkyVisitPlot
-      atools.astromDiffRAScatterVisitPlot: TargetRefCatDeltaRAScatterVisitPlot
-      atools.astromDiffDecScatterVisitPlot: TargetRefCatDeltaDecScatterVisitPlot
-      atools.astromDiffMetrics: TargetRefCatDeltaMetrics
-      atools.astromDiffMetrics.applyContext: VisitContext
-      python: |
-        from lsst.analysis.tools.atools import *
-        from lsst.analysis.tools.contexts import *
+      file: $ANALYSIS_TOOLS_DIR/config/refCatSourceAstrometricCore.py
   photometricMatchVisit:
     class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask
-    config:
-      extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"]
   photometricRefCatSourceVisit:
     class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask
     config:
-      atools.photomDiffPsfSkyVisitPlot: TargetRefCatDeltaPsfSkyVisitPlot
-      atools.photomDiffAp09SkyVisitPlot: TargetRefCatDeltaAp09SkyVisitPlot
-      atools.photoDiffPsfScatterVisitPlot: TargetRefCatDeltaPsfScatterVisitPlot
-      atools.photoDiffCModelScatterVisitPlot: TargetRefCatDeltaAp09ScatterVisitPlot
-      python: from lsst.analysis.tools.atools import *
+      file: $ANALYSIS_TOOLS_DIR/config/refCatSourcePhotometricCore.py
   objectEpochTable:
     class: lsst.analysis.tools.tasks.ObjectEpochTableTask
   sourceObjectMatch:
     class: lsst.analysis.tools.tasks.SourceObjectTableAnalysisTask
-    config:
-      atools.astromColorDiffMetrics: TargetRefCatDeltaColorMetrics
-      python: from lsst.analysis.tools.atools import *
   calexpSummary:
     class: lsst.analysis.tools.tasks.CalexpSummaryAnalysisTask
     config: