From dc7c7aff1724bfee6c9c093379a20048df69ebc6 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 21 Nov 2023 15:43:35 -0800 Subject: [PATCH 1/6] Remove unphysical diaSources from the output of detectAndMeasure --- python/lsst/ip/diffim/detectAndMeasure.py | 34 +++++++++++++++ tests/test_detectAndMeasure.py | 51 ++++++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index a9b71bb2..4dade02f 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -141,6 +141,12 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, target=SkyObjectsTask, doc="Generate sky sources", ) + badSourceFlags = lsst.pex.config.ListField( + dtype=str, + doc="Do not include sources with any of these flags set in the output catalog.", + default=("base_PixelFlags_flag_offimage", + ), + ) idGenerator = DetectorVisitIdGeneratorConfig.make_field() def setDefaults(self): @@ -364,6 +370,7 @@ def processResults(self, science, matchedTemplate, difference, sources, table, self.addSkySources(diaSources, difference.mask, difference.info.id) self.measureDiaSources(diaSources, science, difference, matchedTemplate) + diaSources = self.removeBadSources(diaSources) if self.config.doForcedMeasurement: self.measureForcedSources(diaSources, science, difference.getWcs()) @@ -376,6 +383,33 @@ def processResults(self, science, matchedTemplate, difference, sources, table, return measurementResults + def removeBadSources(self, diaSources): + """Remove bad diaSources from the catalog. + + Parameters + ---------- + diaSources : `lsst.afw.table.SourceCatalog` + The catalog of detected sources. + + Returns + ------- + diaSources : `lsst.afw.table.SourceCatalog` + The updated catalog of detected sources, with any source that has a + flag in ``config.badSourceFlags`` set removed. + """ + flags = np.ones(len(diaSources), dtype=bool) + for flag in self.config.badSourceFlags: + try: + flags &= ~diaSources[flag] + except Exception as e: + self.log.warning("Could not apply source flag: %s", e) + nBad = np.count_nonzero(~flags) + if nBad > 0: + self.log.warning(f"Found and removed {nBad} unphysical sources.") + diaSources = diaSources[flags].copy(deep=True) + self.metadata.add("nRemovedBadFlaggedSources", nBad) + return diaSources + def addSkySources(self, diaSources, mask, seed): """Add sources in empty regions of the difference image for measuring the background. diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 71bd6a68..be202969 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -93,7 +93,7 @@ def _check_values(self, values, minValue=None, maxValue=None): self.assertTrue(np.all(values <= maxValue)) def _setup_detection(self, doApCorr=False, doMerge=False, - doSkySources=False, doForcedMeasurement=False): + doSkySources=False, doForcedMeasurement=False, nSkySources=5, badSourceFlags=[]): """Setup and configure the detection and measurement PipelineTask. Parameters @@ -117,8 +117,9 @@ def _setup_detection(self, doApCorr=False, doMerge=False, config.doMerge = doMerge config.doSkySources = doSkySources config.doForcedMeasurement = doForcedMeasurement + config.badSourceFlags = badSourceFlags if doSkySources: - config.skySources.nSources = 5 + config.skySources.nSources = nSkySources return self.detectionTask(config=config) @@ -189,6 +190,52 @@ def test_measurements_finite(self): self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) self._check_values(output.diaSources.getPsfInstFlux()) + def test_remove_unphysical(self): + """ + """ + + # Set up the simulated images + noiseLevel = 1. + staticSeed = 1 + xSize = 256 + ySize = 256 + kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} + science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, + nSrc=1, **kwargs) + matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, + nSrc=1, **kwargs) + difference = science.clone() + bbox = difference.getBBox() + difference.maskedImage -= matchedTemplate.maskedImage + + # Configure the detection Task, and do not remove unphysical sources + detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, + badSourceFlags=["base_PixelFlags_flag_offimage", ]) + + # Run detection and check the results + diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources + badDiaSrc0 = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBad0 = np.count_nonzero(badDiaSrc0) + # Verify that all sources are physical + self.assertEqual(nBad0, 0) + # Set a few centroids outside the image bounding box + nSetBad = 5 + for src in diaSources[0: nSetBad]: + src["slot_Centroid_x"] += xSize + src["slot_Centroid_y"] += ySize + src["base_PixelFlags_flag_offimage"] = True + # Verify that these sources are outside the image + badDiaSrc1 = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBad1 = np.count_nonzero(badDiaSrc1) + self.assertEqual(nBad1, nSetBad) + diaSources2 = detectionTask.removeBadSources(diaSources) + badDiaSrc2 = ~bbox.contains(diaSources2.getX(), diaSources2.getY()) + nBad2 = np.count_nonzero(badDiaSrc2) + + # Verify that no sources outside the image bounding box remain + self.assertEqual(nBad2, 0) + self.assertEqual(len(diaSources2), len(diaSources) - nSetBad) + def test_detect_transients(self): """Run detection on a difference image containing transients. """ From 2e3396745aa70d15cfc68950cf2fbb5497533db4 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 8 Dec 2023 16:39:55 -0800 Subject: [PATCH 2/6] Respond to review --- python/lsst/ip/diffim/detectAndMeasure.py | 22 ++++++++++------- tests/test_detectAndMeasure.py | 29 +++++++++-------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 4dade02f..92569c48 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -143,7 +143,7 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, ) badSourceFlags = lsst.pex.config.ListField( dtype=str, - doc="Do not include sources with any of these flags set in the output catalog.", + doc="Sources with any of these flags set are removed before writing the output catalog.", default=("base_PixelFlags_flag_offimage", ), ) @@ -397,17 +397,21 @@ def removeBadSources(self, diaSources): The updated catalog of detected sources, with any source that has a flag in ``config.badSourceFlags`` set removed. """ - flags = np.ones(len(diaSources), dtype=bool) + nBadTotal = 0 + selector = np.ones(len(diaSources), dtype=bool) for flag in self.config.badSourceFlags: try: - flags &= ~diaSources[flag] - except Exception as e: + flags = diaSources[flag] + except KeyError as e: self.log.warning("Could not apply source flag: %s", e) - nBad = np.count_nonzero(~flags) - if nBad > 0: - self.log.warning(f"Found and removed {nBad} unphysical sources.") - diaSources = diaSources[flags].copy(deep=True) - self.metadata.add("nRemovedBadFlaggedSources", nBad) + continue + nBad = np.count_nonzero(flags) + if nBad > 0: + self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag) + selector &= ~flags + nBadTotal += nBad + diaSources = diaSources[selector].copy(deep=True) + self.metadata.add("nRemovedBadFlaggedSources", nBadTotal) return diaSources def addSkySources(self, diaSources, mask, seed): diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index be202969..05f7e713 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -92,20 +92,17 @@ def _check_values(self, values, minValue=None, maxValue=None): if maxValue is not None: self.assertTrue(np.all(values <= maxValue)) - def _setup_detection(self, doApCorr=False, doMerge=False, - doSkySources=False, doForcedMeasurement=False, nSkySources=5, badSourceFlags=[]): + def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs): """Setup and configure the detection and measurement PipelineTask. Parameters ---------- - doApCorr : `bool`, optional - Run subtask to apply aperture corrections. - doMerge : `bool`, optional - Merge positive and negative diaSources. doSkySources : `bool`, optional Generate sky sources. - doForcedMeasurement : `bool`, optional - Force photometer diaSource locations on PVI. + nSkySources : `int`, optional + The number of sky sources to add in isolated background regions. + **kwargs + Any additional config parameters to set. Returns ------- @@ -113,13 +110,10 @@ def _setup_detection(self, doApCorr=False, doMerge=False, The configured Task to use for detection and measurement. """ config = self.detectionTask.ConfigClass() - config.doApCorr = doApCorr - config.doMerge = doMerge config.doSkySources = doSkySources - config.doForcedMeasurement = doForcedMeasurement - config.badSourceFlags = badSourceFlags if doSkySources: config.skySources.nSources = nSkySources + config.update(**kwargs) return self.detectionTask(config=config) @@ -191,9 +185,8 @@ def test_measurements_finite(self): self._check_values(output.diaSources.getPsfInstFlux()) def test_remove_unphysical(self): + """Check that sources with specified flags are removed from the catalog. """ - """ - # Set up the simulated images noiseLevel = 1. staticSeed = 1 @@ -249,7 +242,7 @@ def test_detect_transients(self): matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) # Configure the detection Task - detectionTask = self._setup_detection() + detectionTask = self._setup_detection(doMerge=False) kwargs["seed"] = transientSeed kwargs["nSrc"] = 10 kwargs["fluxLevel"] = 1000 @@ -301,7 +294,7 @@ def test_detect_dipoles(self): difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] # Configure the detection Task - detectionTask = self._setup_detection() + detectionTask = self._setup_detection(doMerge=False) # Run detection and check the results output = detectionTask.run(science, matchedTemplate, difference) @@ -509,7 +502,7 @@ def test_detect_transients(self): subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() # Configure the detection Task - detectionTask = self._setup_detection() + detectionTask = self._setup_detection(doMerge=False) kwargs["seed"] = transientSeed kwargs["nSrc"] = 10 kwargs["fluxLevel"] = 1000 @@ -579,7 +572,7 @@ def test_detect_dipoles(self): score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) # Configure the detection Task - detectionTask = self._setup_detection() + detectionTask = self._setup_detection(doMerge=False) # Run detection and check the results output = detectionTask.run(science, matchedTemplate, difference, score) From e60df4511cd058d8add878859219d65e62b94312 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Thu, 14 Dec 2023 15:04:51 -0800 Subject: [PATCH 3/6] Remove diaSources with NaN centroids Downstream code will break in multiple places if there is a single diaSource with a NaN value from .getCentroid() --- python/lsst/ip/diffim/detectAndMeasure.py | 12 +++++- tests/test_detectAndMeasure.py | 49 +++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 92569c48..e59cbadf 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -410,9 +410,17 @@ def removeBadSources(self, diaSources): self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag) selector &= ~flags nBadTotal += nBad - diaSources = diaSources[selector].copy(deep=True) self.metadata.add("nRemovedBadFlaggedSources", nBadTotal) - return diaSources + # Use slot_Centroid_x/y here instead of getX() method, since the former + # works on non-contiguous source tables and the latter does not. + centroidFlag = np.isfinite(diaSources["slot_Centroid_x"]) & np.isfinite(diaSources["slot_Centroid_y"]) + nBad = np.count_nonzero(~centroidFlag) + if nBad > 0: + self.log.info("Found and removed %d unphysical sources with non-finite centroid.", nBad) + self.metadata.add("nRemovedBadCentroidSources", nBadTotal) + nBadTotal += nBad + selector &= centroidFlag + return diaSources[selector].copy(deep=True) def addSkySources(self, diaSources, mask, seed): """Add sources in empty regions of the difference image diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 05f7e713..07e19938 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -229,6 +229,55 @@ def test_remove_unphysical(self): self.assertEqual(nBad2, 0) self.assertEqual(len(diaSources2), len(diaSources) - nSetBad) + def test_remove_nan_centroid(self): + """Check that sources with non-finite centroids are removed from the catalog. + """ + # Set up the simulated images + noiseLevel = 1. + staticSeed = 1 + xSize = 256 + ySize = 256 + kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} + science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, + nSrc=1, **kwargs) + matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, + nSrc=1, **kwargs) + difference = science.clone() + bbox = difference.getBBox() + difference.maskedImage -= matchedTemplate.maskedImage + + # Configure the detection Task, and do not remove unphysical sources + detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, + badSourceFlags=["base_PixelFlags_flag_offimage", ]) + + # Run detection and check the results + diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources + badDiaSrc0 = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBad0 = np.count_nonzero(badDiaSrc0) + # Verify that all sources are physical + self.assertEqual(nBad0, 0) + # Set a few centroids outside the image bounding box + nSetBad = 5 + for i, src in enumerate(diaSources[0: nSetBad]): + if i % 3 == 0: + src["slot_Centroid_x"] = np.nan + elif i % 3 == 1: + src["slot_Centroid_y"] = np.nan + elif i % 3 == 2: + src["slot_Centroid_x"] = np.nan + src["slot_Centroid_y"] = np.nan + # Verify that these sources are outside the image + badDiaSrc1 = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBad1 = np.count_nonzero(badDiaSrc1) + self.assertEqual(nBad1, nSetBad) + diaSources2 = detectionTask.removeBadSources(diaSources) + badDiaSrc2 = ~bbox.contains(diaSources2.getX(), diaSources2.getY()) + nBad2 = np.count_nonzero(badDiaSrc2) + + # Verify that no sources outside the image bounding box remain + self.assertEqual(nBad2, 0) + self.assertEqual(len(diaSources2), len(diaSources) - nSetBad) + def test_detect_transients(self): """Run detection on a difference image containing transients. """ From 2d89abfec39607eb86513bcd28f3edaa7a19ef2e Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 22 Dec 2023 17:47:45 -0800 Subject: [PATCH 4/6] Clean up variable and method names --- python/lsst/ip/diffim/detectAndMeasure.py | 18 ++++++------ tests/test_detectAndMeasure.py | 36 +++++++++++++++-------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index e59cbadf..199b3b74 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -359,18 +359,18 @@ def processResults(self, science, matchedTemplate, difference, sources, table, fpSet = positiveFootprints fpSet.merge(negativeFootprints, self.config.growFootprint, self.config.growFootprint, False) - diaSources = afwTable.SourceCatalog(table) - fpSet.makeSources(diaSources) - self.log.info("Merging detections into %d sources", len(diaSources)) + initialDiaSources = afwTable.SourceCatalog(table) + fpSet.makeSources(initialDiaSources) + self.log.info("Merging detections into %d sources", len(initialDiaSources)) else: - diaSources = sources - self.metadata.add("nMergedDiaSources", len(diaSources)) + initialDiaSources = sources + self.metadata.add("nMergedDiaSources", len(initialDiaSources)) if self.config.doSkySources: - self.addSkySources(diaSources, difference.mask, difference.info.id) + self.addSkySources(initialDiaSources, difference.mask, difference.info.id) - self.measureDiaSources(diaSources, science, difference, matchedTemplate) - diaSources = self.removeBadSources(diaSources) + self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate) + diaSources = self._removeBadSources(initialDiaSources) if self.config.doForcedMeasurement: self.measureForcedSources(diaSources, science, difference.getWcs()) @@ -383,7 +383,7 @@ def processResults(self, science, matchedTemplate, difference, sources, table, return measurementResults - def removeBadSources(self, diaSources): + def _removeBadSources(self, diaSources): """Remove bad diaSources from the catalog. Parameters diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 07e19938..55b58f5d 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -202,15 +202,26 @@ def test_remove_unphysical(self): difference.maskedImage -= matchedTemplate.maskedImage # Configure the detection Task, and do not remove unphysical sources + detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, + badSourceFlags=[]) + + # Run detection and check the results + diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources + badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove) + # Verify that unphysical sources exist + self.assertGreater(nBadNoRemove, 0) + + # Configure the detection Task, and remove unphysical sources detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, badSourceFlags=["base_PixelFlags_flag_offimage", ]) # Run detection and check the results diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources - badDiaSrc0 = ~bbox.contains(diaSources.getX(), diaSources.getY()) - nBad0 = np.count_nonzero(badDiaSrc0) + badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) # Verify that all sources are physical - self.assertEqual(nBad0, 0) + self.assertEqual(nBadDoRemove, 0) # Set a few centroids outside the image bounding box nSetBad = 5 for src in diaSources[0: nSetBad]: @@ -218,16 +229,15 @@ def test_remove_unphysical(self): src["slot_Centroid_y"] += ySize src["base_PixelFlags_flag_offimage"] = True # Verify that these sources are outside the image - badDiaSrc1 = ~bbox.contains(diaSources.getX(), diaSources.getY()) - nBad1 = np.count_nonzero(badDiaSrc1) - self.assertEqual(nBad1, nSetBad) - diaSources2 = detectionTask.removeBadSources(diaSources) - badDiaSrc2 = ~bbox.contains(diaSources2.getX(), diaSources2.getY()) - nBad2 = np.count_nonzero(badDiaSrc2) + badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) + nBad = np.count_nonzero(badDiaSrc) + self.assertEqual(nBad, nSetBad) + diaSourcesNoBad = detectionTask._removeBadSources(diaSources) + badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) # Verify that no sources outside the image bounding box remain - self.assertEqual(nBad2, 0) - self.assertEqual(len(diaSources2), len(diaSources) - nSetBad) + self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) + self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) def test_remove_nan_centroid(self): """Check that sources with non-finite centroids are removed from the catalog. @@ -246,7 +256,7 @@ def test_remove_nan_centroid(self): bbox = difference.getBBox() difference.maskedImage -= matchedTemplate.maskedImage - # Configure the detection Task, and do not remove unphysical sources + # Configure the detection Task, and remove unphysical sources detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, badSourceFlags=["base_PixelFlags_flag_offimage", ]) @@ -270,7 +280,7 @@ def test_remove_nan_centroid(self): badDiaSrc1 = ~bbox.contains(diaSources.getX(), diaSources.getY()) nBad1 = np.count_nonzero(badDiaSrc1) self.assertEqual(nBad1, nSetBad) - diaSources2 = detectionTask.removeBadSources(diaSources) + diaSources2 = detectionTask._removeBadSources(diaSources) badDiaSrc2 = ~bbox.contains(diaSources2.getX(), diaSources2.getY()) nBad2 = np.count_nonzero(badDiaSrc2) From 7ab489cfc6bcb7a24deb70fa56eae5c265bad37b Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 22 Dec 2023 17:48:30 -0800 Subject: [PATCH 5/6] Add consistency check of schema and config in Task init --- python/lsst/ip/diffim/detectAndMeasure.py | 10 +++++----- tests/test_detectAndMeasure.py | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 199b3b74..94fb2124 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -228,6 +228,10 @@ def __init__(self, **kwargs): self.makeSubtask("skySources") self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") + # Check that the schema and config are consistent + for flag in self.config.badSourceFlags: + if flag not in self.schema: + raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag) # initialize InitOutputs self.outputSchema = afwTable.SourceCatalog(self.schema) self.outputSchema.getTable().setMetadata(self.algMetadata) @@ -400,11 +404,7 @@ def _removeBadSources(self, diaSources): nBadTotal = 0 selector = np.ones(len(diaSources), dtype=bool) for flag in self.config.badSourceFlags: - try: - flags = diaSources[flag] - except KeyError as e: - self.log.warning("Could not apply source flag: %s", e) - continue + flags = diaSources[flag] nBad = np.count_nonzero(flags) if nBad > 0: self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag) diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 55b58f5d..47180579 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -25,6 +25,7 @@ import lsst.geom from lsst.ip.diffim import detectAndMeasure, subtractImages from lsst.ip.diffim.utils import makeTestImage +from lsst.pipe.base import InvalidQuantumError import lsst.utils.tests @@ -184,6 +185,13 @@ def test_measurements_finite(self): self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) self._check_values(output.diaSources.getPsfInstFlux()) + def test_raise_config_schema_mismatch(self): + """Check that sources with specified flags are removed from the catalog. + """ + # Configure the detection Task, and and set a config that is not in the schema + with self.assertRaises(InvalidQuantumError): + self._setup_detection(badSourceFlags=["Bogus_flag_42"]) + def test_remove_unphysical(self): """Check that sources with specified flags are removed from the catalog. """ From 3c9f4bd0fa8d6d7a933fe3a022e98ef7fdeb0c4b Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 2 Jan 2024 16:39:18 -0800 Subject: [PATCH 6/6] Remove extra check for NaN centroids. With DM-42313 merged, these should be caught by `base_PixelFlags_flag_offimage` --- python/lsst/ip/diffim/detectAndMeasure.py | 9 ----- tests/test_detectAndMeasure.py | 49 ----------------------- 2 files changed, 58 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 94fb2124..f035ea4a 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -411,15 +411,6 @@ def _removeBadSources(self, diaSources): selector &= ~flags nBadTotal += nBad self.metadata.add("nRemovedBadFlaggedSources", nBadTotal) - # Use slot_Centroid_x/y here instead of getX() method, since the former - # works on non-contiguous source tables and the latter does not. - centroidFlag = np.isfinite(diaSources["slot_Centroid_x"]) & np.isfinite(diaSources["slot_Centroid_y"]) - nBad = np.count_nonzero(~centroidFlag) - if nBad > 0: - self.log.info("Found and removed %d unphysical sources with non-finite centroid.", nBad) - self.metadata.add("nRemovedBadCentroidSources", nBadTotal) - nBadTotal += nBad - selector &= centroidFlag return diaSources[selector].copy(deep=True) def addSkySources(self, diaSources, mask, seed): diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 47180579..7fc3138f 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -247,55 +247,6 @@ def test_remove_unphysical(self): self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) - def test_remove_nan_centroid(self): - """Check that sources with non-finite centroids are removed from the catalog. - """ - # Set up the simulated images - noiseLevel = 1. - staticSeed = 1 - xSize = 256 - ySize = 256 - kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} - science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, - nSrc=1, **kwargs) - matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, - nSrc=1, **kwargs) - difference = science.clone() - bbox = difference.getBBox() - difference.maskedImage -= matchedTemplate.maskedImage - - # Configure the detection Task, and remove unphysical sources - detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, - badSourceFlags=["base_PixelFlags_flag_offimage", ]) - - # Run detection and check the results - diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources - badDiaSrc0 = ~bbox.contains(diaSources.getX(), diaSources.getY()) - nBad0 = np.count_nonzero(badDiaSrc0) - # Verify that all sources are physical - self.assertEqual(nBad0, 0) - # Set a few centroids outside the image bounding box - nSetBad = 5 - for i, src in enumerate(diaSources[0: nSetBad]): - if i % 3 == 0: - src["slot_Centroid_x"] = np.nan - elif i % 3 == 1: - src["slot_Centroid_y"] = np.nan - elif i % 3 == 2: - src["slot_Centroid_x"] = np.nan - src["slot_Centroid_y"] = np.nan - # Verify that these sources are outside the image - badDiaSrc1 = ~bbox.contains(diaSources.getX(), diaSources.getY()) - nBad1 = np.count_nonzero(badDiaSrc1) - self.assertEqual(nBad1, nSetBad) - diaSources2 = detectionTask._removeBadSources(diaSources) - badDiaSrc2 = ~bbox.contains(diaSources2.getX(), diaSources2.getY()) - nBad2 = np.count_nonzero(badDiaSrc2) - - # Verify that no sources outside the image bounding box remain - self.assertEqual(nBad2, 0) - self.assertEqual(len(diaSources2), len(diaSources) - nSetBad) - def test_detect_transients(self): """Run detection on a difference image containing transients. """