[SCSB-174] move DMS requirement <-> test correlations from `@metrics_…

…logger` decorators to `romancal/tests/dms_requirement_tests.json` (#1399)

Co-authored-by: Brett Graham <[email protected]>

changes/1399.general.rst

@@ -0,0 +1 @@
+move DMS requirement <-> test correlations from ``@metrics_logger()`` decorators to ``romancal/tests/dms_requirement_tests.json``

pyproject.toml

@@ -60,7 +60,6 @@ test = [
     "ci-watson >=0.5.0",
     "pytest >=4.6.0, <8.0.0",
     "pytest-astropy",
-    "metrics_logger >= 0.1.0",
     "deepdiff",
 ]
 dev = [

requirements-dev-st.txt

@@ -9,7 +9,6 @@ git+https://github.com/spacetelescope/stpipe.git
 # Other important upstream packages
 git+https://github.com/spacetelescope/crds.git
 git+https://github.com/spacetelescope/gwcs.git
-git+https://github.com/spacetelescope/metrics_logger.git
 git+https://github.com/spacetelescope/tweakwcs.git
 git+https://github.com/spacetelescope/webbpsf.git
 # although not directly used install the dev version of poppy for webbpsf

romancal/regtest/test_catalog.py

@@ -2,7 +2,6 @@
 
 import asdf
 import pytest
-from metrics_logger.decorators import metrics_logger
 
 from romancal.source_catalog.source_catalog_step import SourceCatalogStep
 from romancal.stpipe import RomanStep
@@ -48,7 +47,6 @@ def check_catalog_fields(model, log, modeltype):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS374", "DMS375", "DMS376", "DMS386", "DMS387")
 def test_catalog_l3(rtdata, ignore_asdf_paths):
     # DMS374: positions on ICRF
     # DMS375: fluxes
@@ -76,7 +74,6 @@ def test_catalog_l3(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS374", "DMS375", "DMS376", "DMS386", "DMS387")
 def test_catalog_l2(rtdata, ignore_asdf_paths):
     # DMS374: positions on ICRF
     # DMS375: fluxes

romancal/regtest/test_mos_pipeline.py

@@ -4,7 +4,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.pipeline.mosaic_pipeline import MosaicPipeline
 
@@ -20,7 +19,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS356", "DMS374", "DMS400")
 def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
     """Tests for level 3 processing requirements DMS356"""
     rtdata.get_asn("WFI/image/L3_regtest_asn.json")
@@ -131,7 +129,6 @@ def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS373")
 def test_hlp_mosaic_pipeline(rtdata, ignore_asdf_paths):
     """Tests for level 3 mosaic requirements DMS373"""
     rtdata.get_asn("WFI/image/L3_mosaic_asn.json")

romancal/regtest/test_ramp_fitting.py

@@ -10,7 +10,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.lib.dms import log_result
 from romancal.lib.suffix import replace_suffix
@@ -188,7 +187,6 @@ def test_rampfit_step(rampfit_result, rtdata_module, ignore_asdf_paths):
     except AssertionError as e:
         error_msgs.append(str(e))
 
-    @metrics_logger(requirement)
     def test_success():
         assert not len(error_msgs), "\n".join(error_msgs)
 

romancal/regtest/test_resample.py

@@ -1,6 +1,5 @@
 import numpy as np
 import pytest
-from metrics_logger.decorators import metrics_logger
 from roman_datamodels import datamodels as rdm
 
 from romancal.resample.resample_step import ResampleStep
@@ -9,7 +8,6 @@
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS342", "DMS343", "DMS344", "DMS345")
 @pytest.mark.bigdata
 def test_resample_single_file(rtdata, ignore_asdf_paths):
     output_data = "mosaic_resamplestep.asdf"

romancal/regtest/test_tweakreg.py

@@ -2,7 +2,6 @@
 import numpy as np
 import pytest
 from astropy import units as u
-from metrics_logger.decorators import metrics_logger
 from roman_datamodels import datamodels as rdm
 
 from romancal.stpipe import RomanStep
@@ -11,7 +10,6 @@
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS280", "DMS405", "DMS488")
 @pytest.mark.bigdata
 def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path):
     # N.B.: uncal file is from simulator

romancal/regtest/test_wfi_16resultants.py

@@ -2,7 +2,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.pipeline.exposure_pipeline import ExposurePipeline
 
@@ -16,7 +15,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS413")
 def test_16resultants_image_processing(rtdata, ignore_asdf_paths):
     """Tests for imaging processing requirements for 16 resultants (DMS413)"""
     # The input data is from INS for stress testing at some point this should be generated
@@ -72,7 +70,6 @@ def test_16resultants_image_processing(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS414")
 def test_16resultants_spectral_processing(rtdata, ignore_asdf_paths):
     """Tests for imaging processing requirements for 16 resultants (DMS413)"""
     # The input data is from INS for stress testing at some point this should be generated

romancal/regtest/test_wfi_dq_init.py

@@ -4,15 +4,13 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import DQInitStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS25")
 @pytest.mark.bigdata
 def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     """DMS25 Test: Testing retrieval of best ref file for image data,
@@ -68,7 +66,6 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     assert diff.identical, diff.report()
 
 
-@metrics_logger("DMS25")
 @pytest.mark.bigdata
 def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
     """DMS25 Test: Testing retrieval of best ref file for grism data,

romancal/regtest/test_wfi_flat_field.py

@@ -3,7 +3,6 @@
 import pytest
 import roman_datamodels as rdm
 from crds.core.exceptions import CrdsLookupError
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import FlatFieldStep
 from romancal.stpipe import RomanStep
@@ -69,7 +68,6 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
     assert diff.identical, diff.report()
 
 
-@metrics_logger("DMS79")
 @pytest.mark.bigdata
 @pytest.mark.soctests
 def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):

romancal/regtest/test_wfi_photom.py

@@ -5,15 +5,13 @@
 import pytest
 import roman_datamodels as rdm
 from astropy import units as u
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import PhotomStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS140")
 @pytest.mark.bigdata
 def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     """DMS140 Test: Testing application of photometric correction using

romancal/regtest/test_wfi_pipeline.py

@@ -7,7 +7,6 @@
 import pytest
 import roman_datamodels as rdm
 from gwcs.wcstools import grid_from_bounding_box
-from metrics_logger.decorators import metrics_logger
 from numpy.testing import assert_allclose
 from roman_datamodels.dqflags import pixel
 
@@ -26,7 +25,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS86", "DMS87", "DMS88", "DMS361", "DMS280")
 def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
     """Tests for flat field imaging processing requirements DMS86 & DMS 87"""
     input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
@@ -273,7 +271,6 @@ def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS278", "DMS90", "DMS91", "DMS9", "DMS365")
 def test_level2_grism_processing_pipeline(rtdata, ignore_asdf_paths):
     """Tests for flat field grism processing requirements DMS90, DMS91 and DMS 278"""
     input_data = "r0000201001001001001_01101_0001_WFI01_uncal.asdf"

romancal/tests/dms_requirement_tests.json

@@ -0,0 +1,109 @@
+{
+  "DMS9": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+  ],
+  "DMS25": [
+    "romancal.regtest.test_wfi_dq_init.test_dq_init_grism_step",
+    "romancal.regtest.test_wfi_dq_init.test_dq_init_image_step"
+  ],
+  "DMS79": [
+    "romancal.regtest.test_wfi_flat_field.test_flat_field_crds_match_image_step"
+  ],
+  "DMS86": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+  ],
+  "DMS87": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+  ],
+  "DMS88": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+  ],
+  "DMS90": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+  ],
+  "DMS91": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+  ],
+  "DMS140": [
+    "romancal.regtest.test_wfi_photom.test_absolute_photometric_calibration"
+  ],
+  "DMS278": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+  ],
+  "DMS280": [
+    "romancal.regtest.test_tweakreg.test_tweakreg",
+    "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+  ],
+  "DMS342": [
+    "romancal.regtest.test_resample.test_resample_single_file"
+  ],
+  "DMS343": [
+    "romancal.regtest.test_resample.test_resample_single_file"
+  ],
+  "DMS344": [
+    "romancal.regtest.test_resample.test_resample_single_file"
+  ],
+  "DMS345": [
+    "romancal.regtest.test_resample.test_resample_single_file"
+  ],
+  "DMS356": [
+    "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+  ],
+  "DMS361": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+  ],
+  "DMS362": [
+    "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+  ],
+  "DMS363": [
+    "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+  ],
+  "DMS365": [
+    "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+  ],
+  "DMS366": [
+    "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+  ],
+  "DMS367": [
+    "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+  ],
+  "DMS373": [
+    "romancal.regtest.test_mos_pipeline.test_hlp_mosaic_pipeline"
+  ],
+  "DMS374": [
+    "romancal.regtest.test_catalog.test_catalog_l2",
+    "romancal.regtest.test_catalog.test_catalog_l3",
+    "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+  ],
+  "DMS375": [
+    "romancal.regtest.test_catalog.test_catalog_l2",
+    "romancal.regtest.test_catalog.test_catalog_l3"
+  ],
+  "DMS376": [
+    "romancal.regtest.test_catalog.test_catalog_l2",
+    "romancal.regtest.test_catalog.test_catalog_l3"
+  ],
+  "DMS386": [
+    "romancal.regtest.test_catalog.test_catalog_l2",
+    "romancal.regtest.test_catalog.test_catalog_l3"
+  ],
+  "DMS387": [
+    "romancal.regtest.test_catalog.test_catalog_l2",
+    "romancal.regtest.test_catalog.test_catalog_l3"
+  ],
+  "DMS400": [
+    "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+  ],
+  "DMS405": [
+    "romancal.regtest.test_tweakreg.test_tweakreg"
+  ],
+  "DMS413": [
+    "romancal.regtest.test_wfi_16resultants.test_16resultants_image_processing"
+  ],
+  "DMS414": [
+    "romancal.regtest.test_wfi_16resultants.test_16resultants_spectral_processing"
+  ],
+  "DMS488": [
+    "romancal.regtest.test_tweakreg.test_tweakreg"
+  ]
+}

romancal/tests/test_dms_requirements.py

@@ -0,0 +1,36 @@
+import json
+import re
+from pathlib import Path
+
+TEST_DIRECTORY = Path(__file__).parent.parent
+TEST_REQUIREMENTS_FILENAME = TEST_DIRECTORY / "tests" / "dms_requirement_tests.json"
+
+
+def test_requirements():
+    test_requirements_filename = TEST_REQUIREMENTS_FILENAME.expanduser().absolute()
+    test_directory = TEST_DIRECTORY.expanduser().absolute()
+
+    with open(test_requirements_filename) as test_requirements_file:
+        requirements = json.load(test_requirements_file)
+
+    required_tests = {
+        test
+        for requirement_tests in requirements.values()
+        for test in requirement_tests
+    }
+
+    existing_tests = set()
+    test_regex = re.compile(r"def (test_[^\(]+)\(.*\):")
+    for test_filename in test_directory.glob("**/test_*.py"):
+        with open(test_filename) as test_file:
+            test_file_contents = test_file.read()
+
+        for match in re.finditer(test_regex, test_file_contents):
+            test = f"{test_directory.stem}.{str(test_filename.relative_to(test_directory).parent).replace('/', '.')}.{test_filename.stem}.{match.group(1)}"
+            if test in required_tests:
+                existing_tests.add(test)
+
+    missing_tests = required_tests - existing_tests
+    assert (
+        not missing_tests
+    ), f"could not find the following tests correlated with DMS requirements: {missing_tests}"