From e835fdb6c3a7d146ab8f5f8ef600743fb4fb3cb4 Mon Sep 17 00:00:00 2001 From: D Davis <49163225+ddavis-stsci@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:42:03 -0400 Subject: [PATCH] RCAL-641 Add FOV association generation (#931) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.rst | 5 ++ romancal/associations/generate.py | 11 ++- romancal/associations/lib/dms_base.py | 87 ++++++------------- romancal/associations/lib/rules_elpp_base.py | 84 +++++++++++++++++- romancal/associations/lib/rules_level2.py | 46 ++++++++-- .../associations/tests/test_level2_basics.py | 5 +- .../tests/test_level2_candidates.py | 4 +- 7 files changed, 165 insertions(+), 77 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2f88c6aa7..9de551250 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ 0.13.0 (unreleased) =================== +associations +------------ + +- Add FOV associations to the code [#931] + general ------- diff --git a/romancal/associations/generate.py b/romancal/associations/generate.py index b7184da9d..ed35dff30 100644 --- a/romancal/associations/generate.py +++ b/romancal/associations/generate.py @@ -98,10 +98,13 @@ def generate(pool, rules, version_id=None, finalize=True): # Finalize found associations logger.debug("# associations before finalization: %d", len(associations)) - try: - finalized_asns = rules.callback.reduce("finalize", associations) - except KeyError: - finalized_asns = associations + finalized_asns = associations + if finalize: + logger.debug("Performing association finalization.") + try: + finalized_asns = rules.callback.reduce("finalize", associations) + except KeyError as exception: + logger.debug("Finalization failed for reason: %s", exception) return finalized_asns diff --git a/romancal/associations/lib/dms_base.py b/romancal/associations/lib/dms_base.py index bf10623ec..88bde16e3 100644 --- a/romancal/associations/lib/dms_base.py +++ b/romancal/associations/lib/dms_base.py @@ -21,96 +21,59 @@ # Acquisition and Confirmation images ACQ_EXP_TYPES = ( - "mir_tacq", - "mir_taconfirm", - "nis_taconfirm", - "nis_tacq", "nrc_taconfirm", "nrc_tacq", - "nrs_confirm", - "nrs_msata", - "nrs_taconfirm", - "nrs_tacq", - "nrs_taslit", - "nrs_verify", - "nrs_wata", ) # Exposure EXP_TYPE to Association EXPTYPE mapping # flake8: noqa: E241 EXPTYPE_MAP = { - "mir_darkall": "dark", - "mir_darkimg": "dark", - "mir_darkmrs": "dark", - "mir_flatimage": "flat", - "mir_flatmrs": "flat", - "mir_flatimage-ext": "flat", - "mir_flatmrs-ext": "flat", - "mir_tacq": "target_acquisition", - "mir_taconfirm": "target_acquisition", - "nis_dark": "dark", - "nis_focus": "engineering", - "nis_lamp": "engineering", - "nis_tacq": "target_acquisition", - "nis_taconfirm": "target_acquisition", "nrc_dark": "dark", "nrc_flat": "flat", "nrc_focus": "engineering", "nrc_led": "engineering", "nrc_tacq": "target_acquisition", "nrc_taconfirm": "target_acquisition", - "nrs_autoflat": "autoflat", - "nrs_autowave": "autowave", - "nrs_confirm": "target_acquisition", - "nrs_dark": "dark", - "nrs_focus": "engineering", - "nrs_image": "engineering", - "nrs_lamp": "engineering", - "nrs_msata": "target_acquisition", - "nrs_tacq": "target_acquisition", - "nrs_taconfirm": "target_acquisition", - "nrs_taslit": "target_acquisition", - "nrs_wata": "target_acquisition", } # Coronographic exposures CORON_EXP_TYPES = ["mir_4qpm", "mir_lyot", "nrc_coron"] +# Roman WFI detectors +WFI_DETECTORS = [ + "wfi01", + "wfi02", + "wfi03", + "wfi04", + "wfi05", + "wfi06", + "wfi07", + "wfi08", + "wfi09", + "wfi10", + "wfi11", + "wfi12", + "wfi13", + "wfi14", + "wfi15", + "wfi16", + "wfi17", + "wfi18", +] + # Exposures that get Level2b processing IMAGE2_SCIENCE_EXP_TYPES = [ "wfi_image", - "mir_4qpm", - "mir_image", - "mir_lyot", - "nis_ami", - "nis_image", - "nrc_coron", - "nrc_image", - "nrs_mimf", - "nrc_tsimage", ] IMAGE2_NONSCIENCE_EXP_TYPES = [ - "mir_coroncal", - "nis_focus", - "nrc_focus", - "nrs_focus", - "nrs_image", + "wfi_focus", ] IMAGE2_NONSCIENCE_EXP_TYPES.extend(ACQ_EXP_TYPES) SPEC2_SCIENCE_EXP_TYPES = [ - "mir_lrs-fixedslit", - "mir_lrs-slitless", - "mir_mrs", - "nis_soss", - "nis_wfss", - "nrc_tsgrism", - "nrc_wfss", - "nrs_fixedslit", - "nrs_ifu", - "nrs_msaspec", - "nrs_brightobj", + "wfi_grism", + "wfi_prism", ] SPECIAL_EXPOSURE_MODIFIERS = { diff --git a/romancal/associations/lib/rules_elpp_base.py b/romancal/associations/lib/rules_elpp_base.py index 8cbf7d87d..cf5917ea7 100644 --- a/romancal/associations/lib/rules_elpp_base.py +++ b/romancal/associations/lib/rules_elpp_base.py @@ -1,9 +1,10 @@ """Base classes which define the ELPP Associations""" +import copy import logging import re from collections import defaultdict -from os.path import basename +from os.path import basename, split, splitext from stpipe.format_template import FormatTemplate @@ -18,6 +19,7 @@ IMAGE2_NONSCIENCE_EXP_TYPES, IMAGE2_SCIENCE_EXP_TYPES, SPEC2_SCIENCE_EXP_TYPES, + WFI_DETECTORS, DMSAttrConstraint, DMSBaseMixin, ) @@ -36,6 +38,7 @@ "AsnMixin_AuxData", "AsnMixin_Science", "AsnMixin_Spectrum", + "AsnMixin_Lv2FOV", "AsnMixin_Lv2Image", "AsnMixin_Lv2GBTDSfull", "AsnMixin_Lv2GBTDSpass", @@ -56,6 +59,7 @@ "Constraint_Single_Science", "Constraint_Spectral_Science", "Constraint_Target", + "Constraint_Filename", "DMS_ELPP_Base", "DMSAttrConstraint", "ProcessList", @@ -318,6 +322,43 @@ def make_member(self, item): ) return member + def make_fov_asn(self): + """Take the association with an single exposure with _WFI_ in the name + and expand that to include all 18 detectors. + + Returns + ------- + associations : [association[, ...]] + List of new members to be used in place of + the current one. + """ + results = [] + + # expand the products from _wfi_ to _wfi{det}_ + for product in self["products"]: + for member in product["members"]: + asn = copy.deepcopy(self) + asn.data["products"] = None + product_name = ( + splitext( + split(self.data["products"][0]["members"][0]["expname"])[1] + )[0].rsplit("_", 1)[0] + + "_drzl" + ) + asn.new_product(product_name) + new_members = asn.current_product["members"] + if "_wfi_" in member["expname"]: + # Make and add a member for each detector + for det in WFI_DETECTORS: + new_member = copy.deepcopy(member) + new_member["expname"] = member["expname"].replace("wfi", det) + new_members.append(new_member) + if asn.is_valid: + results.append(asn) + return results + else: + return None + def _init_hook(self, item): """Post-check and pre-add initialization""" super()._init_hook(item) @@ -646,6 +687,16 @@ def __init__(self): ) +class Constraint_Filename(DMSAttrConstraint): + """Select on visit number""" + + def __init__(self): + super().__init__( + name="Filename", + sources=["filename"], + ) + + class Constraint_Expos(DMSAttrConstraint): """Select on exposure number""" @@ -653,8 +704,8 @@ def __init__(self): super().__init__( name="exposure_number", sources=["nexpsur"], - # force_unique=True, - # required=True, + force_unique=True, + required=True, ) @@ -957,6 +1008,33 @@ def _init_hook(self, item): # --------------------------------------------- # Mixins to define the broad category of rules. # --------------------------------------------- +class AsnMixin_Lv2FOV: + """Level 2 Image association base""" + + def _init_hook(self, item): + """Post-check and pre-add initialization""" + + super()._init_hook(item) + self.data["asn_type"] = "FOV" + + def finalize(self): + """Finalize association + + + Returns + ------- + associations: [association[, ...]] or None + List of fully-qualified associations that this association + represents. + `None` if a complete association cannot be produced. + + """ + if self.is_valid: + return self.make_fov_asn() + else: + return None + + class AsnMixin_Lv2Image: """Level 2 Image association base""" diff --git a/romancal/associations/lib/rules_level2.py b/romancal/associations/lib/rules_level2.py index 03fb7dbee..80a324909 100644 --- a/romancal/associations/lib/rules_level2.py +++ b/romancal/associations/lib/rules_level2.py @@ -6,7 +6,14 @@ from romancal.associations.lib.rules_elpp_base import * from romancal.associations.registry import RegistryMarker -__all__ = ["Asn_Lv2Image", "Asn_Lv2GBTDSPass", "Asn_Lv2GBTDSFull", "AsnMixin_Lv2Image"] +__all__ = [ + "Asn_Lv2FOV", + "Asn_Lv2Image", + "Asn_Lv2GBTDSPass", + "Asn_Lv2GBTDSFull", + "AsnMixin_Lv2Image", + "AsnMinxin_Lv2FOV", +] # Configure logging logger = logging.getLogger(__name__) @@ -17,16 +24,40 @@ # -------------------------------- # Start of the User-level rules # -------------------------------- +@RegistryMarker.rule +class Asn_Lv2FOV(AsnMixin_Lv2FOV, DMS_ELPP_Base): + """Level2b Non-TSO Science Image Association + + Characteristics: + - Association type: ``FOV`` + - Pipeline: ``mosaic`` + - Image-based science exposures + - Science exposures for all 18 detectors + """ + + def __init__(self, *args, **kwargs): + # Setup constraints + self.constraints = Constraint( + [ + Constraint_Base(), + Constraint_Target(), + Constraint_Filename(), + ] + ) + + # Now check and continue initialization. + super().__init__(*args, **kwargs) + + @RegistryMarker.rule class Asn_Lv2Image(AsnMixin_Lv2Image, DMS_ELPP_Base): """Level2b Non-TSO Science Image Association Characteristics: - - Association type: ``image2`` - - Pipeline: ``calwebb_image2`` + - Association type: ``image`` + - Pipeline: ``ELPP`` - Image-based science exposures - Single science exposure - - Non-TSO """ def __init__(self, *args, **kwargs): @@ -35,7 +66,12 @@ def __init__(self, *args, **kwargs): [ Constraint_Base(), Constraint_Target(), - Constraint_Expos(), + Constraint( + [ + Constraint_Expos(), + ], + reduce=Constraint.any, + ), Constraint_Optical_Path(), Constraint_Sequence(), Constraint_Pass(), diff --git a/romancal/associations/tests/test_level2_basics.py b/romancal/associations/tests/test_level2_basics.py index 125c6f4d4..663452150 100644 --- a/romancal/associations/tests/test_level2_basics.py +++ b/romancal/associations/tests/test_level2_basics.py @@ -59,7 +59,10 @@ def test_level2_productname(): for member in product["members"] if member["exptype"] == "science" or member["exptype"] == "wfi_image" ] - assert len(science) == 2 + if asn["asn_rule"] == "Asn_Lv2Image": + assert len(science) == 2 + if asn["asn_rule"] == "Asn_Lv2FOV": + assert len(science) == 18 # match = re.match(REGEX_LEVEL2, science[0]['expname']) diff --git a/romancal/associations/tests/test_level2_candidates.py b/romancal/associations/tests/test_level2_candidates.py index 6110114ef..bb85000a5 100644 --- a/romancal/associations/tests/test_level2_candidates.py +++ b/romancal/associations/tests/test_level2_candidates.py @@ -17,11 +17,11 @@ # Basic observation ACIDs (["-i", "o001"], 0), # Whole program - ([], 2), + ([], 5), # Discovered only (["--discover"], 0), # Candidates only - (["--all-candidates"], 2), + (["--all-candidates"], 5), ], ) def test_candidate_observation(partial_args, n_asns):