From 31f1b2dac2e6558af308bfc29e0fc6006fd2091c Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 12:16:02 -0700 Subject: [PATCH 1/7] Add placeholder configuration for embargo SIA Add an initial template for an SIA configuration to use with the embargo repo at USDF. --- configs/embargo-sia.yaml | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 configs/embargo-sia.yaml diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml new file mode 100644 index 0000000..b616c8a --- /dev/null +++ b/configs/embargo-sia.yaml @@ -0,0 +1,71 @@ +facility_name: Rubin-LSST +obs_collection: LSST.Embargo +collections: [] +use_butler_uri: false +dataset_types: + raw: + dataproduct_type: image + dataproduct_subtype: lsst.raw + calib_level: 1 + obs_id_fmt: "{records[exposure].obs_id}-{records[detector].full_name}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + calexp: + dataproduct_type: image + dataproduct_subtype: lsst.calexp + calib_level: 2 + obs_id_fmt: "{records[visit].name}-{records[detector].full_name}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + deepCoadd_calexp: + dataproduct_type: image + dataproduct_subtype: lsst.deepCoadd_calexp + calib_level: 3 + obs_id_fmt: "{skymap}-{tract}-{patch}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + goodSeeingCoadd: + dataproduct_type: image + dataproduct_subtype: lsst.goodSeeingCoadd + calib_level: 3 + obs_id_fmt: "{skymap}-{tract}-{patch}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + goodSeeingDiff_differenceExp: + dataproduct_type: image + dataproduct_subtype: lsst.goodSeeingDiff_differenceExp + calib_level: 3 + obs_id_fmt: "{records[visit].name}-{records[detector].full_name}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" +extra_columns: + lsst_visit: + template: "{visit}" + type: "int" + lsst_detector: + template: "{detector}" + type: "int" + lsst_tract: + template: "{tract}" + type: "int" + lsst_patch: + template: "{patch}" + type: "int" + lsst_band: + template: "{band}" + type: "string" + lsst_filter: + template: "{physical_filter}" + type: "string" +spectral_ranges: + "u": [330.0e-9, 400.0e-9] + "g": [402.0e-9, 552.0e-9] + "r": [552.0e-9, 691.0e-9] + "i": [691.0e-9, 818.0e-9] + "z": [818.0e-9, 922.0e-9] + "y": [970.0e-9, 1060.0e-9] From a3776627527071ad50c5132ec5ab88fc9ecdee53 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 14:18:27 -0700 Subject: [PATCH 2/7] Fix facility name --- configs/embargo-sia.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index b616c8a..b4e1517 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -1,4 +1,8 @@ -facility_name: Rubin-LSST +facility_name: Rubin:Simonyi +facility_map: + LATISS: "Rubin:1.2m" + LSSTComCam: "Rubin:Simonyi" + LSSTCam: "Rubin:Simonyi" obs_collection: LSST.Embargo collections: [] use_butler_uri: false From dae1c1a50f2ac5581ae152b78b1d2a4b29c7f000 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 14:36:33 -0700 Subject: [PATCH 3/7] Add more dataset types for embargo config --- configs/embargo-sia.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index b4e1517..c354386 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -15,6 +15,22 @@ dataset_types: o_ucd: phot.count access_format: "application/x-votable+xml;content=datalink" datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + postISRCCD: + dataproduct_type: image + dataproduct_subtype: lsst.postISRCCD + calib_level: 1 + obs_id_fmt: "{records[exposure].obs_id}-{records[detector].full_name}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + pvi: + dataproduct_type: image + dataproduct_subtype: lsst.pvi + calib_level: 1 + obs_id_fmt: "{records[visit].name}-{records[detector].full_name}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" calexp: dataproduct_type: image dataproduct_subtype: lsst.calexp @@ -23,6 +39,14 @@ dataset_types: o_ucd: phot.count access_format: "application/x-votable+xml;content=datalink" datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" + deepCoadd: + dataproduct_type: image + dataproduct_subtype: lsst.deepCoadd + calib_level: 3 + obs_id_fmt: "{skymap}-{tract}-{patch}" + o_ucd: phot.count + access_format: "application/x-votable+xml;content=datalink" + datalink_url_fmt: "https://usdf-rsp.slac.stanford.edu/api/datalink/links?ID=butler%3A//embargo/{id}" deepCoadd_calexp: dataproduct_type: image dataproduct_subtype: lsst.deepCoadd_calexp From 9578e6b85e20db026946ac2ce56a8802c2dcb5f8 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 14:56:38 -0700 Subject: [PATCH 4/7] Configure embargo collections --- configs/embargo-sia.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index c354386..72558bf 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -4,7 +4,7 @@ facility_map: LSSTComCam: "Rubin:Simonyi" LSSTCam: "Rubin:Simonyi" obs_collection: LSST.Embargo -collections: [] +collections: ["LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*", "LSSTComCam/nightlyValidation"] use_butler_uri: false dataset_types: raw: From bf531e3848977f0cb71d78fc08c0764023f86845 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 15:15:20 -0700 Subject: [PATCH 5/7] Remove collection searches from embargo repo It looks like the SIA code is not currently set up to handle these -- it's not querying the globs at all, and even if it was it is doing a find-first search which is not compatible with glob searches. --- configs/embargo-sia.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index 72558bf..b1e142e 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -4,7 +4,7 @@ facility_map: LSSTComCam: "Rubin:Simonyi" LSSTCam: "Rubin:Simonyi" obs_collection: LSST.Embargo -collections: ["LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*", "LSSTComCam/nightlyValidation"] +collections: [ "LSSTComCam/nightlyValidation" ] # "LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*" use_butler_uri: false dataset_types: raw: From 667ca49e7d6256b3b44afd90240fac71f953c803 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Thu, 12 Dec 2024 15:29:03 -0700 Subject: [PATCH 6/7] Fix formatting --- configs/embargo-sia.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index b1e142e..6f74ba8 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -4,7 +4,7 @@ facility_map: LSSTComCam: "Rubin:Simonyi" LSSTCam: "Rubin:Simonyi" obs_collection: LSST.Embargo -collections: [ "LSSTComCam/nightlyValidation" ] # "LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*" +collections: ["LSSTComCam/nightlyValidation"] # "LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*" use_butler_uri: false dataset_types: raw: From 0e83c561a4792efb9ec22e684f04e4e0dbcb2323 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Wed, 18 Dec 2024 10:05:41 -0700 Subject: [PATCH 7/7] Update collections for embargo SIA --- configs/embargo-sia.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/embargo-sia.yaml b/configs/embargo-sia.yaml index 6f74ba8..a0b573f 100644 --- a/configs/embargo-sia.yaml +++ b/configs/embargo-sia.yaml @@ -4,7 +4,9 @@ facility_map: LSSTComCam: "Rubin:Simonyi" LSSTCam: "Rubin:Simonyi" obs_collection: LSST.Embargo -collections: ["LSSTComCam/nightlyValidation"] # "LSSTComCam/prompt/output-*", "LSSTComCam/runs/DRP/*/w_*/*" +# The campaign management team is manually maintaining this chained collection +# as a pointer to the latest DRP pipeline run. +collections: ["LSSTComCam/SIA"] use_butler_uri: false dataset_types: raw: