From 2986a4e21ca48aa3b21f549398c2fd35add39773 Mon Sep 17 00:00:00 2001
From: Hsin-Fang Chiang <hchiang@lsst.org>
Date: Thu, 3 Oct 2024 14:27:36 -0700
Subject: [PATCH] Ingest LSSTCam photodiode data

LSSTCam is the only instrument with photodiode data that may need to
be ingested.

An example of the photodiode file path is
rubinobs-lfa-cp/MTCamera/photodiode/20240928/MC_C_20240928_002826_photodiode.ecsv
---
 src/info.py   |  5 ++++-
 src/ingest.py | 39 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/src/info.py b/src/info.py
index 1867611..d9281fa 100644
--- a/src/info.py
+++ b/src/info.py
@@ -146,9 +146,12 @@ def __init__(self, path):
                 self.instrument = f"{csc}/{generator}"
             elif len(components) == 6:
                 self.bucket, self.instrument, year, month, day, self.filename = components
+            elif len(components) == 5: # photodiode data
+                self.bucket, self.instrument, data_type, self.obs_day, self.filename = components
             else:
                 raise ValueError(f"Unrecognized number of components: {len(components)}")
-            self.obs_day = f"{year}{month}{day}"
+            if not self.obs_day:
+                self.obs_day = f"{year}{month}{day}"
         except Exception:
             logger.exception("Unable to parse: %s", path)
             raise
diff --git a/src/ingest.py b/src/ingest.py
index 912e351..9a026a4 100644
--- a/src/ingest.py
+++ b/src/ingest.py
@@ -24,13 +24,16 @@
 """
 import json
 import os
+import re
 import socket
 import time
 
 import astropy.io.fits
 import requests
 from lsst.daf.butler import Butler
+from lsst.pipe.base import Instrument
 from lsst.obs.base import DefineVisitsTask, RawIngestTask
+from lsst.obs.lsst import PhotodiodeIngestTask
 from lsst.resources import ResourcePath
 
 from info import Info
@@ -200,7 +203,18 @@ def main():
         on_metadata_failure=on_metadata_failure,
     )
 
-    if not is_lfa:
+    if is_lfa:
+        # LSSTCam photodiode is copy mode only.
+        instrument = Instrument.from_string("LSSTCam", butler.registry)
+        lsstcam_photodiode_ingester = PhotodiodeIngestTask(
+            config=PhotodiodeIngestTask.ConfigClass(),
+            butler=butler,
+            instrument=instrument,
+            on_success=on_success,
+            on_ingest_failure=on_ingest_failure,
+            on_metadata_failure=on_metadata_failure,
+        )
+    else:
         define_visits_config = DefineVisitsTask.ConfigClass()
         define_visits_config.groupExposures = "one-to-one"
         visit_definer = DefineVisitsTask(config=define_visits_config, butler=butler)
@@ -220,6 +234,13 @@ def main():
 
                 logger.info("Ingesting %s", resources)
                 refs = None
+                if is_lfa:
+                    resources_photodiode = [
+                        resource
+                        for resource in resources
+                        if re.search(r"MTCamera/photodiode.*_photodiode.ecsv", resource)
+                    ]
+                    resources = [resource for resource in resources if resource not in resources_photodiode]
                 try:
                     refs = ingester.run(resources)
                 except Exception:
@@ -233,6 +254,22 @@ def main():
                             info = Info.from_path(resource.geturl())
                             r.lrem(worker_queue, 0, info.path)
 
+                if is_lfa and resources_photodiode:
+                    try:
+                        refs = lsstcam_photodiode_ingester.run(resources_photodiode)
+                    except Exception:
+                        logger.exception(
+                            "Error while ingesting %s, retrying one by one", resources_photodiode
+                        )
+                        refs = []
+                        for resource in resources_photodiode:
+                            try:
+                                refs.extend(lsstcam_photodiode_ingester.run([resource]))
+                            except Exception:
+                                logger.exception("Error while ingesting %s", resource)
+                                info = Info.from_path(resource.geturl())
+                                r.lrem(worker_queue, 0, info.path)
+
                 # Define visits if we ingested anything
                 if not is_lfa and refs:
                     ids = [ref.dataId for ref in refs]