diff --git a/prompt-processing/groups.ipynb b/prompt-processing/groups.ipynb index a31434e..d66d2c8 100644 --- a/prompt-processing/groups.ipynb +++ b/prompt-processing/groups.ipynb @@ -154,7 +154,10 @@ "source": [ "from lsst.daf.butler import Butler\n", "\n", - "butler = Butler(\"/repo/embargo\", writeable=False)" + "if instrument == \"LATISS\":\n", + " butler = Butler(\"/repo/embargo\", writeable=False)\n", + "else:\n", + " butler = Butler(\"embargo_or4\", writeable=False)" ] }, { @@ -167,7 +170,7 @@ "results = butler.registry.queryDimensionRecords(\n", " \"exposure\",\n", " where=\"exposure.science_program IN (survey) \"\n", - " \"and instrument=instrument_name and exposure.day_obs=dayobs \",\n", + " \"and instrument=instrument_name and day_obs=dayobs \",\n", " bind={\"dayobs\": dayobs, \"instrument_name\": instrument, \"survey\": survey},\n", ")\n", "logger.info(f\"Found {results.count()} raw exposure records in {butler} for {dayobs}\")\n", @@ -179,7 +182,7 @@ "if embargo_records:\n", " df_butler = pandas.DataFrame.from_records(\n", " [embargo_records[num].toDict() for num in embargo_records]\n", - " ).set_index(\"group_name\")\n", + " ).set_index(\"group\")\n", "else:\n", " df_butler = None\n", " logger.warning(\"No raw records found in /repo/embargo. Notebook will fail.\")" @@ -232,7 +235,7 @@ " df_butler.reset_index(),\n", " how=\"outer\",\n", " left_on=\"groupId\",\n", - " right_on=\"group_name\",\n", + " right_on=\"group\",\n", " suffixes=(\"_efd\", \"_butler\"),\n", " validate=\"one_to_one\",\n", ").set_index(\"groupId\")\n", @@ -386,12 +389,12 @@ " results = butler.registry.queryDimensionRecords(\n", " \"exposure\",\n", " where=\"instrument=instrument_name and exposure.observation_type='science' and\"\n", - " \" exposure.day_obs=dayobs\",\n", + " \" day_obs=dayobs\",\n", " bind={\"dayobs\": dayobs, \"instrument_name\": instrument},\n", " )\n", "\n", " df = pandas.DataFrame(\n", - " [(_.group_name, _.id) for _ in results], columns=[\"groupId\", \"expId\"]\n", + " [(_.group, _.id) for _ in results], columns=[\"groupId\", \"expId\"]\n", " ).set_index(\"groupId\")\n", "\n", " return df" @@ -412,7 +415,11 @@ " -------\n", " df : `pandas.DataFrame`\n", " \"\"\"\n", - " s3_endpoint = \"https://s3dfrgw.slac.stanford.edu\"\n", + " if instrument == \"LATISS\":\n", + " s3_endpoint = \"https://s3dfrgw.slac.stanford.edu\"\n", + " else:\n", + " # embargo_or4\n", + " s3_endpoint = \"https://sdfembs3.sdf.slac.stanford.edu\"\n", " s3client = boto3.client(\"s3\", endpoint_url=s3_endpoint)\n", " refs = butler.registry.queryDatasets(\n", " datasetType=datasetType,\n", @@ -432,7 +439,7 @@ " dimension = \"exposure\"\n", " fits_uri = butler.getURI(ref)\n", " time_written = s3client.head_object(\n", - " Bucket=fits_uri.netloc,\n", + " Bucket=fits_uri.netloc.split(\"@\")[-1],\n", " Key=fits_uri.relativeToPathRoot,\n", " )[\"LastModified\"]\n", " timestamps.append((ref.dataId[dimension], ref.dataId[\"detector\"], time_written))\n", @@ -471,7 +478,7 @@ " butler,\n", " \"raw\",\n", " collections=[f\"{instrument}/raw/all\"],\n", - " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and exposure.day_obs={dayobs}\",\n", + " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and day_obs={dayobs}\",\n", ")" ] }, @@ -671,7 +678,7 @@ " \"prepBtlr\": \"Preparing Butler for visit\",\n", " \"waitSnap\": \"Waiting for snaps\",\n", " \"runPipe1\": \"Running pipeline\",\n", - " \"pipeSucc\": \"Pipeline successfully run\",\n", + " \"pipeSucc\": \"Main pipeline successfully run\",\n", " # There can be more than 1 of this log line per exposure when retry...\n", " \"outputSa\": \"Pipeline products saved to collection\",\n", " \"timeout1\": \"Timed out waiting for image after receiving exposures \",\n", @@ -907,7 +914,7 @@ ")\n", "\n", "# typically a small fraction of seconds before sndStamp\n", - "df_md2[\"ts_group_utc\"] = df_md2[\"group_name\"].apply(\n", + "df_md2[\"ts_group_utc\"] = df_md2[\"group\"].apply(\n", " lambda _: pandas.Timestamp(Time(_, scale=\"tai\").utc.datetime, tz=\"UTC\")\n", ")\n", "\n", diff --git a/prompt-processing/history.ipynb b/prompt-processing/history.ipynb index 3b374c6..5bad2a0 100644 --- a/prompt-processing/history.ipynb +++ b/prompt-processing/history.ipynb @@ -77,8 +77,8 @@ " datasets=\"raw\",\n", " collections=f\"{instrument}/raw/all\",\n", " where=\"exposure.science_program=survey and \"\n", - " \"instrument=instrument_name and exposure.day_obs=day_obs \",\n", - " bind={\"day_obs\": dayobs, \"instrument_name\": instrument, \"survey\": survey},\n", + " \"instrument=instrument_name and day_obs=dayobs \",\n", + " bind={\"dayobs\": dayobs, \"instrument_name\": instrument, \"survey\": survey},\n", " )\n", "\n", " output_chain = f\"{instrument}/prompt/output-{date}\"\n", @@ -88,8 +88,8 @@ " refs = butler.registry.queryDatasets(\n", " datasetType=\"isr_log\",\n", " collections=output_chain,\n", - " where=\"instrument=instrument_name and exposure.day_obs=day_obs\",\n", - " bind={\"day_obs\": dayobs, \"instrument_name\": instrument},\n", + " where=\"instrument=instrument_name and day_obs=dayobs\",\n", + " bind={\"dayobs\": dayobs, \"instrument_name\": instrument},\n", " )\n", " except MissingCollectionError:\n", " refs = None\n", diff --git a/prompt-processing/timing.ipynb b/prompt-processing/timing.ipynb index c775843..0adf377 100644 --- a/prompt-processing/timing.ipynb +++ b/prompt-processing/timing.ipynb @@ -267,7 +267,7 @@ "# butler repo sanity check\n", "count = butler.registry.queryDimensionRecords(\n", " \"exposure\",\n", - " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and exposure.day_obs={dayobs}\",\n", + " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and day_obs={dayobs}\",\n", ").count()\n", "Markdown(\"%i exposure in butler registry records\" % (count,))" ] @@ -285,7 +285,7 @@ " butler,\n", " \"raw\",\n", " collections=[f\"{instrument}/raw/all\"],\n", - " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and exposure.day_obs={dayobs}\",\n", + " where=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and day_obs={dayobs}\",\n", ")" ] },