Merge pull request #25 from lsst-sqre/tickets/DM-44956

DM-44956: Update prompt-processing notebooks for OR4
lsst-sqre · Jun 24, 2024 · 9eafb8e · 9eafb8e
2 parents b3f873f + 2cd175b
commit 9eafb8e
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 12 deletions.
diff --git a/prompt-processing/groups.ipynb b/prompt-processing/groups.ipynb
@@ -618,7 +618,7 @@
    "outputs": [],
    "source": [
     "def get_loki_command_with_phrase(time_start, time_end, phrase, extra=f\" |~ \\\"{instrument}\\\"\"):\n",
-    "    command = f\"\"\"/sdf/data/rubin/user/hchiang2/times_square/logcli-linux-amd64 --output=jsonl --tls-skip-verify query --addr=http://sdfloki.slac.stanford.edu:80 --timezone=UTC -q --limit=2000 --from=\"{time_start}\" --to=\"{time_end}\" --proxy-url=http://sdfproxy.sdf.slac.stanford.edu:3128  '{{namespace=\"vcluster--usdf-prompt-processing\", container=\"user-container\", pod=~\"prompt-proto-service-.+\"}} |~ \"{phrase}\" {extra} ' \"\"\"\n",
+    "    command = f\"\"\"/sdf/data/rubin/user/hchiang2/times_square/logcli-linux-amd64 --output=jsonl --tls-skip-verify query --addr=http://sdfloki.slac.stanford.edu:80 --timezone=UTC -q --limit=10000 --from=\"{time_start}\" --to=\"{time_end}\" --proxy-url=http://sdfproxy.sdf.slac.stanford.edu:3128  '{{namespace=\"vcluster--usdf-prompt-processing\", container=\"user-container\", pod=~\"prompt-proto-service-.+\"}} |~ \"{phrase}\" {extra} ' \"\"\"\n",
     "    return command"
    ]
   },
@@ -787,7 +787,7 @@
    "outputs": [],
    "source": [
     "def get_loki_command_with_phrase_fanout(time_start, time_end, phrase, extra=\"\"):\n",
-    "    command = f\"\"\"/sdf/data/rubin/user/hchiang2/times_square/logcli-linux-amd64 --output=jsonl --tls-skip-verify query --addr=http://sdfloki.slac.stanford.edu:80 --timezone=UTC -q --limit=2000 --from=\"{time_start}\" --to=\"{time_end}\" --proxy-url=http://sdfproxy.sdf.slac.stanford.edu:3128  '{{app=\"vcluster--usdf-prompt-processing\",pod=~\"next-visit-fan-out-.+\"}} |~ \"group\" |~ \"{phrase}\" {extra} ' \"\"\"\n",
+    "    command = f\"\"\"/sdf/data/rubin/user/hchiang2/times_square/logcli-linux-amd64 --output=jsonl --tls-skip-verify query --addr=http://sdfloki.slac.stanford.edu:80 --timezone=UTC -q --limit=10000 --from=\"{time_start}\" --to=\"{time_end}\" --proxy-url=http://sdfproxy.sdf.slac.stanford.edu:3128  '{{app=\"vcluster--usdf-prompt-processing\",pod=~\"next-visit-fan-out-.+\"}} |~ \"group\" |~ \"{phrase}\" {extra} ' \"\"\"\n",
     "    return command"
    ]
   },
@@ -945,12 +945,13 @@
     "# When fan-out handles a group\n",
     "df = get_df_from_loki_fanout(date, \"message deserialized\", f'|~ \"{survey}\"')\n",
     "df.rename(columns={\"ts\": \"ts_fanout\"}, inplace=True)\n",
-    "df_loki = df_loki.merge(\n",
+    "df_loki = pandas.merge(\n",
+    "    df_loki.reset_index(),\n",
     "    df,\n",
-    "    on=\"groupId\",\n",
-    "    how=\"outer\",\n",
-    "    validate=\"one_to_one\",\n",
-    ")"
+    "    on=[\"groupId\"],\n",
+    "    how=\"left\",\n",
+    "    validate=\"many_to_one\",\n",
+    ").set_index([\"groupId\", \"detector\"])\n"
    ]
   },
   {

diff --git a/prompt-processing/history.ipynb b/prompt-processing/history.ipynb
@@ -54,7 +54,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "butler = Butler(\"/repo/embargo\", writeable=False)"
+    "if instrument == \"LATISS\":\n",
+    "    butler = Butler(\"/repo/embargo\", writeable=False)\n",
+    "else:\n",
+    "    butler = Butler(\"embargo_or4\", writeable=False)"
    ]
   },
   {

diff --git a/prompt-processing/timing.ipynb b/prompt-processing/timing.ipynb
@@ -97,7 +97,10 @@
    "source": [
     "from lsst.daf.butler import Butler\n",
     "\n",
-    "butler = Butler(\"/repo/embargo\", writeable=False)"
+    "if instrument == \"LATISS\":\n",
+    "    butler = Butler(\"/repo/embargo\", writeable=False)\n",
+    "else:\n",
+    "    butler = Butler(\"embargo_or4\", writeable=False)"
    ]
   },
   {
@@ -118,8 +121,10 @@
     "    timestamps : `dict` [`int`, datetime]\n",
     "        The keys are the exposure_id\n",
     "    \"\"\"\n",
-    "\n",
-    "    s3_endpoint = \"https://s3dfrgw.slac.stanford.edu\"\n",
+    "    if instrument == \"LATISS\":\n",
+    "        s3_endpoint = \"https://s3dfrgw.slac.stanford.edu\"\n",
+    "    else:\n",
+    "        s3_endpoint = \"https://sdfembs3.sdf.slac.stanford.edu\"\n",
     "    s3client = boto3.client(\"s3\", endpoint_url=s3_endpoint)\n",
     "    refs = butler.registry.queryDatasets(\n",
     "        datasetType=datasetType,\n",
@@ -137,7 +142,7 @@
     "                dimension = \"exposure\"\n",
     "        fits_uri = butler.getURI(ref)\n",
     "        time_written = s3client.head_object(\n",
-    "            Bucket=fits_uri.netloc,\n",
+    "            Bucket=fits_uri.netloc.split(\"@\")[-1],\n",
     "            Key=fits_uri.relativeToPathRoot,\n",
     "        )[\"LastModified\"]\n",
     "        timestamps.append((ref.dataId[dimension], ref.dataId[\"detector\"], time_written))\n",