Update prompt processing notebook

- Name this "Group Summary" for now
- Add group bean counting
- Rotate the box plot
- Show the number of timeout

prompt-processing/groups.ipynb

@@ -5,7 +5,7 @@
    "id": "7c0def35-3c83-4cec-be85-47d6703ee73f",
    "metadata": {},
    "source": [
-    "# Prompt Processing Bean Counting for {{ params.date }}"
+    "# Prompt Processing Group Summary for {{ params.date }}"
    ]
   },
   {
@@ -38,6 +38,8 @@
     "from astropy.time import Time, TimeDelta\n",
     "import boto3\n",
     "import json\n",
+    "%matplotlib inline\n",
+    "import matplotlib.pyplot as plt\n",
     "import pandas\n",
     "from lsst_efd_client import EfdClient"
    ]
@@ -441,24 +443,6 @@
    "outputs": [],
    "source": []
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b4748735-fb63-4b67-ae6b-240591969971",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!ls /sdf/data/rubin -l"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d3705a97-c9a8-437b-95d6-5c163d18ec59",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -513,7 +497,7 @@
     "    \"runPipe1\": \"Running pipeline\",\n",
     "    \"pipeSucc\": \"Pipeline successfully run\",\n",
     "    \"outputSa\": \"Pipeline products saved to collection\",\n",
-    "    #\"timeout1\": \"Timed out waiting for image after receiving exposures \",\n",
+    "    \"timeout1\": \"Timed out waiting for image after receiving exposures \",\n",
     "}"
    ]
   },
@@ -531,6 +515,30 @@
     "df_loki = df_loki.rename(columns={\"group\": \"groupId\"}).set_index(\"groupId\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "034239fc-f95d-4a5e-9484-ab2b7c4e599d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "phases = [\"raw\"] + list(phrases.keys())[:-1]\n",
+    "groups = dict()\n",
+    "groups[\"raw\"] = set(df_md.index)\n",
+    "for ph in ['unpckMsg', 'prepBtlr', 'waitSnap', 'runPipe1', 'pipeSucc', 'outputSa']:\n",
+    "    groups[ph] = set(df_loki[\"ts_\"+ph].dropna().index)\n",
+    "\n",
+    "for i in range(len(groups)):\n",
+    "    no_raw = set(groups[phases[i]]) - set(groups[\"raw\"])\n",
+    "    extra = f\"{len(no_raw)} no files {no_raw}\" if no_raw else \"\"\n",
+    "    logger.info(f\"{len(groups[phases[i]])} groups at {phases[i]}.  {extra} \")\n",
+    "\n",
+    "    if i > 0 :\n",
+    "        groups_diff = set(groups[phases[i-1]]) - set(groups[phases[i]])\n",
+    "        if groups_diff:\n",
+    "            logger.info(\" \"*4 + f\"{len(groups_diff)} {phases[i-1]}-{phases[i]}\\n\" + \" \".join(groups_diff))"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -625,7 +633,7 @@
     "for col_name in df0.columns:\n",
     "    df2[col_name] = (df0[col_name] - df0[ref]).dt.total_seconds()\n",
     "\n",
-    "df2[[\"ts_sndStamp\", \"ts_waitSnap\",\"ts_begin\",\"ts_end\", \"ts_runPipe1\", \"ts_raw\", \"ts_output\"]].plot(\n",
+    "df2[[\"ts_sndStamp\", \"ts_begin\",\"ts_end\", \"ts_runPipe1\", \"ts_raw\", \"ts_output\"]].plot(\n",
     "    kind=\"hist\", title=f\"{date}; ref={ref}\", xlabel=\"seconds\", ylabel=\"\", bins=150,\n",
     "    alpha=0.5, rot=45, \n",
     ")\n",
@@ -640,11 +648,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ax = df2.plot.box(title=f\"{date}; ref={ref}\", ylabel=\"seconds\", figsize=(12,5),\n",
-    "             column=df2.median().sort_values().index.tolist(),\n",
+    "ax = df2.plot.box(title=f\"{date}; ref={ref}\", ylabel=\"seconds\", figsize=(10,6),\n",
+    "             column=df2.median().sort_values(ascending=False).index.tolist(),vert=False, widths=0.3\n",
     ")\n",
     "ax.legend(labels=df2.median().sort_values().to_string().split(\"\\n\"), \n",
-    "          loc=\"lower right\", title=\"median\", handlelength=0)"
+    "          loc=\"lower left\", title=\"median\", handlelength=0)\n",
+    "ax.minorticks_on()\n",
+    "ax.secondary_xaxis('top').minorticks_on()"
    ]
   },
   {

prompt-processing/groups.yaml

@@ -1,4 +1,4 @@
-title: Prompt Processing Bean Counting
+title: Prompt Processing Group Summary
 description: Group-based tables and plots of prompt processing for a given night.
 authors:
   - name: Hsin-Fang Chiang