Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-41368: Make prompt processing notebook work with ComCamSim #20

Merged
merged 5 commits into from
Apr 4, 2024
Merged

DM-41368: Make prompt processing notebook work with ComCamSim #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a95c5de
Consider detector in Loki log records
hsinfang Apr 2, 2024
f112de6
Examine some known errors in the logs
hsinfang Apr 2, 2024
5086acc
Parameterize instrument and survey in the old timing notebook
hsinfang Apr 3, 2024
41c785a
Fix PP timing plots for multiple detectors per exposure
hsinfang Apr 4, 2024
2a672df
Decrease the number of bins in histogram
hsinfang Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 59 additions & 25 deletions prompt-processing/groups.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@
"metadata": {},
"outputs": [],
"source": [
"def get_df_from_loki(date, search_phrase=\"Waiting for snaps\"):\n",
"def get_df_from_loki(date, search_phrase=\"Waiting for snaps\", extra=f\" |~ \\\"{instrument}\\\"\"):\n",
" start = Time(date, scale=\"utc\", format=\"isot\") + TimeDelta(\n",
" 12 * 60 * 60, format=\"sec\"\n",
" )\n",
Expand All @@ -631,11 +631,12 @@
" start.strftime(\"%Y-%m-%dT%H:%M:%SZ\"),\n",
" end.strftime(\"%Y-%m-%dT%H:%M:%SZ\"),\n",
" search_phrase,\n",
" extra,\n",
" )\n",
" results = !{command}\n",
" logger.debug(f\"Got {len(results)} Loki records for {phrase}\")\n",
" logger.debug(f\"Got {len(results)} Loki records for {search_phrase}\")\n",
" if not results:\n",
" return pandas.DataFrame(columns=[\"group\", \"ts\"])\n",
" return pandas.DataFrame(columns=[\"group\", \"detector\", \"ts\"])\n",
"\n",
" data = [json.loads(_) for _ in results]\n",
" df = pandas.json_normalize(data)\n",
Expand All @@ -647,10 +648,15 @@
"\n",
" if \"group\" not in df.columns and \"message\" in df.columns:\n",
" df[\"group\"] = df[\"message\"].str.extract(r\"groupId='([T:.\\d-]*)',\")\n",
"\n",
" df[\"detector\"] = df[\"message\"].str.extract(r\"detector=(\\d*),\")\n",
" if \"detector\" not in df.columns:\n",
" logger.info(f\"Missing detector info in Loki querying {search_phrase}\")\n",
" df[\"detector\"] = -1\n",
" else:\n",
" df[\"detector\"] = df[\"detector\"].astype({'detector': 'int32'})\n",
" df[\"ts\"] = pandas.to_datetime(df[\"timestamp\"])\n",
"\n",
" return df[[\"group\", \"ts\"]]"
" return df[[\"group\", \"detector\", \"ts\"]]"
]
},
{
Expand All @@ -666,60 +672,88 @@
" \"waitSnap\": \"Waiting for snaps\",\n",
" \"runPipe1\": \"Running pipeline\",\n",
" \"pipeSucc\": \"Pipeline successfully run\",\n",
" # There can be more than 1 of this log line per exposure when retry...\n",
" \"outputSa\": \"Pipeline products saved to collection\",\n",
" \"timeout1\": \"Timed out waiting for image after receiving exposures \",\n",
" \"dbConErr\": f'SSL connection has been closed unexpectedly\" |~ \"Traceback',\n",
" \"brokrErr\" : \"Failed to get metadata: Local: Broker transport failure\",\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9afa63c6-359f-486c-805f-684bf86c7ae2",
"id": "d129def6-3a0f-4c28-8553-1eaada2b475c",
"metadata": {},
"outputs": [],
"source": [
"df_loki = pandas.DataFrame(columns=[\"group\"])\n",
"for phrase in phrases:\n",
" df2 = get_df_from_loki(date, phrases[phrase]).rename(columns={\"ts\": \"ts_\" + phrase})\n",
" df_loki = df_loki.merge(\n",
" df2,\n",
" on=\"group\",\n",
" how=\"outer\",\n",
" validate=\"one_to_one\",\n",
" )\n",
"df_loki = df_loki.rename(columns={\"group\": \"groupId\"}).set_index(\"groupId\")"
"df2[df2[\"group\"] == \"2024-03-30T04:11:40.760\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4add2661-4b4e-495e-a5e5-8c780670d73b",
"id": "8d497fac-6992-4568-8c02-a5916c9ce9b0",
"metadata": {},
"outputs": [],
"source": [
"if df_loki.empty and not df_md2.empty:\n",
" logger.warning(\"No Loki query results. Possible data loss or service issue.\")"
"# get_df_from_loki(date, \"SSL connection has been closed unexpectedly\", extra=f\" |~ \\\"Traceback\\\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a9147f89-15f2-4d47-bd7a-bddf1aa368c0",
"id": "26aaf735-24df-430a-a8b2-aa99bba4789c",
"metadata": {},
"outputs": [],
"source": [
"timeouts = df_loki[df_loki[\"ts_timeout1\"].notnull()].index\n",
"logger.debug(f\"{len(timeouts)} timeouts: {timeouts}\")"
"def examine_log(key):\n",
" df2 = get_df_from_loki(date, phrases[key])\n",
" ids = df2[df2[\"ts\"].notnull()][[\"group\", \"detector\"]]\n",
" if len(ids):\n",
" logger.debug(f\"{len(ids)} with error {phrases[key]}: {ids} \")\n",
" phrases.pop(key)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "662b5e62-ab87-4257-9047-2e3768526394",
"id": "bc5bc9c5-bfc1-49cf-81f5-04c34b2743bf",
"metadata": {},
"outputs": [],
"source": [
"df_loki.drop(columns=[\"ts_timeout1\"], inplace=True)"
"for _ in (\"timeout1\", \"dbConErr\", \"brokrErr\"):\n",
" examine_log(_)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9afa63c6-359f-486c-805f-684bf86c7ae2",
"metadata": {},
"outputs": [],
"source": [
"df_loki = pandas.DataFrame(columns=[\"group\", \"detector\"])\n",
"for phrase in phrases:\n",
" df2 = get_df_from_loki(date, phrases[phrase]).rename(columns={\"ts\": \"ts_\" + phrase})\n",
" df_loki = df_loki.merge(\n",
" df2,\n",
" on=[\"group\", \"detector\"],\n",
" how=\"outer\",\n",
" validate=\"one_to_one\",\n",
" )\n",
"df_loki = df_loki.rename(columns={\"group\": \"groupId\"}).set_index([\"groupId\", \"detector\"])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4add2661-4b4e-495e-a5e5-8c780670d73b",
"metadata": {},
"outputs": [],
"source": [
"if df_loki.empty and not df_md2.empty:\n",
" logger.warning(\"No Loki query results. Possible data loss or service issue.\")"
]
},
{
Expand All @@ -733,7 +767,7 @@
"groups = dict()\n",
"groups[\"raw\"] = set(df_md2.index)\n",
"for ph in [\"unpckMsg\", \"prepBtlr\", \"waitSnap\", \"runPipe1\", \"pipeSucc\", \"outputSa\"]:\n",
" groups[ph] = set(df_loki[\"ts_\" + ph].dropna().index)\n",
" groups[ph] = set(df_loki[\"ts_\" + ph].dropna().reset_index(\"detector\").index)\n",
"\n",
"for i in range(len(groups)):\n",
" no_raw = set(groups[phases[i]]) - set(groups[\"raw\"])\n",
Expand Down
Loading
Loading