Updated notebooks to allow qc plot as much as possible.

ARM-Development · Jun 14, 2024 · 09846f4 · 09846f4
1 parent 3531400
commit 09846f4
Show file tree

Hide file tree

Showing 230 changed files with 4,684 additions and 1,829 deletions.
diff --git a/VAPs/quicklook/2DS-AIR/2DS-AIR_tutorial.ipynb b/VAPs/quicklook/2DS-AIR/2DS-AIR_tutorial.ipynb
@@ -510,7 +510,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",

diff --git a/VAPs/quicklook/2DS-AIR/aaf2dsh.c1.ipynb b/VAPs/quicklook/2DS-AIR/aaf2dsh.c1.ipynb
@@ -170,7 +170,7 @@
     "\n",
     "# ds = xr.open_mfdataset(files_list)  # open multiple netCDF files and merge as on dataset. (not always work)\n",
     "ds = xr.open_dataset(files_list[0])  # open the first file for analysis\n",
-    "\n",
+    "ds.clean.cleanup()  #  note: ARM's QC does not work directly with the internal logic. The ARM QC needs to be converted to CF QC before the QC will work.\n",
     "ds\n"
    ]
   },
@@ -190,7 +190,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",
@@ -246,27 +248,44 @@
     "#### Define variable for QC plot"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21c39b16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# existing qc variables\n",
+    "qc_variables = [var for var in list(ds.variables) if \"qc_\" in var]\n",
+    "qc_variables"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "532663a3-4dc0-4497-bda8-018c5f91e1c4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # QC Plot (Not guarantee to work)\n",
+    "# QC Plot \n",
+    "def qc_plot_example(ds, qc_variable):\n",
+    "    # Plot\n",
+    "    qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
+    "    qc_display.add_subplots((2,), figsize = (9.5,10))\n",
+    "    qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
+    "    qc_ax.grid()\n",
+    "    qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
     "\n",
-    "# qc_variable = 'None'\n",
+    "    plt.show()\n",
     "\n",
-    "# ('qc_'+qc_variable) in ds.variables:\n",
     "\n",
-    "# # Plot\n",
-    "# qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
-    "# qc_display.add_subplots((2,), figsize = (9.5,10))\n",
-    "# qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
-    "# qc_ax.grid()\n",
-    "# qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
+    "qc_variable = 'None'\n",
     "\n",
-    "# plt.show()"
+    "if qc_variable:\n",
+    "    try:\n",
+    "        qc_plot_example(ds=ds, qc_variable=qc_variable)\n",
+    "    except Exception as e:\n",
+    "        print(e)"
    ]
   },
   {

diff --git a/VAPs/quicklook/2DS-AIR/aaf2dsv.c1.ipynb b/VAPs/quicklook/2DS-AIR/aaf2dsv.c1.ipynb
@@ -170,7 +170,7 @@
     "\n",
     "# ds = xr.open_mfdataset(files_list)  # open multiple netCDF files and merge as on dataset. (not always work)\n",
     "ds = xr.open_dataset(files_list[0])  # open the first file for analysis\n",
-    "\n",
+    "ds.clean.cleanup()  #  note: ARM's QC does not work directly with the internal logic. The ARM QC needs to be converted to CF QC before the QC will work.\n",
     "ds\n"
    ]
   },
@@ -190,7 +190,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",
@@ -246,27 +248,44 @@
     "#### Define variable for QC plot"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21c39b16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# existing qc variables\n",
+    "qc_variables = [var for var in list(ds.variables) if \"qc_\" in var]\n",
+    "qc_variables"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "532663a3-4dc0-4497-bda8-018c5f91e1c4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # QC Plot (Not guarantee to work)\n",
+    "# QC Plot \n",
+    "def qc_plot_example(ds, qc_variable):\n",
+    "    # Plot\n",
+    "    qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
+    "    qc_display.add_subplots((2,), figsize = (9.5,10))\n",
+    "    qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
+    "    qc_ax.grid()\n",
+    "    qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
     "\n",
-    "# qc_variable = 'None'\n",
+    "    plt.show()\n",
     "\n",
-    "# ('qc_'+qc_variable) in ds.variables:\n",
     "\n",
-    "# # Plot\n",
-    "# qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
-    "# qc_display.add_subplots((2,), figsize = (9.5,10))\n",
-    "# qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
-    "# qc_ax.grid()\n",
-    "# qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
+    "qc_variable = 'None'\n",
     "\n",
-    "# plt.show()"
+    "if qc_variable:\n",
+    "    try:\n",
+    "        qc_plot_example(ds=ds, qc_variable=qc_variable)\n",
+    "    except Exception as e:\n",
+    "        print(e)"
    ]
   },
   {

diff --git a/VAPs/quicklook/ACSMCDCE/ACSMCDCE_tutorial.ipynb b/VAPs/quicklook/ACSMCDCE/ACSMCDCE_tutorial.ipynb
@@ -510,7 +510,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",

diff --git a/VAPs/quicklook/ACSMCDCE/acsmcdce.c1.ipynb b/VAPs/quicklook/ACSMCDCE/acsmcdce.c1.ipynb
@@ -170,7 +170,7 @@
     "\n",
     "# ds = xr.open_mfdataset(files_list)  # open multiple netCDF files and merge as on dataset. (not always work)\n",
     "ds = xr.open_dataset(files_list[0])  # open the first file for analysis\n",
-    "\n",
+    "ds.clean.cleanup()  #  note: ARM's QC does not work directly with the internal logic. The ARM QC needs to be converted to CF QC before the QC will work.\n",
     "ds\n"
    ]
   },
@@ -190,7 +190,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",
@@ -246,27 +248,44 @@
     "#### Define variable for QC plot"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21c39b16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# existing qc variables\n",
+    "qc_variables = [var for var in list(ds.variables) if \"qc_\" in var]\n",
+    "qc_variables"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "532663a3-4dc0-4497-bda8-018c5f91e1c4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # QC Plot (Not guarantee to work)\n",
+    "# QC Plot \n",
+    "def qc_plot_example(ds, qc_variable):\n",
+    "    # Plot\n",
+    "    qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
+    "    qc_display.add_subplots((2,), figsize = (9.5,10))\n",
+    "    qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
+    "    qc_ax.grid()\n",
+    "    qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
     "\n",
-    "# qc_variable = 'total_organics'\n",
+    "    plt.show()\n",
     "\n",
-    "# ('qc_'+qc_variable) in ds.variables:\n",
     "\n",
-    "# # Plot\n",
-    "# qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
-    "# qc_display.add_subplots((2,), figsize = (9.5,10))\n",
-    "# qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
-    "# qc_ax.grid()\n",
-    "# qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
+    "qc_variable = 'total_organics'\n",
     "\n",
-    "# plt.show()"
+    "if qc_variable:\n",
+    "    try:\n",
+    "        qc_plot_example(ds=ds, qc_variable=qc_variable)\n",
+    "    except Exception as e:\n",
+    "        print(e)"
    ]
   },
   {

diff --git a/VAPs/quicklook/ACSMCDCE/acsmcdce.c2.ipynb b/VAPs/quicklook/ACSMCDCE/acsmcdce.c2.ipynb
@@ -170,7 +170,7 @@
     "\n",
     "# ds = xr.open_mfdataset(files_list)  # open multiple netCDF files and merge as on dataset. (not always work)\n",
     "ds = xr.open_dataset(files_list[0])  # open the first file for analysis\n",
-    "\n",
+    "ds.clean.cleanup()  #  note: ARM's QC does not work directly with the internal logic. The ARM QC needs to be converted to CF QC before the QC will work.\n",
     "ds\n"
    ]
   },
@@ -190,7 +190,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",
@@ -246,27 +248,44 @@
     "#### Define variable for QC plot"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21c39b16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# existing qc variables\n",
+    "qc_variables = [var for var in list(ds.variables) if \"qc_\" in var]\n",
+    "qc_variables"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "532663a3-4dc0-4497-bda8-018c5f91e1c4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # QC Plot (Not guarantee to work)\n",
+    "# QC Plot \n",
+    "def qc_plot_example(ds, qc_variable):\n",
+    "    # Plot\n",
+    "    qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
+    "    qc_display.add_subplots((2,), figsize = (9.5,10))\n",
+    "    qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
+    "    qc_ax.grid()\n",
+    "    qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
     "\n",
-    "# qc_variable = 'total_organics_CDCE'\n",
+    "    plt.show()\n",
     "\n",
-    "# ('qc_'+qc_variable) in ds.variables:\n",
     "\n",
-    "# # Plot\n",
-    "# qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
-    "# qc_display.add_subplots((2,), figsize = (9.5,10))\n",
-    "# qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
-    "# qc_ax.grid()\n",
-    "# qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
+    "qc_variable = 'total_organics_CDCE'\n",
     "\n",
-    "# plt.show()"
+    "if qc_variable:\n",
+    "    try:\n",
+    "        qc_plot_example(ds=ds, qc_variable=qc_variable)\n",
+    "    except Exception as e:\n",
+    "        print(e)"
    ]
   },
   {

diff --git a/VAPs/quicklook/ACSMCDCE/acsmtofcdce.c1.ipynb b/VAPs/quicklook/ACSMCDCE/acsmtofcdce.c1.ipynb
@@ -170,7 +170,7 @@
     "\n",
     "# ds = xr.open_mfdataset(files_list)  # open multiple netCDF files and merge as on dataset. (not always work)\n",
     "ds = xr.open_dataset(files_list[0])  # open the first file for analysis\n",
-    "\n",
+    "ds.clean.cleanup()  #  note: ARM's QC does not work directly with the internal logic. The ARM QC needs to be converted to CF QC before the QC will work.\n",
     "ds\n"
    ]
   },
@@ -190,7 +190,9 @@
     "        # if dim_name in [\"time\", \"bound\"]:\n",
     "        #     continue\n",
     "        dim = ds[dim_name].values\n",
-    "        if not any(np.isinf(dim)):\n",
+    "        if not np.issubdtype(ds[dim_name].values.dtype, np.number):  # only works for numerical dtype\n",
+    "            continue\n",
+    "        if not any(np.isinf(dim)):  # skip if there is no inf value\n",
     "            continue\n",
     "        dim_replaced = np.where(np.isinf(dim), np.nanmax(dim[np.isfinite(dim)]) * 1.1, dim)\n",
     "        ds[dim_name] = ((dim_name,), dim_replaced)\n",
@@ -246,27 +248,44 @@
     "#### Define variable for QC plot"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21c39b16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# existing qc variables\n",
+    "qc_variables = [var for var in list(ds.variables) if \"qc_\" in var]\n",
+    "qc_variables"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "532663a3-4dc0-4497-bda8-018c5f91e1c4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # QC Plot (Not guarantee to work)\n",
+    "# QC Plot \n",
+    "def qc_plot_example(ds, qc_variable):\n",
+    "    # Plot\n",
+    "    qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
+    "    qc_display.add_subplots((2,), figsize = (9.5,10))\n",
+    "    qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
+    "    qc_ax.grid()\n",
+    "    qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
     "\n",
-    "# qc_variable = 'chloride'\n",
+    "    plt.show()\n",
     "\n",
-    "# ('qc_'+qc_variable) in ds.variables:\n",
     "\n",
-    "# # Plot\n",
-    "# qc_display = act.plotting.TimeSeriesDisplay(ds)\n",
-    "# qc_display.add_subplots((2,), figsize = (9.5,10))\n",
-    "# qc_ax = qc_display.plot(qc_variable, subplot_index=(0,), set_title=\"QC results on field: \" + qc_variable,)\n",
-    "# qc_ax.grid()\n",
-    "# qc_display.qc_flag_block_plot(qc_variable, subplot_index=(1,))\n",
+    "qc_variable = 'chloride'\n",
     "\n",
-    "# plt.show()"
+    "if qc_variable:\n",
+    "    try:\n",
+    "        qc_plot_example(ds=ds, qc_variable=qc_variable)\n",
+    "    except Exception as e:\n",
+    "        print(e)"
    ]
   },
   {