diff --git a/environment.yml b/environment.yml index fdfceaa..68d9084 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - defaults - pyviz dependencies: - - python=3.9 + - python - arm_pyart - matplotlib - hvplot diff --git a/notebooks/gluing_and_inventory/concatinate_file.ipynb b/notebooks/gluing_and_inventory/concatinate_file.ipynb index bdb4c2f..bd65a1d 100644 --- a/notebooks/gluing_and_inventory/concatinate_file.ipynb +++ b/notebooks/gluing_and_inventory/concatinate_file.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "83599d36", + "id": "b1336072", "metadata": {}, "source": [ "# Gluing and Merging\n", @@ -12,377 +12,571 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "651b9cf0-c8bb-4ccc-90bd-80624845243a", + "execution_count": 1, + "id": "374502ac", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "## You are using the Python ARM Radar Toolkit (Py-ART), an open source\n", + "## library for working with weather radar data. Py-ART is partly\n", + "## supported by the U.S. Department of Energy as part of the Atmospheric\n", + "## Radiation Measurement (ARM) Climate Research Facility, an Office of\n", + "## Science user facility.\n", + "##\n", + "## If you use this software to prepare a publication, please cite:\n", + "##\n", + "## JJ Helmus and SM Collis, JORS 2016, doi: 10.5334/jors.119\n", + "\n" + ] + } + ], "source": [ + "import warnings\n", + "warnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n", + "\n", "import os\n", - "import pyart\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np\n", "import time\n", "import datetime\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e7792af-42a1-42d1-8af7-8f026069eaf5", - "metadata": {}, - "outputs": [], - "source": [ - "data_dir = '/eagle/projects/CPOL/SAIL/202111/'\n", - "all_files = os.listdir(data_dir)\n", - "all_files.sort()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad702eaa-057e-4296-b2a0-707712dbace1", - "metadata": {}, - "outputs": [], - "source": [ - "base_scan_ppi = '1_PPI.nc'\n", - "ppi_pattern = 'PPI.nc'\n", - "base_scans = []\n", - "volumes = []\n", - "ppis = []\n", - "in_volume = False\n", - "for file in all_files:\n", - " if ppi_pattern in file:\n", - " ppis.append(file)\n", - " if base_scan_ppi in file:\n", - " base_scans.append(file)\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f852f1eb-4e96-4989-a461-6bfa7d7d9b85", - "metadata": {}, - "outputs": [], - "source": [ - "n_tilts = 8" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e669b513-4038-4baa-a433-c23a6a15ec9f", - "metadata": {}, - "outputs": [], - "source": [ - "index = np.where(np.array(ppis) == base_scans[0])[0][0]\n", - "print(index)\n", - "print(ppis[index: index+n_tilts])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d9962404-23d2-4932-a674-c341ffcb3eb6", - "metadata": {}, - "outputs": [], - "source": [ - "volumes = []\n", - "for base in base_scans:\n", - " base_scan_index = np.where(np.array(ppis) == base)[0][0]\n", - " #print(base_scan_index)\n", - " volume = ppis[base_scan_index: base_scan_index+n_tilts]\n", - " volumes.append(volume)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e2ce11e5-cbe0-4417-9519-5f79c3a8bbbb", - "metadata": {}, - "outputs": [], - "source": [ - "volumes[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1c0a37e-c88e-4cd9-931b-1ef3d5a50602", - "metadata": {}, - "outputs": [], - "source": [ - "base_radar = pyart.io.read(data_dir+volumes[0][0])\n", - "radars = [pyart.io.read(data_dir+sw) for sw in volumes[0][1::]]" + "import numpy as np\n", + "import xarray as xr\n", + "import tempfile\n", + "from pathlib import Path\n", + "import shutil \n", + "\n", + "from matplotlib import pyplot as plt\n", + "from dask.distributed import Client, LocalCluster, progress\n", + "\n", + "import pyart" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "1d613986-b4b8-460b-98e2-772c092c3658", + "cell_type": "markdown", + "id": "691d7069", "metadata": {}, - "outputs": [], "source": [ - "base_radar.elevation['data'].mean()" + "## Define Helper Functions" ] }, { "cell_type": "code", - "execution_count": null, - "id": "9515ebbe-c020-4745-aee7-bd05dab4bd98", + "execution_count": 2, + "id": "352f8fc6-5d59-45c2-916c-9bc4454910d0", "metadata": {}, "outputs": [], "source": [ - "radars" + "def glue_fix(ds):\n", + " # Define the required encodings for the glue files\n", + " encodings = {'DBZ' : {'_FillValue' : -32768.0},\n", + " 'VEL' : {'_FillValue' : -32768.0},\n", + " 'WIDTH' : {'_FillValue' : -32768.0},\n", + " 'ZDR' : {'_FillValue' : -32768.0},\n", + " 'PHIDP' : {'_FillValue' : -32768.0},\n", + " 'RHOHV' : {'_FillValue' : -32768.0},\n", + " 'NCP' : {'_FillValue' : -32768.0},\n", + " 'DBZhv' : {'_FillValue' : -32768.0},\n", + " } \n", + " \n", + " # Loop over all the variables; update the FillValue and Data Type\n", + " for var in encodings:\n", + " # Convert all values within the DataArray to the correct Fill Value\n", + " # NOTE: xr.where(condition, value_when_condition_is_not_met); so if every index is MVC, check for opposite\n", + " mask = ds[var].where(ds[var] > -99800, encodings[var]['_FillValue'])\n", + " # Append the corrected data to the variable.\n", + " ds[var] = mask\n", + " \n", + " return ds" ] }, { "cell_type": "code", - "execution_count": null, - "id": "bbb380af-132a-4b29-903b-64d0f4c85761", + "execution_count": 3, + "id": "3dd6302a", "metadata": {}, "outputs": [], "source": [ - "for rad in radars:\n", - " print(rad.elevation['data'].mean())\n", - " print(rad.fixed_angle['data'])" + "def radar_glue(b_radar, radar_list):\n", + " if radar_list is not None:\n", + " for rad in radar_list:\n", + " b_radar = pyart.util.join_radar(b_radar, rad)\n", + " del rad\n", + " else:\n", + " b_radar = None\n", + " return b_radar" ] }, { "cell_type": "code", - "execution_count": null, - "id": "d8392dfa-9f31-44a0-9345-7d2422605a2f", + "execution_count": 4, + "id": "b245fb02", "metadata": {}, "outputs": [], "source": [ - "for rad in radars:\n", - " base_radar = pyart.util.join_radar(base_radar, rad)" + "def volume_from_list(base_radar, vlist, base_dir):\n", + " try:\n", + " radars = [pyart.io.read(base_dir+sw) for sw in vlist[1::]]\n", + " except:\n", + " radars = None\n", + " return radar_glue(base_radar, radars)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "221e142c-cab5-4058-88d9-1ac58a91ba2f", + "execution_count": 5, + "id": "127bcf07-ff1f-47b3-a566-ff7ed6840884", "metadata": {}, "outputs": [], "source": [ - "base_radar.nsweeps" + "def fix_times(ds):\n", + " # Determine number of specific times within the file\n", + " specific = set(ds.time.data)\n", + " # Loop through the specific times, and add offset\n", + " for value in specific:\n", + " dt = np.arange(0, len(ds.sel(time=slice(value, value)).time.data))\n", + " dt = dt.astype('timedelta64[ms]')\n", + " # add the time offset to the original times\n", + " new_times = ds.sel(time=slice(value, value)).time.data + dt\n", + " ds.sel(time=slice(value, value)).time.data[:] = new_times\n", + " # Send back to the main\n", + " return ds" ] }, { "cell_type": "code", - "execution_count": null, - "id": "b1c45ce3-2458-4007-bf04-92c8751ae2a0", + "execution_count": 6, + "id": "f1e461bf-7740-45f4-8797-47f1346c66f8", "metadata": {}, "outputs": [], "source": [ - "base_radar.info()" + "def granule(Dvolume):\n", + " n_tilts = 8\n", + " month = \"202203\"\n", + " data_dir = \"/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/nc_files/\" + month + \"_nc/\"\n", + " out_dir = \"/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/glue_files/\" + month + \"_glued/\"\n", + "\n", + " # Read the base scan to determine if it can be read in\n", + " if len(Dvolume) == 8:\n", + " try:\n", + " base_rad = pyart.io.read(data_dir+Dvolume[0])\n", + " except:\n", + " base_rad = None\n", + " # Read all scans and join with base scan\n", + " if base_rad is not None:\n", + " out_radar = volume_from_list(base_rad, Dvolume, data_dir)\n", + " if out_radar is not None:\n", + " # Define the filename time from the radar object\n", + " ff = time.strptime(out_radar.time['units'][14:], '%Y-%m-%dT%H:%M:%SZ')\n", + " dt = datetime.datetime.fromtimestamp(time.mktime(ff)) + datetime.timedelta(seconds= int(out_radar.time['data'][0]))\n", + " strform = dt.strftime(out_dir + 'xprecipradar_guc_volume_%Y%m%d-%H%M%S.b1.nc')\n", + " #FIX for join issue.. to be fixed in Py-ART\n", + " out_radar.sweep_mode['data']=np.tile(base_rad.sweep_mode['data'], n_tilts)\n", + " try:\n", + " pyart.io.write_cfradial(strform, out_radar, arm_time_variables=True)\n", + " print('SUCCESS', strform)\n", + " except:\n", + " print('FAILURE', strform)\n", + " # Delete the radars to free up memory\n", + " del base_rad\n", + " del out_radar\n", + " # Fix the times and encodings of the generated file\n", + " with tempfile.TemporaryDirectory() as tmpdir:\n", + " tmp_path = Path(tmpdir)\n", + " with xr.open_dataset(strform, mask_and_scale=False) as ds:\n", + " ds = ds.load()\n", + " ds = fix_times(ds)\n", + " ds = glue_fix(ds)\n", + " out_path = str(tmp_path) + '/' + strform.split('/')[-1]\n", + " # set time encoding for miliseconds\n", + " ds.time.encoding['units'] = 'milliseconds since 1970-01-01'\n", + " ds.to_netcdf(path=out_path)\n", + " shutil.copy(out_path, strform)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "41046e3f-af62-4c2d-a43d-dd4e5e113f67", + "cell_type": "markdown", + "id": "2bda0c0c", "metadata": {}, - "outputs": [], "source": [ - "base_radar.fixed_angle" + "## Locate all the PPI scans within the Desired Directories" ] }, { "cell_type": "code", - "execution_count": null, - "id": "2d9ae7e5-01e6-4a7d-9a23-48844b85cf93", + "execution_count": 7, + "id": "a405c35d", "metadata": {}, "outputs": [], "source": [ - "def radar_glue(b_radar, radar_list):\n", - " for rad in radar_list:\n", - " b_radar = pyart.util.join_radar(b_radar, rad)\n", - " \n", - " return b_radar" + "month = \"202203\"\n", + "data_dir = \"/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/nc_files/\" + month + \"_nc/\"\n", + "out_dir = \"/gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/glue_files/\" + month + \"_glued/\"" ] }, { "cell_type": "code", - "execution_count": null, - "id": "dcba6886-f6f2-4dfa-bd29-1b46a8c1c156", + "execution_count": 8, + "id": "4e6b5ba6", "metadata": {}, "outputs": [], "source": [ - "def volume_from_list(vlist, base_dir):\n", - " base_radar = pyart.io.read(base_dir+vlist[0])\n", - " radars = [pyart.io.read(base_dir+sw) for sw in vlist[1::]]\n", - " return radar_glue(base_radar, radars)" + "all_files = os.listdir(data_dir)\n", + "all_files.sort()" ] }, { "cell_type": "code", - "execution_count": null, - "id": "4f4cb5cb-c736-4e97-99bc-62d7f4c9e4b7", + "execution_count": 9, + "id": "0648e36d", "metadata": {}, "outputs": [], "source": [ - "fourth_volume = volume_from_list(volumes[400], data_dir)" + "base_scan_ppi = '1_PPI.nc'\n", + "ppi_pattern = 'PPI.nc'\n", + "base_scans = []\n", + "volumes = []\n", + "ppis = []\n", + "in_volume = False\n", + "for file in all_files:\n", + " if ppi_pattern in file:\n", + " ppis.append(file)\n", + " if base_scan_ppi in file:\n", + " base_scans.append(file)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "2b738062-9791-4682-862f-7d891b03b1f5", + "execution_count": 10, + "id": "0b27b11f", "metadata": {}, "outputs": [], "source": [ - "figme = plt.figure(figsize=[15,10])\n", - "display = pyart.graph.RadarMapDisplay(fourth_volume)\n", - " \n", - "# Add our reflectivity (DBZ) field to the plot, including our gatefilter\n", - "display.plot_ppi_map('DBZ', 0, vmin=-20, vmax=40.)\n" + "n_tilts = 8" ] }, { "cell_type": "code", - "execution_count": null, - "id": "92d76feb-83f7-4be7-acc5-b307d62f4b85", + "execution_count": 11, + "id": "36f4ac94", "metadata": {}, "outputs": [], "source": [ - "figme = plt.figure(figsize=[15,10])\n", - "display = pyart.graph.RadarMapDisplay(fourth_volume)\n", - " \n", - "# Add our reflectivity (DBZ) field to the plot, including our gatefilter\n", - "display.plot_ppi_map('DBZ', 7, vmin=-20, vmax=40.)\n" + "volumes = []\n", + "for base in base_scans:\n", + " base_scan_index = np.where(np.array(ppis) == base)[0][0]\n", + " #print(base_scan_index)\n", + " volume = ppis[base_scan_index: base_scan_index+n_tilts]\n", + " volumes.append(volume)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "25973669-21f3-480b-bdd5-34c0549daf96", + "cell_type": "markdown", + "id": "3b34df49", "metadata": {}, - "outputs": [], "source": [ - "for vvl in volumes[0:1]:\n", - " if len(vvl) == n_tilts:\n", - " out_radar = volume_from_list(vvl, data_dir)\n", - " print(out_radar.time['units'][14:])\n", - " ff = time.strptime(out_radar.time['units'][14:], '%Y-%m-%dT%H:%M:%SZ')\n", - " dt = datetime.datetime.fromtimestamp(time.mktime(ff)) + datetime.timedelta(seconds= int(out_radar.time['data'][0]))\n", - " strform = dt.strftime('/eagle/projects/CPOL/SAIL/202111_glued/xprecipradar_guc_volume_%Y%m%d-%H%M%S.b1.nc')\n", - " print(strform)\n", - " #FIX for join issue.. to be fixed in Py-ART\n", - " out_radar.sweep_mode['data']=np.tile(radars[0].sweep_mode['data'], n_tilts)\n", - " pyart.io.write_cfradial(strform, out_radar)\n", - " " + "## Try Single Volume to Verify Process" ] }, { "cell_type": "code", - "execution_count": null, - "id": "d457e2dd-8869-4566-8b34-24c161f32ca9", + "execution_count": 12, + "id": "b6e36298", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['gucxprecipradarS2.00.20220301.004845.raw.csu.sail-20220301-004845_278032_22_1_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.004916.raw.csu.sail-20220301-004916_278034_22_2_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.004949.raw.csu.sail-20220301-004949_278035_22_4_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.005020.raw.csu.sail-20220301-005020_278036_22_6_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.005053.raw.csu.sail-20220301-005053_278037_22_8_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.005124.raw.csu.sail-20220301-005124_278038_22_10_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.005156.raw.csu.sail-20220301-005156_278039_22_12_PPI.nc',\n", + " 'gucxprecipradarS2.00.20220301.005229.raw.csu.sail-20220301-005229_278040_22_15_PPI.nc']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "acompletelynewradar = pyart.io.read('/eagle/projects/CPOL/SAIL/202111_glued/xprecipradar_guc_volume_20211110-000110.b1.nc')" + "volumes[0]" ] }, { "cell_type": "code", - "execution_count": null, - "id": "09e46d5c-f226-45f2-95ae-6cfee57908d1", + "execution_count": 13, + "id": "f6ad63e3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SUCCESS /gpfs/wolf/atm124/proj-shared/gucxprecipradarS2.00/glue_files/202203_glued/xprecipradar_guc_volume_20220301-004845.b1.nc\n", + "CPU times: user 14.4 s, sys: 1.66 s, total: 16 s\n", + "Wall time: 17.1 s\n" + ] + } + ], "source": [ - "figme = plt.figure(figsize=[15,10])\n", - "display = pyart.graph.RadarMapDisplay(acompletelynewradar)\n", - " \n", - "# Add our reflectivity (DBZ) field to the plot, including our gatefilter\n", - "display.plot_ppi_map('DBZ', 7, vmin=-20, vmax=40.)\n" + "%%time\n", + "granule(volumes[0])" ] }, { "cell_type": "code", - "execution_count": null, - "id": "dfdcd3ae-750f-49f3-bcf3-b03097b65f4f", + "execution_count": 17, + "id": "9d7f8f24-a349-4693-9a5d-bda88eef18c8", "metadata": {}, "outputs": [], "source": [ - "def granule(Dvolume):\n", - " data_dir = '/eagle/projects/CPOL/SAIL/202111/'\n", - " if len(Dvolume) == 8:\n", - " base_rad = pyart.io.read(data_dir+Dvolume[0])\n", - " out_radar = volume_from_list(Dvolume, data_dir)\n", - " print(out_radar.time['units'][14:])\n", - " ff = time.strptime(out_radar.time['units'][14:], '%Y-%m-%dT%H:%M:%SZ')\n", - " dt = datetime.datetime.fromtimestamp(time.mktime(ff)) + datetime.timedelta(seconds= int(out_radar.time['data'][0]))\n", - " strform = dt.strftime('/eagle/projects/CPOL/SAIL/202111_glued/xprecipradar_guc_volume_%Y%m%d-%H%M%S.b1.nc')\n", - " print(strform)\n", - " #FIX for join issue.. to be fixed in Py-ART\n", - " out_radar.sweep_mode['data']=np.tile(base_rad.sweep_mode['data'], n_tilts)\n", - " pyart.io.write_cfradial(strform, out_radar)" + "# smaller subset just for testing; for full month map volumes to granule\n", + "subset = volumes[0:16]" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "c9edef51-12e1-4b43-9245-ce7b24767a71", + "cell_type": "markdown", + "id": "45dd0c3a", "metadata": {}, - "outputs": [], "source": [ - "granule(volumes[0])" + "## Start a Dask Cluster" ] }, { "cell_type": "code", - "execution_count": null, - "id": "3c30bfa8-c79b-4b52-a56c-985a9fc32461", + "execution_count": 15, + "id": "8d9383af", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Client

\n", + "

Client-6b38fc35-d1cd-11ee-aa6b-b8cb29b120a2

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + "
\n", + "\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "
\n", + "

Cluster Info

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

LocalCluster

\n", + "

d5f460ae

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Workers: 1\n", + "
\n", + " Total threads: 1\n", + " \n", + " Total memory: 251.48 GiB\n", + "
Status: runningUsing processes: True
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-8ee07010-a59b-4d9e-9820-fab289c28b05

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tcp://127.0.0.1:34455\n", + " \n", + " Workers: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Total threads: 1\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 251.48 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + "
\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 0

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43873\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37303/status\n", + " \n", + " Memory: 251.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:34077\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-6_zfrvvd\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from dask.distributed import Client, LocalCluster\n", "\n", "cluster = LocalCluster()\n", "\n", - "cluster.scale(16) # Sets the number of workers to 10\n", - "cluster.adapt(minimum=8, maximum=16)\n", "client = Client(cluster)\n", "client" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "2a6b9e7c-4d3f-4abb-af31-b0eebeee8e15", + "cell_type": "markdown", + "id": "9a0cee88", "metadata": {}, - "outputs": [], "source": [ - "future = client.map(granule, volumes)" + "## Glue the Files" ] }, { "cell_type": "code", - "execution_count": null, - "id": "f95dd65f-2812-4f53-a550-9ea01ed59e4e", + "execution_count": 16, + "id": "17cca90d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 3.33 s, sys: 2.07 s, total: 5.39 s\n", + "Wall time: 4min 33s\n" + ] + } + ], "source": [ + "%%time\n", + "future = client.map(granule, subset)\n", "my_data = client.gather(future)" ] }, { "cell_type": "code", "execution_count": null, - "id": "08254fae-3631-4b21-8a83-29dc84afebac", - "metadata": {}, - "outputs": [], - "source": [ - "client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb03d974-a65d-43e2-80e7-b778843d1e6e", + "id": "e439c9f8-4e8d-44ac-b45e-4a550f0d2225", "metadata": {}, "outputs": [], "source": [] @@ -390,9 +584,9 @@ ], "metadata": { "kernelspec": { - "display_name": "pyart-simple", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "pyart-simple" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -404,7 +598,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.15" + "version": "3.11.7" } }, "nbformat": 4,