Skip to content

Commit

Permalink
WIP: Extract ROIs from dictionary
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakirkham
jakirkham committed Sep 13, 2017
1 parent 6cdc32a commit 5313bc3
Showing 1 changed file with 211 additions and 18 deletions.
229 changes: 211 additions & 18 deletions nanshe_ipython.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,18 @@
"logging.getLogger(\"nanshe\").setLevel(logging.INFO)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from builtins import (\n",
" map as imap,\n",
" range as irange\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1300,9 +1312,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### ROI Refinement\n",
"### Component Refinement\n",
"\n",
"* `area_min_threshold` (`float`): minimum area required for all ROIs"
"* `area_min_threshold` (`float`): minimum area required for all connected components"
]
},
{
Expand Down Expand Up @@ -1456,6 +1468,80 @@
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Normalize Data\n",
"\n",
"* `block_frames` (`int`): number of frames to work with in each full frame block (run in parallel).\n",
"* `norm_frames` (`int`): number of frames for use during normalization of each full frame block (run in parallel)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"\n",
"block_frames = 40\n",
"norm_frames = 100\n",
"\n",
"\n",
"# Somehow we can't overwrite the file in the container so this is needed.\n",
"io_remove(data_basename + postfix_norm + zarr_ext)\n",
"io_remove(data_basename + postfix_norm + h5_ext)\n",
"\n",
"\n",
"with open_zarr(data_basename + postfix_thrd + zarr_ext, \"r\") as f:\n",
" with get_executor(client) as executor:\n",
" # Load and prep data for computation.\n",
" imgs = f[\"images\"]\n",
" da_imgs = da.from_array(\n",
" imgs, chunks=(block_frames,) + (imgs.ndim - 1) * (block_space,)\n",
" )\n",
"\n",
" da_imgs_flt = da_imgs\n",
" if not (issubclass(da_imgs_flt.dtype.type, np.floating) and \n",
" da_imgs_flt.dtype.itemsize >= 4):\n",
" da_imgs_flt = da_imgs_flt.astype(np.float32)\n",
"\n",
" da_result = normalize_data(da_imgs)\n",
"\n",
" # Store denoised data\n",
" with open_zarr(data_basename + postfix_norm + zarr_ext, \"w\") as f2:\n",
" result = f2.create_dataset(\n",
" \"images\",\n",
" shape=da_result.shape,\n",
" dtype=da_result.dtype,\n",
" chunks=True\n",
" )\n",
" da_result = da_result.rechunk(result.chunks)\n",
" status = executor.compute(da.store(da_result, result, lock=False, compute=False))\n",
" dask.distributed.progress(status, notebook=False)\n",
"\n",
"\n",
"zip_zarr(data_basename + postfix_norm + zarr_ext)\n",
"\n",
"with h5py.File(data_basename + postfix_norm + h5_ext, \"w\") as f2:\n",
" with open_zarr(data_basename + postfix_norm + zarr_ext, \"r\") as f1:\n",
" zarr_to_hdf5(f1, f2)\n",
"\n",
"\n",
"if __IPYTHON__:\n",
" result_image_stack = LazyZarrDataset(data_basename + postfix_norm + zarr_ext, \"images\")\n",
"\n",
" mplsv = plt.figure(FigureClass=MPLViewer)\n",
" mplsv.set_images(\n",
" result_image_stack,\n",
" vmin=par_compute_min_projection(num_frames=norm_frames)(result_image_stack).min(),\n",
" vmax=par_compute_max_projection(num_frames=norm_frames)(result_image_stack).max()\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -1485,7 +1571,7 @@
"n_components = 50\n",
"batchsize = 256\n",
"iters = 100\n",
"lambda1 = 0.2\n",
"lambda1 = 0.5\n",
"lambda2 = 0.0\n",
"\n",
"block_frames = 51\n",
Expand All @@ -1496,7 +1582,7 @@
"io_remove(data_basename + postfix_dict + zarr_ext)\n",
"io_remove(data_basename + postfix_dict + h5_ext)\n",
"\n",
"result = LazyZarrDataset(data_basename + postfix_thrd + zarr_ext, \"images\")\n",
"result = LazyZarrDataset(data_basename + postfix_norm + zarr_ext, \"images\")\n",
"block_shape = (block_frames,) + result.shape[1:]\n",
"with open_zarr(data_basename + postfix_dict + zarr_ext, \"w\") as f2:\n",
" new_result = f2.create_dataset(\"images\", shape=(n_components,) + result.shape[1:], dtype=result.dtype, chunks=True)\n",
Expand Down Expand Up @@ -1541,9 +1627,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### ROI and trace extraction\n",
"### ROI Extraction\n",
"\n",
"* `block_frames` (`int`): number of frames to work with in each block (run in parallel)."
"* `normed_max_threshold` (`float`): minimum required value of maximum value of the normalized basis images.\n",
"* `basis_threshold` (`float`): threshold used on each basis image.\n",
"\n",
"<br>\n",
"\n",
"* `block_frames` (`int`): number of frames to work with in each full frame block (run in parallel)."
]
},
{
Expand All @@ -1552,35 +1643,137 @@
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"#%%time\n",
"\n",
"\n",
"block_frames = 100\n",
"normed_max_threshold = 0.1\n",
"basis_threshold = 0.01\n",
"\n",
"block_frames = 40\n",
"\n",
"\n",
"# Somehow we can't overwrite the file in the container so this is needed.\n",
"io_remove(data_basename + postfix_rois + zarr_ext)\n",
"io_remove(data_basename + postfix_rois + h5_ext)\n",
"\n",
"with open_zarr(data_basename + postfix_rois + zarr_ext, \"w\") as f2:\n",
" with open_zarr(data_basename + postfix_post + zarr_ext, \"r\") as f1:\n",
" f2[\"masks\"] = f1[\"rois/mask\"]\n",
"with open_zarr(data_basename + postfix_dict + zarr_ext, \"r\") as f:\n",
" with get_executor(client) as executor:\n",
" # Load and prep data for computation.\n",
" imgs = f[\"images\"]\n",
" da_imgs = da.from_array(\n",
" imgs, chunks=(block_frames,) + imgs.shape[1:]\n",
" )\n",
"\n",
" da_imgs_pos = da.where(da_imgs > 0, da_imgs, 0)\n",
" da_imgs_norm = normalize_data(da_imgs_pos)\n",
"\n",
" mskimg = f2[\"masks\"]\n",
" mskimg_j = f2.create_dataset(\"masks_j\", shape=mskimg.shape, dtype=numpy.uint8, chunks=True)\n",
" par_norm_layer(num_frames=block_frames)(mskimg, out=mskimg_j)\n",
" da_imgs_thrd = (\n",
" da_imgs_norm *\n",
" (\n",
" da_imgs_norm.max(axis=tuple(irange(1, da_imgs_norm.ndim)), keepdims=True) > normed_max_threshold\n",
" ).astype(da_imgs.dtype)\n",
" )\n",
" da_imgs_thrd = (da_imgs_thrd > basis_threshold)\n",
"\n",
" da_lbl_imgs = []\n",
" da_num_lbls = []\n",
" for i in irange(len(da_imgs_thrd)):\n",
" da_lbl_imgs_i, da_num_lbls_i = dask_ndmeasure.label(da_imgs_thrd[i])\n",
"\n",
" da_lbl_imgs.append(da_lbl_imgs_i)\n",
" da_num_lbls.append(da.asarray([da_num_lbls_i]))\n",
"\n",
" da_lbl_imgs = da.stack(da_lbl_imgs)\n",
" da_num_lbls = da.concatenate(da_num_lbls)\n",
"\n",
" da_adj_lbls = da.concatenate([\n",
" da.asarray(da_num_lbls.dtype.type(0)[None]),\n",
" da.cumsum(da_num_lbls, axis=0)[:-1]\n",
" ])\n",
"\n",
" da_lbl_imgs = da_lbl_imgs + da.where(\n",
" da_lbl_imgs != 0,\n",
" da_adj_lbls[(slice(None),) + (da_lbl_imgs.ndim - da_adj_lbls.ndim) * (None,)],\n",
" da_lbl_imgs.dtype.type(0)\n",
" )\n",
"\n",
" da_lbl_imgs, da_num_lbls, da_adj_lbls = executor.persist([da_lbl_imgs, da_num_lbls, da_adj_lbls])\n",
"\n",
" da_msk_imgs = []\n",
" for i in irange(len(da_lbl_imgs)):\n",
" for l in irange(1 + da_adj_lbls[i].compute(),\n",
" (1 + da_num_lbls + da_adj_lbls)[i].compute()):\n",
" da_msk_imgs.append(\n",
" (da_lbl_imgs[i] == l)[None]\n",
" )\n",
" da_msk_imgs = da.concatenate(da_msk_imgs)\n",
"\n",
" del da_num_lbls, da_adj_lbls\n",
"\n",
" da_lbl_img = (\n",
" da.arange(\n",
" 1, 1 + len(da_msk_imgs), chunks=da_msk_imgs.chunks[0]\n",
" )[(slice(None),) + (da_msk_imgs.ndim - 1) * (None,)] *\n",
" da_msk_imgs.astype(int)\n",
" ).sum(axis=0)\n",
" da_msk_imgs = da_msk_imgs.astype(np.uint8)\n",
"\n",
" # Store data\n",
" with open_zarr(data_basename + postfix_rois + zarr_ext, \"w\") as f2:\n",
" status = []\n",
" for name, da_result in {\"masks\" : da_msk_imgs, \"labels\" : da_lbl_img}.items():\n",
" result = f2.create_dataset(\n",
" name,\n",
" shape=da_result.shape,\n",
" dtype=da_result.dtype,\n",
" chunks=True\n",
" )\n",
" da_result = da_result.rechunk(result.chunks)\n",
" status.append(executor.compute(da.store(\n",
" da_result, result, lock=False, compute=False\n",
" )))\n",
" dask.distributed.progress(status, notebook=False)\n",
"\n",
" lblimg = label_mask_stack(mskimg, np.uint64)\n",
" f2[\"labels\"] = lblimg\n",
" f2[\"labels_j\"] = lblimg.astype(np.uint16)\n",
" lblimg = f2[\"labels\"]\n",
"\n",
"zip_zarr(data_basename + postfix_rois + zarr_ext)\n",
"\n",
"with h5py.File(data_basename + postfix_rois + h5_ext, \"w\") as f2:\n",
" with open_zarr(data_basename + postfix_rois + zarr_ext, \"r\") as f1:\n",
" zarr_to_hdf5(f1, f2)\n",
"\n",
"\n",
"if __IPYTHON__:\n",
" result_image_stack = LazyZarrDataset(data_basename + postfix_rois + zarr_ext, \"masks\")[...][...]\n",
"\n",
" mplsv = plt.figure(FigureClass=MPLViewer)\n",
" mplsv.set_images(\n",
" result_image_stack,\n",
" vmin=result_image_stack.min(),\n",
" vmax=result_image_stack.max()\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Trace extraction\n",
"\n",
"* `block_frames` (`int`): number of frames to work with in each block (run in parallel)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"\n",
"block_frames = 100\n",
"\n",
"\n",
"# Somehow we can't overwrite the file in the container so this is needed.\n",
"io_remove(data_basename + postfix_traces + zarr_ext)\n",
"io_remove(data_basename + postfix_traces + h5_ext)\n",
Expand Down

0 comments on commit 5313bc3

Please sign in to comment.