Major Rework (#13)

* fix typo * test different array layouts * add example nbs * add transform to rad * refactoring * reorder * delete double time step calculation * Add missing packages * Fix data_set.toml * remove date, return to deg * not needed * Git submodule * Add click package * add notebook to test observation times * print active telescopes at start of obs * refactoring * refactoring * Add assert instead of print * Begin testing * Add setup.py for radiosim * Try again * rerun * Try again * Restore ci.yml * Add own data_handler * Update tests * Remove unnecessary submodule * Add h5py to packages * Import missing function * Change expected classes * Add another test * Refactor tests * Prevent 0 scans * Update times notebook * Some refactoring * Fix typo * test num vis for time step 0 * refactoring * array class with lazy property * fix tests * test Array class * more refactoring * add comments * simulation chain tests * Some refactoring * Add missing line * Change test files * Start reworking data structure Slurm step to be done * change tests accordingly * Renaming and deleting * Adjust tests * Fix fits files creation * Remove data_handler and beging rework of gridder * uncorrupted working * direction independent working * corrupted working * Change toml * Fix tests * Delete comments * VLBA geht, VLA nicht * fix vla coords * fix amp phase scaling * delete plot * delete comments and plots * fix multichannel * use corrupted * plot antenna positions * fix vla clean * delete 0 antenna * add num threads * fix test for vla * delete unused keyword * delete unused keyword * fix slurm * really fix slurm * fix typo * delete array, black * Delete unused keywords * Fix file opening now just the right file is loaded. Also black changes * Fix sorting of fits files * Add natsort to packages * hardcode outpath * change multiplicator to 500 * delete seed * change version number, start beta after rework * add second outpath * add outpath for gridded data * fix tests * fix out path * fix another outpath * Comment out prints * add corrupted option * Black changes * Fix failing samples * Add ratio fix * Delete factor * Black changes * include natsort * fix times * Remove and alter prints * Delete comment * Delete empty file Co-authored-by: Kevin Schmidt <[email protected]>
radionets-project · Nov 11, 2022 · feb7a42 · feb7a42
1 parent 90ab645
commit feb7a42
Show file tree

Hide file tree

Showing 24 changed files with 1,493 additions and 346 deletions.
diff --git a/config/data_set.toml b/config/data_set.toml
@@ -1,24 +1,26 @@
 [sampling_options]
 mode = "basic"
-layout = "vlba"
-img_size = 64
-fov_center_ra = [00:00:00.0, 23:59:59.59]
-fov_center_dec = [00:00:00.0, 11:59:59.59]
-fov_size = 0.0064 # max res 0.1
-corr_int_time = 10.0
-scan_start = ["01-01-2020 00:00:01", "31-12-2021 23:59:59"]
-scan_duration = [50, 300]
-scans = [30, 72]
-interval_length = 1200
+layout = "vla"
+img_size = 128
+fov_center_ra = [100, 110]
+fov_center_dec = [30, 40]
+fov_size = 100 # max res 0.1
+corr_int_time = 30.0
+scan_start = ["16-01-2020 00:04:01", "16-01-2020 08:59:59"]
+scan_duration = [60, 90]
+scans = [1, 2]
+interval_length = 360
 base_freq = 15.21e9
 frequsel = [0e8, 0.8e8, 1.44e8, 2.08e8]
 bandwidths = [6.4e7, 6.4e7, 6.4e7, 6.4e7]
+corrupted = true
 
 [bundle_options]
-num_bundles = 5
-size_bundles = 10
-in_path = "../../../test_radiosim/build/test_data"
-out_path = "./build"
-
-[cluster_options]
-num_jobs = 4
+in_path = "build/skies/"
+out_path_fits = "build/uvfits"
+out_path_gridded = "build/gridded"
+num_test_images = 500
+bundle_size = 100
+train_valid_split = 0.2
+grid_size = 128
+amp_phase = true
diff --git a/examples/01_layouts.ipynb b/examples/01_layouts.ipynb
diff --git a/examples/02_times.ipynb b/examples/02_times.ipynb
diff --git a/examples/0X_simulation_chain.ipynb b/examples/0X_simulation_chain.ipynb
@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "42e51422",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pyvisgen.layouts.layouts import get_array_layout"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "719cee21",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7135e0e8",
+   "metadata": {},
+   "source": [
+    "### Define radio interferometer array layout\n",
+    "\n",
+    "Available layouts:\n",
+    "* vla\n",
+    "* vlba\n",
+    "* eht"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "fb73812f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "array_layout = get_array_layout(\"vlba\") # in rc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a4f1f2bd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Stations(st_num=array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), name=array(['MKO', 'OVRO', 'BR', 'NL', 'HC', 'KPN', 'PT', 'FD', 'LA', 'SC'],\n",
+       "      dtype=object), x=array([-5464075.238656, -2409150.471188, -2112065.261576,  -130872.556729,\n",
+       "        1446374.806874, -1995678.891541, -1640953.992891, -1324009.374466,\n",
+       "       -1449752.637707,  2607848.664542]), y=array([-2495247.871441, -4478573.093114, -3705356.502894, -4762317.087045,\n",
+       "       -4447939.68308 , -5037317.693848, -5014816.024485, -5332181.952906,\n",
+       "       -4975298.573645, -5488069.500452]), z=array([2148297.485741, 3838617.326057, 4726813.649034, 4226850.993404,\n",
+       "       4322306.19968 , 3357328.002045, 3575411.772132, 3231962.377936,\n",
+       "       3709123.828301, 1932739.778597]), diam=array([25, 25, 25, 25, 25, 25, 25, 25, 25, 25]), el_low=array([15, 15, 15, 15, 15, 15, 15, 15, 15, 15]), el_high=array([85, 85, 85, 85, 85, 85, 85, 85, 85, 85]), sefd=array([  110, 11900,   560,  4900,  2900,  1600,  7300,  4744,  4744,\n",
+       "        4744]), altitude=array([5030, 3185, 4640, 4205, 2850, 2550, 2800, 3210, 3210, 3210]))"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "array_layout"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6cace9fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from pyvisgen.utils.data import data_handler\n",
+    "from pyvisgen.simulation.data_set import create_sampling_rc\n",
+    "from pyvisgen.simulation.visibility import vis_loop\n",
+    "from pyvisgen.utils.config import read_data_set_conf\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "e2d892fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = \"test_conf.toml\"\n",
+    "conf = read_data_set_conf(config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fb870908",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = data_handler(conf[\"in_path\"])\n",
+    "samp_ops = create_sampling_rc(conf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ff3e2b78",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(16245,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "SI = torch.tensor(data[0][0][0], dtype=torch.cdouble)\n",
+    "vis_data = vis_loop(samp_ops, SI)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bb6d8ab3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/pyvisgen/fits/data.py b/pyvisgen/fits/data.py
@@ -30,7 +30,11 @@ def __getitem__(self, i):
         return self.open_file(i)
 
     def get_files(self, path):
-        return np.sort(np.array([x for x in path.iterdir()]))
+        import natsort
+
+        fits_files = [str(x) for x in path.iterdir()]
+        fits_files_sorted = natsort.natsorted(fits_files)
+        return fits_files_sorted
 
     def get_uv_data(self, i):
         with fits.open(self.files[i]) as hdul:

diff --git a/pyvisgen/fits/writer.py b/pyvisgen/fits/writer.py
@@ -15,9 +15,7 @@ def create_vis_hdu(data, conf, layout="vlba", source_name="sim-source-0"):
 
     w = data.w
 
-    DATE = data.date - int(
-        data.date.min()
-    ) 
+    DATE = data.date - int(data.date.min())
 
     _DATE = data._date  # central time in the integration period
 

diff --git a/pyvisgen/gridding/alt_gridder.py b/pyvisgen/gridding/alt_gridder.py
@@ -98,11 +98,11 @@ def ms2dirty_python_fast(
             # xkernel = kernel(pos - ratposx + xle)
             # ykernel = kernel(pos - ratposy + yle)
             for xx in range(supp):
-                foo = vis #* xkernel[xx]
+                foo = vis  # * xkernel[xx]
                 myxpos = (xle + xx) % ng[0]
                 for yy in range(supp):
                     myypos = (yle + yy) % ng[1]
-                    grid[myxpos, myypos] += foo #* ykernel[yy]
+                    grid[myxpos, myypos] += foo  # * ykernel[yy]
     #     loopim = np.fft.fftshift(np.fft.ifft2(grid)*np.prod(ng))
     #     loopim = loopim[slc0, slc1]
     #     if do_wgridding:
@@ -116,9 +116,9 @@ def ms2dirty_python_fast(
 
 
 def get_npixdirty(uvw, freq, fov_deg, mask):
-    speedOfLight = 299792458.
-    bl = np.sqrt(uvw[:,0]**2+uvw[:,1]**2+uvw[:,2]**2)
-    bluvw = bl.reshape((-1,1))*freq.reshape((1,-1))/speedOfLight
-    maxbluvw = np.max(bluvw*mask)
-    minsize = int((2*fov_deg*np.pi/180*maxbluvw)) + 1
-    return minsize+(minsize%2)  # make even
+    speedOfLight = 299792458.0
+    bl = np.sqrt(uvw[:, 0] ** 2 + uvw[:, 1] ** 2 + uvw[:, 2] ** 2)
+    bluvw = bl.reshape((-1, 1)) * freq.reshape((1, -1)) / speedOfLight
+    maxbluvw = np.max(bluvw * mask)
+    minsize = int((2 * fov_deg * np.pi / 180 * maxbluvw)) + 1
+    return minsize + (minsize % 2)  # make even