From 92f868023ebf89e910e65d2f23a94ffda93c3eb6 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Mon, 12 Aug 2024 17:23:33 +0200 Subject: [PATCH 01/13] init commit of improved srcnet rucio script :cherries: --- karabo/examples/SRCNet_rucio_meta_demo.py | 167 ++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 karabo/examples/SRCNet_rucio_meta_demo.py diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py new file mode 100644 index 00000000..0db80004 --- /dev/null +++ b/karabo/examples/SRCNet_rucio_meta_demo.py @@ -0,0 +1,167 @@ +"""Example script to attach Rucio ObsCore metadata data-products for ingestion. + +Here, we create a simulated visibilities and a cleaned image. This is just an example +script which can be highly adapted, e.g. you custom-sky, simulation params (for larger +data-products, simulation params as user-input, multiple simulations, etc. + +Be aware that API-changes can take place in future Karabo-versions. This script is +based on `SRCNet_rucio_meta.py` but contains a full workflow for a single simulation. + +If not specified further (e.g. through `XDG_CACHE_HOME` or `FileHandler.root_stm`), +Karabo is using /tmp. Thus if you have a script which is producing large and/or many +data products, we suggest to adapt the cache-root to a volume with more space. +""" + +from __future__ import annotations + +import os +from datetime import datetime + +import numpy as np +from astropy import units as u + +from karabo.data.obscore import FitsHeaderAxes, FitsHeaderAxis, ObsCoreMeta +from karabo.data.src import RucioMeta +from karabo.imaging.imager_wsclean import WscleanImageCleaner, WscleanImageCleanerConfig +from karabo.simulation.interferometer import InterferometerSimulation +from karabo.simulation.observation import Observation +from karabo.simulation.sky_model import SkyModel +from karabo.simulation.telescope import Telescope +from karabo.simulator_backend import SimulatorBackend +from karabo.util.helpers import get_rnd_str + + +def main() -> None: + # sky-to-visibilities simulation + # the params for this example are highly adapted to not create large + # data products, because this is not the focus of this script. + sky = SkyModel.get_GLEAM_Sky(min_freq=72e6, max_freq=231e6) # in Hz + phase_center = [250, -80] # RA,DEC in deg + filter_radius_deg = 0.80 + sky = sky.filter_by_radius(0, filter_radius_deg, phase_center[0], phase_center[1]) + tel = Telescope.constructor("ASKAP", backend=SimulatorBackend.OSKAR) + freq_inc_hz = 1e8 + obs = Observation( + start_frequency_hz=76e6, + start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), + phase_centre_ra_deg=phase_center[0], + phase_centre_dec_deg=phase_center[1], + number_of_channels=16 * 2, + frequency_increment_hz=freq_inc_hz, + number_of_time_steps=24 * 2, + ) + interferometer_sim = InterferometerSimulation(channel_bandwidth_hz=freq_inc_hz) + vis = interferometer_sim.run_simulation( + telescope=tel, + sky=sky, + observation=obs, + backend=SimulatorBackend.OSKAR, + ) + # here, I customize the `vis.vis_path` to not have the same file-name for each + # simulation which would be suboptimal for ingestion. Should probably be more + # specific for your use-case. + vis_path = os.path.join( + os.path.split(vis.vis_path)[0], + f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.vis", + ) + os.rename( + vis.vis_path, + vis_path, + ) + vis.vis_path = vis_path + + # create metadata of visibility (currently [08-24], .vis supported, casa .ms not) + vis_ocm = ObsCoreMeta.from_visibility( + vis=vis, + calibrated=False, + tel=tel, + obs=obs, + ) + vis_rm = RucioMeta( + namespace="testing", # needs to be specified by Rucio service + name=os.path.split(vis.vis_path)[-1], # remove path-infos for `name` + lifetime=86400, # 1 day + dataset_name=None, + meta=vis_ocm, + ) + # ObsCore mandatory fields + vis_ocm.obs_collection = "MRO/ASKAP" + obs_sim_id = 0 # nc for new simulation + user_rnd_str = get_rnd_str(k=7, seed=os.environ.get("USER")) + obs_id = f"karabo-{user_rnd_str}-{obs_sim_id}" # unique ID per user & simulation + vis_ocm.obs_id = obs_id + obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults + namespace=vis_rm.namespace, + name=vis_rm.name, + ) + vis_ocm.obs_publisher_did = obs_publisher_did + + # fill/correct other fields of `ObsCoreMeta` here! + # #####START##### + # HERE + # #####END####### + + vis_path_meta = RucioMeta.get_meta_fname(fname=vis.vis_path) + _ = vis_rm.to_dict(fpath=vis_path_meta) + print(f"Created {vis_path_meta=}") + + # -----Imaging----- + + imaging_npixel = 2048 + fov_deg = np.cos(np.deg2rad(45)) * filter_radius_deg * 2 # for squared image(s) + imaging_cellsize = imaging_npixel / fov_deg + + restored = WscleanImageCleaner( + WscleanImageCleanerConfig( + imaging_npixel=imaging_npixel, + imaging_cellsize=imaging_cellsize, + ) + ).create_cleaned_image( # currently, wsclean needs casa .ms, which is also created + ms_file_path=vis.ms_file_path, + ) + + # customize fname for restored image for the same reason as visibilities + restored_image_path = os.path.join( + os.path.split(restored.path)[0], + f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.fits", + ) + os.rename( + restored.path, + restored_image_path, + ) + restored.path = restored_image_path + + # create metadata for restored .fits image + # `FitsHeaderAxes` may need adaption based on the structure of your .fits image + axes = FitsHeaderAxes(freq=FitsHeaderAxis(axis=3, unit=u.Hz)) + restored_ocm = ObsCoreMeta.from_image(img=restored, fits_axes=axes) + restored_rm = RucioMeta( + namespace="testing", # needs to be specified by Rucio service + name=os.path.split(restored.path)[-1], # remove path-infos for `name` + lifetime=86400, # 1 day + dataset_name=None, + meta=restored_ocm, + ) + # ObsCore mandatory fields + # some of the metadata is taken from above, since both data-products originate + # from the same observation + restored_ocm.obs_collection = vis_ocm.obs_collection + restored_ocm.obs_id = vis_ocm.obs_id + obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults + namespace=restored_rm.namespace, + name=restored_rm.name, + ) + vis_ocm.obs_publisher_did = obs_publisher_did + + # fill/correct other fields of `ObsCoreMeta` here! + # #####START##### + # HERE + # #####END####### + + restored_path_meta = RucioMeta.get_meta_fname(fname=restored.path) + _ = restored_rm.to_dict(fpath=restored_path_meta) + print(f"Created {restored_path_meta=}") + + +if __name__ == "__main__": + main() From dc958f741722846c1b0310d4111214617aa39d14 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Tue, 13 Aug 2024 11:13:30 +0200 Subject: [PATCH 02/13] bugfix: calc img-cellsize :oncoming_taxi: --- karabo/examples/SRCNet_rucio_meta_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py index 0db80004..914395cd 100644 --- a/karabo/examples/SRCNet_rucio_meta_demo.py +++ b/karabo/examples/SRCNet_rucio_meta_demo.py @@ -109,7 +109,7 @@ def main() -> None: imaging_npixel = 2048 fov_deg = np.cos(np.deg2rad(45)) * filter_radius_deg * 2 # for squared image(s) - imaging_cellsize = imaging_npixel / fov_deg + imaging_cellsize = fov_deg / imaging_npixel restored = WscleanImageCleaner( WscleanImageCleanerConfig( From 3d0a05d9a6fd5eb11d526bbaa1f9c45c7b515625 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Tue, 13 Aug 2024 17:51:06 +0200 Subject: [PATCH 03/13] minor bugfix and improvements :tennis: --- karabo/data/obscore.py | 6 ++++-- karabo/examples/SRCNet_rucio_meta_demo.py | 13 +++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/karabo/data/obscore.py b/karabo/data/obscore.py index 4c5efba3..53686abd 100644 --- a/karabo/data/obscore.py +++ b/karabo/data/obscore.py @@ -509,7 +509,7 @@ def from_image( cls, img: Image, *, - fits_axes: FitsHeaderAxes = FitsHeaderAxes(), # immutable default + fits_axes: Optional[FitsHeaderAxes] = None, ) -> Self: """Update fields from `Image`. @@ -534,10 +534,12 @@ def from_image( """ file = img.path assert_valid_ending(path=file, ending=".fits") + if fits_axes is None: + fits_axes = FitsHeaderAxes() header = img.header if not header["SIMPLE"]: wmsg = f"{file} doesn't follow .fits standard! Info extraction might fail!" - warn(message=wmsg, category=UserWarning, stacklevel=1) + warn(message=wmsg, category=UserWarning, stacklevel=2) ra_deg = fits_axes.x.crval(header=header).to(u.deg).value # center dec_deg = fits_axes.y.crval(header=header).to(u.deg).value # center freq_center_hz = fits_axes.freq.crval(header=header).to(u.Hz).value # center diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py index 914395cd..c17c7217 100644 --- a/karabo/examples/SRCNet_rucio_meta_demo.py +++ b/karabo/examples/SRCNet_rucio_meta_demo.py @@ -37,7 +37,7 @@ def main() -> None: # data products, because this is not the focus of this script. sky = SkyModel.get_GLEAM_Sky(min_freq=72e6, max_freq=231e6) # in Hz phase_center = [250, -80] # RA,DEC in deg - filter_radius_deg = 0.80 + filter_radius_deg = 0.8 sky = sky.filter_by_radius(0, filter_radius_deg, phase_center[0], phase_center[1]) tel = Telescope.constructor("ASKAP", backend=SimulatorBackend.OSKAR) freq_inc_hz = 1e8 @@ -46,9 +46,9 @@ def main() -> None: start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), phase_centre_ra_deg=phase_center[0], phase_centre_dec_deg=phase_center[1], - number_of_channels=16 * 2, + number_of_channels=16, frequency_increment_hz=freq_inc_hz, - number_of_time_steps=24 * 2, + number_of_time_steps=24, ) interferometer_sim = InterferometerSimulation(channel_bandwidth_hz=freq_inc_hz) vis = interferometer_sim.run_simulation( @@ -107,14 +107,15 @@ def main() -> None: # -----Imaging----- - imaging_npixel = 2048 - fov_deg = np.cos(np.deg2rad(45)) * filter_radius_deg * 2 # for squared image(s) - imaging_cellsize = fov_deg / imaging_npixel + imaging_npixel = 2048 # TODO: calc size on synthesized beam, not fov + fov_deg = filter_radius_deg * 2 # for squared image(s) + imaging_cellsize = np.deg2rad(fov_deg) / imaging_npixel restored = WscleanImageCleaner( WscleanImageCleanerConfig( imaging_npixel=imaging_npixel, imaging_cellsize=imaging_cellsize, + niter=5000, # 10 times less than default ) ).create_cleaned_image( # currently, wsclean needs casa .ms, which is also created ms_file_path=vis.ms_file_path, From f5960da8467f71199992df2f68e54fa7c411304a Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 09:57:49 +0200 Subject: [PATCH 04/13] renamed longest-baseline function to max-baseline :ideograph_advantage: --- karabo/data/obscore.py | 4 ++-- karabo/simulation/telescope.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/karabo/data/obscore.py b/karabo/data/obscore.py index 53686abd..2071af4f 100644 --- a/karabo/data/obscore.py +++ b/karabo/data/obscore.py @@ -500,7 +500,7 @@ def from_visibility( freq_inc_hz = obs.frequency_increment_hz min_freq_hz = obs.start_frequency_hz - freq_inc_hz / 2 end_freq_hz = min_freq_hz + freq_inc_hz * obs.number_of_channels - b = float(tel.longest_baseline()) + b = float(tel.max_baseline()) ocm.s_resolution = tel.ang_res(freq=end_freq_hz, b=b) return ocm @@ -555,7 +555,7 @@ def from_image( f"Pixel-size is not square for `s_pixel_scale`: {x_inc_deg=}, " + f"{y_inc_deg}. `s_pixel_scale` set to {s_pixel_scale=}." ) - warn(message=wmsg, category=UserWarning, stacklevel=1) + warn(message=wmsg, category=UserWarning, stacklevel=2) min_freq_hz = freq_center_hz - freq_inc_hz / 2 c = const.c.value min_wavelength_m = c / min_freq_hz diff --git a/karabo/simulation/telescope.py b/karabo/simulation/telescope.py index 79c08a53..8290dd3f 100644 --- a/karabo/simulation/telescope.py +++ b/karabo/simulation/telescope.py @@ -804,7 +804,7 @@ def get_baselines_dists(self) -> NDArray[np.float64]: ) return dists - def longest_baseline(self) -> np.float64: + def max_baseline(self) -> np.float64: """Gets the longest baseline in meters. Returns: From 3ef7f8784417fde143c11ef8507f44923ab95b03 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 09:58:33 +0200 Subject: [PATCH 05/13] calculated image parameters from simulation observation in demo script :hotsprings: --- karabo/examples/SRCNet_rucio_meta_demo.py | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py index c17c7217..d085445e 100644 --- a/karabo/examples/SRCNet_rucio_meta_demo.py +++ b/karabo/examples/SRCNet_rucio_meta_demo.py @@ -18,6 +18,7 @@ from datetime import datetime import numpy as np +from astropy import constants as const from astropy import units as u from karabo.data.obscore import FitsHeaderAxes, FitsHeaderAxis, ObsCoreMeta @@ -40,13 +41,23 @@ def main() -> None: filter_radius_deg = 0.8 sky = sky.filter_by_radius(0, filter_radius_deg, phase_center[0], phase_center[1]) tel = Telescope.constructor("ASKAP", backend=SimulatorBackend.OSKAR) + start_freq_hz = 76e6 + num_chan = 16 freq_inc_hz = 1e8 + + if num_chan < 1: + err_msg = f"{num_chan=} but must be < 1" + raise ValueError(err_msg) + if num_chan == 1 and freq_inc_hz != 0: + err_msg = f"{freq_inc_hz=} but must be 0 if only one channel is specified" + raise ValueError(err_msg) + obs = Observation( - start_frequency_hz=76e6, + start_frequency_hz=start_freq_hz, start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), phase_centre_ra_deg=phase_center[0], phase_centre_dec_deg=phase_center[1], - number_of_channels=16, + number_of_channels=num_chan, frequency_increment_hz=freq_inc_hz, number_of_time_steps=24, ) @@ -107,10 +118,15 @@ def main() -> None: # -----Imaging----- - imaging_npixel = 2048 # TODO: calc size on synthesized beam, not fov - fov_deg = filter_radius_deg * 2 # for squared image(s) - imaging_cellsize = np.deg2rad(fov_deg) / imaging_npixel + mean_freq = start_freq_hz + freq_inc_hz * (num_chan - 1) / 2 + wavelength = const.c.value / mean_freq # in m + synthesized_beam = wavelength / tel.max_baseline() # in rad + imaging_cellsize = synthesized_beam / 3 # consider nyquist sampling theorem + fov_deg = 2 * filter_radius_deg # angular fov + imaging_npixel_estimate = fov_deg / np.rad2deg(imaging_cellsize) # not even&rounded + imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) + print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...") restored = WscleanImageCleaner( WscleanImageCleanerConfig( imaging_npixel=imaging_npixel, From 73616ce44b75d09e8c14d0a1bf086ad190889d25 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 13:30:31 +0200 Subject: [PATCH 06/13] implemented get-corners-in-world in Image class :kissing_heart: --- karabo/imaging/image.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/karabo/imaging/image.py b/karabo/imaging/image.py index 97c1bd5a..11761d91 100644 --- a/karabo/imaging/image.py +++ b/karabo/imaging/image.py @@ -20,6 +20,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np +from astropy.coordinates.sky_coordinate import SkyCoord from astropy.io import fits from astropy.io.fits.header import Header from astropy.nddata import Cutout2D, NDData @@ -747,6 +748,35 @@ def get_2d_wcs(self, ra_dec_axis: Tuple[int, int] = (1, 2)) -> WCS: wcs_2d = wcs.sub(ra_dec_axis) return wcs_2d + @classmethod + def get_corners_in_world(cls, header: Header) -> NDArray[np.float64]: + wcs = WCS(header) + if wcs.naxis < 2: + err_msg = ( + f"Header must have at least to axis (RA,DEC), but has {wcs.naxis=}" + ) + raise RuntimeError(err_msg) + naxis1 = header["NAXIS1"] + naxis2 = header["NAXIS2"] + corners = np.zeros( + shape=(4, wcs.naxis), + dtype=np.int64, + ) + corners[1, 0] = naxis1 # bottom-right + corners[2, 1] = naxis2 # top-left + corners[3, 0] = naxis1 # top-right + corners[3, 1] = naxis2 # # top-right + + world = wcs.pixel_to_world(*[corners[:, i] for i in range(corners.shape[1])]) + if not isinstance(world, list): + err_msg = f"Unexpected {type(world)=}: {world=}" + raise TypeError(err_msg) + sky_coords: SkyCoord = world[0] + world_coords: NDArray[np.float64] = ( + sky_coords.transform_to("icrs").to_table().to_pandas().to_numpy() + ) + return world_coords + class ImageMosaicker: """ From 475d767aff2e2ad60c48a98944bdf5113bd6ae59 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 14:02:21 +0200 Subject: [PATCH 07/13] bugfix: calc ObsCore region coverage on sphere :peach: --- karabo/data/obscore.py | 13 ++++--------- karabo/imaging/image.py | 23 ++++++++++++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/karabo/data/obscore.py b/karabo/data/obscore.py index 2071af4f..d319b1f3 100644 --- a/karabo/data/obscore.py +++ b/karabo/data/obscore.py @@ -540,8 +540,6 @@ def from_image( if not header["SIMPLE"]: wmsg = f"{file} doesn't follow .fits standard! Info extraction might fail!" warn(message=wmsg, category=UserWarning, stacklevel=2) - ra_deg = fits_axes.x.crval(header=header).to(u.deg).value # center - dec_deg = fits_axes.y.crval(header=header).to(u.deg).value # center freq_center_hz = fits_axes.freq.crval(header=header).to(u.Hz).value # center x_inc_deg = fits_axes.x.cdelt(header=header).to(u.deg).value # inc at center y_inc_deg = fits_axes.y.cdelt(header=header).to(u.deg).value # inc at center @@ -564,14 +562,11 @@ def from_image( fov_deg = np.sqrt( (s_pixel_scale * x_pixel) ** 2 + (y_inc_deg * y_pixel) ** 2 ) # circular fov of flattened image - half_width_deg = s_pixel_scale * x_pixel / 2 - half_height_deg = abs(y_inc_deg) * y_pixel / 2 - bottom_left = (ra_deg - half_width_deg, dec_deg - half_height_deg) - top_left = (ra_deg - half_width_deg, dec_deg + half_height_deg) - top_right = (ra_deg + half_width_deg, dec_deg + half_height_deg) - bottom_right = (ra_deg + half_width_deg, dec_deg - half_height_deg) + world_coords = Image.get_corners_in_world(header=header) + # assuming `get_corners_in_world` has circling corner order + corners = [(world_coord[0], world_coord[1]) for world_coord in world_coords] spoly = cls.spoly( - poly=(bottom_left, top_left, top_right, bottom_right), + poly=corners, ndigits=3, suffix="d", ) diff --git a/karabo/imaging/image.py b/karabo/imaging/image.py index 11761d91..99a24ecc 100644 --- a/karabo/imaging/image.py +++ b/karabo/imaging/image.py @@ -750,6 +750,16 @@ def get_2d_wcs(self, ra_dec_axis: Tuple[int, int] = (1, 2)) -> WCS: @classmethod def get_corners_in_world(cls, header: Header) -> NDArray[np.float64]: + """Get the image corners RA,DEC of bl, br, tr & tl from `header` in deg. + + Assumes that `header` has at least two axis RA & DEC. + + Args: + header: Header to extract image-infos. + + Returns: + Corners in world coordinates [deg] with shape 4x2. + """ wcs = WCS(header) if wcs.naxis < 2: err_msg = ( @@ -763,13 +773,16 @@ def get_corners_in_world(cls, header: Header) -> NDArray[np.float64]: dtype=np.int64, ) corners[1, 0] = naxis1 # bottom-right - corners[2, 1] = naxis2 # top-left - corners[3, 0] = naxis1 # top-right - corners[3, 1] = naxis2 # # top-right + corners[2, 0] = naxis1 # top-right + corners[2, 1] = naxis2 # # top-right + corners[3, 1] = naxis2 # top-left world = wcs.pixel_to_world(*[corners[:, i] for i in range(corners.shape[1])]) - if not isinstance(world, list): - err_msg = f"Unexpected {type(world)=}: {world=}" + if not isinstance(world, list): # typeguard would be safer, but was lazy + err_msg = ( + f"Expected list[{SkyCoord.__name__}], " + + f"but got {type(world)=} of {world=}" + ) raise TypeError(err_msg) sky_coords: SkyCoord = world[0] world_coords: NDArray[np.float64] = ( From e5ae79d85c55346a9df2b48da3b779c2a54e0210 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 14:06:35 +0200 Subject: [PATCH 08/13] bugfix: added obs-publisher-did to restored obscore :fish: --- karabo/examples/SRCNet_rucio_meta_demo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py index d085445e..642c99ec 100644 --- a/karabo/examples/SRCNet_rucio_meta_demo.py +++ b/karabo/examples/SRCNet_rucio_meta_demo.py @@ -118,12 +118,13 @@ def main() -> None: # -----Imaging----- + # calc imaging params mean_freq = start_freq_hz + freq_inc_hz * (num_chan - 1) / 2 wavelength = const.c.value / mean_freq # in m synthesized_beam = wavelength / tel.max_baseline() # in rad imaging_cellsize = synthesized_beam / 3 # consider nyquist sampling theorem fov_deg = 2 * filter_radius_deg # angular fov - imaging_npixel_estimate = fov_deg / np.rad2deg(imaging_cellsize) # not even&rounded + imaging_npixel_estimate = fov_deg / np.rad2deg(imaging_cellsize) # not even|rounded imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...") @@ -168,7 +169,7 @@ def main() -> None: namespace=restored_rm.namespace, name=restored_rm.name, ) - vis_ocm.obs_publisher_did = obs_publisher_did + restored_ocm.obs_publisher_did = obs_publisher_did # fill/correct other fields of `ObsCoreMeta` here! # #####START##### From 62a59c74c9b9942c32c4b62fb4c6a764ad4062c1 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Wed, 14 Aug 2024 14:14:47 +0200 Subject: [PATCH 09/13] replaced SRCNet-rucio-meta with demo-script :handbag: --- karabo/examples/SRCNet_rucio_meta.py | 224 ++++++++++++++-------- karabo/examples/SRCNet_rucio_meta_demo.py | 185 ------------------ 2 files changed, 144 insertions(+), 265 deletions(-) delete mode 100644 karabo/examples/SRCNet_rucio_meta_demo.py diff --git a/karabo/examples/SRCNet_rucio_meta.py b/karabo/examples/SRCNet_rucio_meta.py index d10734c1..b544aea2 100644 --- a/karabo/examples/SRCNet_rucio_meta.py +++ b/karabo/examples/SRCNet_rucio_meta.py @@ -1,119 +1,183 @@ """Example script to attach Rucio ObsCore metadata data-products for ingestion. -This script probably needs adaption for your use-case. The parameters e.g. are not - customizable through an API. It also assumes that there's already a visibility - and image file available. Otherwise, you have to create them first. The manually - added values in this script are arbitrary to some extent and should be set (or not) - by yourself. - -An end-to-end workflow would add the needed parts at the end of its simulation. - However, we just operate on existing files to avoid example-duplication. +Here, we create a simulated visibilities and a cleaned image. This is just an example +script which should be highly adapted, e.g. you custom-sky, simulation params (for +larger data-products, simulation params as user-input, multiple simulations, etc. + +Be aware that API-changes can take place in future Karabo-versions. + +If not specified further (e.g. through `XDG_CACHE_HOME` or `FileHandler.root_stm`), +Karabo is using /tmp. Thus if you have a script which is producing large and/or many +data products, we suggest to adapt the cache-root to a volume with more space. """ from __future__ import annotations import os -from argparse import ArgumentParser from datetime import datetime +import numpy as np +from astropy import constants as const from astropy import units as u from karabo.data.obscore import FitsHeaderAxes, FitsHeaderAxis, ObsCoreMeta from karabo.data.src import RucioMeta -from karabo.imaging.image import Image +from karabo.imaging.imager_wsclean import WscleanImageCleaner, WscleanImageCleanerConfig +from karabo.simulation.interferometer import InterferometerSimulation from karabo.simulation.observation import Observation +from karabo.simulation.sky_model import SkyModel from karabo.simulation.telescope import Telescope -from karabo.simulation.visibility import Visibility +from karabo.simulator_backend import SimulatorBackend from karabo.util.helpers import get_rnd_str def main() -> None: - parser = ArgumentParser() - parser.add_argument( - "--dp-path", - required=True, - type=str, - help="Path to data product inode (most likely file).", + # sky-to-visibilities simulation + # the params for this example are highly adapted to not create large + # data products, because this is not the focus of this script. + sky = SkyModel.get_GLEAM_Sky(min_freq=72e6, max_freq=231e6) # in Hz + phase_center = [250, -80] # RA,DEC in deg + filter_radius_deg = 0.8 + sky = sky.filter_by_radius(0, filter_radius_deg, phase_center[0], phase_center[1]) + tel = Telescope.constructor("ASKAP", backend=SimulatorBackend.OSKAR) + start_freq_hz = 76e6 + num_chan = 16 + freq_inc_hz = 1e8 + + if num_chan < 1: + err_msg = f"{num_chan=} but must be < 1" + raise ValueError(err_msg) + if num_chan == 1 and freq_inc_hz != 0: + err_msg = f"{freq_inc_hz=} but must be 0 if only one channel is specified" + raise ValueError(err_msg) + + obs = Observation( + start_frequency_hz=start_freq_hz, + start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), + phase_centre_ra_deg=phase_center[0], + phase_centre_dec_deg=phase_center[1], + number_of_channels=num_chan, + frequency_increment_hz=freq_inc_hz, + number_of_time_steps=24, ) - parser.add_argument( - "--dp-type", - required=True, - type=str, - choices=["image", "visibility"], - help="Data product type. See `ObsCoreMeta` which file-formats are supported.", + interferometer_sim = InterferometerSimulation(channel_bandwidth_hz=freq_inc_hz) + vis = interferometer_sim.run_simulation( + telescope=tel, + sky=sky, + observation=obs, + backend=SimulatorBackend.OSKAR, ) - args = parser.parse_args() - dp_path: str = args.dp_path - dp_type: str = args.dp_type - - if not os.path.exists(dp_path): - err_msg = f"Inode {dp_path=} doesn't exist!" - raise RuntimeError(err_msg) - dp_path_meta = RucioMeta.get_meta_fname(fname=dp_path) - if os.path.exists(dp_path_meta): - err_msg = f"{dp_path_meta=} already exists!" - raise FileExistsError(err_msg) - if dp_type == "image": - image = Image(path=dp_path) - # `FitsHeaderAxes` may need adaption based on the structure of your .fits image - axes = FitsHeaderAxes(freq=FitsHeaderAxis(axis=4, unit=u.Hz)) - ocm = ObsCoreMeta.from_image(img=image, fits_axes=axes) - elif dp_type == "visibility": - vis = Visibility(vis_path=dp_path) # .vis supported, .ms not atm [07/2024] - # To extract additional information, `Telescope` & `Observation` should be - # provided with the same settings as `vis` was created. As mentioned in the - # module docstring, this is only necessary because we don't show the whole - # workflow here. - telescope = Telescope.constructor("ASKAP") - observation = Observation( # settings from notebook, of `minimal_visibility` - start_frequency_hz=100e6, - start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), - phase_centre_ra_deg=250.0, - phase_centre_dec_deg=-80.0, - number_of_channels=16, - frequency_increment_hz=1e6, - number_of_time_steps=24, - ) - ocm = ObsCoreMeta.from_visibility( - vis=vis, - calibrated=False, - tel=telescope, - obs=observation, + # here, I customize the `vis.vis_path` to not have the same file-name for each + # simulation which would be suboptimal for ingestion. Should probably be more + # specific for your use-case. + vis_path = os.path.join( + os.path.split(vis.vis_path)[0], + f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.vis", + ) + os.rename( + vis.vis_path, + vis_path, + ) + vis.vis_path = vis_path + + # create metadata of visibility (currently [08-24], .vis supported, casa .ms not) + vis_ocm = ObsCoreMeta.from_visibility( + vis=vis, + calibrated=False, + tel=tel, + obs=obs, + ) + vis_rm = RucioMeta( + namespace="testing", # needs to be specified by Rucio service + name=os.path.split(vis.vis_path)[-1], # remove path-infos for `name` + lifetime=86400, # 1 day + dataset_name=None, + meta=vis_ocm, + ) + # ObsCore mandatory fields + vis_ocm.obs_collection = "MRO/ASKAP" + obs_sim_id = 0 # nc for new simulation + user_rnd_str = get_rnd_str(k=7, seed=os.environ.get("USER")) + obs_id = f"karabo-{user_rnd_str}-{obs_sim_id}" # unique ID per user & simulation + vis_ocm.obs_id = obs_id + obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults + namespace=vis_rm.namespace, + name=vis_rm.name, + ) + vis_ocm.obs_publisher_did = obs_publisher_did + + # fill/correct other fields of `ObsCoreMeta` here! + # #####START##### + # HERE + # #####END####### + + vis_path_meta = RucioMeta.get_meta_fname(fname=vis.vis_path) + _ = vis_rm.to_dict(fpath=vis_path_meta) + print(f"Created {vis_path_meta=}") + + # -----Imaging----- + + # calc imaging params + mean_freq = start_freq_hz + freq_inc_hz * (num_chan - 1) / 2 + wavelength = const.c.value / mean_freq # in m + synthesized_beam = wavelength / tel.max_baseline() # in rad + imaging_cellsize = synthesized_beam / 3 # consider nyquist sampling theorem + fov_deg = 2 * filter_radius_deg # angular fov + imaging_npixel_estimate = fov_deg / np.rad2deg(imaging_cellsize) # not even|rounded + imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) + + print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...", flush=True) + restored = WscleanImageCleaner( + WscleanImageCleanerConfig( + imaging_npixel=imaging_npixel, + imaging_cellsize=imaging_cellsize, + niter=5000, # 10 times less than default ) - else: - err_msg = f"Unexpected {dp_type=}, allowed are only `dp-type` choices." - raise RuntimeError(err_msg) + ).create_cleaned_image( # currently, wsclean needs casa .ms, which is also created + ms_file_path=vis.ms_file_path, + ) - # adapt each field according to your needs + # customize fname for restored image for the same reason as visibilities + restored_image_path = os.path.join( + os.path.split(restored.path)[0], + f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.fits", + ) + os.rename( + restored.path, + restored_image_path, + ) + restored.path = restored_image_path - # be sure that name & namespace together are unique, e.g. by having different fnames - name = os.path.split(dp_path)[-1] - rm = RucioMeta( + # create metadata for restored .fits image + # `FitsHeaderAxes` may need adaption based on the structure of your .fits image + axes = FitsHeaderAxes(freq=FitsHeaderAxis(axis=3, unit=u.Hz)) + restored_ocm = ObsCoreMeta.from_image(img=restored, fits_axes=axes) + restored_rm = RucioMeta( namespace="testing", # needs to be specified by Rucio service - name=name, + name=os.path.split(restored.path)[-1], # remove path-infos for `name` lifetime=86400, # 1 day dataset_name=None, - meta=ocm, + meta=restored_ocm, ) - # ObsCore mandatory fields - ocm.obs_collection = "MRO/ASKAP" - obs_sim_id = 0 # unique observation-simulation ID of `USER` - user_rnd_str = get_rnd_str(k=7, seed=os.environ.get("USER")) - ocm.obs_id = f"karabo-{user_rnd_str}-{obs_sim_id}" + # some of the metadata is taken from above, since both data-products originate + # from the same observation + restored_ocm.obs_collection = vis_ocm.obs_collection + restored_ocm.obs_id = vis_ocm.obs_id obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults - namespace=rm.namespace, - name=rm.name, + namespace=restored_rm.namespace, + name=restored_rm.name, ) - ocm.obs_publisher_did = obs_publisher_did + restored_ocm.obs_publisher_did = obs_publisher_did - # fill other fields of `ObsCoreMeta` here! + # fill/correct other fields of `ObsCoreMeta` here! # #####START##### # HERE # #####END####### - _ = rm.to_dict(fpath=dp_path_meta) - print(f"Created {dp_path_meta}") + restored_path_meta = RucioMeta.get_meta_fname(fname=restored.path) + _ = restored_rm.to_dict(fpath=restored_path_meta) + print(f"Created {restored_path_meta=}") if __name__ == "__main__": diff --git a/karabo/examples/SRCNet_rucio_meta_demo.py b/karabo/examples/SRCNet_rucio_meta_demo.py deleted file mode 100644 index 642c99ec..00000000 --- a/karabo/examples/SRCNet_rucio_meta_demo.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Example script to attach Rucio ObsCore metadata data-products for ingestion. - -Here, we create a simulated visibilities and a cleaned image. This is just an example -script which can be highly adapted, e.g. you custom-sky, simulation params (for larger -data-products, simulation params as user-input, multiple simulations, etc. - -Be aware that API-changes can take place in future Karabo-versions. This script is -based on `SRCNet_rucio_meta.py` but contains a full workflow for a single simulation. - -If not specified further (e.g. through `XDG_CACHE_HOME` or `FileHandler.root_stm`), -Karabo is using /tmp. Thus if you have a script which is producing large and/or many -data products, we suggest to adapt the cache-root to a volume with more space. -""" - -from __future__ import annotations - -import os -from datetime import datetime - -import numpy as np -from astropy import constants as const -from astropy import units as u - -from karabo.data.obscore import FitsHeaderAxes, FitsHeaderAxis, ObsCoreMeta -from karabo.data.src import RucioMeta -from karabo.imaging.imager_wsclean import WscleanImageCleaner, WscleanImageCleanerConfig -from karabo.simulation.interferometer import InterferometerSimulation -from karabo.simulation.observation import Observation -from karabo.simulation.sky_model import SkyModel -from karabo.simulation.telescope import Telescope -from karabo.simulator_backend import SimulatorBackend -from karabo.util.helpers import get_rnd_str - - -def main() -> None: - # sky-to-visibilities simulation - # the params for this example are highly adapted to not create large - # data products, because this is not the focus of this script. - sky = SkyModel.get_GLEAM_Sky(min_freq=72e6, max_freq=231e6) # in Hz - phase_center = [250, -80] # RA,DEC in deg - filter_radius_deg = 0.8 - sky = sky.filter_by_radius(0, filter_radius_deg, phase_center[0], phase_center[1]) - tel = Telescope.constructor("ASKAP", backend=SimulatorBackend.OSKAR) - start_freq_hz = 76e6 - num_chan = 16 - freq_inc_hz = 1e8 - - if num_chan < 1: - err_msg = f"{num_chan=} but must be < 1" - raise ValueError(err_msg) - if num_chan == 1 and freq_inc_hz != 0: - err_msg = f"{freq_inc_hz=} but must be 0 if only one channel is specified" - raise ValueError(err_msg) - - obs = Observation( - start_frequency_hz=start_freq_hz, - start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), - phase_centre_ra_deg=phase_center[0], - phase_centre_dec_deg=phase_center[1], - number_of_channels=num_chan, - frequency_increment_hz=freq_inc_hz, - number_of_time_steps=24, - ) - interferometer_sim = InterferometerSimulation(channel_bandwidth_hz=freq_inc_hz) - vis = interferometer_sim.run_simulation( - telescope=tel, - sky=sky, - observation=obs, - backend=SimulatorBackend.OSKAR, - ) - # here, I customize the `vis.vis_path` to not have the same file-name for each - # simulation which would be suboptimal for ingestion. Should probably be more - # specific for your use-case. - vis_path = os.path.join( - os.path.split(vis.vis_path)[0], - f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.vis", - ) - os.rename( - vis.vis_path, - vis_path, - ) - vis.vis_path = vis_path - - # create metadata of visibility (currently [08-24], .vis supported, casa .ms not) - vis_ocm = ObsCoreMeta.from_visibility( - vis=vis, - calibrated=False, - tel=tel, - obs=obs, - ) - vis_rm = RucioMeta( - namespace="testing", # needs to be specified by Rucio service - name=os.path.split(vis.vis_path)[-1], # remove path-infos for `name` - lifetime=86400, # 1 day - dataset_name=None, - meta=vis_ocm, - ) - # ObsCore mandatory fields - vis_ocm.obs_collection = "MRO/ASKAP" - obs_sim_id = 0 # nc for new simulation - user_rnd_str = get_rnd_str(k=7, seed=os.environ.get("USER")) - obs_id = f"karabo-{user_rnd_str}-{obs_sim_id}" # unique ID per user & simulation - vis_ocm.obs_id = obs_id - obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults - namespace=vis_rm.namespace, - name=vis_rm.name, - ) - vis_ocm.obs_publisher_did = obs_publisher_did - - # fill/correct other fields of `ObsCoreMeta` here! - # #####START##### - # HERE - # #####END####### - - vis_path_meta = RucioMeta.get_meta_fname(fname=vis.vis_path) - _ = vis_rm.to_dict(fpath=vis_path_meta) - print(f"Created {vis_path_meta=}") - - # -----Imaging----- - - # calc imaging params - mean_freq = start_freq_hz + freq_inc_hz * (num_chan - 1) / 2 - wavelength = const.c.value / mean_freq # in m - synthesized_beam = wavelength / tel.max_baseline() # in rad - imaging_cellsize = synthesized_beam / 3 # consider nyquist sampling theorem - fov_deg = 2 * filter_radius_deg # angular fov - imaging_npixel_estimate = fov_deg / np.rad2deg(imaging_cellsize) # not even|rounded - imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) - - print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...") - restored = WscleanImageCleaner( - WscleanImageCleanerConfig( - imaging_npixel=imaging_npixel, - imaging_cellsize=imaging_cellsize, - niter=5000, # 10 times less than default - ) - ).create_cleaned_image( # currently, wsclean needs casa .ms, which is also created - ms_file_path=vis.ms_file_path, - ) - - # customize fname for restored image for the same reason as visibilities - restored_image_path = os.path.join( - os.path.split(restored.path)[0], - f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.fits", - ) - os.rename( - restored.path, - restored_image_path, - ) - restored.path = restored_image_path - - # create metadata for restored .fits image - # `FitsHeaderAxes` may need adaption based on the structure of your .fits image - axes = FitsHeaderAxes(freq=FitsHeaderAxis(axis=3, unit=u.Hz)) - restored_ocm = ObsCoreMeta.from_image(img=restored, fits_axes=axes) - restored_rm = RucioMeta( - namespace="testing", # needs to be specified by Rucio service - name=os.path.split(restored.path)[-1], # remove path-infos for `name` - lifetime=86400, # 1 day - dataset_name=None, - meta=restored_ocm, - ) - # ObsCore mandatory fields - # some of the metadata is taken from above, since both data-products originate - # from the same observation - restored_ocm.obs_collection = vis_ocm.obs_collection - restored_ocm.obs_id = vis_ocm.obs_id - obs_publisher_did = RucioMeta.get_ivoid( # rest args are defaults - namespace=restored_rm.namespace, - name=restored_rm.name, - ) - restored_ocm.obs_publisher_did = obs_publisher_did - - # fill/correct other fields of `ObsCoreMeta` here! - # #####START##### - # HERE - # #####END####### - - restored_path_meta = RucioMeta.get_meta_fname(fname=restored.path) - _ = restored_rm.to_dict(fpath=restored_path_meta) - print(f"Created {restored_path_meta=}") - - -if __name__ == "__main__": - main() From bac640e1f84d2e6bb52439adebd25ba211170067 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Thu, 22 Aug 2024 17:51:18 +0200 Subject: [PATCH 10/13] addressed PR requests :herb: --- karabo/examples/SRCNet_rucio_meta.py | 45 +++++++--------------------- karabo/simulation/observation.py | 10 +++++++ 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/karabo/examples/SRCNet_rucio_meta.py b/karabo/examples/SRCNet_rucio_meta.py index b544aea2..9c438aac 100644 --- a/karabo/examples/SRCNet_rucio_meta.py +++ b/karabo/examples/SRCNet_rucio_meta.py @@ -2,11 +2,11 @@ Here, we create a simulated visibilities and a cleaned image. This is just an example script which should be highly adapted, e.g. you custom-sky, simulation params (for -larger data-products, simulation params as user-input, multiple simulations, etc. +larger data-products, simulation params as user-input, multiple simulations, etc.). Be aware that API-changes can take place in future Karabo-versions. -If not specified further (e.g. through `XDG_CACHE_HOME` or `FileHandler.root_stm`), +If not specified further (e.g. using `XDG_CACHE_HOME` or `FileHandler.root_stm`), Karabo is using /tmp. Thus if you have a script which is producing large and/or many data products, we suggest to adapt the cache-root to a volume with more space. """ @@ -28,6 +28,7 @@ from karabo.simulation.sky_model import SkyModel from karabo.simulation.telescope import Telescope from karabo.simulator_backend import SimulatorBackend +from karabo.util.file_handler import FileHandler from karabo.util.helpers import get_rnd_str @@ -44,13 +45,6 @@ def main() -> None: num_chan = 16 freq_inc_hz = 1e8 - if num_chan < 1: - err_msg = f"{num_chan=} but must be < 1" - raise ValueError(err_msg) - if num_chan == 1 and freq_inc_hz != 0: - err_msg = f"{freq_inc_hz=} but must be 0 if only one channel is specified" - raise ValueError(err_msg) - obs = Observation( start_frequency_hz=start_freq_hz, start_date_and_time=datetime(2024, 3, 15, 10, 46, 0), @@ -60,25 +54,17 @@ def main() -> None: frequency_increment_hz=freq_inc_hz, number_of_time_steps=24, ) - interferometer_sim = InterferometerSimulation(channel_bandwidth_hz=freq_inc_hz) + # define any unique (required for ingestion) output-file-path + vis_path = os.path.join(FileHandler.stm(), "my-unique-vis-fname.vis") + interferometer_sim = InterferometerSimulation( + vis_path=vis_path, channel_bandwidth_hz=freq_inc_hz + ) vis = interferometer_sim.run_simulation( telescope=tel, sky=sky, observation=obs, backend=SimulatorBackend.OSKAR, ) - # here, I customize the `vis.vis_path` to not have the same file-name for each - # simulation which would be suboptimal for ingestion. Should probably be more - # specific for your use-case. - vis_path = os.path.join( - os.path.split(vis.vis_path)[0], - f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.vis", - ) - os.rename( - vis.vis_path, - vis_path, - ) - vis.vis_path = vis_path # create metadata of visibility (currently [08-24], .vis supported, casa .ms not) vis_ocm = ObsCoreMeta.from_visibility( @@ -96,7 +82,7 @@ def main() -> None: ) # ObsCore mandatory fields vis_ocm.obs_collection = "MRO/ASKAP" - obs_sim_id = 0 # nc for new simulation + obs_sim_id = 0 # inc/change for new simulation user_rnd_str = get_rnd_str(k=7, seed=os.environ.get("USER")) obs_id = f"karabo-{user_rnd_str}-{obs_sim_id}" # unique ID per user & simulation vis_ocm.obs_id = obs_id @@ -127,6 +113,7 @@ def main() -> None: imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...", flush=True) + restored_path = os.path.join(FileHandler.stm(), "my-unique-image.fits") restored = WscleanImageCleaner( WscleanImageCleanerConfig( imaging_npixel=imaging_npixel, @@ -135,19 +122,9 @@ def main() -> None: ) ).create_cleaned_image( # currently, wsclean needs casa .ms, which is also created ms_file_path=vis.ms_file_path, + output_fits_path=restored_path, ) - # customize fname for restored image for the same reason as visibilities - restored_image_path = os.path.join( - os.path.split(restored.path)[0], - f"gleam-ra{phase_center[0]}-dec{phase_center[1]}.fits", - ) - os.rename( - restored.path, - restored_image_path, - ) - restored.path = restored_image_path - # create metadata for restored .fits image # `FitsHeaderAxes` may need adaption based on the structure of your .fits image axes = FitsHeaderAxes(freq=FitsHeaderAxis(axis=3, unit=u.Hz)) diff --git a/karabo/simulation/observation.py b/karabo/simulation/observation.py index 3b3a20db..58225520 100644 --- a/karabo/simulation/observation.py +++ b/karabo/simulation/observation.py @@ -68,6 +68,16 @@ def __init__( Defaults to 1. """ + if number_of_channels < 1: + err_msg = f"{number_of_channels=} but must be < 1" + raise ValueError(err_msg) + if number_of_channels == 1 and frequency_increment_hz != 0: + err_msg = ( + f"{frequency_increment_hz=} but must be 0 if only one " + + "channel is specified" + ) + raise ValueError(err_msg) + self.start_frequency_hz = start_frequency_hz if isinstance(start_date_and_time, str): From 65f2e1ccde789cc7dbf4859c7283bcfd90737918 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Thu, 22 Aug 2024 17:59:06 +0200 Subject: [PATCH 11/13] addressed PR requests :raised_hand: --- karabo/examples/SRCNet_rucio_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karabo/examples/SRCNet_rucio_meta.py b/karabo/examples/SRCNet_rucio_meta.py index 9c438aac..a1db6db0 100644 --- a/karabo/examples/SRCNet_rucio_meta.py +++ b/karabo/examples/SRCNet_rucio_meta.py @@ -113,7 +113,7 @@ def main() -> None: imaging_npixel = int(np.floor((imaging_npixel_estimate + 1) / 2.0) * 2.0) print(f"Imaging: {imaging_npixel=}, {imaging_cellsize=} ...", flush=True) - restored_path = os.path.join(FileHandler.stm(), "my-unique-image.fits") + restored_path = os.path.join(FileHandler.stm(), "my-unique-image-fname.fits") restored = WscleanImageCleaner( WscleanImageCleanerConfig( imaging_npixel=imaging_npixel, From a17e4abd6f1402826f1bfed12fa253c21c6413ee Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Fri, 23 Aug 2024 10:13:26 +0200 Subject: [PATCH 12/13] removed inc-freq-hz check in Observation init :zzz: --- karabo/simulation/observation.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/karabo/simulation/observation.py b/karabo/simulation/observation.py index 58225520..3b3a20db 100644 --- a/karabo/simulation/observation.py +++ b/karabo/simulation/observation.py @@ -68,16 +68,6 @@ def __init__( Defaults to 1. """ - if number_of_channels < 1: - err_msg = f"{number_of_channels=} but must be < 1" - raise ValueError(err_msg) - if number_of_channels == 1 and frequency_increment_hz != 0: - err_msg = ( - f"{frequency_increment_hz=} but must be 0 if only one " - + "channel is specified" - ) - raise ValueError(err_msg) - self.start_frequency_hz = start_frequency_hz if isinstance(start_date_and_time, str): From 2feefbae11bc8f967186eef6de3cb9ab39ef9a70 Mon Sep 17 00:00:00 2001 From: "lukas.gehrig" Date: Thu, 12 Sep 2024 15:30:22 +0200 Subject: [PATCH 13/13] specified mypy ignore comment in download :arrow_backward: --- karabo/data/external_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karabo/data/external_data.py b/karabo/data/external_data.py index d2c76959..de98b6bf 100644 --- a/karabo/data/external_data.py +++ b/karabo/data/external_data.py @@ -83,7 +83,7 @@ def download( os.makedirs(download_dir, exist_ok=True) desc = f"Downloading {url} to {local_file_path}" - response.raw.read = functools.partial( # type: ignore + response.raw.read = functools.partial( # type: ignore[method-assign] response.raw.read, decode_content=True ) with tqdm.wrapattr(