diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f131e13..171e7dd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,9 +1,9 @@ version: 2 build: - os: "ubuntu-20.04" + os: 'ubuntu-20.04' tools: - python: "mambaforge-4.10" + python: 'mambaforge-4.10' # Build documentation in the docs/ directory with Sphinx sphinx: @@ -15,4 +15,4 @@ conda: python: install: - method: pip - path: . \ No newline at end of file + path: . diff --git a/environment-nocuda.yml b/environment-nocuda.yml index a183d63..a150962 100644 --- a/environment-nocuda.yml +++ b/environment-nocuda.yml @@ -21,4 +21,5 @@ dependencies: - astra-toolbox - bqplot - pip: - - bqplot-image-gl \ No newline at end of file + - bqplot-image-gl + - tomoscan diff --git a/environment.yml b/environment.yml index 0b1e4b0..f4a174d 100644 --- a/environment.yml +++ b/environment.yml @@ -24,3 +24,4 @@ dependencies: - bqplot - pip: - bqplot-image-gl + - tomoscan diff --git a/examples/data/nxtomo/create_tomo.py b/examples/data/nxtomo/create_tomo.py new file mode 100644 index 0000000..ffc7a1e --- /dev/null +++ b/examples/data/nxtomo/create_tomo.py @@ -0,0 +1,99 @@ +import os +import pathlib + +import dxchange +import numpy +from nxtomo import NXtomo +from nxtomo.nxobject.nxdetector import ImageKey +from tomoscan.esrf.scan.nxtomoscan import NXtomoScan + +THIS_PATH = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) +DATA_PATH = pathlib.Path(THIS_PATH / "tomo_00077.h5") # from tomobank + +# I took most of this from the tomoscan tutorial, but I had to modify it. +# I am not sure how you are storing your "projections" data. Basically, we want +# projection images along x and y in a [Z, Y, X] array, like is output by dxchange.read_aps_32id. +# You can modify your data import method in tomopyui.backend.io to get it into the right format. +proj, flat, dark, theta = dxchange.read_aps_32id(fname=DATA_PATH, proj=(0, 477)) + +binning = 8 +proj_binned = proj[:, ::binning, ::binning] +dark_binned = dark[:, ::binning, ::binning] +flat_binned = flat[:, ::binning, ::binning] +assert proj_binned.shape[2] == dark_binned.shape[2] == flat_binned.shape[2] +assert proj_binned.shape[1] == dark_binned.shape[1] == flat_binned.shape[1] +proj_rotation_angles = theta * 180 / numpy.pi +assert len(proj_rotation_angles) == len(proj_binned) + +my_nxtomo = NXtomo() + +# create the array +data = numpy.concatenate( + [ + dark_binned, + flat_binned, + proj_binned, + ] +) +assert data.ndim == 3 +print(data.shape) +# then register the data to the detector +my_nxtomo.instrument.detector.data = data + +image_key_control = numpy.concatenate( + [ + [ImageKey.DARK_FIELD] * len(dark_binned), + [ImageKey.FLAT_FIELD] * len(flat_binned), + [ImageKey.PROJECTION] * len(proj_binned), + ] +) + +# insure with have the same number of frames and image key +assert len(image_key_control) == len(data) +# print position of flats in the sequence +print("flats indexes are", numpy.where(image_key_control == ImageKey.FLAT_FIELD)) +# then register the image keys to the detector +my_nxtomo.instrument.detector.image_key_control = image_key_control + +rotation_angle = numpy.concatenate( + [ + [0 for x in range(len(dark_binned))], + [0 for x in range(len(flat_binned))], + proj_rotation_angles, + ] +) +assert len(rotation_angle) == len(data) +# register rotation angle to the sample +my_nxtomo.sample.rotation_angle = rotation_angle + +my_nxtomo.instrument.detector.field_of_view = "Full" + +my_nxtomo.instrument.detector.x_pixel_size = ( + my_nxtomo.instrument.detector.y_pixel_size +) = 1e-7 # pixel size must be provided in SI: meter +my_nxtomo.instrument.detector.x_pixel_size = ( + my_nxtomo.instrument.detector.y_pixel_size +) = 0.1 +my_nxtomo.instrument.detector.x_pixel_size.unit = ( + my_nxtomo.instrument.detector.y_pixel_size.unit +) = "micrometer" + +nx_tomo_file_path = pathlib.Path(THIS_PATH / "tomo_00077.nx") +my_nxtomo.save(file_path=str(nx_tomo_file_path), data_path="entry", overwrite=True) + +has_tomoscan = False +try: + import tomoscan +except ImportError: + has_tomoscan = False + from tomoscan.esrf import NXtomoScan + from tomoscan.validator import ReconstructionValidator + + has_tomoscan = True + +if has_tomoscan: + scan = NXtomoScan(nx_tomo_file_path, entry="entry") + validator = ReconstructionValidator( + scan, check_phase_retrieval=False, check_values=True + ) + assert validator.is_valid() diff --git a/examples/data/nxtomo/tomo_00077.nx b/examples/data/nxtomo/tomo_00077.nx new file mode 100644 index 0000000..d2ad2cc Binary files /dev/null and b/examples/data/nxtomo/tomo_00077.nx differ diff --git a/tomopyui/backend/io.py b/tomopyui/backend/io.py index 32e6f7b..74e28fc 100644 --- a/tomopyui/backend/io.py +++ b/tomopyui/backend/io.py @@ -1,10 +1,10 @@ import copy import datetime import json +import multiprocessing import os import pathlib import re -import multiprocessing import time from abc import ABC, abstractmethod @@ -17,12 +17,14 @@ import olefile import pandas as pd import scipy.ndimage as ndi +import silx.io import tifffile as tf import tomopy.prep.normalize as tomopy_normalize from ipywidgets import * from skimage.exposure import rescale_intensity from skimage.util import img_as_float32 from tomopy.sim.project import angles as angle_maker +from tomoscan.esrf.scan.nxtomoscan import NXtomoScan from tomopyui.backend.util.dask_downsample import pyramid_reduce_gaussian from tomopyui.backend.util.dxchange.reader import read_ole_metadata, read_txrm, read_xrm @@ -1970,6 +1972,147 @@ def save_normalized_metadata(self, import_time=None, parent_metadata=None): return metadata +class RawProjections_NXTomo(RawProjectionsBase): + + def __init__(self): + super().__init__() + self.allowed_extensions = [".nx"] + self.metadata = Metadata_NXTomo_Raw() + + def import_filedir_all(self, filedir): + pass + + def import_filedir_projections(self, filedir): + pass + + def import_filedir_flats(self, filedir): + pass + + def import_filedir_darks(self, filedir): + pass + + def import_nxtomo( + self, scan: NXtomoScan + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + projs = [] + if scan.projections is None: + raise ValueError("No projections found in the scan") + for proj in scan.projections.values(): + # print(proj) + projs.append(silx.io.utils.get_data(proj)) + + projs = np.array(projs) + darks = [] + if scan.darks is None: + raise ValueError("No projections found in the scan") + for dark in scan.darks.values(): + darks.append(silx.io.utils.get_data(dark)) + + darks = np.array(darks) + flats = [] + + if scan.flats is None: + raise ValueError("No projections found in the scan") + for flat in scan.flats.values(): + flats.append(silx.io.utils.get_data(flat)) + + flats = np.array(flats) + + return projs, flats, darks + + def import_file_all(self, Uploader): + self.import_status_label = Uploader.import_status_label + self.tic = time.perf_counter() + self.filedir = Uploader.filedir + self.filename = Uploader.filename + self.filepath = self.filedir / self.filename + self.metadata = Uploader.reset_metadata_to() + self.metadata.load_metadata_h5(self.filepath) + self.metadata.set_attributes_from_metadata(self) + self.import_status_label.value = "Importing" + scan = NXtomoScan(scan=self.filepath) + + self._data, self.flats, self.darks = self.import_nxtomo(scan) + + self.data = self._data + self.metadata.set_metadata(self) + self.metadata.save_metadata() + self.imported = True + self.make_import_savedir("tomopyui-" + str(self.filepath.stem)) + self.import_status_label.value = "Normalizing" + + # Here is the normaliation. You can adjust this depending on what + # you do for normalization. + self.normalize_nf() + self._data[self._data < 0] = 0.0 + self._data[np.isnan(self._data)] = 0.0 + self._data[np.isinf(self._data)] = 0.0 + self.data = self._data + self.import_status_label.value = "Calculating histogram of raw data and saving." + + # What we want here in self.data is an array of projection images. + # These should be [Z, Y, X], where Z is the rotation direction, and X and Y + # are projection image dimensions. + self.data = da.from_array(self.data, chunks={0: "auto", 1: -1, 2: -1}) + self.import_status_label.value = "Saving metadata." + self.save_data_and_metadata(Uploader) + self.data_hierarchy_level = 1 + self.metadata.set_metadata(self) + self.metadata.filedir = self.import_savedir + self.metadata.filename = "import_metadata.json" + self.metadata.save_metadata() + self._close_hdf_file() + + def import_metadata(self, filepath=None): + if filepath is None: + filepath = self.filepath + self.metadata.load_metadata_h5(filepath) + self.metadata.set_attributes_from_metadata(self) + + def import_file_projections(self, filepath): + pass + + def import_file_flats(self, filepath): + pass + + def import_file_darks(self, filepath): + pass + + def import_file_angles(self, filepath): + pass + + def save_normalized_metadata(self, import_time=None, parent_metadata=None): + metadata = Metadata_NXTomo_Prenorm() + metadata.filedir = self.filedir + metadata.metadata = parent_metadata.copy() + if parent_metadata is not None: + metadata.metadata["parent_metadata"] = parent_metadata.copy() + if import_time is not None: + metadata.metadata["import_time"] = import_time + metadata.set_metadata(self) + metadata.save_metadata() + return metadata + + def save_data_and_metadata(self, Uploader): + """ + Saves current data and metadata in import_savedir. + """ + self.filedir = self.import_savedir + self._dask_hist_and_save_data() + self.saved_as_tiff = False + _metadata = self.metadata.metadata.copy() + if Uploader.save_tiff_on_import_checkbox.value: + Uploader.import_status_label.value = "Saving projections as .tiff." + self.saved_as_tiff = True + self.save_normalized_as_tiff() + self.metadata.metadata["saved_as_tiff"] = True + self.metadata.filedir = self.filedir + self.toc = time.perf_counter() + self.metadata = self.save_normalized_metadata(self.toc - self.tic, _metadata) + Uploader.import_status_label.value = "Checking for downsampled data." + self._check_downsampled_data(label=Uploader.import_status_label) + + class Metadata(ABC): """ Base class for all metadatas. @@ -2066,6 +2209,12 @@ def parse_metadata_type(filepath: pathlib.Path = None, metadata=None): if metadata["metadata_type"] == "ALS832_Raw": metadata_instance = Metadata_ALS_832_Raw() + # Nxtomo + if metadata["metadata_type"] == "nxtomo_raw": + metadata_instance = Metadata_NXTomo_Raw() + if metadata["metadata_type"] == "nxtomo_prenorm": + metadata_instance = Metadata_NXTomo_Prenorm() + # Metadata through rest of processing pipeline if metadata["metadata_type"] == "Prep": metadata_instance = Metadata_Prep() @@ -3610,6 +3759,193 @@ def metadata_to_DataFrame(self): self.dataframe = s +class Metadata_NXTomo_Raw(Metadata): + def __init__(self): + super().__init__() + self.filename = "raw_metadata.json" + self.metadata["metadata_type"] = "nxtomo_raw" + self.metadata["data_hierarchy_level"] = 0 + self.table_label.value = "NXtomo Raw Metadata" + + def load_metadata_h5(self, filepath): + scan = NXtomoScan(scan=filepath) + self.filedir = filepath.parent + self.filepath = filepath + + frame = None + if scan.projections is None: + raise ValueError("No projections found in the file.") + for key in scan.projections: + frame = silx.io.utils.get_data(scan.projections[key]) + if frame is not None: + break + + if not isinstance(frame, np.ndarray): + raise ValueError("Frames are not numpy arrays.") + + # You can customize these, but you will need the pixels in X/Y, plus + # the flats locations for normalize_nf (tomopy's version of normalization). + # You also need the angles... + self.metadata["flats_locations"] = [loc for loc in scan.flats.keys()] + self.metadata["darks_locations"] = [loc for loc in scan.darks.keys()] + self.metadata["pxY"] = frame.shape[0] + self.metadata["pxX"] = frame.shape[1] + self.metadata["pxZ"] = len(scan.projections) + self.metadata["pxsize"] = scan.pixel_size * 1e6 if scan.pixel_size else 1 + self.metadata["px_size_units"] = "um" + self.metadata["energy_float"] = float(scan.energy) if scan.energy else 8.0 + self.metadata["kev"] = self.metadata["energy_float"] + self.metadata["energy_str"] = str(self.metadata["energy_float"]) + self.metadata["energy_units"] = "keV" + proj_keys = list(scan.projections.keys()) + self.metadata["angles_deg"] = [scan.rotation_angle[x] for x in proj_keys] + self.metadata["angles_rad"] = angle_maker( + len(self.metadata["angles_deg"]), + self.metadata["angles_deg"][0], + self.metadata["angles_deg"][-1], + ) + + def set_metadata(self, projections): + """ + Sets metadata from the APS h5 filetype + """ + self.metadata["pxX"] = projections.pxX + self.metadata["pxY"] = projections.pxY + self.metadata["pxZ"] = projections.pxZ + self.metadata["pxsize"] = projections.px_size + self.metadata["px_size_units"] = "um" + if projections.energy is not None: + self.metadata["kev"] = projections.energy + self.metadata["energy_units"] = "keV" + else: + self.metadata["kev"] = 8000.0 + self.metadata["energy_units"] = "keV" + if projections.angles_deg is not None: + self.metadata["angles_deg"] = list(projections.angles_deg) + self.metadata["angles_rad"] = list(projections.angles_rad) + + def set_attributes_from_metadata(self, projections): + projections.pxX = self.metadata["pxX"] + projections.pxY = self.metadata["pxY"] + projections.pxZ = self.metadata["pxZ"] + projections.px_size = self.metadata["pxsize"] + projections.px_size_units = self.metadata["px_size_units"] + projections.energy = self.metadata["kev"] + projections.angles_rad = self.metadata["angles_rad"] + projections.angles_deg = self.metadata["angles_deg"] + projections.flats_ind = self.metadata["flats_locations"] + + # You can customize the metadata display here. + def metadata_to_DataFrame(self): + + # create headers and data for table + top_headers = [] + middle_headers = [] + data = [] + # Image information + top_headers.append(["Image Information"]) + middle_headers.append(["X Pixels", "Y Pixels", "Num. θ"]) + data.append( + [ + self.metadata["pxX"], + self.metadata["pxY"], + self.metadata["pxZ"], + ] + ) + + top_headers.append(["Experiment Settings"]) + middle_headers.append(["Energy (keV)"]) + data.append( + [ + self.metadata["kev"], + ] + ) + + # create dataframe with the above settings + df = pd.DataFrame( + [data[0]], + columns=pd.MultiIndex.from_product([top_headers[0], middle_headers[0]]), + ) + for i in range(len(middle_headers)): + if i == 0: + continue + else: + newdf = pd.DataFrame( + [data[i]], + columns=pd.MultiIndex.from_product( + [top_headers[i], middle_headers[i]] + ), + ) + df = df.join(newdf) + + s = df.style.hide(axis="index") + s.set_table_styles( + { + ("Experiment Settings", "Energy (keV)"): [ + {"selector": "td", "props": "border-left: 1px solid white"}, + {"selector": "th", "props": "border-left: 1px solid white"}, + ], + }, + overwrite=False, + ) + + s.set_table_styles( + [ + {"selector": "th.col_heading", "props": "text-align: center;"}, + {"selector": "th.col_heading.level0", "props": "font-size: 1.2em;"}, + {"selector": "td", "props": "text-align: center;" "font-size: 1.2em;"}, + { + "selector": "th:not(.index_name)", + "props": "background-color: #0F52BA; color: white;", + }, + ], + overwrite=False, + ) + + self.dataframe = s + + +class Metadata_NXTomo_Prenorm(Metadata_NXTomo_Raw): + def __init__(self): + super().__init__() + self.filename = "import_metadata.json" + self.metadata["metadata_type"] = "nxtomo_prenorm" + self.metadata["data_hierarchy_level"] = 1 + self.data_hierarchy_level = self.metadata["data_hierarchy_level"] + self.table_label.value = "" + + def set_metadata(self, projections): + super().set_metadata(projections) + self.filename = "import_metadata.json" + self.metadata["metadata_type"] = "nxtomo_prenorm" + # set to 1 + self.metadata["data_hierarchy_level"] = 1 + + # You can again change this. Some of them might not be accurate here, so double check... + def set_attributes_from_metadata(self, projections): + projections.pxY = self.metadata["pxX"] + projections.pxX = self.metadata["pxY"] + projections.pxZ = self.metadata["pxZ"] + projections.px_size = self.metadata["pxsize"] + projections.px_size_units = self.metadata["px_size_units"] + projections.energy = self.metadata["kev"] * 1000 + projections.units = "eV" + projections.angles_deg = self.metadata["angles_deg"] + projections.angles_rad = self.metadata["angles_rad"] + projections.angle_start = projections.angles_rad[0] + projections.angle_end = projections.angles_rad[-1] + + def metadata_to_DataFrame(self): + self.dataframe = None + + def create_metadata_box(self): + """ + Method overloaded because the metadata table is the same as the superclass. + This avoids a space between tables during display. + """ + self.metadata_vbox = Output() + + class Metadata_APS_Prenorm(Metadata_APS_Raw): """ Prenormalized metadata class. The table produced by this function may look nearly diff --git a/tomopyui/widgets/imports.py b/tomopyui/widgets/imports.py index 0fd5065..c03d697 100644 --- a/tomopyui/widgets/imports.py +++ b/tomopyui/widgets/imports.py @@ -13,16 +13,16 @@ from ipyfilechooser import FileChooser from ipywidgets import * -from tomopyui._sharedvars import ( - extend_description_style, -) +from tomopyui._sharedvars import extend_description_style from tomopyui.backend.io import ( Metadata, Metadata_Align, Metadata_ALS_832_Raw, Metadata_APS_Raw, Metadata_General_Prenorm, + Metadata_NXTomo_Raw, Projections_Prenormalized, + RawProjections_NXTomo, RawProjectionsHDF5_ALS832, RawProjectionsHDF5_APS, RawProjectionsTiff_SSRL62B, @@ -319,6 +319,66 @@ def __init__(self): super().__init__() self.raw_uploader = RawUploader_APS(self) self.make_tab() + + +class Import_NxTomo(ImportBase): + """""" + + def __init__(self): + super().__init__() + self.raw_uploader = RawUploader_NxTomo(self) + self.make_tab() + + def make_tab(self): + + self.switch_data_buttons = HBox( + [self.use_raw_button.button, self.use_prenorm_button.button], + layout=Layout(justify_content="center"), + ) + + # raw_import = HBox([item for sublist in raw_import for item in sublist]) + self.raw_accordion = Accordion( + children=[ + VBox( + [ + HBox( + [self.raw_uploader.metadata_table_output], + layout=Layout(justify_content="center"), + ), + HBox( + [self.raw_uploader.progress_output], + layout=Layout(justify_content="center"), + ), + self.raw_uploader.app, + ] + ), + ], + selected_index=None, + titles=("Import and Normalize Raw Data",), + ) + + self.prenorm_accordion = Accordion( + children=[ + VBox( + [ + HBox( + [self.prenorm_uploader.metadata_table_output], + layout=Layout(justify_content="center"), + ), + self.prenorm_uploader.app, + ] + ), + ], + selected_index=None, + titles=("Import Prenormalized Data",), + ) + + self.tab = VBox( + [ + self.raw_accordion, + self.prenorm_accordion, + ] + ) class UploaderBase(ABC): @@ -1510,6 +1570,94 @@ def create_app(self): ) +class RawUploader_NxTomo(UploaderBase): + """ + + """ + + def __init__(self, Import): + super().__init__() # look at UploaderBase __init__() + self._init_widgets() + self.projections = RawProjections_NXTomo() + self.reset_metadata_to = Metadata_NXTomo_Raw + self.Import = Import + self.filechooser.title = "Import Raw nx File" + self.filetypes_to_look_for = [".nx"] + self.files_not_found_str = "Choose a directory with an nx file." + + # Creates the app that goes into the Import object + self.create_app() + + def _init_widgets(self): + """ + You can make your widgets more fancy with this function. See the example in + RawUploader_SSRL62C. + """ + pass + + def import_data(self): + """ + This is what is called when you click the blue import button on the frontend. + """ + with self.progress_output: + self.progress_output.clear_output() + display(self.import_status_label) + tic = time.perf_counter() + self.projections.import_file_all(self) + toc = time.perf_counter() + self.projections.metadatas = Metadata.get_metadata_hierarchy( + self.projections.metadata.filedir / self.projections.metadata.filename + ) + self.import_status_label.value = f"Import and normalization took {toc-tic:.0f}s" + self.viewer.plot(self.projections) + + def update_filechooser_from_quicksearch(self, h5files): + """ + This is what is called when you update the quick path search bar. Right now, + this is very basic. If you want to see a more complex version of this you can + look at the example in PrenormUploader. + + This is called after _update_filechooser_from_quicksearch in UploaderBase. + """ + if len(h5files) == 1: + self.filename = h5files[0] + elif len(h5files) > 1 and self.filename is None: + self.find_metadata_status_label.value = ( + "Multiple h5 files found in this" + + " directory. Choose one with the file browser." + ) + self.import_button.disable() + return + self.projections.metadata = self.reset_metadata_to() + self.projections.import_metadata(self.filedir / self.filename) + self.projections.metadata.metadata_to_DataFrame() + with self.metadata_table_output: + self.metadata_table_output.clear_output(wait=True) + display(self.projections.metadata.dataframe) + self.import_button.enable() + + def create_app(self): + self.app = HBox( + [ + VBox( + [ + self.quick_path_label, + HBox( + [ + self.quick_path_search, + self.import_button.button, + ] + ), + self.filechooser, + ], + ), + self.viewer.app, + ], + layout=Layout(justify_content="center"), + ) + + + class RawUploader_ALS832(UploaderBase): """ Raw uploaders are the way you get your raw data (projections, flats, dark fields) diff --git a/tomopyui/widgets/main.py b/tomopyui/widgets/main.py index 0940125..645e93e 100644 --- a/tomopyui/widgets/main.py +++ b/tomopyui/widgets/main.py @@ -3,14 +3,15 @@ from tomopyui.widgets.analysis import Align, Recon from tomopyui.widgets.center import Center from tomopyui.widgets.dataexplorer import DataExplorerTab +from tomopyui.widgets.helpers import check_cuda_gpus_with_cupy from tomopyui.widgets.imports import ( Import_ALS832, Import_APS, + Import_NxTomo, Import_SSRL62B, Import_SSRL62C, ) from tomopyui.widgets.prep import Prep -from tomopyui.widgets.helpers import check_cuda_gpus_with_cupy def create_dashboard(institution: str): @@ -46,6 +47,8 @@ def create_dashboard(institution: str): file_import = Import_SSRL62B() if institution == "APS": file_import = Import_APS() + if institution == "nxtomo": + file_import = Import_NxTomo() prep = Prep(file_import) center = Center(file_import) align = Align(file_import, center)