dataset updated

pyproject.toml

@@ -4,23 +4,20 @@ version = "1.0.0a2"
 description = ""
 homepage = "https://kartezio.com"
 repository = "https://github.com/KevinCortacero/Kartezio"
-authors = ["Kevin Cortacero <kevin.cortacero@inserm.fr>"]
+authors = ["Kevin Cortacero <kevin.cortacero@protonmail.com>"]
 readme = "README.md"
 packages = [{include = "kartezio", from = "src"}]
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-opencv-python = "*"
+numpy = ">=1.24.0"
 pandas = "*"
+scikit-image = "*"
+opencv-python = "*"
+numba = "*"
 czifile = "^2019.7.2"
 tabulate = "^0.9.0"
-numpy = ">=1.24.0"
-numba = "*"
-tomli = "^2.0.1"
-tomli-w = "^1.0.0"
 roifile = ">=2023.2.12"
-scikit-image = "*"
-
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.0"
@@ -30,6 +27,7 @@ flake8 = ">=5.0.4"
 pytest-cov = "^5.0.0"
 isort = "^5.13.2"
 pre-commit = "*"
+ipykernel = "^6.29.5"
 
 [build-system]
 requires = ["poetry-core"]

src/kartezio/core/components.py

@@ -162,8 +162,8 @@ def inner(item_cls):
         name = item_cls.__name__
         if Components._contains(fundamental_name, name):
             if not replace:
-                raise KeyError(
-                    f"""Error registering {fundamental_name} called '{name}'.
+                print(
+                    f"""Warning registering {fundamental_name} called '{name}'.
                     Here is the list of all registered {fundamental_name} components:
                     \n{Components._registry[fundamental_name].keys()}.
                     \n > Replace it using 'replace=True' in @register, or use another name.

src/kartezio/data/dataset.py

@@ -1,19 +1,13 @@
+from abc import abstractmethod
 from dataclasses import dataclass, field
 from typing import List, Tuple
-from abc import abstractmethod
-
-from kartezio.core.components import Components
-from kartezio.utils.directory import Directory
-from kartezio.vision.common import draw_overlay
-from kartezio.utils.json_handler import json_read, json_write
-
-
 
 import numpy as np
 
+from kartezio.utils.directory import Directory
+from kartezio.vision.common import draw_overlay
 
 CSV_DATASET = "dataset.csv"
-JSON_META = "meta.json"
 
 
 class Dataset:
@@ -39,13 +33,9 @@ def xy(self):
         def xyv(self):
             return self.x, self.y, self.v
 
-    def __init__(
-        self, train_set, test_set, name, label_name, inputs, indices=None
-    ):
+    def __init__(self, train_set, test_set, inputs, indices=None):
         self.train_set = train_set
         self.test_set = test_set
-        self.name = name
-        self.label_name = label_name
         self.inputs = inputs
         self.indices = indices
 
@@ -94,73 +84,42 @@ def split(self):
         return self.train_x, self.train_y, self.test_x, self.test_y
 
 
+class DataReader:
+    def __init__(self, directory, scale=1.0):
+        self.scale = scale
+        self.directory = directory
 
-class DatasetMeta:
-    @staticmethod
-    def write(
-        filepath,
-        name,
-        input_type,
-        input_format,
-        label_type,
-        label_format,
-        label_name,
-        scale=1.0,
-        mode="dataframe",
-        meta_filename=JSON_META,
-    ):
-        json_data = {
-            "name": name,
-            "scale": scale,
-            "label_name": label_name,
-            "mode": mode,
-            "input": {"type": input_type, "format": input_format},
-            "label": {"type": label_type, "format": label_format},
-        }
-        json_write(filepath + "/" + meta_filename, json_data)
+    def read(self, filename, shape=None):
+        if str(filename) == "nan":
+            filepath = ""
+        else:
+            filepath = f"{self.directory}/{filename}"
+        return self._read(filepath, shape)
 
-    @staticmethod
-    def read(filepath, meta_filename):
-        return json_read(filepath / meta_filename)
+    @abstractmethod
+    def _read(self, filepath, shape=None):
+        pass
 
 
 @dataclass
 class DatasetReader(Directory):
-    counting: bool = False
+    x_reader: DataReader
+    y_reader: DataReader
     preview: bool = False
     preview_dir: Directory = field(init=False)
+    color_preview = (51, 152, 75)
 
     def __post_init__(self, path):
         super().__post_init__(path)
         if self.preview:
             self.preview_dir = self.next("__preview__")
 
-    def _read_meta(self, meta_filename):
-        from kartezio.readers import ImageRGBReader, RoiPolygonReader
-        meta = DatasetMeta.read(self._path, meta_filename=meta_filename)
-        self.name = meta["name"]
-        self.scale = meta["scale"]
-        self.mode = meta["mode"]
-        self.label_name = meta["label_name"]
-        input_reader_name = (
-            f"{meta['input']['type']}_{meta['input']['format']}"
-        )
-        label_reader_name = (
-            f"{meta['label']['type']}_{meta['label']['format']}"
-        )
-        self.input_reader = ImageRGBReader(self)  # Components.instantiate("DataReader", input_reader_name, directory=self, scale=self.scale)
-        self.label_reader = RoiPolygonReader(self)  # Components.instantiate("DataReader", label_reader_name, directory=self, scale=self.scale)
-
     def read_dataset(
         self,
         dataset_filename=CSV_DATASET,
-        meta_filename=JSON_META,
         indices=None,
     ):
-        self._read_meta(meta_filename)
-        if self.mode == "dataframe":
-            return self._read_from_dataframe(dataset_filename, indices)
-        raise AttributeError(f"{self.mode} is not handled yet")
+        return self._read_from_dataframe(dataset_filename, indices)
 
     def _read_from_dataframe(self, dataset_filename, indices):
         dataframe = self.read(dataset_filename)
@@ -184,14 +143,13 @@ def _read_from_dataframe(self, dataset_filename, indices):
             )
 
         if self.preview:
-            color = [98, 36, 97]
             for i in range(len(training.x)):
                 visual = training.v[i]
                 label = training.y[i][0]
                 preview = draw_overlay(
                     visual,
                     label.astype(np.uint8),
-                    color=color,
+                    color=self.color_preview,
                     alpha=0.5,
                     thickness=3,
                 )
@@ -202,14 +160,12 @@ def _read_from_dataframe(self, dataset_filename, indices):
                 preview = draw_overlay(
                     visual,
                     label.astype(np.uint8),
-                    color=color,
+                    color=self.color_preview,
                     alpha=0.5,
                     thickness=3,
                 )
                 self.preview_dir.write(f"test_{i}.png", preview)
-        return Dataset(
-            training, testing, self.name, self.label_name, inputs, indices
-        )
+        return Dataset(training, testing, inputs, indices)
 
     def _read_auto(self, dataset):
         pass
@@ -220,14 +176,10 @@ def _read_dataset(self, dataframe, indices=None):
         if indices:
             dataframe = dataframe.loc[indices]
         for row in dataframe.itertuples():
-            x = self.input_reader.read(row.input, shape=None)
-            y = self.label_reader.read(row.label, shape=x.shape)
-            if self.counting:
-                y = [y.datalist[0], y.count]
-            else:
-                y = y.datalist
+            x = self.x_reader.read(row.input, shape=None)
+            y = self.y_reader.read(row.label, shape=x.shape)
             dataset.n_inputs = x.size
-            dataset.add_item(x.datalist, y)
+            dataset.add_item(x.datalist, y.datalist)
             visual_from_table = False
             if "visual" in dataframe.columns:
                 if str(row.visual) != "nan":
@@ -238,23 +190,6 @@ def _read_dataset(self, dataframe, indices=None):
         return dataset
 
 
-class DataReader:
-    def __init__(self, directory, scale=1.0):
-        self.scale = scale
-        self.directory = directory
-
-    def read(self, filename, shape=None):
-        if str(filename) == "nan":
-            filepath = ""
-        else:
-            filepath = str(self.directory / filename)
-        return self._read(filepath, shape)
-
-    @abstractmethod
-    def _read(self, filepath, shape=None):
-        pass
-
-
 @dataclass
 class DataItem:
     datalist: List
@@ -269,18 +204,35 @@ def size(self):
 
 def read_dataset(
     dataset_path,
+    x_reader,
+    y_reader,
     filename=CSV_DATASET,
-    meta_filename=JSON_META,
     indices=None,
-    counting=False,
     preview=False,
-    reader=None,
 ):
+    from kartezio.readers import (
+        ImageGrayscaleReader,
+        ImageLabels,
+        ImageRGBReader,
+        RoiPolygonReader,
+    )
+
+    if isinstance(x_reader, str):
+        match x_reader:
+            case "rgb":
+                x_reader = ImageRGBReader(dataset_path)
+            case "grayscale":
+                x_reader = ImageGrayscaleReader(dataset_path)
+    if isinstance(y_reader, str):
+        match y_reader:
+            case "labels":
+                y_reader = ImageLabels(dataset_path)
+            case "imagej":
+                y_reader = RoiPolygonReader(dataset_path)
+
     dataset_reader = DatasetReader(
-        dataset_path, counting=counting, preview=preview
+        dataset_path, x_reader, y_reader, preview=preview
     )
-    if reader is not None:
-        dataset_reader.add_reader(reader)
     return dataset_reader.read_dataset(
-        dataset_filename=filename, meta_filename=meta_filename, indices=indices
-    )
+        dataset_filename=filename, indices=indices
+    )

src/kartezio/preprocessing.py

@@ -212,7 +212,8 @@ def __to_dict__(self) -> Dict:
             "args": {"channels": self.channels},
         }
 
-@register(Preprocessing,"format_3d")
+
+@register(Preprocessing)
 class Format3D(Preprocessing):
     def __init__(self, channels=None, z_range=None):
         super().__init__()
@@ -226,10 +227,14 @@ def preprocess(self, x):
             if self.channels:
                 if self.z_range:
                     for z in self.z_range:
-                        one_item.append([x[i][channel][z] for channel in self.channels])
+                        one_item.append(
+                            [x[i][channel][z] for channel in self.channels]
+                        )
                 else:
                     for z in range(len(x[i][0])):
-                        one_item.append([x[i][channel][z] for channel in self.channels])
+                        one_item.append(
+                            [x[i][channel][z] for channel in self.channels]
+                        )
             else:
                 if self.z_range:
                     for z in self.z_range:
@@ -241,4 +246,4 @@ def preprocess(self, x):
         return new_x
 
     def _to_json_kwargs(self) -> dict:
-        pass
+        pass

src/kartezio/primitives/array.py

@@ -13,7 +13,6 @@
     convolution,
     gradient_magnitude,
     morph_fill,
-    threshold_binary,
     threshold_tozero,
 )
 from kartezio.vision.kernel import (
@@ -293,19 +292,6 @@ def call(self, x: List[np.ndarray], args: List[int]):
         return gradient_magnitude(gx.astype(np.float32), gy.astype(np.float32))
 
 
-@register(Primitive)
-class Deriche(Primitive):
-    def __init__(self):
-        super().__init__([TypeArray], TypeArray, 0)
-        self.alpha = 1.0
-        self.omega = 1.0
-
-    def call(self, x: List[np.ndarray], args: List[int]):
-        gx = cv2.ximgproc.GradientDericheX(x[0], self.alpha, self.omega)
-        gy = cv2.ximgproc.GradientDericheY(x[0], self.alpha, self.omega)
-        return gradient_magnitude(gx, gy)
-
-
 @register(Primitive)
 class Roberts(Primitive):
     def __init__(self):
@@ -866,7 +852,6 @@ def create_array_lib(use_scalars=False):
     library_opencv.add_primitive(Log())
     library_opencv.add_primitive(Laplacian())
     library_opencv.add_primitive(Sobel())
-    library_opencv.add_primitive(Deriche())
     library_opencv.add_primitive(Roberts())
     library_opencv.add_primitive(Canny())
     library_opencv.add_primitive(Sharpen())

src/kartezio/readers.py

@@ -3,7 +3,6 @@
 import cv2
 import numpy as np
 
-from kartezio.core.components import register
 from kartezio.data.dataset import DataItem, DataReader
 from kartezio.utils.image import imread_gray, imread_rgb, imread_tiff
 from kartezio.utils.imagej import read_polygons_from_roi

src/kartezio/vision/watershed.py

@@ -226,7 +226,7 @@ def marker_controlled_watershed(image, markers, watershed_line):
     np.ndarray
         Integer-labeled segmented image.
     """
-    # markers = _connected_components(markers)
+    markers = _connected_components(markers)
     return watershed_transform(image, markers, watershed_line)
 
 
@@ -363,12 +363,13 @@ def double_threshold_watershed(image, threshold1, threshold2, watershed_line):
     np.ndarray
         Labeled segmentation from watershed.
     """
-    """
+
     image = threshold_tozero(image, threshold1)
     markers = threshold_tozero(image, threshold2)
     """
     markers = np.zeros_like(image)
     markers[image < threshold1] = 1
     markers[image > threshold2] = 2
-
+    """
+
     return marker_controlled_watershed(image, markers, watershed_line)