better null handling + req fixes (#37)

pyproject.toml

@@ -14,7 +14,7 @@ authors = [
     { name = "Forrest Collman" },
 ]
 dependencies = [
-    "caveclient>=5.14.0",
+    "caveclient>=5.22.0",
     "ipython",
     "neuroglancer",
     "numpy>=1.11.0",
@@ -23,6 +23,7 @@ dependencies = [
     "six",
     "tables",
     "webcolors",
+    "attrs>=23.2.0",
 ]
 classifiers = [
     "License :: OSI Approved :: MIT License",

src/nglui/segmentprops/base.py

@@ -40,6 +40,10 @@ def sort_tag_arrays(x: list) -> list:
     return [sorted(y) for y in x]
 
 
+def zero_null_strings(x: list) -> list:
+    return [y if not is_null_value(y) else "" for y in x]
+
+
 def is_null_value(value):
     if value is None:
         return True
@@ -51,6 +55,10 @@ def is_null_value(value):
         return False
 
 
+def preprocess_string_column(x: list) -> list:
+    return space_to_underscore(zero_null_strings(x))
+
+
 @attrs.define
 class InlineProperties:
     ids = attrs.field(type=list[int], converter=list_of_strings, kw_only=True)
@@ -209,18 +217,22 @@ def _make_tag_property(df, value_columns, bool_columns, tag_descriptions, name="
         unique_tags = df[col].unique()
         # df.unique works differently for categorical dtype columns and does not return an ndarray so we have to check
         if isinstance(unique_tags, np.ndarray):
-            unique_tags = sorted([x for x in unique_tags.tolist() if x is not None])
+            unique_tags = [x for x in unique_tags.tolist() if not is_null_value(x)]
+            unique_tags = sorted(unique_tags)
         else:
             unique_tags = unique_tags.sort_values().tolist()
+            unique_tags = [x for x in unique_tags if not is_null_value(x)]
+        print("is Null:", is_null_value(""))
+        print("unique_tags:", unique_tags)
         if np.any(np.isin(tags, unique_tags)):
             raise ValueError("Tags across columns are not unique")
         tags.extend(unique_tags)
     tags.extend(bool_columns)
-    tag_map = {tag: i for i, tag in enumerate(tags) if tag is not None}
+    tag_map = {tag: i for i, tag in enumerate(tags) if not is_null_value(tag)}
     tag_values = []
     for _, row in df.iterrows():
         tag_values.append(
-            [tag_map[tag] for tag in row[value_columns] if tag is not None]
+            [tag_map[tag] for tag in row[value_columns] if not is_null_value(tag)]
         )
     for tv in bool_columns:
         for loc in np.flatnonzero(df[tv]):

src/nglui/statebuilder/layers.py

@@ -346,7 +346,7 @@ def _render_single_segment_prop(self, data, seg_prop_map, client):
         return client.state.build_neuroglancer_url(
             pid,
             target_site="cave-explorer",
-            format_propeties=True,
+            format_properties=True,
         )
 
     def _render_segment_property_map(self, data, client, target_site=None):

tests/test_segment_props.py

@@ -20,6 +20,21 @@ def test_df():
     )
 
 
+@pytest.fixture
+def test_null_df():
+    return pd.DataFrame(
+        {
+            "seg_id": np.arange(0, 100),
+            "cell_type": 30 * ["ct_a"] + 30 * [""] + 40 * [None],
+            "category": np.nan,
+            "number_int": np.arange(300, 400),
+            "number_float": np.arange(300, 400) + 0.1,
+            "tag_a": 90 * [False] + 10 * [True],
+            "tag_b": 50 * [True] + 50 * [False],
+        }
+    )
+
+
 @pytest.fixture
 def test_segprops():
     return {
@@ -115,6 +130,21 @@ def test_segment_props(test_df):
     assert len(rh_props) == 100
 
 
+def test_segment_props_nulls(test_null_df):
+    props = SegmentProperties.from_dataframe(
+        test_null_df,
+        id_col="seg_id",
+        tag_value_cols="cell_type",
+        tag_bool_cols=["tag_a", "tag_b"],
+        tag_descriptions={"tag_a": "The first tag", "tag_b": "The second tag"},
+    )
+
+    assert tuple(props.tag_properties.tags) == ("ct_a", "tag_a", "tag_b")
+    p_dict = props.to_dict()
+    assert 2 in p_dict["inline"]["properties"][0]["values"][0]
+    assert len(p_dict["inline"]["properties"][0]["values"][50]) == 0
+
+
 def test_property_conversion(test_segprops):
     props = SegmentProperties.from_dict(test_segprops)
     assert len(props) == 10