Skip to content

Commit

Permalink
better null handling + req fixes (#37)
Browse files Browse the repository at this point in the history
ceesem authored Jul 15, 2024
1 parent 99cfba8 commit 7c9c0b6
Showing 4 changed files with 48 additions and 5 deletions.
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ authors = [
{ name = "Forrest Collman" },
]
dependencies = [
"caveclient>=5.14.0",
"caveclient>=5.22.0",
"ipython",
"neuroglancer",
"numpy>=1.11.0",
@@ -23,6 +23,7 @@ dependencies = [
"six",
"tables",
"webcolors",
"attrs>=23.2.0",
]
classifiers = [
"License :: OSI Approved :: MIT License",
18 changes: 15 additions & 3 deletions src/nglui/segmentprops/base.py
Original file line number Diff line number Diff line change
@@ -40,6 +40,10 @@ def sort_tag_arrays(x: list) -> list:
return [sorted(y) for y in x]


def zero_null_strings(x: list) -> list:
return [y if not is_null_value(y) else "" for y in x]


def is_null_value(value):
if value is None:
return True
@@ -51,6 +55,10 @@ def is_null_value(value):
return False


def preprocess_string_column(x: list) -> list:
return space_to_underscore(zero_null_strings(x))


@attrs.define
class InlineProperties:
ids = attrs.field(type=list[int], converter=list_of_strings, kw_only=True)
@@ -209,18 +217,22 @@ def _make_tag_property(df, value_columns, bool_columns, tag_descriptions, name="
unique_tags = df[col].unique()
# df.unique works differently for categorical dtype columns and does not return an ndarray so we have to check
if isinstance(unique_tags, np.ndarray):
unique_tags = sorted([x for x in unique_tags.tolist() if x is not None])
unique_tags = [x for x in unique_tags.tolist() if not is_null_value(x)]
unique_tags = sorted(unique_tags)
else:
unique_tags = unique_tags.sort_values().tolist()
unique_tags = [x for x in unique_tags if not is_null_value(x)]
print("is Null:", is_null_value(""))
print("unique_tags:", unique_tags)
if np.any(np.isin(tags, unique_tags)):
raise ValueError("Tags across columns are not unique")
tags.extend(unique_tags)
tags.extend(bool_columns)
tag_map = {tag: i for i, tag in enumerate(tags) if tag is not None}
tag_map = {tag: i for i, tag in enumerate(tags) if not is_null_value(tag)}
tag_values = []
for _, row in df.iterrows():
tag_values.append(
[tag_map[tag] for tag in row[value_columns] if tag is not None]
[tag_map[tag] for tag in row[value_columns] if not is_null_value(tag)]
)
for tv in bool_columns:
for loc in np.flatnonzero(df[tv]):
2 changes: 1 addition & 1 deletion src/nglui/statebuilder/layers.py
Original file line number Diff line number Diff line change
@@ -346,7 +346,7 @@ def _render_single_segment_prop(self, data, seg_prop_map, client):
return client.state.build_neuroglancer_url(
pid,
target_site="cave-explorer",
format_propeties=True,
format_properties=True,
)

def _render_segment_property_map(self, data, client, target_site=None):
30 changes: 30 additions & 0 deletions tests/test_segment_props.py
Original file line number Diff line number Diff line change
@@ -20,6 +20,21 @@ def test_df():
)


@pytest.fixture
def test_null_df():
return pd.DataFrame(
{
"seg_id": np.arange(0, 100),
"cell_type": 30 * ["ct_a"] + 30 * [""] + 40 * [None],
"category": np.nan,
"number_int": np.arange(300, 400),
"number_float": np.arange(300, 400) + 0.1,
"tag_a": 90 * [False] + 10 * [True],
"tag_b": 50 * [True] + 50 * [False],
}
)


@pytest.fixture
def test_segprops():
return {
@@ -115,6 +130,21 @@ def test_segment_props(test_df):
assert len(rh_props) == 100


def test_segment_props_nulls(test_null_df):
props = SegmentProperties.from_dataframe(
test_null_df,
id_col="seg_id",
tag_value_cols="cell_type",
tag_bool_cols=["tag_a", "tag_b"],
tag_descriptions={"tag_a": "The first tag", "tag_b": "The second tag"},
)

assert tuple(props.tag_properties.tags) == ("ct_a", "tag_a", "tag_b")
p_dict = props.to_dict()
assert 2 in p_dict["inline"]["properties"][0]["values"][0]
assert len(p_dict["inline"]["properties"][0]["values"][50]) == 0


def test_property_conversion(test_segprops):
props = SegmentProperties.from_dict(test_segprops)
assert len(props) == 10

0 comments on commit 7c9c0b6

Please sign in to comment.