From a1ba80efeb54dda271920ecccee321f88606ac6a Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 4 Feb 2025 18:21:18 -0500 Subject: [PATCH] [Backport release-1.15] [python] Resolve issue with #3611 and `pyarrow<17` (#3671) * [Backport release-1.15] [python] Resolve issue with #3611 and `pyarrow<17` * Fix issue with pyarrow<17 --- apis/python/tests/test_dataframe.py | 11 +++++++---- apis/python/tests/test_sparse_nd_array.py | 13 ++++++++----- libtiledbsoma/src/soma/column_buffer.cc | 8 ++++++-- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/apis/python/tests/test_dataframe.py b/apis/python/tests/test_dataframe.py index 947615b2a0..a5cd24428a 100644 --- a/apis/python/tests/test_dataframe.py +++ b/apis/python/tests/test_dataframe.py @@ -2092,10 +2092,13 @@ def test_arrow_table_validity_with_slicing(tmp_path): ) table = pa.Table.from_pydict(pydict) - with soma.DataFrame.open(uri, "w") as A: - with raises_no_typeguard(soma.SOMAError): - # soma_joinid cannot be nullable - A.write(table) + # As of version 1.15.6 we were throwing in this case. However, we found + # a compatibility issue with pyarrow versions below 17. Thus this is + # now non-fatal. + # with soma.DataFrame.open(uri, "w") as A: + # with raises_no_typeguard(soma.SOMAError): + # # soma_joinid cannot be nullable + # A.write(table) pydict["soma_joinid"] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] table = pa.Table.from_pydict(pydict) diff --git a/apis/python/tests/test_sparse_nd_array.py b/apis/python/tests/test_sparse_nd_array.py index d933ba5e83..776734aca4 100644 --- a/apis/python/tests/test_sparse_nd_array.py +++ b/apis/python/tests/test_sparse_nd_array.py @@ -2022,11 +2022,14 @@ def test_sparse_nd_array_null(tmp_path): soma.SparseNDArray.create(uri, type=pa.int64(), shape=(10,)) - with soma.SparseNDArray.open(uri, "w") as A: - with raises_no_typeguard(soma.SOMAError): - # soma_joinid cannot be nullable - A.write(table[:5]) - A.write(table[5:]) + # As of version 1.15.6 we were throwing in this case. However, we found + # a compatibility issue with pyarrow versions below 17. Thus this is + # now non-fatal. + # with soma.SparseNDArray.open(uri, "w") as A: + # with raises_no_typeguard(soma.SOMAError): + # # soma_joinid cannot be nullable + # A.write(table[:5]) + # A.write(table[5:]) pydict["soma_dim_0"] = pa.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) table = pa.Table.from_pydict(pydict) diff --git a/libtiledbsoma/src/soma/column_buffer.cc b/libtiledbsoma/src/soma/column_buffer.cc index cf7b943f16..64c9ed08bf 100644 --- a/libtiledbsoma/src/soma/column_buffer.cc +++ b/libtiledbsoma/src/soma/column_buffer.cc @@ -157,9 +157,13 @@ void ColumnBuffer::attach(Query& query, std::optional subarray) { "to attach to Query"); } + // As of version 1.15.6 we were throwing here. However, we found a + // compatibility issue with pyarrow versions below 17. Thus we log and + // continue. if (!validity_.empty() && is_dim) { - throw TileDBSOMAError(fmt::format( - "[ColumnBuffer::attach] Validity buffer passed for dimension '{}'", + LOG_DEBUG(fmt::format( + "[ColumnBuffer::attach] Validity buffer passed for dimension '{}' " + "is being ignored", name_)); }