diff --git a/CMakeLists.txt b/CMakeLists.txt index dea8954..51eefba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ string(TOLOWER ${PROJECT_NAME} PROJECT_NAME_LOWER) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") enable_language(CXX) -set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD 17) if ( CMAKE_COMPILER_IS_GNUCC ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -DMPICH_SKIP_MPICXX=1 -DOMPI_SKIP_MPICXX=1 -DH5_USE_110_API") @@ -15,7 +15,7 @@ endif() include(${PROJECT_SOURCE_DIR}/cmake/neuroh5_utils.cmake) -set(NEUROH5_VERSION 0.1.9) +set(NEUROH5_VERSION 0.1.15) cmake_policy(SET CMP0074 NEW) # enables use of HDF5_ROOT variable diff --git a/include/data/attr_val.hh b/include/data/attr_val.hh index 0301f51..b8396b0 100644 --- a/include/data/attr_val.hh +++ b/include/data/attr_val.hh @@ -72,7 +72,10 @@ namespace neuroh5 void resize (size_t size); template - const std::vector& attr_vec (size_t i) const; + const std::vector& const_attr_vec (size_t i) const; + + template + std::vector& attr_vec (size_t i); template size_t size_attr (size_t i) const; diff --git a/include/graph/edge_attributes.hh b/include/graph/edge_attributes.hh index 66e59d9..eb02618 100644 --- a/include/graph/edge_attributes.hh +++ b/include/graph/edge_attributes.hh @@ -283,7 +283,7 @@ namespace neuroh5 { size_t i = attr_index.attr_index(attr_name); string path = hdf5::edge_attribute_path(src_pop_name, dst_pop_name, attr_namespace, attr_name); - graph::write_edge_attribute(comm, file, path, edge_attr_values.attr_vec(i)); + graph::write_edge_attribute(comm, file, path, edge_attr_values.const_attr_vec(i)); } } else @@ -368,7 +368,7 @@ namespace neuroh5 size_t i = attr_index.attr_index(attr_name); graph::append_edge_attribute(comm, file, src_pop_name, dst_pop_name, attr_namespace, attr_name, - edge_attr_values.attr_vec(i), + edge_attr_values.const_attr_vec(i), chunk_size); } } diff --git a/include/neuroh5_types.hh b/include/neuroh5_types.hh index 3e2f2ad..e7727ff 100644 --- a/include/neuroh5_types.hh +++ b/include/neuroh5_types.hh @@ -205,8 +205,10 @@ namespace neuroh5 typedef std::map edge_map_t; - typedef edge_map_t::const_iterator edge_map_iter_t; - + typedef edge_map_t::const_iterator edge_map_const_iter_t; + + typedef edge_map_t::iterator edge_map_iter_t; + typedef std::map rank_edge_map_t; typedef rank_edge_map_t::const_iterator rank_edge_map_iter_t; diff --git a/python/neuroh5/CMakeLists.txt b/python/neuroh5/CMakeLists.txt index edce92d..be1206b 100644 --- a/python/neuroh5/CMakeLists.txt +++ b/python/neuroh5/CMakeLists.txt @@ -1,6 +1,6 @@ set(NEUROH5_IO_SRCS - "iomodule.cc" + "iomodule.cc" ) include_directories(${MPI_C_INCLUDE_DIRS}) diff --git a/python/neuroh5/iomodule.cc b/python/neuroh5/iomodule.cc index 45e282f..2803347 100644 --- a/python/neuroh5/iomodule.cc +++ b/python/neuroh5/iomodule.cc @@ -67,6 +67,7 @@ #include "edge_attributes.hh" #include "serialize_data.hh" #include "split_intervals.hh" +#include "shared_array.hh" #if PY_MAJOR_VERSION >= 3 #define Py_TPFLAGS_HAVE_ITER ((Py_ssize_t)0) @@ -1074,7 +1075,7 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, npy_intp dims[1]; dims[0] = num_nodes; py_section_topology = PyDict_New(); - py_section_vector = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); + py_section_vector = create_typed_shared_array(dims[0]); SECTION_IDX_T *section_vector_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_vector, &ind); size_t sections_ptr=0; SECTION_IDX_T section_idx = 0; @@ -1091,7 +1092,7 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, npy_intp nodes_dims[1], nodes_ind = 0; nodes_dims[0] = num_section_nodes; py_section_key = PyLong_FromLong((long)section_idx); - py_section_nodes = (PyObject *)PyArray_SimpleNew(1, nodes_dims, NPY_UINT32); + py_section_nodes = create_typed_shared_array(nodes_dims[0]); NODE_IDX_T *py_section_nodes_ptr = (NODE_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_nodes, &nodes_ind); sections_ptr++; for (size_t p = 0; p < num_section_nodes; p++) @@ -1120,16 +1121,14 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, npy_intp topology_dims[1]; topology_dims[0] = src_vector.size(); - py_section_src = (PyObject *)PyArray_SimpleNew(1, topology_dims, NPY_UINT16); - SECTION_IDX_T *section_src_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_src, &ind); - py_section_dst = (PyObject *)PyArray_SimpleNew(1, topology_dims, NPY_UINT16); - SECTION_IDX_T *section_dst_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_dst, &ind); - py_section_loc = (PyObject *)PyArray_SimpleNew(1, topology_dims, NPY_UINT32); + + py_section_src = create_shared_array_from_deque(std::move(src_vector)); + py_section_dst = create_shared_array_from_deque(std::move(dst_vector)); + + py_section_loc = create_typed_shared_array(topology_dims[0]); NODE_IDX_T *section_loc_ptr = (NODE_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_loc, &ind); for (size_t s = 0; s < src_vector.size(); s++) { - section_src_ptr[s] = src_vector[s]; - section_dst_ptr[s] = dst_vector[s]; auto node_map_it = section_node_map.find(src_vector[s]); throw_assert (node_map_it != section_node_map.end(), "py_build_tree_value: invalid section index in tree source vector"); @@ -1186,25 +1185,14 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { npy_intp topology_dims[1]; topology_dims[0] = src_vector.size(); - py_section_src = (PyObject *)PyArray_SimpleNew(1, topology_dims, NPY_UINT16); - SECTION_IDX_T *section_src_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_src, &ind); - py_section_dst = (PyObject *)PyArray_SimpleNew(1, topology_dims, NPY_UINT16); - SECTION_IDX_T *section_dst_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_section_dst, &ind); - for (size_t s = 0; s < src_vector.size(); s++) - { - section_src_ptr[s] = src_vector[s]; - section_dst_ptr[s] = dst_vector[s]; - } - + py_section_src = create_shared_array_from_deque(std::move(src_vector)); + py_section_dst = create_shared_array_from_deque(std::move(dst_vector)); + npy_intp sections_dims[1]; sections_dims[0] = sections.size(); - py_sections = (PyObject *)PyArray_SimpleNew(1, sections_dims, NPY_UINT16); - SECTION_IDX_T *sections_ptr = (SECTION_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_sections, &ind); - for (size_t p = 0; p < sections.size(); p++) - { - sections_ptr[p] = sections[p]; - } + + py_sections = create_shared_array_from_deque(std::move(sections)); } @@ -1212,31 +1200,14 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, npy_intp dims[1]; dims[0] = num_nodes; - PyObject *py_xcoords = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *xcoords_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_xcoords, &ind); - PyObject *py_ycoords = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *ycoords_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_ycoords, &ind); - PyObject *py_zcoords = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *zcoords_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_zcoords, &ind); - PyObject *py_radiuses = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *radiuses_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_radiuses, &ind); - PyObject *py_layers = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - LAYER_IDX_T *layers_ptr = (LAYER_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_layers, &ind); - PyObject *py_parents = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - PARENT_NODE_IDX_T *parents_ptr = (PARENT_NODE_IDX_T *)PyArray_GetPtr((PyArrayObject *)py_parents, &ind); - PyObject *py_swc_types = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - SWC_TYPE_T *swc_types_ptr = (SWC_TYPE_T *)PyArray_GetPtr((PyArrayObject *)py_swc_types, &ind); - for (size_t j = 0; j < xcoords.size(); j++) - { - xcoords_ptr[j] = xcoords[j]; - ycoords_ptr[j] = ycoords[j]; - zcoords_ptr[j] = zcoords[j]; - radiuses_ptr[j] = radiuses[j]; - layers_ptr[j] = layers[j]; - parents_ptr[j] = parents[j]; - swc_types_ptr[j] = swc_types[j]; - } - + PyObject *py_xcoords = create_shared_array_from_deque(std::move(xcoords)); + PyObject *py_ycoords = create_shared_array_from_deque(std::move(ycoords)); + PyObject *py_zcoords = create_shared_array_from_deque(std::move(zcoords)); + PyObject *py_radiuses = create_shared_array_from_deque(std::move(radiuses)); + PyObject *py_layers = create_shared_array_from_deque(std::move(layers)); + PyObject *py_parents = create_shared_array_from_deque(std::move(parents)); + PyObject *py_swc_types = create_shared_array_from_deque(std::move(swc_types)); + PyObject *py_treeval = PyDict_New(); PyDict_SetItemString(py_treeval, "x", py_xcoords); Py_DECREF(py_xcoords); @@ -1310,12 +1281,7 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = float_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *py_value_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_float][i]).c_str(), @@ -1327,12 +1293,7 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = uint8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *py_value_ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_uint8][i]).c_str(), @@ -1343,13 +1304,8 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = int8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *py_value_ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_int8][i]).c_str(), py_value); @@ -1359,13 +1315,8 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = uint16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *py_value_ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } - + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_uint16][i]).c_str(), py_value); @@ -1375,13 +1326,8 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = uint32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *py_value_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_uint32][i]).c_str(), py_value); @@ -1391,13 +1337,8 @@ PyObject* py_build_tree_value(const CELL_IDX_T key, const neurotree_t &tree, { const deque &attr_value = int32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *py_value_ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_namespace_dict, (attr_names[AttrMap::attr_index_int32][i]).c_str(), py_value); @@ -1593,12 +1534,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = float_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *py_value_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_float][i]).c_str(), @@ -1610,12 +1547,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = uint8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *py_value_ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_uint8][i]).c_str(), @@ -1627,12 +1560,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = int8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *py_value_ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_int8][i]).c_str(), @@ -1645,13 +1574,9 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = uint16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *py_value_ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } - + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); + PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_uint16][i]).c_str(), py_value); @@ -1664,12 +1589,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = int16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT16); - int16_t *py_value_ptr = (int16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_int16][i]).c_str(), @@ -1682,12 +1603,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = uint32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *py_value_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_uint32][i]).c_str(), @@ -1700,12 +1617,8 @@ PyObject* py_build_cell_attr_values_dict(const CELL_IDX_T key, { const deque &attr_value = int32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *py_value_ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyDict_SetItemString(py_attrval, (attr_names[AttrMap::attr_index_int32][i]).c_str(), @@ -1819,12 +1732,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = float_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *py_value_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1833,12 +1742,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = uint8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *py_value_ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1846,13 +1751,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, for (size_t i=0; i &attr_value = int8_attrs[i]; - dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *py_value_ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1861,12 +1761,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = uint16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *py_value_ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1875,12 +1771,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = int16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT16); - int16_t *py_value_ptr = (int16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1889,12 +1781,8 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = uint32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *py_value_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -1903,13 +1791,9 @@ PyObject* py_build_cell_attr_values_tuple(const CELL_IDX_T key, { const deque &attr_value = int32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *py_value_ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); + PyTuple_SetItem(py_attrval, attr_pos++, py_value); } @@ -2025,12 +1909,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = float_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *py_value_ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2039,12 +1919,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = uint8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *py_value_ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2053,12 +1929,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = int8_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *py_value_ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2067,12 +1939,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = uint16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *py_value_ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2081,12 +1949,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = int16_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT16); - int16_t *py_value_ptr = (int16_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2095,12 +1959,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = uint32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *py_value_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2109,12 +1969,8 @@ PyObject* py_build_cell_attr_values_struct(const CELL_IDX_T key, { const deque &attr_value = int32_attrs[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *py_value_ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_deque(std::move(attr_value)); PyStructSequence_SetItem(py_attrval, attr_pos++, py_value); } @@ -2368,7 +2224,7 @@ void build_edge_attr_vec (const AttrVal& attr_val, template -void py_build_edge_attr_value (const vector < vector >& edge_attr_values, +void py_build_edge_attr_value (vector < vector >& edge_attr_values, const NPY_TYPES numpy_type, const size_t attr_index, const vector >& attr_names, @@ -2381,12 +2237,8 @@ void py_build_edge_attr_value (const vector < vector >& edge_attr_values, npy_intp dims[1]; npy_intp ind = 0; const vector &attr_value = edge_attr_values[i]; dims[0] = attr_value.size(); - PyObject *py_value = (PyObject *)PyArray_SimpleNew(1, dims, numpy_type); - T *py_value_ptr = (T *)PyArray_GetPtr((PyArrayObject *)py_value, &ind); - for (size_t j = 0; j < attr_value.size(); j++) - { - py_value_ptr[j] = attr_value[j]; - } + + PyObject *py_value = create_shared_array_from_vector(attr_value); PyDict_SetItemString(py_attrval, (attr_names[attr_index][i]).c_str(), @@ -2477,13 +2329,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, npy_intp dims[1], ind = 0; dims[0] = adj_vector.size(); - PyObject *adj_arr = PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *adj_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)adj_arr, &ind); - - for (size_t j = 0; j < adj_vector.size(); j++) - { - adj_ptr[j] = adj_vector[j]; - } + PyObject *adj_arr = create_shared_array_from_vector(adj_vector); PyObject *py_attrmap = PyDict_New(); size_t namespace_index=0; @@ -2493,12 +2339,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2507,12 +2348,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2520,12 +2356,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2533,12 +2364,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2546,12 +2372,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2559,12 +2380,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT16); - int16_t *ptr = (int16_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2572,12 +2388,7 @@ PyObject* py_build_edge_tuple_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyList_Append(py_attrval, py_arr); throw_assert(status == 0, "py_build_edge_tuple_value: unable to append to list"); @@ -2611,14 +2422,8 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, npy_intp dims[1], ind = 0; dims[0] = adj_vector.size(); - PyObject *adj_arr = PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *adj_ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)adj_arr, &ind); - - for (size_t j = 0; j < adj_vector.size(); j++) - { - adj_ptr[j] = adj_vector[j]; - } - + PyObject *adj_arr = create_shared_array_from_vector(adj_vector); + PyObject *py_attrmap = PyDict_New(); size_t namespace_index=0; for (auto const & edge_attr_values : edge_attr_vector) @@ -2628,12 +2433,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_FLOAT); - float *ptr = (float *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_float][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2641,12 +2441,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT8); - uint8_t *ptr = (uint8_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_uint8][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2654,12 +2449,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT16); - uint16_t *ptr = (uint16_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_uint16][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2667,12 +2457,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_UINT32); - uint32_t *ptr = (uint32_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_uint32][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2680,12 +2465,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT8); - int8_t *ptr = (int8_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_int8][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2693,12 +2473,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT16); - int16_t *ptr = (int16_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_int16][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2706,12 +2481,7 @@ PyObject* py_build_edge_array_dict_value (const NODE_IDX_T key, } for (size_t i = 0; i < edge_attr_values.size_attr_vec(); i++) { - PyObject *py_arr = (PyObject *)PyArray_SimpleNew(1, dims, NPY_INT32); - int32_t *ptr = (int32_t *)PyArray_GetPtr((PyArrayObject *)py_arr, &ind); - for (size_t j = 0; j < adj_vector.size(); j++) - { - ptr[j] = edge_attr_values.at(i,j); - } + PyObject *py_arr = create_shared_array_from_vector(edge_attr_values.const_attr_vec(i)); status = PyDict_SetItemString(py_attrvalmap, attr_names.at(attr_namespace)[AttrVal::attr_index_int32][i].c_str(), py_arr); throw_assert(status == 0, "py_build_edge_array_dict_value: unable to set dictionary item"); @@ -2773,7 +2543,7 @@ PyObject* NeuroH5EdgeIter_iternext(PyObject *self) if (py_state->state->it_edge != py_state->state->edge_map.cend()) { const NODE_IDX_T key = py_state->state->it_edge->first; - const edge_tuple_t& et = py_state->state->it_edge->second; + edge_tuple_t& et = py_state->state->it_edge->second; PyObject* py_edge_tuple_value = py_build_edge_tuple_value (key, et, py_state->state->edge_attr_name_spaces); throw_assert(py_edge_tuple_value != NULL, @@ -2849,7 +2619,7 @@ NeuroH5EdgeIter_FromMap(const edge_map_t& prj_edge_map, p->state->count = prj_edge_map.size(); p->state->edge_map = std::move(prj_edge_map); p->state->edge_attr_name_spaces = edge_attr_name_spaces; - p->state->it_edge = p->state->edge_map.cbegin(); + p->state->it_edge = p->state->edge_map.begin(); return (PyObject *)p; } @@ -3262,10 +3032,10 @@ extern "C" if (prj_edge_map.size() > 0) { - for (auto const& it : prj_edge_map) + for (auto & it : prj_edge_map) { const NODE_IDX_T key_node = it.first; - const edge_tuple_t& et = it.second; + edge_tuple_t& et = it.second; PyObject* py_edge_tuple_value = py_build_edge_tuple_value (key_node, et, edge_attr_name_spaces); @@ -7449,7 +7219,7 @@ extern "C" py_ngg->state->pop_search_ranges = pop_search_ranges; py_ngg->state->pop_pairs = pop_pairs; py_ngg->state->pop_labels = pop_labels; - py_ngg->state->edge_map_iter = py_ngg->state->edge_map.cbegin(); + py_ngg->state->edge_map_iter = py_ngg->state->edge_map.begin(); py_ngg->state->edge_map_type = edge_map_type; py_ngg->state->edge_attr_name_spaces = attr_name_spaces; py_ngg->state->total_num_nodes = total_num_nodes; @@ -8253,7 +8023,7 @@ extern "C" py_ngg->state->edge_map = prj_vector[0]; //throw_assert(py_ngg->state->edge_map.size() > 0); - py_ngg->state->edge_map_iter = py_ngg->state->edge_map.cbegin(); + py_ngg->state->edge_map_iter = py_ngg->state->edge_map.begin(); py_ngg->state->block_index += py_ngg->state->total_read_blocks; status = MPI_Barrier(py_ngg->state->comm); diff --git a/python/neuroh5/shared_array.cc b/python/neuroh5/shared_array.cc index c77ace9..b5973f6 100644 --- a/python/neuroh5/shared_array.cc +++ b/python/neuroh5/shared_array.cc @@ -1,327 +1,9 @@ -// shared_array.cpp -#define PY_SSIZE_T_CLEAN -#include -#include -#include -#include - -// Type mapping struct to convert C++ types to NumPy types -template -struct NumpyTypeMap { - static constexpr int type_num = -1; // Invalid default -}; - -// Specializations for supported types -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_FLOAT; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT32; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT16; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT8; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT32; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT16; }; -template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT8; }; - -// Structure to hold type information -struct TypeInfo { - int numpy_type; - size_t item_size; -}; - -// Template class to manage array memory -template -class SharedArrayHolder { -private: - std::unique_ptr data; - size_t size; - -public: - SharedArrayHolder(size_t n) : size(n) { - data = std::make_unique(n); - // Initialize array with sequential values - for (size_t i = 0; i < n; i++) { - data[i] = static_cast(i); - } - } - - // Approach 1: Constructor that takes ownership of vector's data - explicit SharedArrayHolder(std::vector&& vec) : size(vec.size()) - { - // Get the vector's allocator - auto alloc = vec.get_allocator(); - - // Get pointer to vector's data - T* ptr = vec.data(); - size = vec.size(); - - // Release ownership from vector (C++17) - vec.release(); - - // Take ownership of the pointer - data.reset(ptr); - } - - // Approach 2: Construct array from pointer and size - SharedArrayHolder(T* ptr, size_t n) : data(ptr), size(n) {} - - // Approach 3: Constructor that copies vector's buffer - explicit SharedArrayHolder(const std::vector& vec) : size(vec.size()) - { - // Allocate new buffer - data = std::make_unique(size); - // Use memcpy for POD types (more efficient than std::copy) - if (std::is_trivially_copyable_v) - { - std::memcpy(data.get(), vec.data(), size * sizeof(T)); - } else - { - std::copy(vec.begin(), vec.end(), data.get()); - } - } - - // Constructor for deque input - explicit SharedArrayHolder(const std::deque& deq) : size(deq.size()) - { - data = std::make_unique(size); - - if constexpr (std::is_trivially_copyable_v) { - // For POD types, copy each chunk efficiently - size_t pos = 0; - for (auto it = deq.begin(); it != deq.end(); ++it) { - data[pos++] = *it; - } - } else { - // For non-POD types, use standard copy - std::copy(deq.begin(), deq.end(), data.get()); - } - } - - // Move constructor for deque - explicit SharedArrayHolder(std::deque&& deq) : size(deq.size()) - { - data = std::make_unique(size); - - if constexpr (std::is_trivially_copyable_v) { - // For POD types, move each element efficiently - size_t pos = 0; - while (!deq.empty()) { - data[pos++] = std::move(deq.front()); - deq.pop_front(); - } - } else { - // For non-POD types, use move iterator - std::move(deq.begin(), deq.end(), data.get()); - deq.clear(); - } - } - - T* get_data() { return data.get(); } - size_t get_size() const { return size; } -}; - - - -// Cleanup function template -template -static void shared_array_dealloc(PyObject* capsule) { - auto* holder = static_cast*>( - PyCapsule_GetPointer(capsule, "array_memory") - ); - delete holder; -} - -// Helper function to create shared numpy array of specific type -template -static PyObject* create_typed_shared_array(Py_ssize_t size) -{ - // Create the array holder - auto holder = new SharedArrayHolder(size); - - // Create dimensions for the numpy array - npy_intp dims[1] = {static_cast(size)}; - - // Create a capsule to own the memory - PyObject* capsule = PyCapsule_New( - holder, - "array_memory", - shared_array_dealloc - ); - - if (!capsule) { - delete holder; - return nullptr; - } - - // Create the numpy array - PyObject* array = PyArray_NewFromDescr( - &PyArray_Type, - PyArray_DescrFromType(NumpyTypeMap::type_num), - 1, // nd - dims, // dimensions - nullptr, // strides - holder->get_data(), // data - NPY_ARRAY_WRITEABLE, // flags - nullptr // obj - ); - - if (!array) { - Py_DECREF(capsule); - return nullptr; - } - - // Set the array's base object to our capsule - if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) - { - Py_DECREF(capsule); - Py_DECREF(array); - return nullptr; - } - - return array; -} - - -// Create shared array from deque (by const reference) -template -static PyObject* create_shared_array_from_deque(const std::deque& deq) -{ - auto holder = new SharedArrayHolder(deq); - - npy_intp dims[1] = {static_cast(holder->get_size())}; - - PyObject* capsule = PyCapsule_New( - holder, - "array_memory", - shared_array_dealloc - ); - - if (!capsule) { - delete holder; - return nullptr; - } - - PyObject* array = PyArray_NewFromDescr( - &PyArray_Type, - PyArray_DescrFromType(NumpyTypeMap::type_num), - 1, - dims, - nullptr, - holder->get_data(), - NPY_ARRAY_WRITEABLE, - nullptr - ); - - if (!array) - { - Py_DECREF(capsule); - return nullptr; - } - - if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) - { - Py_DECREF(capsule); - Py_DECREF(array); - return nullptr; - } - - return array; -} - - - -// Create shared array from deque (by rvalue reference) -template -static PyObject* create_shared_array_from_deque(std::deque&& deq) -{ - auto holder = new SharedArrayHolder(std::move(deq)); - - npy_intp dims[1] = {static_cast(holder->get_size())}; - - PyObject* capsule = PyCapsule_New( - holder, - "array_memory", - shared_array_dealloc - ); - - if (!capsule) - { - delete holder; - return nullptr; - } - - PyObject* array = PyArray_NewFromDescr( - &PyArray_Type, - PyArray_DescrFromType(NumpyTypeMap::type_num), - 1, - dims, - nullptr, - holder->get_data(), - NPY_ARRAY_WRITEABLE, - nullptr - ); - - if (!array) { - Py_DECREF(capsule); - return nullptr; - } - - if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) { - Py_DECREF(capsule); - Py_DECREF(array); - return nullptr; - } - - return array; -} - -// Function to create shared numpy array from existing C++ vector using move semantics -template -static PyObject* create_shared_array_from_vector(std::vector&& vec) -{ - - // Create the array holder using move constructor - auto holder = new SharedArrayHolder(std::move(vec)); - - npy_intp dims[1] = {static_cast(holder->get_size())}; - - PyObject* capsule = PyCapsule_New( - holder, - "array_memory", - shared_array_dealloc - ); - - if (!capsule) { - delete holder; - return nullptr; - } - - PyObject* array = PyArray_NewFromDescr( - &PyArray_Type, - PyArray_DescrFromType(NumpyTypeMap::type_num), - 1, - dims, - nullptr, - holder->get_data(), - NPY_ARRAY_WRITEABLE, - nullptr - ); - - if (!array) { - Py_DECREF(capsule); - return nullptr; - } - - if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) { - Py_DECREF(capsule); - Py_DECREF(array); - return nullptr; - } - - return array; -} - +#include "shared_array.hh" // Function to create shared numpy array -static PyObject* create_shared_array(PyObject* self, PyObject* args) { +static PyObject* create_shared_array(PyObject* self, PyObject* args) +{ Py_ssize_t size; const char* dtype_str; diff --git a/python/neuroh5/shared_array.hh b/python/neuroh5/shared_array.hh new file mode 100644 index 0000000..5752c2d --- /dev/null +++ b/python/neuroh5/shared_array.hh @@ -0,0 +1,379 @@ +#ifndef NEUROH5_SHARED_ARRAY_HH +#define NEUROH5_SHARED_ARRAY_HH + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +// Type mapping struct to convert C++ types to NumPy types +template +struct NumpyTypeMap { + static constexpr int type_num = -1; // Invalid default +}; + +// Specializations for supported types +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_FLOAT; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT32; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT16; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_INT8; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT32; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT16; }; +template<> struct NumpyTypeMap { static constexpr int type_num = NPY_UINT8; }; + +// Structure to hold type information +struct TypeInfo { + int numpy_type; + size_t item_size; +}; + +// Template class to manage array memory +template +class SharedArrayHolder { +private: + std::unique_ptr data; + size_t size; + +public: + SharedArrayHolder(size_t n) : size(n) { + data = std::make_unique(n); + // Initialize array with sequential values + for (size_t i = 0; i < n; i++) { + data[i] = static_cast(i); + } + } + + // Approach 1: Constructor that takes ownership of vector's data + explicit SharedArrayHolder(std::vector&& vec) : size(vec.size()) + { + // Get the vector's allocator + auto alloc = vec.get_allocator(); + + // Get pointer to vector's data + T* ptr = vec.data(); + size = vec.size(); + + // Release ownership from vector (C++17) + vec.release(); + + // Take ownership of the pointer + data.reset(ptr); + } + + // Approach 2: Construct array from pointer and size + SharedArrayHolder(T* ptr, size_t n) : data(ptr), size(n) {} + + // Approach 3: Constructor that copies vector's buffer + explicit SharedArrayHolder(const std::vector& vec) : size(vec.size()) + { + // Allocate new buffer + data = std::make_unique(size); + // Use memcpy for POD types (more efficient than std::copy) + if (std::is_trivially_copyable_v) + { + std::memcpy(data.get(), vec.data(), size * sizeof(T)); + } else + { + std::copy(vec.begin(), vec.end(), data.get()); + } + } + + // Constructor for deque input + explicit SharedArrayHolder(const std::deque& deq) : size(deq.size()) + { + data = std::make_unique(size); + + if constexpr (std::is_trivially_copyable_v) { + // For POD types, copy each chunk efficiently + size_t pos = 0; + for (auto it = deq.begin(); it != deq.end(); ++it) { + data[pos++] = *it; + } + } else { + // For non-POD types, use standard copy + std::copy(deq.begin(), deq.end(), data.get()); + } + } + + // Move constructor for deque + explicit SharedArrayHolder(std::deque&& deq) : size(deq.size()) + { + data = std::make_unique(size); + + if constexpr (std::is_trivially_copyable_v) { + // For POD types, move each element efficiently + size_t pos = 0; + while (!deq.empty()) { + data[pos++] = std::move(deq.front()); + deq.pop_front(); + } + } else { + // For non-POD types, use move iterator + std::move(deq.begin(), deq.end(), data.get()); + deq.clear(); + } + } + + T* get_data() { return data.get(); } + size_t get_size() const { return size; } +}; + + + +// Cleanup function template +template +static void shared_array_dealloc(PyObject* capsule) { + auto* holder = static_cast*>( + PyCapsule_GetPointer(capsule, "array_memory") + ); + delete holder; +} + +// Helper function to create shared numpy array of specific type +template +static PyObject* create_typed_shared_array(Py_ssize_t size) +{ + // Create the array holder + auto holder = new SharedArrayHolder(size); + + // Create dimensions for the numpy array + npy_intp dims[1] = {static_cast(size)}; + + // Create a capsule to own the memory + PyObject* capsule = PyCapsule_New( + holder, + "array_memory", + shared_array_dealloc + ); + + if (!capsule) { + delete holder; + return nullptr; + } + + // Create the numpy array + PyObject* array = PyArray_NewFromDescr( + &PyArray_Type, + PyArray_DescrFromType(NumpyTypeMap::type_num), + 1, // nd + dims, // dimensions + nullptr, // strides + holder->get_data(), // data + NPY_ARRAY_WRITEABLE, // flags + nullptr // obj + ); + + if (!array) { + Py_DECREF(capsule); + return nullptr; + } + + // Set the array's base object to our capsule + if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) + { + Py_DECREF(capsule); + Py_DECREF(array); + return nullptr; + } + + return array; +} + + +// Create shared array from deque (by const reference) +template +static PyObject* create_shared_array_from_deque(const std::deque& deq) +{ + auto holder = new SharedArrayHolder(deq); + + npy_intp dims[1] = {static_cast(holder->get_size())}; + + PyObject* capsule = PyCapsule_New( + holder, + "array_memory", + shared_array_dealloc + ); + + if (!capsule) { + delete holder; + return nullptr; + } + + PyObject* array = PyArray_NewFromDescr( + &PyArray_Type, + PyArray_DescrFromType(NumpyTypeMap::type_num), + 1, + dims, + nullptr, + holder->get_data(), + NPY_ARRAY_WRITEABLE, + nullptr + ); + + if (!array) + { + Py_DECREF(capsule); + return nullptr; + } + + if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) + { + Py_DECREF(capsule); + Py_DECREF(array); + return nullptr; + } + + return array; +} + + + +// Create shared array from deque (by rvalue reference) +template +static PyObject* create_shared_array_from_deque(std::deque&& deq) +{ + auto holder = new SharedArrayHolder(std::move(deq)); + + npy_intp dims[1] = {static_cast(holder->get_size())}; + + PyObject* capsule = PyCapsule_New( + holder, + "array_memory", + shared_array_dealloc + ); + + if (!capsule) + { + delete holder; + return nullptr; + } + + PyObject* array = PyArray_NewFromDescr( + &PyArray_Type, + PyArray_DescrFromType(NumpyTypeMap::type_num), + 1, + dims, + nullptr, + holder->get_data(), + NPY_ARRAY_WRITEABLE, + nullptr + ); + + if (!array) { + Py_DECREF(capsule); + return nullptr; + } + + if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) { + Py_DECREF(capsule); + Py_DECREF(array); + return nullptr; + } + + return array; +} + +// Function to create shared numpy array from existing C++ vector using move semantics +template +static PyObject* create_shared_array_from_vector(std::vector&& vec) +{ + + // Create the array holder using move constructor + auto holder = new SharedArrayHolder(std::move(vec)); + + npy_intp dims[1] = {static_cast(holder->get_size())}; + + PyObject* capsule = PyCapsule_New( + holder, + "array_memory", + shared_array_dealloc + ); + + if (!capsule) { + delete holder; + return nullptr; + } + + PyObject* array = PyArray_NewFromDescr( + &PyArray_Type, + PyArray_DescrFromType(NumpyTypeMap::type_num), + 1, + dims, + nullptr, + holder->get_data(), + NPY_ARRAY_WRITEABLE, + nullptr + ); + + if (!array) { + Py_DECREF(capsule); + return nullptr; + } + + if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) { + Py_DECREF(capsule); + Py_DECREF(array); + return nullptr; + } + + return array; +} + + +// Function to create shared numpy array from existing C++ vector by const reference +template +static PyObject* create_shared_array_from_vector(const std::vector& vec) +{ + + // Create the array holder using move constructor + auto holder = new SharedArrayHolder(vec); + + npy_intp dims[1] = {static_cast(holder->get_size())}; + + PyObject* capsule = PyCapsule_New( + holder, + "array_memory", + shared_array_dealloc + ); + + if (!capsule) { + delete holder; + return nullptr; + } + + PyObject* array = PyArray_NewFromDescr( + &PyArray_Type, + PyArray_DescrFromType(NumpyTypeMap::type_num), + 1, + dims, + nullptr, + holder->get_data(), + NPY_ARRAY_WRITEABLE, + nullptr + ); + + if (!array) { + Py_DECREF(capsule); + return nullptr; + } + + if (PyArray_SetBaseObject((PyArrayObject*)array, capsule) < 0) { + Py_DECREF(capsule); + Py_DECREF(array); + return nullptr; + } + + return array; +} + +#endif + diff --git a/setup.py b/setup.py index 7fa67b0..102aacb 100644 --- a/setup.py +++ b/setup.py @@ -141,7 +141,7 @@ def build_extensions(self): name="NeuroH5", package_dir={"": "python"}, packages=["neuroh5"], - version="0.1.14", + version="0.1.15", maintainer="Ivan Raikov", maintainer_email="ivan.g.raikov@gmail.com", description="A parallel HDF5-based library for storage and processing of large-scale graphs and neural cell model attributes.", diff --git a/src/data/attr_val.cc b/src/data/attr_val.cc index 978edd6..c456e0e 100644 --- a/src/data/attr_val.cc +++ b/src/data/attr_val.cc @@ -250,40 +250,79 @@ namespace neuroh5 } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->float_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->uint8_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->int8_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->uint16_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->int16_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const { return this->uint32_values[i]; } template<> - const std::vector& AttrVal::attr_vec (size_t i) const + const std::vector& AttrVal::const_attr_vec (size_t i) const + { + return this->int32_values[i]; + } + + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->float_values[i]; + } + + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->uint8_values[i]; + } + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->int8_values[i]; + } + + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->uint16_values[i]; + } + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->int16_values[i]; + } + + template<> + std::vector& AttrVal::attr_vec (size_t i) + { + return this->uint32_values[i]; + } + template<> + std::vector& AttrVal::attr_vec (size_t i) { return this->int32_values[i]; } diff --git a/tests/test_read_trees.py b/tests/test_read_trees.py index a3775a9..a047110 100644 --- a/tests/test_read_trees.py +++ b/tests/test_read_trees.py @@ -1,7 +1,7 @@ import pprint from mpi4py import MPI from neuroh5.io import read_trees -(g,_) = read_trees("data/dentatenet_Full_Scale_GC_Exc_Sat_SLN_extent_arena_margin_20201223_compressed.h5", "GC", topology=True, validate=True) +(g,_) = read_trees("data/dentatenet_Full_Scale_GC_Exc_Sat_SLN_extent_arena_margin_20201221_compressed.h5", "GC", topology=True, validate=True) #(g,_) = read_trees("data/GC_tree_syns_connections_20200214.h5", "GC", topology=False) (gid,t) = next(g) print(gid) diff --git a/tests/test_scatter_read_graph.py b/tests/test_scatter_read_graph.py index 30a6c94..e35858c 100644 --- a/tests/test_scatter_read_graph.py +++ b/tests/test_scatter_read_graph.py @@ -5,18 +5,17 @@ rank = comm.rank input_file='./data/dentate_test.h5' -input_file='/oasis/scratch/comet/iraikov/temp_project/dentate/Full_Scale_Control/DG_Connections_Full_Scale_20180722.h5' -input_file = '/scratch1/03320/iraikov/striped/dentate/Test_GC_1000/DG_Test_GC_1000_connections_20190625_compressed.h5' #(g,a) = scatter_read_graph(comm, input_file, io_size=1, namespaces=["Attributes"]) -(graph, a) = scatter_read_graph(input_file,io_size=8) +(graph, a) = scatter_read_graph(input_file, io_size=2) #projections=[('GC', 'MC'), ('MC', 'MC'), ('AAC', 'MC')], #namespaces=['Synapses','Connections']) -#print graph.keys() edge_dict = {} edge_iter = graph['GC']['MC'] +print(f"edge_iter = {edge_iter}") for (gid,edges) in edge_iter: + print(f"gid = {gid}") edge_dict[gid] = edges print("rank %d: %s" % (rank, str(edge_dict))) diff --git a/tests/test_scatter_read_trees.py b/tests/test_scatter_read_trees.py index a0ce5a6..91f485e 100644 --- a/tests/test_scatter_read_trees.py +++ b/tests/test_scatter_read_trees.py @@ -8,8 +8,9 @@ size = comm.Get_size() path = "/scratch1/03320/iraikov/striped2/MiV/Microcircuit/MiV_Cells_Microcircuit_20220412.h5" +path = "./data/DGC_forest_append_test_20180116.h5" -(g,n) = scatter_read_trees(path, "PYR", io_size=12, comm=comm) +(g,n) = scatter_read_trees(path, "GC", io_size=1, comm=comm) for (gid, tree) in g: print (f"rank {rank}: gid {gid} = {np.sum(tree['section'])}")