From 70a5e59d3333f6484478dab0351a3e073ca66a97 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Fri, 26 Jul 2024 16:34:42 -0500 Subject: [PATCH 1/9] Implementation for upper level APIs as documented in RFC:Programming model to Support Sparse Data in HDF5: --APIs to handle sparse data --APIs to support direct chunk I/O --APIs for structured chunk filtering --test/sparse_storage.c to trigger the coding path for the APIs --- doxygen/examples/tables/propertyLists.dox | 20 + src/H5D.c | 359 +++++++ src/H5Dchunk.c | 149 +++ src/H5Dint.c | 72 ++ src/H5Dpkg.h | 11 + src/H5Dpublic.h | 337 +++++++ src/H5Pocpl.c | 512 ++++++++++ src/H5Ppublic.h | 592 ++++++++++++ src/H5VLconnector.h | 18 +- src/H5VLnative.h | 61 ++ src/H5VLnative_dataset.c | 162 +++- src/H5VLnative_introspect.c | 4 + src/H5Zpublic.h | 7 + test/CMakeLists.txt | 1 + test/Makefile.am | 2 +- test/sparse_storage.c | 1065 +++++++++++++++++++++ 16 files changed, 3366 insertions(+), 6 deletions(-) create mode 100644 test/sparse_storage.c diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 340e13c26a5..75cc522fccd 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -905,6 +905,26 @@ encoding for object names. #H5Pset_fletcher32 Sets up use of the Fletcher32 checksum filter. + +#H5Pset_filter2/#H5Pget_filter3 +Adds/gets a filter to/from the filter pipeline for a specified section of structured chunk. + + +#H5Pget_filter_by_id3 +Returns information about a filter in the pipeline for a specified section of structured chunk. + + +#H5Pget_nfilters2 +Returns the number of filters in the pipeline for a specified section of structured chunk. + + +#H5Premove_filter2 +Remove a filter in the filter pipeline for a specified section of structured chunk. + + +#H5Pmodify_filter2 +Modifies a filter in the filter pipeline for a specified section of structured chunk. + //! [ocpl_table] * diff --git a/src/H5D.c b/src/H5D.c index 7416405e7f9..1e28f7f78a1 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2447,3 +2447,362 @@ H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t op, void *op_dat done: FUNC_LEAVE_API(ret_value) } /* end H5Dchunk_iter() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_defined + * + * Purpose: Retrieves a dataspace with a selection containing all defined + * elements that are also selected in the input argument + * "file_space_id". + * + * Return: Success: For H5D_SPARSE_CHUNK layout, this function returns + * a dataspace ID with a selection containing all + * defined elements that are also selected in + * "file_space_id". + * For other layouts, this function returns a copy of + * "file_space_id". + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_dataset_get_args_t vol_cb_args; /* Arguments to VOL callback */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + + /* Check args */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid dataset identifier"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + vol_cb_args.op_type = H5VL_DATASET_GET_DEFINED; + vol_cb_args.args.get_defined.file_space_id = file_space_id; + vol_cb_args.args.get_defined.space_id = H5I_INVALID_HID; + + /* Get the dataspace with defined elements */ + if (H5VL_dataset_get(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, H5I_INVALID_HID, "can't get dataspce with defined elements"); + + /* Set return value */ + ret_value = vol_cb_args.args.get_defined.space_id; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_defined() */ + +/*------------------------------------------------------------------------- + * Function: H5Derase + * + * Purpose: Deletes elements from a dataset causing them to no longer be defined + * This function is only useful for datasets with H5D_SPARSE_CHUNK layout. + * For other layouts this function will return an error. + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_dataset_specific_args_t vol_cb_args; /* Arguments to VOL callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid dataset identifier"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + vol_cb_args.op_type = H5VL_DATASET_ERASE; + vol_cb_args.args.erase.file_space_id = file_space_id; + + /* Deletes elements from a dataset causing them to no longer be defined */ + if (H5VL_dataset_specific(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to erase elements in dataset"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Derase() */ + +/*------------------------------------------------------------------------- + * Function: H5Dwrite_struct_chunk + * + * Purpose: Writes an entire structured chunk to the file directly. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]) +{ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID"); + if (!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL"); + if (!chunk_info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "chunk_info cannot be NULL"); + if (!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf array not provided"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.write_struct_chunk.offset = offset; + dset_opt_args.write_struct_chunk.chunk_info = chunk_info; + dset_opt_args.write_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; + + /* Write chunk */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dwrite_struct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: H5Dread_struct_chunk + * + * Purpose: Reads an entire structured chunk from the file directly. + * + * Return: Non-negative on success/Negative on failure + * + *--------------------------------------------------------------------------- + */ +herr_t +H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[] /*out*/) +{ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID"); + if (!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL"); + if (!chunk_info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "chunk_info cannot be NULL"); + if (!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf array cannot be NULL"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.read_struct_chunk.offset = offset; + dset_opt_args.read_struct_chunk.chunk_info = chunk_info; + dset_opt_args.read_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; + + /* Read the raw chunk */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dread_struct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_struct_chunk_info + * + * Purpose: Retrieves structured chunk information using chunk index + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hid_t fspace_id; IN: File dataspace selection ID + * hsize_t chunk_idx; IN: Chunk index + * hsize_t *offset OUT: Logical position of the chunk's first + * element in the array + * H5D_struct_chunk_info_t *chunk_info OUT: Information about the structured chunk + * haddr_t *addr OUT: Chunk address in the file + * hsize_t *chunk_size OUT: Chunk size in bytes + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset /*out*/, + H5D_struct_chunk_info_t *chunk_info/*out*/, haddr_t *addr /*out*/, hsize_t *chunk_size /*out*/) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == offset && NULL == chunk_info && NULL == addr && NULL == chunk_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid arguments, must have at least one non-null output argument"); + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + + /* Set up VOL callback arguments */ + dset_opt_args.get_struct_chunk_info_by_idx.fspace_id = fspace_id; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_idx = chunk_idx; + dset_opt_args.get_struct_chunk_info_by_idx.offset = offset; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_idx.addr = addr; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX; + vol_cb_args.args = &dset_opt_args; + + /* Call private function to get the chunk info given the chunk's index */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by index"); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Dget_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_struct_chunk_info_by_coord + * + * Purpose: Retrieves information about a chunk specified by its logical + * coordinates. + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hsize_t *offset IN: Logical position of the chunk's first + * element in the array + * H5D_struct_chunk_info_t *chunk_info OUT: Information about the structured chunk + * haddr_t *addr OUT: Chunk address in the file + * hsize_t *chunk_size OUT: Chunk size in bytes + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info/*out*/, + haddr_t *addr /*out*/, hsize_t *chunk_size /*out*/) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + if (NULL == chunk_info && NULL == addr && NULL == chunk_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid arguments, must have at least one non-null output argument"); + if (NULL == offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)"); + + /* Set up VOL callback arguments */ + dset_opt_args.get_struct_chunk_info_by_coord.offset = offset; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_coord.addr = addr; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD; + vol_cb_args.args = &dset_opt_args; + + /* Call private function to get the chunk info given the chunk's index */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by its logical coordinates"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_struct_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5Dstruct_chunk_iter + * + * Purpose: Iterates over all structured chunks in dataset calling + * the user specified callback function, "cb", and the + * callback's required data, "op_data". + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hid_t dxpl_id; IN: Dataset transfer property list ID + * H5D_struct_chunk_iter_op_t cb IN: Callback function provided by user; called for every chunk + * void *op_data IN/OUT: User-defined data passed on to user callback. + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t op, void *op_data) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + if (NULL == op) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid callback to chunk iteration"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.struct_chunk_iter.op = op; + dset_opt_args.struct_chunk_iter.op_data = op_data; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER; + vol_cb_args.args = &dset_opt_args; + + /* Iterate over the chunks */ + if ((ret_value = H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL)) < 0) + HERROR(H5E_DATASET, H5E_BADITER, "error iterating over dataset chunks"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dstruct_chunk_iter() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 6419a52700b..e399f5f3482 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8253,3 +8253,152 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_get_offset_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5D__write_struct_chunk_direct + * + * Purpose: Internal routine to write a structured chunk directly into the file. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + /* TBD: check for H5D_SPARSE_CHUNK */ + /* assert(layout->type == H5D_SPARSE_CHUNK); */ + + /* TBD: set up and call routine to write the structured chunk */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__write_struct_chunk_direct() */ + +/*------------------------------------------------------------------------- + * Function: H5D__read_struct_chunk_direct + * + * Purpose: Internal routine to read a structured chunk directly from the file. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + /* TBD: check for H5D_SPARSE_CHUNK */ + /* assert(dset && H5D_SPARSE_CHUNK == layout->type); */ + assert(offset); + assert(chunk_info); + assert(buf); + + /* TBD: set up and call routine to read the structured chunk */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__read_struct_chunk_direct() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_struct_chunk_info + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk. + * + * TBD: Is the following also true for structured chunk? + * Note: Currently, the domain of the index in this function is of all + * the written chunks, regardless the dataspace. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space, + hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, + hsize_t H5_ATTR_UNUSED *chunk_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + assert(dset); + assert(dset->shared); + assert(space); + + /* TBD: go get structured chunk information using chunk index */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_struct_chunk_info_by_coord + * + * Purpose: Iterate over the structured chunks in the dataset to get the info + * of the desired chunk, given by its offset coordinates. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, + hsize_t H5_ATTR_UNUSED *chunk_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + assert(dset); + assert(dset->shared); + assert(offset); + + /* TBD: go get structured chunk information using chunk coordinates */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_struct_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5D__struct_chunk_iter + * + * Purpose: Iterate over all the structured chunks in the dataset + * with given callback and the callback's required data. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, + void H5_ATTR_UNUSED *op_data) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + assert(dset); + assert(dset->shared); + + /* TBD: iterate over all the structured chunks in the dataset */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_iter() */ + diff --git a/src/H5Dint.c b/src/H5Dint.c index 700d8306604..52253904f55 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -4010,3 +4010,75 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d) FUNC_LEAVE_NOAPI(d->dcpl_id); } /* end H5D_get_dcpl_id() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_defined + * + * Purpose: Returns the dataspace ID with selection of defined elements + * + * Return: Success: ID for dataspace + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +hid_t +H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) +{ + H5S_t *space = NULL; + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_PACKAGE + + /* TBD: + if (dset->shared->layout.type == H5D_SPARSE_CHUNK) + call routine to get defined elements + else + if (NULL == (space = H5S_copy(fspace, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); + */ + + /* FOR NOW: return copy of fspace */ + if (NULL == (space = H5S_copy(fspace, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); + + /* Create an ID */ + if ((ret_value = H5I_register(H5I_DATASPACE, space, true)) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, FAIL, "unable to register dataspace"); + +done: + if (ret_value < 0) + if (space != NULL) + if (H5S_close(space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5D__get_defined() */ + +/*------------------------------------------------------------------------- + * + * Function: H5D__erase + * + * Purpose: Deletes elements from a dataset + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* TBD: + if (dset->shared->layout.type == H5D_SPARSE_CHUNK) + call routine to delete elements + else + return error + */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__erase() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index c6c50c2eef3..75272a2eaa6 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -755,6 +755,17 @@ H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_ H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); #endif /* H5D_CHUNK_DEBUG */ +/* Functions that operate on H5D_SPARSE_CHUNK storage */ +H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace); +H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace); +H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chunk_idx, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); +H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + haddr_t *addr, hsize_t *chunk_size); +H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data); + /* format convert */ H5_DLL herr_t H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info, H5D_chk_idx_info_t *new_idx_info); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 19b9266556a..d224f40843e 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -128,6 +128,25 @@ typedef enum H5D_vds_view_t { } H5D_vds_view_t; //! + +//! +/** + * Used by APIs for direct chunk chunk I/O on filtered and non-filtered structured chunks + */ +typedef struct H5D_struct_chunk_info_t { + H5D_layout_t type; /* Type of the structured chunk */ + uint8_t num_sections; /* Number of sections in the structured chunk */ + uint16_t *filter_mask; /* Array of "num_sections" size */ + /* Contains filter mask for each section */ + size_t *section_size; /* Array of "num_sections" size */ + /* Contains the size of each section */ + size_t *section_orig_size; /* Array of "num_sections" size */ + /* Contains the original size of each section */ +} H5D_struct_chunk_info_t; +//! + + + //! /** * \brief Callback for H5Pset_append_flush() @@ -250,6 +269,29 @@ typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, void *op_data); //! +//! +/** + * \brief Callback for H5Dstruct_chunk_iter() + * + * \param[in] offset Logical position of the chunk's first element in the array + * \param[in] chunk_info Information about the structured chunk + * \param[in] addr Chunk address in the file, taking the user block (if any) into account + * \param[in] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * \param[in,out] op_data User-defined pointer to data required by the callback function + * \return \li Zero (#H5_ITER_CONT) causes the iterator to continue, returning + * zero when all elements have been processed. + * \li A positive value (#H5_ITER_STOP) causes the iterator to + * immediately return that value, indicating short-circuit success. + * \li A negative (#H5_ITER_ERROR) causes the iterator to immediately + * return that value, indicating failure. + * + * \since 1.x.x + * + */ +typedef int (*H5D_struct_chunk_iter_op_t)(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + haddr_t *addr, hsize_t *chunk_size, void *op_data); +//! + /********************/ /* Public Variables */ /********************/ @@ -1637,6 +1679,301 @@ H5_DLL herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, */ H5_DLL herr_t H5Dclose(hid_t dset_id); +/** + * -------------------------------------------------------------------------- + *\ingroup H5D + * + * \brief Retrieves a dataspace selection containing defined elements + * + * \dset_id to get the selection of defined elements from + * \param[in] file_space_id Dataspace identifier containing the selection of elements to + * be queried if they are defined + * \dxpl_id + * + * \return Returns a dataspace with a selection containing all defined elements + * that are also selected in \p file_space_id if successful; + * otherwise returns #H5I_INVALID_HID. + * + * \details H5Dget_defined() retrieves a dataspace with only the defined + * elements of a (subset of) dataset selected. The dataset is specified + * by its identifier \p dset_id, and data transfer properties are defined + * by \p dxpl_id. The subset of the dataset to search for defined values is + * given by the selection in \p file_space_id. Setting \p file_space_id to + * #H5S_ALL causes this function to return a selection containing all defined + * values in the dataset. + * + * This function is only useful for datasets with layout TBD: H5D_SPARSE_CHUNK. + * For other layouts this function will simply return a copy of \p file_space_id, + * as all elements are defined for non-sparse datasets. + * + * \since 1.x.x + * + * \par Example: TBD + * \snippet H5D_examples.c update + * + * \see .... + * + */ +H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); + +/** + * -------------------------------------------------------------------------- + *\ingroup H5D + * + * \brief Deletes elements from a dataset causing them to no longer be defined + * + * \dset_id to erase elements from + * \param[in] file_space_id Dataspace identifier containing the selection of elements to + * be erased + * \dxpl_id + * + * \return \herr_t + * + * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, + * causing them to no longer be defined. The part of the dataset to erase is defined + * by \p file_space_id. After this operation, reading from these elements + * will return fill values, and the elements will no longer be included in the + * selection returned by H5Dget_defined(). + * + * This function is only useful for datasets with layout TBD:H5D_SPARSE_CHUNK. + * For other layouts this function will return an error. + * + * \since 1.x.x + * + * \par Example: TBD + * \snippet H5D_examples.c update + * + * \see .... + * + */ +H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Writes a structured chunk from a buffer directly to a dataset in a file + * + * \dset_id + * \dxpl_id + * \param[in] offset Logical position of the chunk's first element in the + * dataspace + * \param[in] chunk_info Information about the structured chunk + * \param[in] buf Array of pointers to the sections of the structured + * chunk; the size of the array is equal to the number + * of sections in the structured chunk + * + * \return \herr_t + * + \ \details H5Dwrite_struct_chunk() writes a structured chunk specified by its + * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles + * the structured chunk according to the information provided in the + * \p chunk_info parameter and using data pointed by \p buf. + * \p buf is an array of pointers to the buffers containing data for each + * section of the structured chunk. + * + * Initially, this function will support only sparse chunks of the + * fixed-size data. Such chunks have only two sections: one for the + * encoded selection and the second one for data elements. + * + * \attention Exercise caution when using H5Dread_struct_chunk() and + * H5Dwrite_struct_chunk(), as they read and write data chunks directly + * in a file. H5Dwrite_struct_chunk() bypasses hyperslab selection, the + * conversion of data from one datatype to another, and the filter + * pipeline to write the chunk. Developers should have experience + * with these processes before using this function. + * + * \note H5Dread_struct_chunk() and H5Dwrite_struct_chunk() are currently not supported + * with parallel HDF5 and do not support variable-length types. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Reads a structured chunk directly from a dataset in a file into + * the provided buffer. + * + * \dset_id + * \dxpl_id + * \param[in] offset Logical position of the chunk's first element in the + * dataspace + * \param[in,out] chunk_info Information about the structured chunk + * \param[out] buf Array of pointers to the sections of structured chunk; + * the size of the array is equal to the number of + * sections in the structured chunk + * + * \return \herr_t + * + * \details H5Dread_struct_chunk() reads a structured chunk as specified by its logical + * offset \p offset in a chunked dataset \p dset_id and places data into the provided + * buffers pointed by \p buf. Information about the structured chunk is returned via + * the \p chunk_info parameter. \p buf is an array of pointers to the buffers into which + * data for each section of the structured chunk will be read into. It is + * application’s responsibility to allocate buffers of the appropriate size. + * + * Initially, this function will support only sparse chunks of the fixed-size data. + * Such chunk has only two sections: one for the encoded selection and the second + * one for data elements. + * + * \attention Exercise caution when using H5Dread_struct_chunk() and + * H5Dwrite_struct_chunk(), as they read and write data chunks directly + * in a file. H5Dwrite_struct_chunk() bypasses hyperslab selection, the + * conversion of data from one datatype to another, and the filter + * pipeline to write the chunk. Developers should have experience + * with these processes before using this function. Please see + * Using the Direct Chunk Write Function for more information. + * + * \note H5Dread_struct_chunk() and H5Dwrite_struct_chunk() are currently not supported + * with parallel HDF5 and do not support variable-length datatypes. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Retrieves structured chunk information using chunk index + * + * \dset_id + * \param[in] fspace_id File dataspace selection identifier + * \param[in] chunk_idx Chunk index + * \param[out] offset Logical position of the chunk's first element in the array + * \param[out] chunk_info Information about the structured chunk + * \param[out] addr Chunk address in the file, taking the user block (if any) into account + * \param[out] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * + * \return \herr_t + * + * \details H5Dget_struct_chunk_info() retrieves the offset coordinates, + * \p offset, structured chunk information, \p chunk_info, + * chunk's address, \p addr, and the size, \p chunk_size, + * for the dataset specified by the identifier \p dset_id and + * the chunk specified by the index, \p chunk_idx. + * The chunk belongs to a set of chunks in the selection + * specified by \p fspace_id. If the queried chunk + * does not exist in the file, the size will be set to 0 and + * address to #HADDR_UNDEF. \c NULL can be passed in for + * any \p out parameters. + * + * \note Prior to HDF5 1.14.4, the reported address did not take the + * user block into account. + * + * \p chunk_idx is the chunk index in the selection. + * The index value may have a value of 0 up + * to the number of chunks stored in the file that + * have a nonempty intersection with the file dataspace. + * + * \note As of 1.10.5, the dataspace intersection is not yet + * supported. Hence, the index is of all the written chunks. + * + * \p fspace_id specifies the file dataspace selection. It is + * intended to take #H5S_ALL to specify the current selection. + * + * \note Please be aware that this function currently does not + * support non-trivial selections; thus \p fspace_id has no + * effect. Also, the implementation does not handle the #H5S_ALL + * macro correctly. As a workaround, an application can get + * the dataspace for the dataset using H5Dget_space() and pass that + * in for \p fspace_id. This will be fixed in a future release. + * + * \par + * \note TBD: need to verify the above notes info are true for + * structured chunk. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Retrieves structured chunk information using its coordinates + * + * \dset_id + * \param[in] offset Logical position of the chunk's first element in the array + * \param[out] chunk_info Information about the structured chunk + * \param[out] addr Chunk address in the file, taking the user block + * (if any) into account + * \param[out] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * + * \return \herr_t + * + * \details H5Dget_struct_chunk_info_by_coord() retrieves the structured + * information \p chunk_info, chunk's address, \p addr and the + * size, \p chunk_size, for the dataset specified by + * the identifier \p dset_id and the chunk specified by its + * coordinates \p offset. + * + * \p offset is a pointer to a one-dimensional array with a size + * equal to the dataset's rank. Each element is the logical + * position of the chunk's first element in a dimension. + * + * If the queried chunk does not exist in the file, the size will + * be set to 0 and address to \c HADDR_UNDEF. The value pointed + * to by \p chunk_info will not be modified. \c NULL can be passed + * in for any \p out parameters. + * + * \note Prior to HDF5 1.14.4, the reported address did not take the + * user block into account. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Iterate over all chunks of a chunked dataset + * + * \dset_id + * \param[in] dxpl_id Identifier of a transfer property list + * \param[in] cb User callback function, called for every chunk. + * \param[in] op_data User-defined pointer to data required by op + * + * \return \herr_t + * + * \details H5Dstruct_chunk_iter iterates over all structured chunks in the dataset, + * calling the user supplied callback \p cb, and the callback's required + * data \p op_data. + * + * \note Prior to HDF5 1.14.4, the address passed to the callback did not take + * the user block into account. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \par Example: TBD + * For each chunk, print the allocated chunk size (0 for unallocated chunks). + * \snippet H5D_examples.c H5Dchunk_iter_cb + * Iterate over all structured chunked datasets and chunks in a file. + * \snippet H5D_examples.c H5Ovisit_cb + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t cb, void *op_data); + /** * -------------------------------------------------------------------------- * \ingroup ASYNC diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index f9d89ab4156..232691ba8dc 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -93,6 +93,8 @@ static herr_t H5P__ocrt_pipeline_close(const char *name, size_t size, void *valu /* Local routines */ static herr_t H5P__set_filter(H5P_genplist_t *plist, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/]); +static herr_t H5P__set_filter2(H5P_genplist_t *plist, uint64_t section_number, H5Z_filter_t filter, + uint64_t flags, size_t buf_size, const void *buf); /*********************/ /* Package Variables */ @@ -562,6 +564,233 @@ H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t FUNC_LEAVE_API(ret_value) } /* end H5Pmodify_filter() */ +/*------------------------------------------------------------------------- + * Function: H5Pmodify_filter2 + * + * Purpose: Modifies the specified FILTER in the filter pipeline for a + * specified SECTION_NUMBER of the structured chunk. + * + * The PLIST_ID argument is either a dataset or group creation + * creation property list. + * + * The SECTION_NUMBER argument is the specified section of the + * structured chunk + * + * The FILTER arguments specifies the filter identifier + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF_SIZE is the size in bytes of BUF which points to + * a buffer with auxiliary data for the filter. The values + * will be stored in the dataset object header as part of the + * filter information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. + * If the filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * If this bit is clear and the filter fails then the entire + * I/O operation fails. + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (filter < 0 || filter > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identifier"); + if (flags & ~((unsigned)H5Z_FLAG_DEFMASK)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + if (buf_size > 0 && !buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied"); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + Call new H5P_modify_filter() with buf_size and buf instead of cd_nelmts and cd_values + + /* Modify the filter parameters of the I/O pipeline */ + if (H5P_modify_filter(plist, filter, flags, cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't modify filter"); +#endif + + /* For now, just return SUCCEED */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pmodify_filter2() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_filter2 + * + * Purpose: Adds the specified FILTER and corresponding properties to the + * end of the data or link output filter pipeline + * + * The PLIST_ID argument is a dataset creation or group + * creation property list. + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * The FILTER argument is the filter identifier for the filter + * to be added to the pipeline. + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF argument points to a buffer of BUF_SIZE bytes which + * contains auxiliary data for the filter. The values will be + * stored in the dataset object header as part of the filter + * information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. If the + * filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * + * If this bit is clear and the filter fails then the entire I/O + * operation fails. + * + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (filter > 255) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid section number"); + if (filter < 0 || filter > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identifier"); + if (flags & ~((unsigned)H5Z_FLAG_DEFMASK)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + if (buf_size > 0 && !buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied"); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Call the private function */ + if (H5P__set_filter2(plist, section_number, filter, flags, buf_size, buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "failed to call private function"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_filter2() */ + +/*------------------------------------------------------------------------- + * Function: H5P__set_filter2 + * + * Purpose: Adds the specified FILTER and corresponding properties to the + * end of the data or link output filter pipeline. + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * The FILTER argument is the filter identifier for the filter + * to be added to the pipeline. + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF argument points to a buffer of BUF_SIZE bytes which + * contains auxiliary data for the filter. The values will be + * stored in the dataset object header as part of the filter + * information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. If the + * filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * + * If this bit is clear and the filter fails then the entire I/O + * operation fails. + * + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * If the filter is not registered, this function tries to load + * it dynamically during run time. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__set_filter2(H5P_genplist_t H5_ATTR_UNUSED *plist, uint64_t H5_ATTR_UNUSED section_number, + H5Z_filter_t H5_ATTR_UNUSED filter, uint64_t H5_ATTR_UNUSED flags, + size_t H5_ATTR_UNUSED buf_size, const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + +#ifdef TBD + Need to set up new H5O_pline_t message + Need to do something similar like H5P__set_filter1() as below: + + /* Check if filter is already available */ + if ((filter_avail = H5Z_filter_avail(filter)) < 0) + + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't check filter availability"); + + /* Get the pipeline property to append to */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Add the filter to the I/O pipeline */ + if (H5Z_append(&pline, filter, flags, cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to add filter to pipeline"); + + /* Put the I/O pipeline information back into the property list */ + if (H5P_poke(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set pipeline"); +#endif + + /* For now: just return SUCCEED */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__set_filter2() */ + + /*------------------------------------------------------------------------- * Function: H5Pset_filter * @@ -723,6 +952,59 @@ H5Pget_nfilters(hid_t plist_id) FUNC_LEAVE_API(ret_value) } /* end H5Pget_nfilters */ +/*------------------------------------------------------------------------- + * Function: H5Pget_nfilters2 + * + * Purpose: Returns the number of filters in the data or link + * pipeline depending on whether PLIST_ID is a dataset creation + * or group creation property list. + * In each pipeline the filters are numbered from zero through + * N-1 where N is the value returned by this function. + * During output to the file the filters of a pipeline are applied + * in increasing order (the inverse is true for input). + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * Return: Success: Number of filters or zero if there are none. + * + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5Pget_nfilters2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number) +{ + H5P_genplist_t *plist; /* Property list */ + int ret_value; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + Need to get the new pline with pline.nused for the specified section + + /* Get the pipeline property to query */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Set return value */ + ret_value = (int)(pline.nused); +#endif + + /* For now just return 0 */ + ret_value = 0; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_nfilters2 */ + + + /*------------------------------------------------------------------------- * Function: H5Pget_filter2 * @@ -804,6 +1086,103 @@ H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t FUNC_LEAVE_API(ret_value) } /* end H5Pget_filter2() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_filter3 + * + * Purpose: This is the query counterpart of H5Pset_filter2(). + * It retrieves information for a filter in the pipeline for a + * specified section of the structured chunk. + * The information returned is about a particular filter number in + * the filter pipeline depending on whether PLIST_ID is a + * dataset creation or group creation property list. + * + * On input: + * SECTION_NUMBER is the specified section of the structured chunk + * + * IDX should be a value between zero and N-1 as described + * for H5Pget_nfilters(). + * + * BUF_SIZE indicates the size in bytes of the buffer pointed to + * by BUF which is allocated by the caller. On exit, it contains the + * values defined by the filter. + * + * FILTER_CONFIG is a bit field containing encode/decode flags from H5Zpublic.h. + * + * The function will return failure if the filter number is out of range. + * + * Return: Success: Filter identification number. + * Failure: H5Z_FILTER_ERROR (Negative) + * + *------------------------------------------------------------------------- + */ +H5Z_filter_t +H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, unsigned H5_ATTR_UNUSED idx, + uint64_t H5_ATTR_UNUSED *flags /*out*/, size_t H5_ATTR_UNUSED *buf_size /*in,out*/, + void H5_ATTR_UNUSED *buf /*out*/, size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, + unsigned H5_ATTR_UNUSED *filter_config /*out*/) +{ + H5Z_filter_t ret_value; /* return value */ + + FUNC_ENTER_API(H5Z_FILTER_ERROR) + +#ifdef TBD + + --Check arguments buf_size and buf instead of cd_nelmts and cd_values + --Obtain new pline info for structured chunk + --Call new H5P__get_filter() routine with buf_size and buf in place of cd_nelemts and cd_values + + /* Check args */ + if (cd_nelmts || cd_values) { + /* + * It's likely that users forget to initialize this on input, so + * we'll check that it has a reasonable value. The actual number + * is unimportant because the H5O layer will detect when a message + * is too large. + */ + if (cd_nelmts && *cd_nelmts > 256) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, + "probable uninitialized *cd_nelmts argument"); + if (cd_nelmts && *cd_nelmts > 0 && !cd_values) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, "client data values not supplied"); + + /* + * If cd_nelmts is null but cd_values is non-null then just ignore + * cd_values + */ + if (!cd_nelmts) + cd_values = NULL; + } /* end if */ + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_FILTER_ERROR, "can't find object for ID"); + + /* Get the pipeline property to query */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5Z_FILTER_ERROR, "can't get pipeline"); + + /* Check index */ + if (idx >= pline.nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, "filter number is invalid"); + + /* Set pointer to particular filter to query */ + filter = &pline.filter[idx]; + + /* Get filter information */ + if (H5P__get_filter(filter, flags, cd_nelmts, cd_values, namelen, name, filter_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5Z_FILTER_ERROR, "can't get filter info"); + + /* Set return value */ + ret_value = filter->id; +#endif + + /* For new just return 0 */ + ret_value = H5Z_FILTER_NONE; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_filter3() */ + /*------------------------------------------------------------------------- * Function: H5P_get_filter_by_id * @@ -914,6 +1293,89 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* FUNC_LEAVE_API(ret_value) } /* end H5Pget_filter_by_id2() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_filter_by_id3 + * + * Purpose: This is an additional query counterpart of H5Pset_filter2(). + * It retrieves information for a filter specified by its identifier + * in the pipeline for a specified section of the structured chunk. + * The information returned is about about a particular filter + * identifier in the filter pipeline depending on whether + * PLIST_ID is a dataset creation or group creation property list. + * + * On input: + * SECTION_NUMBER is the specified section of the structured chunk + * + * ID is the filter ID to retrieve the parameters for + * + * BUF_SIZE indicates the size in bytes of the buffer pointed to + * by BUF which is allocated by the caller. On exit, it contains + * size of the values defined by the filter. + * + * FILTER_CONFIG is a bit field containing encode/decode flags + * from H5Zpublic.h. + * + * If the filter is not set for the property list, an error will be returned. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, + H5Z_filter_t id, uint64_t H5_ATTR_UNUSED *flags /*out*/, + size_t H5_ATTR_UNUSED *buf_size /*in,out*/, void H5_ATTR_UNUSED *buf /*out*/, + size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, + unsigned H5_ATTR_UNUSED *filter_config /*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range"); + +#ifdef TBD + --Check arguments buf_size and buf instead of cd_nelmts and cd_values + --Call new H5P_get_filter_by_id() routine with buf_size and buf in place + of cd_nelemts and cd_values + + if (cd_nelmts || cd_values) { + /* + * It's likely that users forget to initialize this on input, so + * we'll check that it has a reasonable value. The actual number + * is unimportant because the H5O layer will detect when a message + * is too large. + */ + if (cd_nelmts && *cd_nelmts > 256) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "probable uninitialized *cd_nelmts argument"); + if (cd_nelmts && *cd_nelmts > 0 && !cd_values) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "client data values not supplied"); + + /* + * If cd_nelmts is null but cd_values is non-null then just ignore + * cd_values + */ + if (!cd_nelmts) + cd_values = NULL; + } /* end if */ + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Get filter information */ + if (H5P_get_filter_by_id(plist, id, flags, cd_nelmts, cd_values, namelen, name, filter_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get filter info"); +#endif + + /* For now just return SUCCEED */ +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_filter_by_id3() */ + /*------------------------------------------------------------------------- * Function: H5Pall_filters_avail * @@ -1025,6 +1487,56 @@ H5Premove_filter(hid_t plist_id, H5Z_filter_t filter) FUNC_LEAVE_API(ret_value) } /* end H5Premove_filter() */ +/*------------------------------------------------------------------------- + * Function: H5Premove_filter2 + * + * Purpose: Deletes a filter for a specified section of the structured chunk + * from the dataset creation or group creation property list; + * deletes all filters if FILTER is H5Z_FILTER_ALL + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Premove_filter2(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, + H5Z_filter_t H5_ATTR_UNUSED filter) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + --Obtain new pline info for structured chunk then delete the filter + + /* Get the pipeline property to modify */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Check if there are any filters */ + if (pline.filter) { + /* Delete filter */ + if (H5Z_delete(&pline, filter) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't delete filter"); + + /* Put the I/O pipeline information back into the property list */ + if (H5P_poke(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set pipeline"); + } /* end if */ +#endif + + /* For now, just return SUCCEED */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Premove_filter2() */ + /*------------------------------------------------------------------------- * Function: H5Pset_deflate * diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 724893484a2..643c7fbc782 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -2069,6 +2069,7 @@ H5_DLL herr_t H5Pget_attr_creation_order(hid_t plist_id, unsigned *crt_order_fla * */ H5_DLL herr_t H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact, unsigned *min_dense); + /** * \ingroup OCPL * @@ -2133,6 +2134,75 @@ H5_DLL herr_t H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact, un H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, char name[], unsigned *filter_config /*out*/); +/** + * \ingroup OCPL + * + * \brief Returns information for a filter in the pipeline for a specified section + * of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] idx Sequence number within the filter pipeline of + * the filter for which information is sought + * \param[out] flags Bit vector specifying certain general properties + * of the filter + * \param[in,out] buf_size Size in bytes of \p buf buffer + * \param[out] buf Buffer with auxiliary data for the filter + * \param[in] namelen Anticipated number of characters in \p name + * \param[out] name Name of the filter + * \param[out] filter_config Bit field, as described in H5Zget_filter_info() + * + * \return Returns a negative value on failure, and the filter identifier + * if successful (see #H5Z_filter_t): + * - #H5Z_FILTER_DEFLATE Data compression filter, + * employing the gzip algorithm + * - #H5Z_FILTER_SHUFFLE Data shuffling filter + * - #H5Z_FILTER_FLETCHER32 Error detection filter, employing the + * Fletcher32 checksum algorithm + * - #H5Z_FILTER_SZIP Data compression filter, employing the + * SZIP algorithm + * - #H5Z_FILTER_NBIT Data compression filter, employing the + * N-bit algorithm + * - #H5Z_FILTER_SCALEOFFSET Data compression filter, employing the + * scale-offset algorithm + * + * \details H5Pget_filter3() returns information about a filter for a specified + * section in the structured chunk. The filter is specified + * by its filter number, in a filter pipeline specified by the property + * list with which it is associated. + * + * \p plist_id must be a dataset or group creation property list. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * \p idx is a value between zero and N-1, as described in + * H5Pget_nfilters(). The function will return a negative value if + * the filter number is out of range. + * + * The structure of the \p flags argument is discussed in + * H5Pset_filter2(). + * + * On input, \p buf_size indicates the size of the buffer pointerd + * to by \p buf, as allocated by the caller; on return, + * \p buf_size contains the size of the values defined by the filter. + * + * If \p name is a pointer to an array of at least \p namelen bytes, + * the filter name will be copied into that array. The name will be + * null terminated if \p namelen is large enough. The filter name + * returned will be the name appearing in the file, the name + * registered for the filter, or an empty string. + * + * \p filter_config is the bit field described in + * H5Zget_filter_info(). + * + * \since 1.x.x + * + */ +H5_DLL H5Z_filter_t H5Pget_filter3(hid_t plist_id, uint64_t section_number, unsigned idx, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, unsigned *filter_config /*out*/); + /** * \ingroup OCPL * @@ -2191,6 +2261,71 @@ H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *f H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, char name[] /*out*/, unsigned *filter_config /*out*/); +/** + * \ingroup OCPL + * + * \brief Returns information for a filter specified by its identifier in the + * pipeline for a specified section of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter_id Filter identifier + * \param[out] flags Bit vector specifying certain general + * properties of the filter + * \param[in,out] buf_size Size in bytes of \p buf buffer + * \param[out] buf Auxiliary data for the filter + * \param[in] namelen Length of filter name and number of + * elements in \p name + * \param[out] name[] Name of filter + * \param[out] filter_config Bit field, as described in + * H5Zget_filter_info() + * + * \return \herr_t + * + * \details H5Pget_filter_by_id3() returns information about a filter + * for a specified section in the structured chunk with the + * filter identifier specified in \p filter_id. + * + * \p plist_id must be a dataset or group creation property list + * and \p filter_id must be in the associated filter pipeline. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p filter_id and \p flags parameters are used in the same + * manner as described in the discussion of H5Pset_filter2(). + * + * Aside from the fact that they are used for output, the + * parameters \p buf_size and \p buf are used in the same + * manner as described in the discussion of H5Pset_filter2(). + * On input, the \p buf_size parameter indicates the size in + * bytes of the buffer pointed to by \p buf which is allocated + * by the calling program; on exit it contains the size of the + * values defined by the filter. + * + * On input, the \p namelen parameter indicates the number of + * characters allocated for the filter name by the calling program + * in the array \p name[]. On exit \p name[] contains the name of the + * filter with one character of the name in each element of the + * array. + * + * \p filter_config is the bit field described in + * H5Zget_filter_info(). + * + * If the filter specified in \p filter_id is not set for the + * property list, an error will be returned and + * H5Pget_filter_by_id3() will fail. + * + * \version 1.8.5 Function extended to work with group creation property + * lists. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Pget_filter_by_id3(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter_id, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, unsigned *filter_config /*out*/); + /** * \ingroup OCPL * @@ -2216,6 +2351,36 @@ H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsig * */ H5_DLL int H5Pget_nfilters(hid_t plist_id); + + +/** + * \ingroup OCPL + * + * \brief Returns the number of filters in the pipeline for a section of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * + * \return Returns the number of filters in the pipeline if successful; + * otherwise returns a negative value. + * + * \details H5Pget_nfilters2() returns the number of filters defined in the + * filter pipeline for the section \p section_number that is + * associated with the property list \p plist_id. + * + * In each pipeline, the filters are numbered from 0 through \Code{N-1}, + * where \c N is the value returned by this function. During output to + * the file, the filters are applied in increasing order; during + * input from the file, they are applied in decreasing order. + * + * H5Pget_nfilters() returns the number of filters in the pipeline + * for the section \p section_number, including zero (0) if there are none. + * + * \since 1.0.0 + * + */ +H5_DLL int H5Pget_nfilters2(hid_t plist_id, uint64_t section_number); + /** * \ingroup OCPL * @@ -2248,6 +2413,7 @@ H5_DLL int H5Pget_nfilters(hid_t plist_id); * */ H5_DLL herr_t H5Pget_obj_track_times(hid_t plist_id, hbool_t *track_times); + /** * \ingroup OCPL * @@ -2277,6 +2443,43 @@ H5_DLL herr_t H5Pget_obj_track_times(hid_t plist_id, hbool_t *track_times); */ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/]); + +/** + * \ingroup OCPL + * + * \brief Modifies a filter in the filter pipeline for a specified section of the + * structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter to be modified + * \param[in] flags Bit vector specifying certain general properties + * of the filter + * \param[in] buf_size Size in bytes of \p buf buffer + * \param[in] buf Auxiliary data for the filter + * + * \return \herr_t + * + * \details H5Pmodify_filter2() modifies the specified \p filter in the + * filter pipeline for a specified section of the structured + * chunk. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p plist_id parameter must be a dataset or group creation + * property list. + * + * The \p filter, \p flags \p buf_size, and \p buf + * parameters are used in the same manner and accept the same + * values as described in the discussion of H5Pset_filter3(). + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Pmodify_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, + uint64_t flags, size_t buf_size, const void *buf); + /** * \ingroup OCPL * @@ -2343,6 +2546,77 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int * */ H5_DLL herr_t H5Premove_filter(hid_t plist_id, H5Z_filter_t filter); + +/** + * \ingroup OCPL + * + * \brief Removes a filter in the filter pipeline for a specified section + * of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter to be deleted + * + * \return \herr_t + * + * \details H5Premove_filter2() removes the specified \p filter for a specified + * \p section_number from the filter pipeline in the dataset or + * group creation property list \p plist_id. + * + * The \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p filter parameter specifies the filter to be removed. + * Valid values for use in \p filter are as follows: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FILTER_ALLRemoves all filters from the filter pipeline
#H5Z_FILTER_DEFLATEData compression filter, employing the gzip + * algorithm
#H5Z_FILTER_SHUFFLEData shuffling filter
#H5Z_FILTER_FLETCHER32Error detection filter, employing the Fletcher32 + * checksum algorithm
#H5Z_FILTER_SZIPData compression filter, employing the SZIP + * algorithm
#H5Z_FILTER_NBITData compression filter, employing the N-Bit + * algorithm
#H5Z_FILTER_SCALEOFFSETData compression filter, employing the scale-offset + * algorithm
+ * + * Additionally, user-defined filters can be removed with this + * routine by passing the filter identifier with which they were + * registered with the HDF5 library. + * + * Attempting to remove a filter that is not in the filter + * pipeline is an error. + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Premove_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter); + /** * \ingroup OCPL * @@ -2489,6 +2763,7 @@ H5_DLL herr_t H5Pset_attr_phase_change(hid_t plist_id, unsigned max_compact, uns * */ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); + /** * \ingroup OCPL * @@ -2757,6 +3032,323 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); */ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, const unsigned int c_values[]); + +/** + * \ingroup OCPL + * + * \brief Adds a filter to the filter pipeline for a specified section of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter identifier for the filter to be added to the + * pipeline + * \param[in] flags Bit vector specifying certain general properties of + * the filter + * \param[in] buf_size Size in bytes of \p buf buffer + * \param[in] buf Buffer with an auxiliary data for the filter + * + * \return \herr_t + * + * \details H5Pset_filter2() adds a filter and corresponding properties to the end + * of an output filter pipeline. This function can be used with + * both current chunked storage and structured chunk storage + * including sparse chunk. It also addresses deficiency of + * H5Pset_filter1() in passing the filter’s data as described below. + * + * Note the following differences with H5Pset_filter1(): + * - This function accepts a new parameter \p section_number that + * specifies the section of the structured chunk to which the filter is applied. + * For chunked storage, there is just one section. + * - Data type for the \p flags parameter is changed to uint64_t to provide + * more flexibility to the VOL connectors that use the function. + * - This function passes the filter’s data by using a void pointer to a buffer + * with auxiliary data for the filter instead of unsigned int c_values[]. + * + * \p plist_id must be either a dataset creation property list or + * group creation property list identifier. If \p plist_id is a + * dataset creation property list identifier, the filter is added + * to the raw data filter pipeline. + * + * If \p plist_id is a group creation property list identifier, + * the filter is added to the link filter pipeline, which filters + * the fractal heap used to store most of the link metadata in + * certain types of groups. The only predefined filters that can + * be set in a group creation property list are the gzip filter + * (#H5Z_FILTER_DEFLATE) and the Fletcher32 error detection filter + * (#H5Z_FILTER_FLETCHER32). + * + * \p section_number specifies the section number. The value + * is 0 to 255 when native HDF5 file format is used. + * For sparse chunk, the convenience flag can be used to specify + * a section of the structured chunk to be filtered as described below: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FLAG_SPARSE_SELECTIONAdds the filter to the filter pipeline for the encoded + * selection section of the sparse chunk. It has the same + * effect as passing 0. The flag will be ignored if the + * structured chunk is not sparse. + *
#H5Z_FLAG_SPARSE_FIXED_DATAAdds the filter to the filter pipeline for section 1 of + * the sparse chunk. It has the same effect as passing 1. + *
#H5Z_FLAG_SPARSE_VL_DATAAdds the filter to the filter pipeline for section 2 of + * the sparse chunk if data has variable-length datatype. + * It has the same effect as passing 2. + *
+ * + * \p buf points to \p buf_size bytes of buffer + * which are auxiliary data for the filter. The values are typically + * used as parameters to control the filter. In a filter's + * \p set_local method (called from \p H5Dcreate), the values are + * interpreted and possibly modified before they are used to control + * the filter. These, possibly modified values, are then stored in + * the dataset object header as auxiliary data for the filter. + * + * The \p flags argument is a bit vector with the following + * fields specifying certain general properties of the filter: + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FLAG_OPTIONALIf this bit is set then the filter is optional. If the + * filter fails (see below) during an H5Dwrite() operation + * then the filter is just excluded from the pipeline for + * the chunk for which it failed; the filter will not + * participate in the pipeline during an H5Dread() of the + * chunk. This is commonly used for compression filters: + * if the filter result would be larger than the input, + * then the compression filter returns failure and the + * uncompressed data is stored in the file.

+ * This flag should not be set for the Fletcher32 checksum + * filter as it will bypass the checksum filter without + * reporting checksum errors to an application.
#H5Z_FLAG_MANDATORYIf the filter is required, that is, set to mandatory, + * and the filter fails, the library's behavior depends + * on whether the chunk cache is in use: + * \li If the chunk cache is enabled, data chunks will + * be flushed to the file during H5Dclose() and the + * library will return the failure in H5Dclose(). + * \li When the chunk cache is disabled or not big enough, + * or the chunk is being evicted from the cache, the + * failure will happen during H5Dwrite(). + * + * In each case, the library will still write to the file + * all data chunks that were processed by the filter + * before the failure occurred.

+ * For example, assume that an application creates a + * dataset of four chunks, the chunk cache is enabled and + * is big enough to hold all four chunks, and the filter + * fails when it tries to write the fourth chunk. The + * actual flush of the chunks will happen during + * H5Dclose(), not H5Dwrite(). By the time H5Dclose() + * fails, the first three chunks will have been written + * to the file. Even though H5Dclose() fails, all the + * resources will be released and the file can be closed + * properly.

+ * If, however, the filter fails on the second chunk, only + * the first chunk will be written to the file as nothing + * further can be written once the filter fails.
+ * + * The \p filter parameter specifies the filter to be set. Valid + * pre-defined filter identifiers are as follows: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FILTER_DEFLATEData compression filter, employing the gzip + * algorithm
#H5Z_FILTER_SHUFFLEData shuffling filter
#H5Z_FILTER_FLETCHER32Error detection filter, employing the Fletcher32 + * checksum algorithm
#H5Z_FILTER_SZIPData compression filter, employing the SZIP + * algorithm
#H5Z_FILTER_NBITData compression filter, employing the N-Bit + * algorithm
#H5Z_FILTER_SCALEOFFSETData compression filter, employing the scale-offset + * algorithm
+ * Also see H5Pset_edc_check() and H5Pset_filter_callback(). + * TBD: The behavior is the same for H5Pset_edc_check() but the + * functionality is not implemented for varible-length data. + * + * \note TBD: review/modify/add the info in the following note section + * for structured chunk v.s. dense chunks. + * + * \note When a non-empty filter pipeline is used with a group creation + * property list, the group will be created with the new group file + * format. The filters will come into play only when dense storage + * is used (see H5Pset_link_phase_change()) and will be applied to + * the group's fractal heap. The fractal heap will contain most of + * the group's link metadata, including link names. + * + * \note When working with group creation property lists, if you are + * adding a filter that is not in HDF5's set of predefined filters, + * i.e., a user-defined or third-party filter, you must first + * determine that the filter will work for a group. See the + * discussion of the set local and can apply callback functions + * in H5Zregister(). + * + * \note If multiple filters are set for a property list, they will be + * applied to each chunk of raw data for datasets or each block + * of the fractal heap for groups in the order in which they were + * set. + * + * \note Filters can be applied only to chunked datasets; + * they cannot be used with other dataset storage methods, such as + * contiguous, compact, or external datasets. + * + * \note Dataset elements of variable-length and dataset region + * reference datatypes are stored in separate structures in the + * file called heaps. Filters cannot currently be applied to + * these heaps. + * + * \note Filter Behavior in HDF5:
+ * Filters can be inserted into the HDF5 pipeline to perform + * functions such as compression and conversion. As such, they are + * a very flexible aspect of HDF5; for example, a user-defined + * filter could provide encryption for an HDF5 dataset. + * + * \note A filter can be declared as either required or optional. + * Required is the default status; optional status must be + * explicitly declared. + * + * \note A required filter that fails or is not defined causes an + * entire output operation to fail; if it was applied when the + * data was written, such a filter will cause an input operation + * to fail. + * + * \note The following table summarizes required filter behavior. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Required FILTER_X not availableFILTER_X available
H5Pset_Will fail.Will succeed.
H5Dwrite with FILTER_X setWill fail.Will succeed; FILTER_X will be applied to + * the data.
H5Dread with FILTER_X setWill fail.Will succeed.
+ * \note An optional filter can be set for an HDF5 dataset even when + * the filter is not available. Such a filter can then be + * applied to the dataset when it becomes available on the + * original system or when the file containing the dataset is + * processed on a system on which it is available. + * + * \note A filter can be declared as optional through the use of the + * #H5Z_FLAG_OPTIONAL flag with H5Pset_filter(). + * + * \note Consider a situation where one is creating files that will + * normally be used only on systems where the optional (and + * fictional) filter FILTER_Z is routinely available. One can + * create those files on system A, which lacks FILTER_Z, create + * chunked datasets in the files with FILTER_Z defined in the + * dataset creation property list, and even write data to those + * datasets. The dataset object header will indicate that FILTER_Z + * has been associated with this dataset. But since system A does + * not have FILTER_Z, dataset chunks will be written without it + * being applied. + * + * \note HDF5 has a mechanism for determining whether chunks are + * actually written with the filters specified in the object + * header, so while the filter remains unavailable, system A will + * be able to read the data. Once the file is moved to system B, + * where FILTER_Z is available, HDF5 will apply FILTER_Z to any + * data rewritten or new data written in these datasets. Dataset + * chunks that have been written on system B will then be + * unreadable on system A; chunks that have not been re-written + * since being written on system A will remain readable on system + * A. All chunks will be readable on system B. + * + * \note The following table summarizes optional filter behavior. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
FILTER_Z not availableFILTER_Z available
with encode and decode		FILTER_Z available decode only
H5Pset_Will succeed.Will succeed.Will succeed.
H5Dread with FILTER_Z setWill succeed if FILTER_Z has not actually
+ * been applied to data.		Will succeed.Will succeed.
H5Dwrite with FILTER_Z setWill succeed;
of the structured chunto k + * FILTER_Z will not be applied to the data.		Will succeed;
+ * FILTER_Z will be applied to the data.		Will succeed;
+ * FILTER_Z will not be applied to the data.
+ * \note The above principles apply generally in the use of HDF5 + * optional filters insofar as HDF5 does as much as possible to + * complete an operation when an optional filter is unavailable. + * (The SZIP filter is an exception to this rule; see H5Pset_szip() + * for details.) + * + * \see \ref_filter_pipe, \ref_group_impls + * + * \version 1.8.5 Function applied to group creation property lists. + * \since 1.6.0 + * + */ +H5_DLL herr_t H5Pset_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf); + /** * \ingroup OCPL * diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index 4bf082397f8..c5399d65c6a 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -221,7 +221,8 @@ typedef enum H5VL_dataset_get_t { H5VL_DATASET_GET_SPACE, /* dataspace */ H5VL_DATASET_GET_SPACE_STATUS, /* space status */ H5VL_DATASET_GET_STORAGE_SIZE, /* storage size */ - H5VL_DATASET_GET_TYPE /* datatype */ + H5VL_DATASET_GET_TYPE, /* datatype */ + H5VL_DATASET_GET_DEFINED /* dataspace selection of defined elements */ } H5VL_dataset_get_t; /* Parameters for dataset 'get' operations */ @@ -259,6 +260,13 @@ typedef struct H5VL_dataset_get_args_t { struct { hid_t type_id; /* Datatype ID (OUT) */ } get_type; + + /* H5VL_DATASET_GET_DEFINED */ + struct { + hid_t file_space_id; /* file dataspace selection to be queried (IN) */ + hid_t space_id; /* dataspace selection of defined elements (OUT) */ + } get_defined; + } args; } H5VL_dataset_get_args_t; @@ -266,7 +274,8 @@ typedef struct H5VL_dataset_get_args_t { typedef enum H5VL_dataset_specific_t { H5VL_DATASET_SET_EXTENT, /* H5Dset_extent */ H5VL_DATASET_FLUSH, /* H5Dflush */ - H5VL_DATASET_REFRESH /* H5Drefresh */ + H5VL_DATASET_REFRESH, /* H5Drefresh */ + H5VL_DATASET_ERASE, /* H5Derase */ } H5VL_dataset_specific_t; /* Parameters for dataset 'specific' operations */ @@ -289,6 +298,11 @@ typedef struct H5VL_dataset_specific_args_t { struct { hid_t dset_id; /* Dataset ID (IN) */ } refresh; + + /* H5VL_DATASET_ERASE */ + struct { + hid_t file_space_id; /* file dataspace selection to be erased (IN) */ + } erase; } args; } H5VL_dataset_specific_args_t; diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 6ca484ae329..84213fff872 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -64,6 +64,11 @@ typedef union H5VL_native_attr_optional_args_t { /* NOTE: If new values are added here, the H5VL__native_introspect_opt_query * routine must be updated. */ +/* TBD: the existing H5VL_NATIVE_DATASET_CHUNK_ITER is not added to + * H5VL__native_introspect_opt_query, therefore I didn't add + * H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER to H5VL__native_introspect_opt_query either. + * Check for sure later. + */ #define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ #define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ #define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ @@ -75,6 +80,11 @@ typedef union H5VL_native_attr_optional_args_t { #define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ #define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ #define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ +#define H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK 11 /* H5Dread_struct_chunk */ +#define H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK 12 /* H5Dwrite_struct_chunk */ +#define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX 13 /* H5Dget_struct_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD 14 /* H5Dget_struct_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER 15 /* H5Dstruct_chunk_iter */ /* NOTE: If values over 1023 are added, the H5VL_RESERVED_NATIVE_OPTIONAL macro * must be updated. */ @@ -94,6 +104,21 @@ typedef struct H5VL_native_dataset_chunk_write_t { const void *buf; } H5VL_native_dataset_chunk_write_t; +/* TBD: can we use the same struct for both read and write operations? */ +/* Parameters for native connector's dataset 'read struct chunk' operation */ +typedef struct H5VL_native_dataset_read_struct_chunk_t { + const hsize_t *offset; + H5D_struct_chunk_info_t *chunk_info; + void **buf; +} H5VL_native_dataset_read_struct_chunk_t; + +/* Parameters for native connector's dataset 'write struct chunk' operation */ +typedef struct H5VL_native_dataset_write_struct_chunk_t { + const hsize_t *offset; + H5D_struct_chunk_info_t *chunk_info; + void **buf; +} H5VL_native_dataset_write_struct_chunk_t; + /* Parameters for native connector's dataset 'get vlen buf size' operation */ typedef struct H5VL_native_dataset_get_vlen_buf_size_t { hid_t type_id; @@ -131,6 +156,24 @@ typedef struct H5VL_native_dataset_get_chunk_info_by_coord_t { hsize_t *size; /* Size of chunk in file (OUT) */ } H5VL_native_dataset_get_chunk_info_by_coord_t; +/* Parameters for native connector's dataset 'get struct chunk info by idx' operation */ +typedef struct H5VL_native_dataset_get_struct_chunk_info_by_idx_t { + hid_t fspace_id; /* File dataspace selection */ + hsize_t chunk_idx; /* Chunk index */ + hsize_t *offset; /* Logical position of the chunk's first element in the array (OUT) */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ +} H5VL_native_dataset_get_struct_chunk_info_by_idx_t; + +/* Parameters for native connector's dataset 'get struct chunk info by coord' operation */ +typedef struct H5VL_native_dataset_get_struct_chunk_info_by_coord_t { + const hsize_t *offset; /* Logical position of the chunk's first element in the array */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ +} H5VL_native_dataset_get_struct_chunk_info_by_coord_t; + /* Parameters for native connector's dataset 'optional' operations */ typedef union H5VL_native_dataset_optional_args_t { /* H5VL_NATIVE_DATASET_FORMAT_CONVERT */ @@ -173,6 +216,24 @@ typedef union H5VL_native_dataset_optional_args_t { void *op_data; /* Context to pass to iteration callback */ } chunk_iter; + /* H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK */ + H5VL_native_dataset_write_struct_chunk_t write_struct_chunk; + + /* H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK */ + H5VL_native_dataset_read_struct_chunk_t read_struct_chunk; + + /* H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX */ + H5VL_native_dataset_get_struct_chunk_info_by_idx_t get_struct_chunk_info_by_idx; + + /* H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD */ + H5VL_native_dataset_get_struct_chunk_info_by_coord_t get_struct_chunk_info_by_coord; + + /* H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER */ + struct { + H5D_struct_chunk_iter_op_t op; /* Structured chunk iteration callback */ + void *op_data; /* Context to pass to iteration callback */ + } struct_chunk_iter; + } H5VL_native_dataset_optional_args_t; /* Values for native VOL connector file optional VOL operations */ diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index ced4569d8d1..e118a7d1bee 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -205,7 +205,7 @@ H5VL__native_dataset_io_setup(size_t count, void *obj[], hid_t mem_type_id[], hi * Purpose: Frees memory allocated by H5VL__native_dataset_io_setup() * * Return: SUCCEED/FAIL - * + *------------------------------------------------------------------------- */ static herr_t @@ -497,6 +497,27 @@ H5VL__native_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t H5_ATTR break; } + /* H5Dget_defined */ + case H5VL_DATASET_GET_DEFINED: { + const H5S_t *fspace = NULL; + + assert(dset->shared); + assert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if (args->args.get_defined.file_space_id == H5S_ALL) + fspace = dset->shared->space; + else /* otherwise, use the given space ID */ + if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.get_defined.file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + /* Call private function */ + if ((args->args.get_defined.space_id = H5D__get_defined(dset, fspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace of defined elements"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't get this type of information from dataset"); } /* end switch */ @@ -547,6 +568,19 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid break; } + /* H5Derase */ + case H5VL_DATASET_ERASE: { + const H5S_t *fspace = NULL; + + if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.erase.file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + if (H5D__erase(dset, fspace) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to erase dataset elements"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid specific operation"); } /* end switch */ @@ -631,7 +665,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ case H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE: { H5VL_native_dataset_get_chunk_storage_size_t *gcss_args = &opt_args->get_chunk_storage_size; - /* Make sure the dataset is chunked */ + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dget_chunk_storage_storage() is supposed to work for both dense and sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -657,7 +692,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ if (NULL == (space = (const H5S_t *)H5I_object_verify(gnc_args->space_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); - /* Make sure the dataset is chunked */ + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dget_num_chunks() is supposed to work for both dense and sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -722,6 +758,9 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dread_chunk() is supposed to work for sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -747,6 +786,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -798,6 +838,122 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ break; } + /* H5Dread_struct_chunk */ + case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: { + H5VL_native_dataset_read_struct_chunk_t *read_struct_chunk_args = &opt_args->read_struct_chunk; + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + + /* Check arguments */ + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + /* TBD: check for H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). + */ + if (H5D__chunk_get_offset_copy(dset, read_struct_chunk_args->offset, offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + + /* Read the structured chunk */ + if (H5D__read_struct_chunk_direct(dset, offset_copy, read_struct_chunk_args->chunk_info, + read_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); + + break; + } + + /* H5Dwrite_struct_chunk */ + case H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK: { + H5VL_native_dataset_write_struct_chunk_t *write_struct_chunk_args = &opt_args->write_struct_chunk; + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + + /* Check arguments */ + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + /* TBD: check for H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). + */ + if (H5D__chunk_get_offset_copy(dset, write_struct_chunk_args->offset, offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + + /* Write the structured chunk */ + if (H5D__write_struct_chunk_direct(dset, offset_copy, write_struct_chunk_args->chunk_info, + write_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); + + break; + } + + /* H5Dget_struct_chunk_info */ + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX: { + H5VL_native_dataset_get_struct_chunk_info_by_idx_t *gcibi_args = &opt_args->get_struct_chunk_info_by_idx; + const H5S_t *space; + + assert(dset->shared); + assert(dset->shared->space); + + /* TBD: is this true for structured chunk */ + /* When default dataspace is given, use the dataset's dataspace */ + if (gcibi_args->fspace_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if (NULL == (space = (const H5S_t *)H5I_object_verify(gcibi_args->fspace_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if (H5D__get_struct_chunk_info(dset, space, gcibi_args->chunk_idx, gcibi_args->offset, + gcibi_args->chunk_info, gcibi_args->addr, gcibi_args->chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get struct chunk info by index"); + + break; + } + + /* H5Dget_struct_chunk_info_by_coord */ + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD: { + H5VL_native_dataset_get_struct_chunk_info_by_coord_t *gcibc_args = &opt_args->get_struct_chunk_info_by_coord; + + assert(dset->shared); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if (H5D__get_struct_chunk_info_by_coord(dset, gcibc_args->offset, gcibc_args->chunk_info, + gcibc_args->addr, gcibc_args->chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't get struct chunk info by its logical coordinates"); + + break; + } + + /* H5Dstruct_chunk_iter */ + case H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER: { + /* Sanity check */ + assert(dset->shared); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if ((ret_value = H5D__struct_chunk_iter(dset, opt_args->struct_chunk_iter.op, opt_args->struct_chunk_iter.op_data)) < + 0) + HERROR(H5E_DATASET, H5E_BADITER, "struct chunk iteration failed"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid optional operation"); } /* end switch */ diff --git a/src/H5VLnative_introspect.c b/src/H5VLnative_introspect.c index 4c1cb53c0db..d6b8dfb8384 100644 --- a/src/H5VLnative_introspect.c +++ b/src/H5VLnative_introspect.c @@ -117,16 +117,20 @@ H5VL__native_introspect_opt_query(void H5_ATTR_UNUSED *obj, H5VL_subclass_t subc case H5VL_NATIVE_DATASET_GET_NUM_CHUNKS: case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX: case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD: + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX: + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD: case H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE: case H5VL_NATIVE_DATASET_GET_OFFSET: *flags |= H5VL_OPT_QUERY_QUERY_METADATA; break; case H5VL_NATIVE_DATASET_CHUNK_READ: + case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: *flags |= H5VL_OPT_QUERY_READ_DATA; break; case H5VL_NATIVE_DATASET_CHUNK_WRITE: + case H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK: *flags |= H5VL_OPT_QUERY_WRITE_DATA; break; diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index bd557fc8630..ee71f6c82a7 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -263,6 +263,13 @@ typedef enum H5Z_cb_return_t { typedef H5Z_cb_return_t (*H5Z_filter_func_t)(H5Z_filter_t filter, void *buf, size_t buf_size, void *op_data); //! + +/* TBD: */ +/* Convenience flag used by H5Pset_filter2() to specify a section of the sparse chunk to be filtered */ +#define H5Z_FLAG_SPARSE_SELECTION 0 +#define H5Z_FLAG_SPARSE_FIXED_DATA 1 +#define H5Z_FLAG_SPARSE_VL_DATA 2 + #ifdef __cplusplus extern "C" { #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fe860d5ae77..921a08756fd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -378,6 +378,7 @@ set (H5_TESTS page_buffer dtypes dsets + sparse_storage select_io_dset chunk_info # compression lib link cmpd_dset diff --git a/test/Makefile.am b/test/Makefile.am index fdd83e5bdeb..9019da5f76a 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -70,7 +70,7 @@ TEST_PROG= testhdf5 \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ unregister cache_logging cork swmr thread_id vol timer event_set \ - onion + onion sparse_storage # List programs to be built when testing here # diff --git a/test/sparse_storage.c b/test/sparse_storage.c new file mode 100644 index 00000000000..8d067c53131 --- /dev/null +++ b/test/sparse_storage.c @@ -0,0 +1,1065 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Tests sparse storage based on the concept of structured chunk + */ +#define H5D_FRIEND /*suppress error about including H5Dpkg */ +#define H5D_TESTING +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5FD_TESTING + +#define H5Z_FRIEND /*suppress error about including H5Zpkg */ + +#include "testhdf5.h" +#include "H5srcdir.h" + +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Iprivate.h" +#include "H5Pprivate.h" + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" /* File access */ + +#define H5S_FRIEND /*suppress error about including H5Spkg */ +#include "H5Spkg.h" /* Dataspace */ + +#define H5T_FRIEND /*suppress error about including H5Tpkg */ +#include "H5Tpkg.h" /* Datatype */ + +#define H5A_FRIEND /*suppress error about including H5Apkg */ +#include "H5Apkg.h" /* Attributes */ + +/* Use in version bound test */ +#define H5O_FRIEND /*suppress error about including H5Opkg */ +#include "H5Opkg.h" /* Object headers */ + +#include "H5Dpkg.h" +#include "H5FDpkg.h" +#include "H5VMprivate.h" +#include "H5Zpkg.h" + +static const char *FILENAME[] = {"sparse", /* 0 */ + "sparse_data", /* 1 */ + "sparse_direct_chunk", /* 2 */ + "sparse_query_direct_chunk", /* 3 */ + "sparse_filter", /* 4 */ + "sparse_dense_api", /* 5 */ + NULL}; +#define FILENAME_BUF_SIZE 1024 + +#define SPARSE_DSET "sparse_dset" + +#define RANK 2 +#define NX 10 +#define NY 10 +#define CHUNK_NX 5 +#define CHUNK_NY 5 + +/* Size of a chunk */ +#define CHK_SIZE (CHUNK_NX * CHUNK_NY * sizeof(int)) + +/*------------------------------------------------------------------------- + * Function: test_sparse_data + * + * Purpose: Verify APIs for handling sparse data: + * --H5Dget_defined() + * --H5Derase() + * + * Return: Success: 0 + * Failure: -1 + *------------------------------------------------------------------------- + */ +static herr_t +test_sparse_data(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t sid1 = H5I_INVALID_HID; + hid_t sid2 = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hsize_t dim[1] = {50}; /* 1-d dataspace */ + hsize_t chunk_dim[1] = {5}; /* Chunk size */ + int wbuf[50]; /* Write buffer */ + + TESTING("APIs for handling sparse data"); + + /* Create a file */ + h5_fixname(FILENAME[1], fapl, filename, sizeof filename); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(1, dim, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Create property list for compact dataset creation */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, 1, chunk_dim) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Write sparse data to the dataset */ + memset(wbuf, 0, sizeof(wbuf)); + + /* Initialize and write sparse data to the dataset */ + wbuf[1] = 1; + wbuf[12] = 12; wbuf[13] = 13; wbuf[14] = 14; + wbuf[22] = 22; wbuf[23] = 23; wbuf[24] = 24; + wbuf[48] = 48; wbuf[49] = 49; + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + /* Get defined elements */ + /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ + if ((sid1 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* TBD: Verify defined elements in sid1 are as expected */ + + /* Erase all defined elements */ + /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ + if (H5Derase(did, sid1, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + + /* Call H5Dget_defined() again after H5Derase() */ + if ((sid2 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* TBD: Verify nothing is defined in sid2 */ + + if (H5Sclose(sid1) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid2) < 0) + FAIL_STACK_ERROR; + + /* Closing */ + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Sclose(sid1); + H5Sclose(sid2); + H5Pclose(dcpl); + H5Dclose(did); + H5Fclose(fid); + } + H5E_END_TRY + + return FAIL; +} /* end test_sparse_data() */ + +/*------------------------------------------------------------------------- + * Function: test_sparse_direct_chunk + * + * Purpose: Verify APIs for direct chunk I/O on structured chunk: + * --H5Dwrite_struct_chunk() + * --H5Dread_struct_chunk() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static herr_t +test_sparse_direct_chunk(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; + + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; + + int buf[NX][NY]; + + TESTING("APIs for direct chunk I/O on structured chunks"); + + /* Create a file */ + h5_fixname(FILENAME[2], fapl, filename, sizeof filename); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* + * Create the data space with unlimited dimensions. + */ + if ((sid = H5Screate_simple(RANK, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* + * Create a new dataset within the file using dcpl + */ + if ((did = + H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + FAIL_STACK_ERROR; + + { + /* TBD: for H5D_SPARSE_CHUNK */ + + H5D_struct_chunk_info_t wr_chk_info; + uint16_t wr_filter_mask[2] = {0,0}; + hsize_t wr_offset[2] = {0, 0}; + size_t wr_section_size[2] = {32,48}; + void *wr_buf[2]; + hsize_t wr_buf0[4] = {3,2,4,4}; /* Encoded coordinates: [3,2] - [4,4] */ + hsize_t wr_buf1[6] = {66,69,72,96,99,102}; /* Data: 66,69,72,96,99,102 */ + + wr_buf[0] = wr_buf0; + wr_buf[1] = wr_buf1; + + H5D_struct_chunk_info_t rd_chk_info; + uint16_t rd_filter_mask[2] = {0,0}; + hsize_t rd_offset[2] = {0, 0}; + hsize_t rd_section_size[2]; + void *rd_buf[2]; + hsize_t rd_buf0[20]; /* coordinates */ + hsize_t rd_buf1[20]; /* Data */ + + rd_buf[0] = rd_buf0; + rd_buf[1] = rd_buf1; + + wr_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + wr_chk_info.num_sections = 2; + wr_chk_info.filter_mask = wr_filter_mask; + wr_chk_info.section_size = wr_section_size; + wr_chk_info.section_orig_size = wr_section_size; + + /* Write the structured chunk at offset [0,0] */ + if (H5Dwrite_struct_chunk(did, dxpl, wr_offset, &wr_chk_info, wr_buf) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, buf) < 0) + FAIL_STACK_ERROR; + + /* TBD: Verify buf read has data as in wr_buf1[] at location wr_buf0[0] */ + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + + rd_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + rd_chk_info.num_sections = 2; + rd_chk_info.filter_mask = rd_filter_mask; + rd_chk_info.section_size = rd_section_size; + rd_chk_info.section_orig_size = rd_section_size; + + /* Read the structured chunk at offset [0,0] */ + if (H5Dread_struct_chunk(did, dxpl, rd_offset, &rd_chk_info, rd_buf) < 0) + FAIL_STACK_ERROR; + + /* Verify rd_chk_info and rd_buf are the same as wr_chk_info and wr_buf */ + } + + /* + * Close/release resources. + */ + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(dxpl) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Pclose(dxpl); + H5Fclose(fid); + + } + H5E_END_TRY + + H5_FAILED(); + return 1; + +} /* test_sparse_direct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: verify_get_struct_chunk_info (helper function) + * + * Purpose: Verifies that H5Dget_struct_chunk_info returns correct + * values for a chunk. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +verify_get_struct_chunk_info(hid_t did, hid_t sid, hsize_t chk_index, const hsize_t H5_ATTR_UNUSED *exp_offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], hsize_t H5_ATTR_UNUSED exp_chk_size) +{ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + H5D_struct_chunk_info_t out_chunk_info[50]; + + /* Get info of the chunk specified by chk_index */ + if (H5Dget_struct_chunk_info(did, sid, chk_index, out_offset, out_chunk_info, &out_addr, &out_chk_size) < 0) + TEST_ERROR; + +#ifdef TBD + + /* Verify info from H5Dget_struct_chunk_info() with expected chunk info */ + + if (out_offset[0] != exp_offset[0]) + FAIL_PUTS_ERROR("unexpected offset[0]"); + if (out_offset[1] != exp_offset[1]) + FAIL_PUTS_ERROR("unexpected offset[1]"); + + Compare out_chunk_info with exp_chunk_info + + if (HADDR_UNDEF == out_addr) + FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); + + if (out_chk_size != exp_chk_size) + FAIL_PUTS_ERROR("unexpected chunk size"); + +#endif + + /* For now, just return SUCCEED */ + + return SUCCEED; + +error: + return FAIL; +} /* verify_get_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * + * Function: verify_get_struct_chunk_info_by_coord (helper function) + * + * Purpose: Verifies that H5Dget_struct_chunk_info_by_coord returns correct + * values for a chunk. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], + hsize_t H5_ATTR_UNUSED exp_chk_size) +{ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + H5D_struct_chunk_info_t out_chunk_info[50]; + + /* Get info of the chunk at logical coordinates specified by offset */ + if (H5Dget_struct_chunk_info_by_coord(did, offset, out_chunk_info, &out_addr, &out_chk_size) < 0) + TEST_ERROR; + +#ifdef TBD + { + /* Verify info from H5Dget_struct_chunk_info_by_coord() with expected chunk info */ + + if (HADDR_UNDEF == out_addr) + FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); + + Compare out_chunk_info with exp_chunk_info + + if (out_chk_size != exp_chk_size) + FAIL_PUTS_ERROR("unexpected chunk size"); + } +#endif + + /* For now, just return SUCCEED */ + + return SUCCEED; + +error: + return FAIL; +} /* verify_get_struct_chunk_info_by_coord() */ + +typedef struct struct_chunk_iter_info_t { + hsize_t offset[2]; + H5D_struct_chunk_info_t *chunk_info; + haddr_t addr; + hsize_t chunk_size; +} struct_chunk_iter_info_t; + +typedef struct struct_chunk_iter_udata_t { + struct_chunk_iter_info_t *struct_chunk_info; + int last_index; +} struct_chunk_iter_udata_t; + + +static int +iter_cb_struct(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size, void *op_data) +{ + struct_chunk_iter_udata_t *cidata = (struct_chunk_iter_udata_t *)op_data; + int idx = cidata->last_index + 1; + + cidata->struct_chunk_info[idx].offset[0] = offset[0]; + cidata->struct_chunk_info[idx].offset[1] = offset[1]; + cidata->struct_chunk_info[idx].chunk_info = chunk_info; + cidata->struct_chunk_info[idx].addr = *addr; + cidata->struct_chunk_info[idx].chunk_size = *chunk_size; + + cidata->last_index++; + + return H5_ITER_CONT; +} /* iter_cb_struct() */ + +/*------------------------------------------------------------------------- + * Function: test_sparse_direct_chunk_query + * + * Purpose: Verify APIs for direct chunk I/O query on structured chunk: + * --H5Dget_struct_chunk_info() + * --H5Dget_struct_chunk_info_by_coord() + * --H5Dstruct_chunk_iter() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_sparse_direct_chunk_query(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + + struct_chunk_iter_info_t chunk_infos[2]; /* Chunk infos filled up by iterator */ + struct_chunk_iter_udata_t udata; /* udata for iteration */ + H5D_struct_chunk_info_t chk_info; + + uint16_t filter_mask[2] = {0,0}; + hsize_t offset[2] = {0, 0}; + size_t section_size[2] = {32,48}; + void *write_buf[2]; + hsize_t in0[4] = {3,2,4,4}; /* Encoded coordinates: [3,2] - [4,4] */ + hsize_t in1[6] = {66,69,72,96,99,102}; /* Data: 66,69,72,96,99,102 */ + + TESTING("APIs for direct chunk I/O query on structured chunk"); + + /* Create the file */ + h5_fixname(FILENAME[3], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT); + if (did < 0) + TEST_ERROR; + + write_buf[0] = in0; + write_buf[1] = in1; + + chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + chk_info.num_sections = 2; + chk_info.filter_mask = filter_mask; + chk_info.section_size = section_size; + chk_info.section_orig_size = section_size; + + /* Write the structured chunk at offset */ + if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) + FAIL_STACK_ERROR; + + /* Verify info of the first and only chunk via H5Dget_struct_chunk_info() */ + if (verify_get_struct_chunk_info(did, H5S_ALL, 0, offset, &chk_info, CHK_SIZE) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_struct_chunk_info failed\n"); + + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + /* Write the structured chunk at offset */ + if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) + FAIL_STACK_ERROR; + + /* Verify info of the chunk at offset [CHUNK_NX,CHUNK_NY] via H5Dget_struct_chunk_info_by_coord() */ + if (verify_get_struct_chunk_info_by_coord(did, offset, &chk_info, CHK_SIZE) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_struct_chunk_info_by_coord failed\n"); + + /* For now, H5Dstruct_chunk_iter() just returns SUCCEED without actual iteration */ + udata.struct_chunk_info = chunk_infos; + udata.last_index = -1; + if (H5Dstruct_chunk_iter(did, H5P_DEFAULT, &iter_cb_struct, &udata) < 0) + TEST_ERROR; + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_sparse_direct_chunk_query() */ + + +/*------------------------------------------------------------------------- + * + * Function: test_sparse_filter() + * + * Purpose: Verify the following APIs for structured chunk filtering: + * --H5Pset_filter2() + * --H5Pget_nfilters2() + * --H5Pget_filters3() + * --H5Pget_filter_by_id3() + * --H5Premove_filter2() + * --H5pmodify_filter2() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_sparse_filter(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int nfilters; + H5Z_filter_t filtn; + uint64_t flags; + + + TESTING("APIs for structured chunk filtering"); + + /* Create the file */ + h5_fixname(FILENAME[4], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + if (H5Pset_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_MANDATORY, 0, NULL) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT); + + /* TBD: verify nfilters is correct; for now it is 0 */ + nfilters = H5Pget_nfilters2(dcpl, H5Z_FLAG_SPARSE_SELECTION); + if (nfilters) + TEST_ERROR; + + if (did < 0) + TEST_ERROR; + + if (H5Dclose(did) < 0) + TEST_ERROR; + + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + + if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Dget_create_plist(did)) < 0) + FAIL_STACK_ERROR; + + /* TBD: verify that filtn is H5Z_FILTER_DEFLATE and flags is H5Z_FLAG_MANDATORY */ + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, &flags, NULL, NULL, (size_t)0, NULL, NULL); + + /* For now: just verify it's 0 */ + if (filtn != H5Z_FILTER_NONE) + TEST_ERROR; + + /* TBD: Modify the filter's flags to optional */ + if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < 0) + FAIL_STACK_ERROR; + + /* TBD: verify that flags is H5Z_FLAG_OPTIONAL */ + if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, NULL, + NULL) < 0) + FAIL_STACK_ERROR; + + /* TBD: Remove the filter */ + if (H5Premove_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE) < 0) + FAIL_STACK_ERROR; + + /* TBD: verify that filtn is H5Z_FILTER_NONE */ + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, NULL, NULL, NULL, (size_t)0, NULL, NULL); + + /* For now: just verify it's 0 */ + if (filtn != H5Z_FILTER_NONE) + TEST_ERROR; + + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_sparse_filter() */ + +typedef struct chunk_iter_info_t { + hsize_t offset[2]; + unsigned filter_mask; + haddr_t addr; + hsize_t size; +} chunk_iter_info_t; + +typedef struct chunk_iter_udata_t { + chunk_iter_info_t *chunk_info; + int last_index; +} chunk_iter_udata_t; + +static int +iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) +{ + chunk_iter_udata_t *cidata = (chunk_iter_udata_t *)op_data; + int idx = cidata->last_index + 1; + + cidata->chunk_info[idx].offset[0] = offset[0]; + cidata->chunk_info[idx].offset[1] = offset[1]; + cidata->chunk_info[idx].filter_mask = filter_mask; + cidata->chunk_info[idx].addr = addr; + cidata->chunk_info[idx].size = size; + + cidata->last_index++; + + return H5_ITER_CONT; +} /* iter_cb() */ + +/*------------------------------------------------------------------------- + * Function: test_dense_chunk_api_on_sparse() + * + * Purpose: Verify the following dense chunk APIs will fail for + * H5D_SPARSE_CHUNK layout: + * --H5Dwrite_chunk() + * --H5Dget_chunk_info() + * --H5Dget_chunk_info_by_coord() + * --H5Dchunk_iter() + * Verify the following dense chunk APIs will succeed for + * H5D_SPARSE_CHUNK layout: + * --H5Dread_chunk() + * --H5Dget_chunk_storage_size() + * --H5Dget_num_chunks() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_dense_chunk_api_on_sparse(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + chunk_iter_info_t chunk_infos[2]; + chunk_iter_udata_t udata; + hsize_t nchunks = 0; + hsize_t chunk_nbytes; + hsize_t offset[2] = {0, 0}; + int direct_buf[CHUNK_NX][CHUNK_NY]; + haddr_t addr = 0; + uint32_t filters = 0; + + TESTING("APIs for direct chunk I/O: dense chunk functions on sparse layout"); + + /* Create the file */ + h5_fixname(FILENAME[3], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + /* The layout is set to H5D_CHUNKED as a side-effect */ + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT); + if (did < 0) + TEST_ERROR; + + H5E_BEGIN_TRY + { + H5Dwrite_chunk(did, H5P_DEFAULT, 0, offset, CHK_SIZE, direct_buf); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + H5Dget_chunk_info(did, H5S_ALL, 0, NULL, NULL, &addr, NULL); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + H5Dget_chunk_info_by_coord(did, offset, NULL, &addr, NULL); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + udata.chunk_info = chunk_infos; + udata.last_index = -1; + H5Dchunk_iter(did, H5P_DEFAULT, &iter_cb, &udata); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5Dread_chunk(did, H5P_DEFAULT, offset, &filters, direct_buf); + /* TBD: should succeed */ + + H5Dget_num_chunks(did, sid, &nchunks); + /* TBD: should succeed */ + + H5Dget_chunk_storage_size(did, offset, &chunk_nbytes); + /* TBD: should succeed */ + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_dense_chunk_api_on_sparse() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests for sparse data + * Tests are copied and modified from: + * --test/dsets.c + * --test/direct_chunk.c + * --test/chunk_info.c + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + char filename[FILENAME_BUF_SIZE]; + hid_t file, grp, fapl, fapl2; + hid_t fcpl = H5I_INVALID_HID, fcpl2 = H5I_INVALID_HID; + unsigned new_format; + unsigned paged; + unsigned minimized_ohdr; + int mdc_nelmts; + size_t rdcc_nelmts; + size_t rdcc_nbytes; + double rdcc_w0; + int nerrors = 0; + const char *driver_name; + bool contig_addr_vfd; /* Whether VFD used has a contiguous address space */ + bool driver_is_default_compatible; + + /* Don't run this test using certain file drivers */ + driver_name = h5_get_test_driver_name(); + + /* Current VFD that does not support contiguous address space */ + contig_addr_vfd = (bool)(strcmp(driver_name, "split") != 0 && strcmp(driver_name, "multi") != 0); + + /* Set the random # seed */ + srand((unsigned)time(NULL)); + + /* Testing setup */ + h5_test_init(); + fapl = h5_fileaccess(); + + if (h5_driver_is_default_vfd_compatible(fapl, &driver_is_default_compatible) < 0) + TEST_ERROR; + + /* Turn off the chunk cache, so all the chunks are immediately written to disk */ + if (H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) + goto error; + rdcc_nbytes = 0; + if (H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) + goto error; + + /* Copy the file access property list */ + if ((fapl2 = H5Pcopy(fapl)) < 0) + TEST_ERROR; + + /* Set the "use the latest version of the format" bounds for creating objects in the file */ + if (H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if ((fcpl2 = H5Pcopy(fcpl)) < 0) + TEST_ERROR; + + /* Set file space strategy to paged aggregation and persisting free-space */ + if (H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, true, (hsize_t)1) < 0) + TEST_ERROR; + + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + /* Test with paged aggregation enabled or not */ + for (paged = false; paged <= true; paged++) { + + /* Temporary: skip testing for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ + if (!contig_addr_vfd && paged) + continue; + + for (minimized_ohdr = false; minimized_ohdr <= true; minimized_ohdr++) { + + /* Test with old & new format groups */ + for (new_format = false; new_format <= true; new_format++) { + hid_t my_fapl, my_fcpl; + + /* Set the FAPL for the type of format */ + if (new_format) { + my_fapl = fapl2; + if (paged) { + my_fcpl = fcpl2; + puts("\nTesting with new file format and paged aggregation"); + } + else { + my_fcpl = fcpl; + puts("\nTesting with new file format and non-paged aggregation"); + } + } /* end if */ + else { + my_fapl = fapl; + if (paged) { + my_fcpl = fcpl2; + puts("Testing with old file format and paged aggregation:"); + } + else { + my_fcpl = fcpl; + puts("Testing with old file format and non-paged aggregation:"); + } + } /* end else */ + + /* Create the file for this test */ + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, my_fcpl, my_fapl)) < 0) + goto error; + + if (true == minimized_ohdr) { + if (0 > H5Fset_dset_no_attrs_hint(file, true)) + goto error; + puts("(minimized dataset object headers with file setting)"); + } + + /* Cause the library to emit initial messages */ + if ((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) + goto error; + if (H5Gclose(grp) < 0) + goto error; + + /* Create its own testfile */ + nerrors += (test_sparse_data(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_direct_chunk(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_direct_chunk_query(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_filter(my_fapl) < 0 ? 1 : 0); + nerrors += (test_dense_chunk_api_on_sparse(my_fapl) < 0 ? 1 : 0); + + if (H5Fclose(file) < 0) + goto error; + } /* end for new_format */ + } /* end for minimized_ohdr */ + } /* end for paged */ + + /* Close property lists */ + if (H5Pclose(fapl2) < 0) + TEST_ERROR; + if (H5Pclose(fcpl) < 0) + TEST_ERROR; + if (H5Pclose(fcpl2) < 0) + TEST_ERROR; + + /* Verify symbol table messages are cached */ + nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0); + + if (nerrors) + goto error; + printf("All sparse storage tests passed.\n"); + + h5_cleanup(FILENAME, fapl); + + exit(EXIT_SUCCESS); + +error: + nerrors = MAX(1, nerrors); + printf("***** %d SPARSE STORAGE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); + exit(EXIT_FAILURE); +} /* end main() */ From 5e58f08c0511d79ac19c66f83f6348252a43108f Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 21:37:43 +0000 Subject: [PATCH 2/9] Committing clang-format changes --- src/H5D.c | 78 +++++----- src/H5Dchunk.c | 31 ++-- src/H5Dint.c | 10 +- src/H5Dpkg.h | 21 +-- src/H5Dpublic.h | 142 +++++++++---------- src/H5Pocpl.c | 113 +++++++-------- src/H5Ppublic.h | 75 +++++----- src/H5VLconnector.h | 8 +- src/H5VLnative.h | 64 ++++----- src/H5VLnative_dataset.c | 31 ++-- src/H5Zpublic.h | 7 +- test/sparse_storage.c | 299 +++++++++++++++++++-------------------- 12 files changed, 442 insertions(+), 437 deletions(-) diff --git a/src/H5D.c b/src/H5D.c index 1e28f7f78a1..9b15be851c7 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2486,9 +2486,9 @@ H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); /* Set up VOL callback arguments */ - vol_cb_args.op_type = H5VL_DATASET_GET_DEFINED; + vol_cb_args.op_type = H5VL_DATASET_GET_DEFINED; vol_cb_args.args.get_defined.file_space_id = file_space_id; - vol_cb_args.args.get_defined.space_id = H5I_INVALID_HID; + vol_cb_args.args.get_defined.space_id = H5I_INVALID_HID; /* Get the dataspace with defined elements */ if (H5VL_dataset_get(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) @@ -2515,7 +2515,7 @@ H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) { - H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ H5VL_dataset_specific_args_t vol_cb_args; /* Arguments to VOL callback */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2532,7 +2532,7 @@ H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); /* Set up VOL callback arguments */ - vol_cb_args.op_type = H5VL_DATASET_ERASE; + vol_cb_args.op_type = H5VL_DATASET_ERASE; vol_cb_args.args.erase.file_space_id = file_space_id; /* Deletes elements from a dataset causing them to no longer be defined */ @@ -2553,12 +2553,12 @@ H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) *------------------------------------------------------------------------- */ herr_t -H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, +H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]) { - H5VL_object_t *vol_obj; /* Dataset for this operation */ - H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ - H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -2580,11 +2580,11 @@ H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); /* Set up VOL callback arguments */ - dset_opt_args.write_struct_chunk.offset = offset; + dset_opt_args.write_struct_chunk.offset = offset; dset_opt_args.write_struct_chunk.chunk_info = chunk_info; - dset_opt_args.write_struct_chunk.buf = buf; - vol_cb_args.op_type = H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK; - vol_cb_args.args = &dset_opt_args; + dset_opt_args.write_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; /* Write chunk */ if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) @@ -2604,8 +2604,8 @@ H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, *--------------------------------------------------------------------------- */ herr_t -H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, - H5D_struct_chunk_info_t *chunk_info, void *buf[] /*out*/) +H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + void *buf[] /*out*/) { H5VL_object_t *vol_obj; /* Dataset for this operation */ H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ @@ -2631,11 +2631,11 @@ H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); /* Set up VOL callback arguments */ - dset_opt_args.read_struct_chunk.offset = offset; + dset_opt_args.read_struct_chunk.offset = offset; dset_opt_args.read_struct_chunk.chunk_info = chunk_info; - dset_opt_args.read_struct_chunk.buf = buf; - vol_cb_args.op_type = H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK; - vol_cb_args.args = &dset_opt_args; + dset_opt_args.read_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; /* Read the raw chunk */ if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) @@ -2666,7 +2666,8 @@ H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, */ herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset /*out*/, - H5D_struct_chunk_info_t *chunk_info/*out*/, haddr_t *addr /*out*/, hsize_t *chunk_size /*out*/) + H5D_struct_chunk_info_t *chunk_info /*out*/, haddr_t *addr /*out*/, + hsize_t *chunk_size /*out*/) { H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ @@ -2683,14 +2684,14 @@ H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsiz HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); /* Set up VOL callback arguments */ - dset_opt_args.get_struct_chunk_info_by_idx.fspace_id = fspace_id; - dset_opt_args.get_struct_chunk_info_by_idx.chunk_idx = chunk_idx; - dset_opt_args.get_struct_chunk_info_by_idx.offset = offset; - dset_opt_args.get_struct_chunk_info_by_idx.chunk_info = chunk_info; - dset_opt_args.get_struct_chunk_info_by_idx.addr = addr; - dset_opt_args.get_struct_chunk_info_by_idx.chunk_size = chunk_size; - vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX; - vol_cb_args.args = &dset_opt_args; + dset_opt_args.get_struct_chunk_info_by_idx.fspace_id = fspace_id; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_idx = chunk_idx; + dset_opt_args.get_struct_chunk_info_by_idx.offset = offset; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_idx.addr = addr; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX; + vol_cb_args.args = &dset_opt_args; /* Call private function to get the chunk info given the chunk's index */ if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) @@ -2719,8 +2720,9 @@ H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsiz *------------------------------------------------------------------------- */ herr_t -H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info/*out*/, - haddr_t *addr /*out*/, hsize_t *chunk_size /*out*/) +H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info /*out*/, haddr_t *addr /*out*/, + hsize_t *chunk_size /*out*/) { H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ @@ -2739,12 +2741,12 @@ H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, H5D_stru HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)"); /* Set up VOL callback arguments */ - dset_opt_args.get_struct_chunk_info_by_coord.offset = offset; - dset_opt_args.get_struct_chunk_info_by_coord.chunk_info = chunk_info; - dset_opt_args.get_struct_chunk_info_by_coord.addr = addr; - dset_opt_args.get_struct_chunk_info_by_coord.chunk_size = chunk_size; - vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD; - vol_cb_args.args = &dset_opt_args; + dset_opt_args.get_struct_chunk_info_by_coord.offset = offset; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_coord.addr = addr; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD; + vol_cb_args.args = &dset_opt_args; /* Call private function to get the chunk info given the chunk's index */ if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) @@ -2758,7 +2760,7 @@ H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, H5D_stru * Function: H5Dstruct_chunk_iter * * Purpose: Iterates over all structured chunks in dataset calling - * the user specified callback function, "cb", and the + * the user specified callback function, "cb", and the * callback's required data, "op_data". * * Parameters: @@ -2796,8 +2798,8 @@ H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t op /* Set up VOL callback arguments */ dset_opt_args.struct_chunk_iter.op = op; dset_opt_args.struct_chunk_iter.op_data = op_data; - vol_cb_args.op_type = H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER; - vol_cb_args.args = &dset_opt_args; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER; + vol_cb_args.args = &dset_opt_args; /* Iterate over the chunks */ if ((ret_value = H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL)) < 0) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index e399f5f3482..9c60b2be009 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8264,10 +8264,10 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of *------------------------------------------------------------------------- */ herr_t -H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, +H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -8291,10 +8291,10 @@ H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSE *------------------------------------------------------------------------- */ herr_t -H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, +H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -8325,12 +8325,12 @@ H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_ *------------------------------------------------------------------------- */ herr_t -H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space, - hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, +H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space, + hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -8356,11 +8356,11 @@ H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR *------------------------------------------------------------------------- */ herr_t -H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, - hsize_t H5_ATTR_UNUSED *chunk_size) +H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, + haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -8378,17 +8378,17 @@ H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsiz /*------------------------------------------------------------------------- * Function: H5D__struct_chunk_iter * - * Purpose: Iterate over all the structured chunks in the dataset + * Purpose: Iterate over all the structured chunks in the dataset * with given callback and the callback's required data. * * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ herr_t -H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, +H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, void H5_ATTR_UNUSED *op_data) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR @@ -8401,4 +8401,3 @@ H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5 FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_iter() */ - diff --git a/src/H5Dint.c b/src/H5Dint.c index 52253904f55..57311a3968c 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -4014,14 +4014,14 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d) /*------------------------------------------------------------------------- * Function: H5D__get_defined * - * Purpose: Returns the dataspace ID with selection of defined elements + * Purpose: Returns the dataspace ID with selection of defined elements * * Return: Success: ID for dataspace * Failure: FAIL * *------------------------------------------------------------------------- */ -hid_t +hid_t H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) { H5S_t *space = NULL; @@ -4052,7 +4052,7 @@ H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); FUNC_LEAVE_NOAPI(ret_value) - + } /* H5D__get_defined() */ /*------------------------------------------------------------------------- @@ -4065,10 +4065,10 @@ H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) * *------------------------------------------------------------------------- */ -herr_t +herr_t H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 75272a2eaa6..0087c35b05e 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -756,15 +756,20 @@ H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); #endif /* H5D_CHUNK_DEBUG */ /* Functions that operate on H5D_SPARSE_CHUNK storage */ -H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace); +H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace); H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace); -H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); -H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); -H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chunk_idx, hsize_t *offset, - H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); -H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, - haddr_t *addr, hsize_t *chunk_size); -H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data); +H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, + hsize_t chunk_idx, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); +H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); +H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data); /* format convert */ H5_DLL herr_t H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info, diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index d224f40843e..26c23319836 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -128,25 +128,22 @@ typedef enum H5D_vds_view_t { } H5D_vds_view_t; //! - //! /** * Used by APIs for direct chunk chunk I/O on filtered and non-filtered structured chunks */ typedef struct H5D_struct_chunk_info_t { - H5D_layout_t type; /* Type of the structured chunk */ - uint8_t num_sections; /* Number of sections in the structured chunk */ - uint16_t *filter_mask; /* Array of "num_sections" size */ - /* Contains filter mask for each section */ - size_t *section_size; /* Array of "num_sections" size */ - /* Contains the size of each section */ - size_t *section_orig_size; /* Array of "num_sections" size */ - /* Contains the original size of each section */ + H5D_layout_t type; /* Type of the structured chunk */ + uint8_t num_sections; /* Number of sections in the structured chunk */ + uint16_t *filter_mask; /* Array of "num_sections" size */ + /* Contains filter mask for each section */ + size_t *section_size; /* Array of "num_sections" size */ + /* Contains the size of each section */ + size_t *section_orig_size; /* Array of "num_sections" size */ + /* Contains the original size of each section */ } H5D_struct_chunk_info_t; //! - - //! /** * \brief Callback for H5Pset_append_flush() @@ -288,8 +285,8 @@ typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, * \since 1.x.x * */ -typedef int (*H5D_struct_chunk_iter_op_t)(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, - haddr_t *addr, hsize_t *chunk_size, void *op_data); +typedef int (*H5D_struct_chunk_iter_op_t)(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + haddr_t *addr, hsize_t *chunk_size, void *op_data); //! /********************/ @@ -1688,22 +1685,22 @@ H5_DLL herr_t H5Dclose(hid_t dset_id); * \dset_id to get the selection of defined elements from * \param[in] file_space_id Dataspace identifier containing the selection of elements to * be queried if they are defined - * \dxpl_id + * \dxpl_id * - * \return Returns a dataspace with a selection containing all defined elements - * that are also selected in \p file_space_id if successful; + * \return Returns a dataspace with a selection containing all defined elements + * that are also selected in \p file_space_id if successful; * otherwise returns #H5I_INVALID_HID. * - * \details H5Dget_defined() retrieves a dataspace with only the defined - * elements of a (subset of) dataset selected. The dataset is specified - * by its identifier \p dset_id, and data transfer properties are defined - * by \p dxpl_id. The subset of the dataset to search for defined values is - * given by the selection in \p file_space_id. Setting \p file_space_id to - * #H5S_ALL causes this function to return a selection containing all defined + * \details H5Dget_defined() retrieves a dataspace with only the defined + * elements of a (subset of) dataset selected. The dataset is specified + * by its identifier \p dset_id, and data transfer properties are defined + * by \p dxpl_id. The subset of the dataset to search for defined values is + * given by the selection in \p file_space_id. Setting \p file_space_id to + * #H5S_ALL causes this function to return a selection containing all defined * values in the dataset. * - * This function is only useful for datasets with layout TBD: H5D_SPARSE_CHUNK. - * For other layouts this function will simply return a copy of \p file_space_id, + * This function is only useful for datasets with layout TBD: H5D_SPARSE_CHUNK. + * For other layouts this function will simply return a copy of \p file_space_id, * as all elements are defined for non-sparse datasets. * * \since 1.x.x @@ -1724,19 +1721,19 @@ H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * * \dset_id to erase elements from * \param[in] file_space_id Dataspace identifier containing the selection of elements to - * be erased + * be erased * \dxpl_id * * \return \herr_t * - * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, - * causing them to no longer be defined. The part of the dataset to erase is defined - * by \p file_space_id. After this operation, reading from these elements - * will return fill values, and the elements will no longer be included in the - * selection returned by H5Dget_defined(). + * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, + * causing them to no longer be defined. The part of the dataset to erase is defined + * by \p file_space_id. After this operation, reading from these elements + * will return fill values, and the elements will no longer be included in the + * selection returned by H5Dget_defined(). * - * This function is only useful for datasets with layout TBD:H5D_SPARSE_CHUNK. - * For other layouts this function will return an error. + * This function is only useful for datasets with layout TBD:H5D_SPARSE_CHUNK. + * For other layouts this function will return an error. * * \since 1.x.x * @@ -1765,16 +1762,16 @@ H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * * \return \herr_t * - \ \details H5Dwrite_struct_chunk() writes a structured chunk specified by its - * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles - * the structured chunk according to the information provided in the - * \p chunk_info parameter and using data pointed by \p buf. - * \p buf is an array of pointers to the buffers containing data for each - * section of the structured chunk. + \ \details H5Dwrite_struct_chunk() writes a structured chunk specified by its + * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles + * the structured chunk according to the information provided in the + * \p chunk_info parameter and using data pointed by \p buf. + * \p buf is an array of pointers to the buffers containing data for each + * section of the structured chunk. * - * Initially, this function will support only sparse chunks of the - * fixed-size data. Such chunks have only two sections: one for the - * encoded selection and the second one for data elements. + * Initially, this function will support only sparse chunks of the + * fixed-size data. Such chunks have only two sections: one for the + * encoded selection and the second one for data elements. * * \attention Exercise caution when using H5Dread_struct_chunk() and * H5Dwrite_struct_chunk(), as they read and write data chunks directly @@ -1807,21 +1804,21 @@ H5_DLL herr_t H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * dataspace * \param[in,out] chunk_info Information about the structured chunk * \param[out] buf Array of pointers to the sections of structured chunk; - * the size of the array is equal to the number of + * the size of the array is equal to the number of * sections in the structured chunk * * \return \herr_t * - * \details H5Dread_struct_chunk() reads a structured chunk as specified by its logical - * offset \p offset in a chunked dataset \p dset_id and places data into the provided - * buffers pointed by \p buf. Information about the structured chunk is returned via - * the \p chunk_info parameter. \p buf is an array of pointers to the buffers into which - * data for each section of the structured chunk will be read into. It is - * application’s responsibility to allocate buffers of the appropriate size. + * \details H5Dread_struct_chunk() reads a structured chunk as specified by its logical + * offset \p offset in a chunked dataset \p dset_id and places data into the provided + * buffers pointed by \p buf. Information about the structured chunk is returned via + * the \p chunk_info parameter. \p buf is an array of pointers to the buffers into which + * data for each section of the structured chunk will be read into. It is + * application’s responsibility to allocate buffers of the appropriate size. * - * Initially, this function will support only sparse chunks of the fixed-size data. - * Such chunk has only two sections: one for the encoded selection and the second - * one for data elements. + * Initially, this function will support only sparse chunks of the fixed-size data. + * Such chunk has only two sections: one for the encoded selection and the second + * one for data elements. * * \attention Exercise caution when using H5Dread_struct_chunk() and * H5Dwrite_struct_chunk(), as they read and write data chunks directly @@ -1839,7 +1836,7 @@ H5_DLL herr_t H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * \since 1.x.x * */ -H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, +H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); /** @@ -1859,23 +1856,23 @@ H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * * \return \herr_t * * \details H5Dget_struct_chunk_info() retrieves the offset coordinates, - * \p offset, structured chunk information, \p chunk_info, - * chunk's address, \p addr, and the size, \p chunk_size, + * \p offset, structured chunk information, \p chunk_info, + * chunk's address, \p addr, and the size, \p chunk_size, * for the dataset specified by the identifier \p dset_id and - * the chunk specified by the index, \p chunk_idx. - * The chunk belongs to a set of chunks in the selection - * specified by \p fspace_id. If the queried chunk - * does not exist in the file, the size will be set to 0 and - * address to #HADDR_UNDEF. \c NULL can be passed in for + * the chunk specified by the index, \p chunk_idx. + * The chunk belongs to a set of chunks in the selection + * specified by \p fspace_id. If the queried chunk + * does not exist in the file, the size will be set to 0 and + * address to #HADDR_UNDEF. \c NULL can be passed in for * any \p out parameters. * * \note Prior to HDF5 1.14.4, the reported address did not take the * user block into account. * - * \p chunk_idx is the chunk index in the selection. - * The index value may have a value of 0 up + * \p chunk_idx is the chunk index in the selection. + * The index value may have a value of 0 up * to the number of chunks stored in the file that - * have a nonempty intersection with the file dataspace. + * have a nonempty intersection with the file dataspace. * * \note As of 1.10.5, the dataspace intersection is not yet * supported. Hence, the index is of all the written chunks. @@ -1898,7 +1895,8 @@ H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * * */ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset, - H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); /** * -------------------------------------------------------------------------- @@ -1915,9 +1913,9 @@ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t c * * \return \herr_t * - * \details H5Dget_struct_chunk_info_by_coord() retrieves the structured - * information \p chunk_info, chunk's address, \p addr and the - * size, \p chunk_size, for the dataset specified by + * \details H5Dget_struct_chunk_info_by_coord() retrieves the structured + * information \p chunk_info, chunk's address, \p addr and the + * size, \p chunk_size, for the dataset specified by * the identifier \p dset_id and the chunk specified by its * coordinates \p offset. * @@ -1927,7 +1925,7 @@ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t c * * If the queried chunk does not exist in the file, the size will * be set to 0 and address to \c HADDR_UNDEF. The value pointed - * to by \p chunk_info will not be modified. \c NULL can be passed + * to by \p chunk_info will not be modified. \c NULL can be passed * in for any \p out parameters. * * \note Prior to HDF5 1.14.4, the reported address did not take the @@ -1938,8 +1936,9 @@ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t c * \since 1.x.x * */ -H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, - H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); +H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); /** * -------------------------------------------------------------------------- @@ -1954,7 +1953,7 @@ H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *of * * \return \herr_t * - * \details H5Dstruct_chunk_iter iterates over all structured chunks in the dataset, + * \details H5Dstruct_chunk_iter iterates over all structured chunks in the dataset, * calling the user supplied callback \p cb, and the callback's required * data \p op_data. * @@ -1972,7 +1971,8 @@ H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *of * \since 1.x.x * */ -H5_DLL herr_t H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t cb, void *op_data); +H5_DLL herr_t H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t cb, + void *op_data); /** * -------------------------------------------------------------------------- diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index 232691ba8dc..7f179dbe8de 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -567,13 +567,13 @@ H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t /*------------------------------------------------------------------------- * Function: H5Pmodify_filter2 * - * Purpose: Modifies the specified FILTER in the filter pipeline for a + * Purpose: Modifies the specified FILTER in the filter pipeline for a * specified SECTION_NUMBER of the structured chunk. - * + * * The PLIST_ID argument is either a dataset or group creation - * creation property list. + * creation property list. * - * The SECTION_NUMBER argument is the specified section of the + * The SECTION_NUMBER argument is the specified section of the * structured chunk * * The FILTER arguments specifies the filter identifier @@ -589,12 +589,12 @@ H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t * The FLAGS argument is a bit vector of the following fields: * * H5Z_FLAG_OPTIONAL(0x0001) - * If this bit is set then the filter is optional. + * If this bit is set then the filter is optional. * If the filter fails during an H5Dwrite() operation then the filter * is just excluded from the pipeline for the chunk for which it * failed; the filter will not participate in the pipeline - * during an H5Dread() of the chunk. - * If this bit is clear and the filter fails then the entire + * during an H5Dread() of the chunk. + * If this bit is clear and the filter fails then the entire * I/O operation fails. * If this bit is set but encoding is disabled for a filter, * attempting to write will generate an error. @@ -628,9 +628,9 @@ H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_fi Call new H5P_modify_filter() with buf_size and buf instead of cd_nelmts and cd_values - /* Modify the filter parameters of the I/O pipeline */ - if (H5P_modify_filter(plist, filter, flags, cd_nelmts, cd_values) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't modify filter"); + /* Modify the filter parameters of the I/O pipeline */ + if (H5P_modify_filter(plist, filter, flags, cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't modify filter"); #endif /* For now, just return SUCCEED */ @@ -646,7 +646,7 @@ H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_fi * end of the data or link output filter pipeline * * The PLIST_ID argument is a dataset creation or group - * creation property list. + * creation property list. * * The SECTION_NUMBER argument specified the section of the * structured chunk to which the filter is applied. @@ -669,9 +669,9 @@ H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_fi * filter fails during an H5Dwrite() operation then the filter * is just excluded from the pipeline for the chunk for which it * failed; the filter will not participate in the pipeline - * during an H5Dread() of the chunk. + * during an H5Dread() of the chunk. * - * If this bit is clear and the filter fails then the entire I/O + * If this bit is clear and the filter fails then the entire I/O * operation fails. * * If this bit is set but encoding is disabled for a filter, @@ -682,7 +682,7 @@ H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_fi *------------------------------------------------------------------------- */ herr_t -H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, +H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, size_t buf_size, const void *buf) { H5P_genplist_t *plist; /* Property list */ @@ -739,10 +739,10 @@ H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filte * filter fails during an H5Dwrite() operation then the filter * is just excluded from the pipeline for the chunk for which it * failed; the filter will not participate in the pipeline - * during an H5Dread() of the chunk. + * during an H5Dread() of the chunk. * - * If this bit is clear and the filter fails then the entire I/O - * operation fails. + * If this bit is clear and the filter fails then the entire I/O + * operation fails. * * If this bit is set but encoding is disabled for a filter, * attempting to write will generate an error. @@ -755,22 +755,21 @@ H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filte *------------------------------------------------------------------------- */ static herr_t -H5P__set_filter2(H5P_genplist_t H5_ATTR_UNUSED *plist, uint64_t H5_ATTR_UNUSED section_number, +H5P__set_filter2(H5P_genplist_t H5_ATTR_UNUSED *plist, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t H5_ATTR_UNUSED filter, uint64_t H5_ATTR_UNUSED flags, size_t H5_ATTR_UNUSED buf_size, const void H5_ATTR_UNUSED *buf) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR #ifdef TBD - Need to set up new H5O_pline_t message - Need to do something similar like H5P__set_filter1() as below: + Need to set up new H5O_pline_t message Need to do something similar like H5P__set_filter1() as below : - /* Check if filter is already available */ - if ((filter_avail = H5Z_filter_avail(filter)) < 0) + /* Check if filter is already available */ + if ((filter_avail = H5Z_filter_avail(filter)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't check filter availability"); + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't check filter availability"); /* Get the pipeline property to append to */ if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) @@ -790,7 +789,6 @@ H5P__set_filter2(H5P_genplist_t H5_ATTR_UNUSED *plist, uint64_t H5_ATTR_UNUSED s FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__set_filter2() */ - /*------------------------------------------------------------------------- * Function: H5Pset_filter * @@ -957,10 +955,10 @@ H5Pget_nfilters(hid_t plist_id) * * Purpose: Returns the number of filters in the data or link * pipeline depending on whether PLIST_ID is a dataset creation - * or group creation property list. - * In each pipeline the filters are numbered from zero through - * N-1 where N is the value returned by this function. - * During output to the file the filters of a pipeline are applied + * or group creation property list. + * In each pipeline the filters are numbered from zero through + * N-1 where N is the value returned by this function. + * During output to the file the filters of a pipeline are applied * in increasing order (the inverse is true for input). * * The SECTION_NUMBER argument specified the section of the @@ -1003,8 +1001,6 @@ H5Pget_nfilters2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number) FUNC_LEAVE_API(ret_value) } /* end H5Pget_nfilters2 */ - - /*------------------------------------------------------------------------- * Function: H5Pget_filter2 * @@ -1089,24 +1085,24 @@ H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t /*------------------------------------------------------------------------- * Function: H5Pget_filter3 * - * Purpose: This is the query counterpart of H5Pset_filter2(). + * Purpose: This is the query counterpart of H5Pset_filter2(). * It retrieves information for a filter in the pipeline for a * specified section of the structured chunk. * The information returned is about a particular filter number in * the filter pipeline depending on whether PLIST_ID is a - * dataset creation or group creation property list. + * dataset creation or group creation property list. * * On input: * SECTION_NUMBER is the specified section of the structured chunk * - * IDX should be a value between zero and N-1 as described + * IDX should be a value between zero and N-1 as described * for H5Pget_nfilters(). * * BUF_SIZE indicates the size in bytes of the buffer pointed to - * by BUF which is allocated by the caller. On exit, it contains the - * values defined by the filter. + * by BUF which is allocated by the caller. On exit, it contains the + * values defined by the filter. * - * FILTER_CONFIG is a bit field containing encode/decode flags from H5Zpublic.h. + * FILTER_CONFIG is a bit field containing encode/decode flags from H5Zpublic.h. * * The function will return failure if the filter number is out of range. * @@ -1116,12 +1112,13 @@ H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t *------------------------------------------------------------------------- */ H5Z_filter_t -H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, unsigned H5_ATTR_UNUSED idx, - uint64_t H5_ATTR_UNUSED *flags /*out*/, size_t H5_ATTR_UNUSED *buf_size /*in,out*/, - void H5_ATTR_UNUSED *buf /*out*/, size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, +H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, + unsigned H5_ATTR_UNUSED idx, uint64_t H5_ATTR_UNUSED *flags /*out*/, + size_t H5_ATTR_UNUSED *buf_size /*in,out*/, void H5_ATTR_UNUSED *buf /*out*/, + size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, unsigned H5_ATTR_UNUSED *filter_config /*out*/) { - H5Z_filter_t ret_value; /* return value */ + H5Z_filter_t ret_value; /* return value */ FUNC_ENTER_API(H5Z_FILTER_ERROR) @@ -1132,7 +1129,8 @@ H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_nu --Call new H5P__get_filter() routine with buf_size and buf in place of cd_nelemts and cd_values /* Check args */ - if (cd_nelmts || cd_values) { + if (cd_nelmts || cd_values) + { /* * It's likely that users forget to initialize this on input, so * we'll check that it has a reasonable value. The actual number @@ -1300,8 +1298,8 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* * It retrieves information for a filter specified by its identifier * in the pipeline for a specified section of the structured chunk. * The information returned is about about a particular filter - * identifier in the filter pipeline depending on whether - * PLIST_ID is a dataset creation or group creation property list. + * identifier in the filter pipeline depending on whether + * PLIST_ID is a dataset creation or group creation property list. * * On input: * SECTION_NUMBER is the specified section of the structured chunk @@ -1310,9 +1308,9 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* * * BUF_SIZE indicates the size in bytes of the buffer pointed to * by BUF which is allocated by the caller. On exit, it contains - * size of the values defined by the filter. + * size of the values defined by the filter. * - * FILTER_CONFIG is a bit field containing encode/decode flags + * FILTER_CONFIG is a bit field containing encode/decode flags * from H5Zpublic.h. * * If the filter is not set for the property list, an error will be returned. @@ -1323,13 +1321,12 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* *------------------------------------------------------------------------- */ herr_t -H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, - H5Z_filter_t id, uint64_t H5_ATTR_UNUSED *flags /*out*/, - size_t H5_ATTR_UNUSED *buf_size /*in,out*/, void H5_ATTR_UNUSED *buf /*out*/, - size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, - unsigned H5_ATTR_UNUSED *filter_config /*out*/) +H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t id, + uint64_t H5_ATTR_UNUSED *flags /*out*/, size_t H5_ATTR_UNUSED *buf_size /*in,out*/, + void H5_ATTR_UNUSED *buf /*out*/, size_t H5_ATTR_UNUSED namelen, + char H5_ATTR_UNUSED name[] /*out*/, unsigned H5_ATTR_UNUSED *filter_config /*out*/) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1337,12 +1334,12 @@ H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED sect if (id < 0 || id > H5Z_FILTER_MAX) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range"); -#ifdef TBD - --Check arguments buf_size and buf instead of cd_nelmts and cd_values - --Call new H5P_get_filter_by_id() routine with buf_size and buf in place - of cd_nelemts and cd_values +#ifdef TBD + --Check arguments buf_size and buf instead of cd_nelmts and cd_values-- Call new H5P_get_filter_by_id() + routine with buf_size and buf in place of cd_nelemts and cd_values - if (cd_nelmts || cd_values) { + if (cd_nelmts || cd_values) + { /* * It's likely that users forget to initialize this on input, so * we'll check that it has a reasonable value. The actual number @@ -1499,7 +1496,7 @@ H5Premove_filter(hid_t plist_id, H5Z_filter_t filter) *------------------------------------------------------------------------- */ herr_t -H5Premove_filter2(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, +H5Premove_filter2(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t H5_ATTR_UNUSED filter) { H5P_genplist_t *plist; /* Property list */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 643c7fbc782..c21a0a3841d 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -2144,7 +2144,7 @@ H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *f * \param[in] section_number An integer specifying section number of the structured chunk * \param[in] idx Sequence number within the filter pipeline of * the filter for which information is sought - * \param[out] flags Bit vector specifying certain general properties + * \param[out] flags Bit vector specifying certain general properties * of the filter * \param[in,out] buf_size Size in bytes of \p buf buffer * \param[out] buf Buffer with auxiliary data for the filter @@ -2199,9 +2199,10 @@ H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *f * \since 1.x.x * */ -H5_DLL H5Z_filter_t H5Pget_filter3(hid_t plist_id, uint64_t section_number, unsigned idx, - uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, - size_t namelen /*in*/, char name[] /*out*/, unsigned *filter_config /*out*/); +H5_DLL H5Z_filter_t H5Pget_filter3(hid_t plist_id, uint64_t section_number, unsigned idx, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, + unsigned *filter_config /*out*/); /** * \ingroup OCPL @@ -2283,7 +2284,7 @@ H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsig * \return \herr_t * * \details H5Pget_filter_by_id3() returns information about a filter - * for a specified section in the structured chunk with the + * for a specified section in the structured chunk with the * filter identifier specified in \p filter_id. * * \p plist_id must be a dataset or group creation property list @@ -2322,9 +2323,10 @@ H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsig * \since 1.x.x * */ -H5_DLL herr_t H5Pget_filter_by_id3(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter_id, - uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, - size_t namelen /*in*/, char name[] /*out*/, unsigned *filter_config /*out*/); +H5_DLL herr_t H5Pget_filter_by_id3(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter_id, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, + unsigned *filter_config /*out*/); /** * \ingroup OCPL @@ -2352,7 +2354,6 @@ H5_DLL herr_t H5Pget_filter_by_id3(hid_t plist_id, uint64_t section_number, H5Z_ */ H5_DLL int H5Pget_nfilters(hid_t plist_id); - /** * \ingroup OCPL * @@ -2447,7 +2448,7 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int /** * \ingroup OCPL * - * \brief Modifies a filter in the filter pipeline for a specified section of the + * \brief Modifies a filter in the filter pipeline for a specified section of the * structured chunk * * \ocpl_id{plist_id} @@ -2462,12 +2463,12 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int * * \details H5Pmodify_filter2() modifies the specified \p filter in the * filter pipeline for a specified section of the structured - * chunk. + * chunk. * * \p section_number is an integer specifying a section in the * structured chunk. * - * The \p plist_id parameter must be a dataset or group creation + * The \p plist_id parameter must be a dataset or group creation * property list. * * The \p filter, \p flags \p buf_size, and \p buf @@ -2477,8 +2478,8 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int * \since 1.x.x * */ -H5_DLL herr_t H5Pmodify_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, - uint64_t flags, size_t buf_size, const void *buf); +H5_DLL herr_t H5Pmodify_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf); /** * \ingroup OCPL @@ -2560,7 +2561,7 @@ H5_DLL herr_t H5Premove_filter(hid_t plist_id, H5Z_filter_t filter); * \return \herr_t * * \details H5Premove_filter2() removes the specified \p filter for a specified - * \p section_number from the filter pipeline in the dataset or + * \p section_number from the filter pipeline in the dataset or * group creation property list \p plist_id. * * The \p section_number is an integer specifying a section in the @@ -3049,19 +3050,19 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * * \return \herr_t * - * \details H5Pset_filter2() adds a filter and corresponding properties to the end - * of an output filter pipeline. This function can be used with - * both current chunked storage and structured chunk storage - * including sparse chunk. It also addresses deficiency of + * \details H5Pset_filter2() adds a filter and corresponding properties to the end + * of an output filter pipeline. This function can be used with + * both current chunked storage and structured chunk storage + * including sparse chunk. It also addresses deficiency of * H5Pset_filter1() in passing the filter’s data as described below. - * + * * Note the following differences with H5Pset_filter1(): - * - This function accepts a new parameter \p section_number that - * specifies the section of the structured chunk to which the filter is applied. + * - This function accepts a new parameter \p section_number that + * specifies the section of the structured chunk to which the filter is applied. * For chunked storage, there is just one section. - * - Data type for the \p flags parameter is changed to uint64_t to provide - * more flexibility to the VOL connectors that use the function. - * - This function passes the filter’s data by using a void pointer to a buffer + * - Data type for the \p flags parameter is changed to uint64_t to provide + * more flexibility to the VOL connectors that use the function. + * - This function passes the filter’s data by using a void pointer to a buffer * with auxiliary data for the filter instead of unsigned int c_values[]. * * \p plist_id must be either a dataset creation property list or @@ -3077,7 +3078,7 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * (#H5Z_FILTER_DEFLATE) and the Fletcher32 error detection filter * (#H5Z_FILTER_FLETCHER32). * - * \p section_number specifies the section number. The value + * \p section_number specifies the section number. The value * is 0 to 255 when native HDF5 file format is used. * For sparse chunk, the convenience flag can be used to specify * a section of the structured chunk to be filtered as described below: @@ -3085,22 +3086,22 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * * * - * * * * - * * * * - * * @@ -3224,8 +3225,8 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * of the fractal heap for groups in the order in which they were * set. * - * \note Filters can be applied only to chunked datasets; - * they cannot be used with other dataset storage methods, such as + * \note Filters can be applied only to chunked datasets; + * they cannot be used with other dataset storage methods, such as * contiguous, compact, or external datasets. * * \note Dataset elements of variable-length and dataset region @@ -3346,8 +3347,8 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * \since 1.6.0 * */ -H5_DLL herr_t H5Pset_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, - size_t buf_size, const void *buf); +H5_DLL herr_t H5Pset_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf); /** * \ingroup OCPL diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index c5399d65c6a..c2263730214 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -221,7 +221,7 @@ typedef enum H5VL_dataset_get_t { H5VL_DATASET_GET_SPACE, /* dataspace */ H5VL_DATASET_GET_SPACE_STATUS, /* space status */ H5VL_DATASET_GET_STORAGE_SIZE, /* storage size */ - H5VL_DATASET_GET_TYPE, /* datatype */ + H5VL_DATASET_GET_TYPE, /* datatype */ H5VL_DATASET_GET_DEFINED /* dataspace selection of defined elements */ } H5VL_dataset_get_t; @@ -263,8 +263,8 @@ typedef struct H5VL_dataset_get_args_t { /* H5VL_DATASET_GET_DEFINED */ struct { - hid_t file_space_id; /* file dataspace selection to be queried (IN) */ - hid_t space_id; /* dataspace selection of defined elements (OUT) */ + hid_t file_space_id; /* file dataspace selection to be queried (IN) */ + hid_t space_id; /* dataspace selection of defined elements (OUT) */ } get_defined; } args; @@ -301,7 +301,7 @@ typedef struct H5VL_dataset_specific_args_t { /* H5VL_DATASET_ERASE */ struct { - hid_t file_space_id; /* file dataspace selection to be erased (IN) */ + hid_t file_space_id; /* file dataspace selection to be erased (IN) */ } erase; } args; } H5VL_dataset_specific_args_t; diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 84213fff872..060c957cdb4 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -64,27 +64,27 @@ typedef union H5VL_native_attr_optional_args_t { /* NOTE: If new values are added here, the H5VL__native_introspect_opt_query * routine must be updated. */ -/* TBD: the existing H5VL_NATIVE_DATASET_CHUNK_ITER is not added to - * H5VL__native_introspect_opt_query, therefore I didn't add +/* TBD: the existing H5VL_NATIVE_DATASET_CHUNK_ITER is not added to + * H5VL__native_introspect_opt_query, therefore I didn't add * H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER to H5VL__native_introspect_opt_query either. * Check for sure later. */ -#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ -#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ -#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ -#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ -#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ -#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ -#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ -#define H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK 11 /* H5Dread_struct_chunk */ -#define H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK 12 /* H5Dwrite_struct_chunk */ +#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ +#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ +#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ +#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ +#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ +#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ +#define H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK 11 /* H5Dread_struct_chunk */ +#define H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK 12 /* H5Dwrite_struct_chunk */ #define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX 13 /* H5Dget_struct_chunk_info */ #define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD 14 /* H5Dget_struct_chunk_info_by_coord */ -#define H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER 15 /* H5Dstruct_chunk_iter */ +#define H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER 15 /* H5Dstruct_chunk_iter */ /* NOTE: If values over 1023 are added, the H5VL_RESERVED_NATIVE_OPTIONAL macro * must be updated. */ @@ -107,16 +107,16 @@ typedef struct H5VL_native_dataset_chunk_write_t { /* TBD: can we use the same struct for both read and write operations? */ /* Parameters for native connector's dataset 'read struct chunk' operation */ typedef struct H5VL_native_dataset_read_struct_chunk_t { - const hsize_t *offset; + const hsize_t *offset; H5D_struct_chunk_info_t *chunk_info; - void **buf; + void **buf; } H5VL_native_dataset_read_struct_chunk_t; /* Parameters for native connector's dataset 'write struct chunk' operation */ typedef struct H5VL_native_dataset_write_struct_chunk_t { - const hsize_t *offset; + const hsize_t *offset; H5D_struct_chunk_info_t *chunk_info; - void **buf; + void **buf; } H5VL_native_dataset_write_struct_chunk_t; /* Parameters for native connector's dataset 'get vlen buf size' operation */ @@ -158,20 +158,20 @@ typedef struct H5VL_native_dataset_get_chunk_info_by_coord_t { /* Parameters for native connector's dataset 'get struct chunk info by idx' operation */ typedef struct H5VL_native_dataset_get_struct_chunk_info_by_idx_t { - hid_t fspace_id; /* File dataspace selection */ - hsize_t chunk_idx; /* Chunk index */ - hsize_t *offset; /* Logical position of the chunk's first element in the array (OUT) */ - H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ - haddr_t *addr; /* Chunk address in the file (OUT) */ - hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ + hid_t fspace_id; /* File dataspace selection */ + hsize_t chunk_idx; /* Chunk index */ + hsize_t *offset; /* Logical position of the chunk's first element in the array (OUT) */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ } H5VL_native_dataset_get_struct_chunk_info_by_idx_t; /* Parameters for native connector's dataset 'get struct chunk info by coord' operation */ typedef struct H5VL_native_dataset_get_struct_chunk_info_by_coord_t { - const hsize_t *offset; /* Logical position of the chunk's first element in the array */ - H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ - haddr_t *addr; /* Chunk address in the file (OUT) */ - hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ + const hsize_t *offset; /* Logical position of the chunk's first element in the array */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ } H5VL_native_dataset_get_struct_chunk_info_by_coord_t; /* Parameters for native connector's dataset 'optional' operations */ @@ -230,8 +230,8 @@ typedef union H5VL_native_dataset_optional_args_t { /* H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER */ struct { - H5D_struct_chunk_iter_op_t op; /* Structured chunk iteration callback */ - void *op_data; /* Context to pass to iteration callback */ + H5D_struct_chunk_iter_op_t op; /* Structured chunk iteration callback */ + void *op_data; /* Context to pass to iteration callback */ } struct_chunk_iter; } H5VL_native_dataset_optional_args_t; diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index e118a7d1bee..7ecdbaf85eb 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -205,7 +205,7 @@ H5VL__native_dataset_io_setup(size_t count, void *obj[], hid_t mem_type_id[], hi * Purpose: Frees memory allocated by H5VL__native_dataset_io_setup() * * Return: SUCCEED/FAIL - + *------------------------------------------------------------------------- */ static herr_t @@ -499,7 +499,7 @@ H5VL__native_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t H5_ATTR /* H5Dget_defined */ case H5VL_DATASET_GET_DEFINED: { - const H5S_t *fspace = NULL; + const H5S_t *fspace = NULL; assert(dset->shared); assert(dset->shared->space); @@ -508,7 +508,8 @@ H5VL__native_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t H5_ATTR if (args->args.get_defined.file_space_id == H5S_ALL) fspace = dset->shared->space; else /* otherwise, use the given space ID */ - if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.get_defined.file_space_id, H5I_DATASPACE))) + if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.get_defined.file_space_id, + H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); /* Call private function */ @@ -570,9 +571,10 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid /* H5Derase */ case H5VL_DATASET_ERASE: { - const H5S_t *fspace = NULL; + const H5S_t *fspace = NULL; - if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.erase.file_space_id, H5I_DATASPACE))) + if (NULL == + (fspace = (const H5S_t *)H5I_object_verify(args->args.erase.file_space_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); if (H5D__erase(dset, fspace) < 0) @@ -857,7 +859,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); /* Read the structured chunk */ - if (H5D__read_struct_chunk_direct(dset, offset_copy, read_struct_chunk_args->chunk_info, + if (H5D__read_struct_chunk_direct(dset, offset_copy, read_struct_chunk_args->chunk_info, read_struct_chunk_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); @@ -892,8 +894,9 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dget_struct_chunk_info */ case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX: { - H5VL_native_dataset_get_struct_chunk_info_by_idx_t *gcibi_args = &opt_args->get_struct_chunk_info_by_idx; - const H5S_t *space; + H5VL_native_dataset_get_struct_chunk_info_by_idx_t *gcibi_args = + &opt_args->get_struct_chunk_info_by_idx; + const H5S_t *space; assert(dset->shared); assert(dset->shared->space); @@ -912,7 +915,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Call private function */ if (H5D__get_struct_chunk_info(dset, space, gcibi_args->chunk_idx, gcibi_args->offset, - gcibi_args->chunk_info, gcibi_args->addr, gcibi_args->chunk_size) < 0) + gcibi_args->chunk_info, gcibi_args->addr, + gcibi_args->chunk_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get struct chunk info by index"); break; @@ -920,7 +924,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dget_struct_chunk_info_by_coord */ case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD: { - H5VL_native_dataset_get_struct_chunk_info_by_coord_t *gcibc_args = &opt_args->get_struct_chunk_info_by_coord; + H5VL_native_dataset_get_struct_chunk_info_by_coord_t *gcibc_args = + &opt_args->get_struct_chunk_info_by_coord; assert(dset->shared); @@ -930,7 +935,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Call private function */ if (H5D__get_struct_chunk_info_by_coord(dset, gcibc_args->offset, gcibc_args->chunk_info, - gcibc_args->addr, gcibc_args->chunk_size) < 0) + gcibc_args->addr, gcibc_args->chunk_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get struct chunk info by its logical coordinates"); @@ -947,8 +952,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); /* Call private function */ - if ((ret_value = H5D__struct_chunk_iter(dset, opt_args->struct_chunk_iter.op, opt_args->struct_chunk_iter.op_data)) < - 0) + if ((ret_value = H5D__struct_chunk_iter(dset, opt_args->struct_chunk_iter.op, + opt_args->struct_chunk_iter.op_data)) < 0) HERROR(H5E_DATASET, H5E_BADITER, "struct chunk iteration failed"); break; diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index ee71f6c82a7..65d470817f1 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -263,12 +263,11 @@ typedef enum H5Z_cb_return_t { typedef H5Z_cb_return_t (*H5Z_filter_func_t)(H5Z_filter_t filter, void *buf, size_t buf_size, void *op_data); //! - /* TBD: */ /* Convenience flag used by H5Pset_filter2() to specify a section of the sparse chunk to be filtered */ -#define H5Z_FLAG_SPARSE_SELECTION 0 -#define H5Z_FLAG_SPARSE_FIXED_DATA 1 -#define H5Z_FLAG_SPARSE_VL_DATA 2 +#define H5Z_FLAG_SPARSE_SELECTION 0 +#define H5Z_FLAG_SPARSE_FIXED_DATA 1 +#define H5Z_FLAG_SPARSE_VL_DATA 2 #ifdef __cplusplus extern "C" { diff --git a/test/sparse_storage.c b/test/sparse_storage.c index 8d067c53131..30b3b5c4bc8 100644 --- a/test/sparse_storage.c +++ b/test/sparse_storage.c @@ -58,7 +58,7 @@ static const char *FILENAME[] = {"sparse", /* 0 */ NULL}; #define FILENAME_BUF_SIZE 1024 -#define SPARSE_DSET "sparse_dset" +#define SPARSE_DSET "sparse_dset" #define RANK 2 #define NX 10 @@ -73,8 +73,8 @@ static const char *FILENAME[] = {"sparse", /* 0 */ * Function: test_sparse_data * * Purpose: Verify APIs for handling sparse data: - * --H5Dget_defined() - * --H5Derase() + * --H5Dget_defined() + * --H5Derase() * * Return: Success: 0 * Failure: -1 @@ -83,16 +83,16 @@ static const char *FILENAME[] = {"sparse", /* 0 */ static herr_t test_sparse_data(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t fid = H5I_INVALID_HID; - hid_t sid = H5I_INVALID_HID; - hid_t sid1 = H5I_INVALID_HID; - hid_t sid2 = H5I_INVALID_HID; - hid_t dcpl = H5I_INVALID_HID; - hid_t did = H5I_INVALID_HID; - hsize_t dim[1] = {50}; /* 1-d dataspace */ - hsize_t chunk_dim[1] = {5}; /* Chunk size */ - int wbuf[50]; /* Write buffer */ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t sid1 = H5I_INVALID_HID; + hid_t sid2 = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hsize_t dim[1] = {50}; /* 1-d dataspace */ + hsize_t chunk_dim[1] = {5}; /* Chunk size */ + int wbuf[50]; /* Write buffer */ TESTING("APIs for handling sparse data"); @@ -116,18 +116,22 @@ test_sparse_data(hid_t fapl) if (H5Pset_chunk(dcpl, 1, chunk_dim) < 0) FAIL_STACK_ERROR; - if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT)) < 0) + if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; /* Write sparse data to the dataset */ memset(wbuf, 0, sizeof(wbuf)); /* Initialize and write sparse data to the dataset */ - wbuf[1] = 1; - wbuf[12] = 12; wbuf[13] = 13; wbuf[14] = 14; - wbuf[22] = 22; wbuf[23] = 23; wbuf[24] = 24; - wbuf[48] = 48; wbuf[49] = 49; + wbuf[1] = 1; + wbuf[12] = 12; + wbuf[13] = 13; + wbuf[14] = 14; + wbuf[22] = 22; + wbuf[23] = 23; + wbuf[24] = 24; + wbuf[48] = 48; + wbuf[49] = 49; if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) TEST_ERROR; @@ -199,18 +203,18 @@ test_sparse_data(hid_t fapl) static herr_t test_sparse_direct_chunk(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t fid = H5I_INVALID_HID; - hid_t did = H5I_INVALID_HID; - hid_t sid = H5I_INVALID_HID; - hid_t dcpl = H5I_INVALID_HID; - hid_t dxpl = H5I_INVALID_HID; + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dxpl = H5I_INVALID_HID; hsize_t dims[2] = {NX, NY}; hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; - int buf[NX][NY]; + int buf[NX][NY]; TESTING("APIs for direct chunk I/O on structured chunks"); @@ -238,8 +242,7 @@ test_sparse_direct_chunk(hid_t fapl) /* * Create a new dataset within the file using dcpl */ - if ((did = - H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) @@ -249,31 +252,31 @@ test_sparse_direct_chunk(hid_t fapl) /* TBD: for H5D_SPARSE_CHUNK */ H5D_struct_chunk_info_t wr_chk_info; - uint16_t wr_filter_mask[2] = {0,0}; - hsize_t wr_offset[2] = {0, 0}; - size_t wr_section_size[2] = {32,48}; - void *wr_buf[2]; - hsize_t wr_buf0[4] = {3,2,4,4}; /* Encoded coordinates: [3,2] - [4,4] */ - hsize_t wr_buf1[6] = {66,69,72,96,99,102}; /* Data: 66,69,72,96,99,102 */ + uint16_t wr_filter_mask[2] = {0, 0}; + hsize_t wr_offset[2] = {0, 0}; + size_t wr_section_size[2] = {32, 48}; + void *wr_buf[2]; + hsize_t wr_buf0[4] = {3, 2, 4, 4}; /* Encoded coordinates: [3,2] - [4,4] */ + hsize_t wr_buf1[6] = {66, 69, 72, 96, 99, 102}; /* Data: 66,69,72,96,99,102 */ wr_buf[0] = wr_buf0; wr_buf[1] = wr_buf1; H5D_struct_chunk_info_t rd_chk_info; - uint16_t rd_filter_mask[2] = {0,0}; - hsize_t rd_offset[2] = {0, 0}; - hsize_t rd_section_size[2]; - void *rd_buf[2]; - hsize_t rd_buf0[20]; /* coordinates */ - hsize_t rd_buf1[20]; /* Data */ + uint16_t rd_filter_mask[2] = {0, 0}; + hsize_t rd_offset[2] = {0, 0}; + hsize_t rd_section_size[2]; + void *rd_buf[2]; + hsize_t rd_buf0[20]; /* coordinates */ + hsize_t rd_buf1[20]; /* Data */ rd_buf[0] = rd_buf0; rd_buf[1] = rd_buf1; - wr_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ - wr_chk_info.num_sections = 2; - wr_chk_info.filter_mask = wr_filter_mask; - wr_chk_info.section_size = wr_section_size; + wr_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + wr_chk_info.num_sections = 2; + wr_chk_info.filter_mask = wr_filter_mask; + wr_chk_info.section_size = wr_section_size; wr_chk_info.section_orig_size = wr_section_size; /* Write the structured chunk at offset [0,0] */ @@ -292,11 +295,10 @@ test_sparse_direct_chunk(hid_t fapl) if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; - - rd_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ - rd_chk_info.num_sections = 2; - rd_chk_info.filter_mask = rd_filter_mask; - rd_chk_info.section_size = rd_section_size; + rd_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + rd_chk_info.num_sections = 2; + rd_chk_info.filter_mask = rd_filter_mask; + rd_chk_info.section_size = rd_section_size; rd_chk_info.section_orig_size = rd_section_size; /* Read the structured chunk at offset [0,0] */ @@ -329,7 +331,6 @@ test_sparse_direct_chunk(hid_t fapl) H5Pclose(dcpl); H5Pclose(dxpl); H5Fclose(fid); - } H5E_END_TRY @@ -348,34 +349,36 @@ test_sparse_direct_chunk(hid_t fapl) *------------------------------------------------------------------------- */ static herr_t -verify_get_struct_chunk_info(hid_t did, hid_t sid, hsize_t chk_index, const hsize_t H5_ATTR_UNUSED *exp_offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], hsize_t H5_ATTR_UNUSED exp_chk_size) +verify_get_struct_chunk_info(hid_t did, hid_t sid, hsize_t chk_index, + const hsize_t H5_ATTR_UNUSED *exp_offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], + hsize_t H5_ATTR_UNUSED exp_chk_size) { - hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ - hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ - haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ H5D_struct_chunk_info_t out_chunk_info[50]; /* Get info of the chunk specified by chk_index */ - if (H5Dget_struct_chunk_info(did, sid, chk_index, out_offset, out_chunk_info, &out_addr, &out_chk_size) < 0) + if (H5Dget_struct_chunk_info(did, sid, chk_index, out_offset, out_chunk_info, &out_addr, &out_chk_size) < + 0) TEST_ERROR; #ifdef TBD - /* Verify info from H5Dget_struct_chunk_info() with expected chunk info */ + /* Verify info from H5Dget_struct_chunk_info() with expected chunk info */ - if (out_offset[0] != exp_offset[0]) - FAIL_PUTS_ERROR("unexpected offset[0]"); - if (out_offset[1] != exp_offset[1]) - FAIL_PUTS_ERROR("unexpected offset[1]"); + if (out_offset[0] != exp_offset[0]) + FAIL_PUTS_ERROR("unexpected offset[0]"); + if (out_offset[1] != exp_offset[1]) + FAIL_PUTS_ERROR("unexpected offset[1]"); - Compare out_chunk_info with exp_chunk_info + Compare out_chunk_info with exp_chunk_info - if (HADDR_UNDEF == out_addr) - FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); + if (HADDR_UNDEF == out_addr) FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); - if (out_chk_size != exp_chk_size) - FAIL_PUTS_ERROR("unexpected chunk size"); + if (out_chk_size != exp_chk_size) + FAIL_PUTS_ERROR("unexpected chunk size"); #endif @@ -399,12 +402,12 @@ verify_get_struct_chunk_info(hid_t did, hid_t sid, hsize_t chk_index, const hsiz *------------------------------------------------------------------------- */ static herr_t -verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], - hsize_t H5_ATTR_UNUSED exp_chk_size) +verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], + hsize_t H5_ATTR_UNUSED exp_chk_size) { - hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ - haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ H5D_struct_chunk_info_t out_chunk_info[50]; /* Get info of the chunk at logical coordinates specified by offset */ @@ -420,8 +423,7 @@ verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, Compare out_chunk_info with exp_chunk_info - if (out_chk_size != exp_chk_size) - FAIL_PUTS_ERROR("unexpected chunk size"); + if (out_chk_size != exp_chk_size) FAIL_PUTS_ERROR("unexpected chunk size"); } #endif @@ -434,29 +436,29 @@ verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, } /* verify_get_struct_chunk_info_by_coord() */ typedef struct struct_chunk_iter_info_t { - hsize_t offset[2]; + hsize_t offset[2]; H5D_struct_chunk_info_t *chunk_info; - haddr_t addr; - hsize_t chunk_size; + haddr_t addr; + hsize_t chunk_size; } struct_chunk_iter_info_t; typedef struct struct_chunk_iter_udata_t { struct_chunk_iter_info_t *struct_chunk_info; - int last_index; + int last_index; } struct_chunk_iter_udata_t; - static int -iter_cb_struct(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size, void *op_data) +iter_cb_struct(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size, + void *op_data) { struct_chunk_iter_udata_t *cidata = (struct_chunk_iter_udata_t *)op_data; - int idx = cidata->last_index + 1; + int idx = cidata->last_index + 1; - cidata->struct_chunk_info[idx].offset[0] = offset[0]; - cidata->struct_chunk_info[idx].offset[1] = offset[1]; - cidata->struct_chunk_info[idx].chunk_info = chunk_info; - cidata->struct_chunk_info[idx].addr = *addr; - cidata->struct_chunk_info[idx].chunk_size = *chunk_size; + cidata->struct_chunk_info[idx].offset[0] = offset[0]; + cidata->struct_chunk_info[idx].offset[1] = offset[1]; + cidata->struct_chunk_info[idx].chunk_info = chunk_info; + cidata->struct_chunk_info[idx].addr = *addr; + cidata->struct_chunk_info[idx].chunk_size = *chunk_size; cidata->last_index++; @@ -478,24 +480,24 @@ iter_cb_struct(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr static int test_sparse_direct_chunk_query(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ - hid_t did = H5I_INVALID_HID; /* Dataset ID */ - hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ - hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ - - struct_chunk_iter_info_t chunk_infos[2]; /* Chunk infos filled up by iterator */ - struct_chunk_iter_udata_t udata; /* udata for iteration */ - H5D_struct_chunk_info_t chk_info; - - uint16_t filter_mask[2] = {0,0}; - hsize_t offset[2] = {0, 0}; - size_t section_size[2] = {32,48}; - void *write_buf[2]; - hsize_t in0[4] = {3,2,4,4}; /* Encoded coordinates: [3,2] - [4,4] */ - hsize_t in1[6] = {66,69,72,96,99,102}; /* Data: 66,69,72,96,99,102 */ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + + struct_chunk_iter_info_t chunk_infos[2]; /* Chunk infos filled up by iterator */ + struct_chunk_iter_udata_t udata; /* udata for iteration */ + H5D_struct_chunk_info_t chk_info; + + uint16_t filter_mask[2] = {0, 0}; + hsize_t offset[2] = {0, 0}; + size_t section_size[2] = {32, 48}; + void *write_buf[2]; + hsize_t in0[4] = {3, 2, 4, 4}; /* Encoded coordinates: [3,2] - [4,4] */ + hsize_t in1[6] = {66, 69, 72, 96, 99, 102}; /* Data: 66,69,72,96,99,102 */ TESTING("APIs for direct chunk I/O query on structured chunk"); @@ -522,18 +524,17 @@ test_sparse_direct_chunk_query(hid_t fapl) TEST_ERROR; /* Create a new dataset using dcpl creation properties */ - did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT); + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); if (did < 0) TEST_ERROR; write_buf[0] = in0; write_buf[1] = in1; - chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ - chk_info.num_sections = 2; - chk_info.filter_mask = filter_mask; - chk_info.section_size = section_size; + chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + chk_info.num_sections = 2; + chk_info.filter_mask = filter_mask; + chk_info.section_size = section_size; chk_info.section_orig_size = section_size; /* Write the structured chunk at offset */ @@ -557,7 +558,7 @@ test_sparse_direct_chunk_query(hid_t fapl) /* For now, H5Dstruct_chunk_iter() just returns SUCCEED without actual iteration */ udata.struct_chunk_info = chunk_infos; - udata.last_index = -1; + udata.last_index = -1; if (H5Dstruct_chunk_iter(did, H5P_DEFAULT, &iter_cb_struct, &udata) < 0) TEST_ERROR; @@ -590,7 +591,6 @@ test_sparse_direct_chunk_query(hid_t fapl) return 1; } /* test_sparse_direct_chunk_query() */ - /*------------------------------------------------------------------------- * * Function: test_sparse_filter() @@ -610,17 +610,16 @@ test_sparse_direct_chunk_query(hid_t fapl) static int test_sparse_filter(hid_t fapl) { - char filename[FILENAME_BUF_SIZE]; /* File name */ - hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ - hid_t did = H5I_INVALID_HID; /* Dataset ID */ - hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ - hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ - int nfilters; - H5Z_filter_t filtn; - uint64_t flags; - + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int nfilters; + H5Z_filter_t filtn; + uint64_t flags; TESTING("APIs for structured chunk filtering"); @@ -650,8 +649,7 @@ test_sparse_filter(hid_t fapl) TEST_ERROR; /* Create a new dataset using dcpl creation properties */ - did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT); + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); /* TBD: verify nfilters is correct; for now it is 0 */ nfilters = H5Pget_nfilters2(dcpl, H5Z_FLAG_SPARSE_SELECTION); @@ -681,12 +679,13 @@ test_sparse_filter(hid_t fapl) TEST_ERROR; /* TBD: Modify the filter's flags to optional */ - if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < 0) + if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < + 0) FAIL_STACK_ERROR; /* TBD: verify that flags is H5Z_FLAG_OPTIONAL */ if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, NULL, - NULL) < 0) + NULL) < 0) FAIL_STACK_ERROR; /* TBD: Remove the filter */ @@ -700,7 +699,6 @@ test_sparse_filter(hid_t fapl) if (filtn != H5Z_FILTER_NONE) TEST_ERROR; - /* Release resource */ if (H5Dclose(did) < 0) TEST_ERROR; @@ -768,7 +766,7 @@ iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, * --H5Dget_chunk_info() * --H5Dget_chunk_info_by_coord() * --H5Dchunk_iter() - * Verify the following dense chunk APIs will succeed for + * Verify the following dense chunk APIs will succeed for * H5D_SPARSE_CHUNK layout: * --H5Dread_chunk() * --H5Dget_chunk_storage_size() @@ -788,14 +786,14 @@ test_dense_chunk_api_on_sparse(hid_t fapl) hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ - chunk_iter_info_t chunk_infos[2]; - chunk_iter_udata_t udata; - hsize_t nchunks = 0; - hsize_t chunk_nbytes; - hsize_t offset[2] = {0, 0}; - int direct_buf[CHUNK_NX][CHUNK_NY]; - haddr_t addr = 0; - uint32_t filters = 0; + chunk_iter_info_t chunk_infos[2]; + chunk_iter_udata_t udata; + hsize_t nchunks = 0; + hsize_t chunk_nbytes; + hsize_t offset[2] = {0, 0}; + int direct_buf[CHUNK_NX][CHUNK_NY]; + haddr_t addr = 0; + uint32_t filters = 0; TESTING("APIs for direct chunk I/O: dense chunk functions on sparse layout"); @@ -823,12 +821,11 @@ test_dense_chunk_api_on_sparse(hid_t fapl) TEST_ERROR; /* Create a new dataset using dcpl creation properties */ - did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT); + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); if (did < 0) TEST_ERROR; - H5E_BEGIN_TRY + H5E_BEGIN_TRY { H5Dwrite_chunk(did, H5P_DEFAULT, 0, offset, CHK_SIZE, direct_buf); } @@ -836,25 +833,25 @@ test_dense_chunk_api_on_sparse(hid_t fapl) /* TBD: set return status and verify that it should fail */ H5E_BEGIN_TRY - { - H5Dget_chunk_info(did, H5S_ALL, 0, NULL, NULL, &addr, NULL); - } + { + H5Dget_chunk_info(did, H5S_ALL, 0, NULL, NULL, &addr, NULL); + } H5E_END_TRY /* TBD: set return status and verify that it should fail */ H5E_BEGIN_TRY - { - H5Dget_chunk_info_by_coord(did, offset, NULL, &addr, NULL); - } + { + H5Dget_chunk_info_by_coord(did, offset, NULL, &addr, NULL); + } H5E_END_TRY /* TBD: set return status and verify that it should fail */ H5E_BEGIN_TRY - { - udata.chunk_info = chunk_infos; - udata.last_index = -1; - H5Dchunk_iter(did, H5P_DEFAULT, &iter_cb, &udata); - } + { + udata.chunk_info = chunk_infos; + udata.last_index = -1; + H5Dchunk_iter(did, H5P_DEFAULT, &iter_cb, &udata); + } H5E_END_TRY /* TBD: set return status and verify that it should fail */ @@ -901,7 +898,7 @@ test_dense_chunk_api_on_sparse(hid_t fapl) * * Purpose: Tests for sparse data * Tests are copied and modified from: - * --test/dsets.c + * --test/dsets.c * --test/direct_chunk.c * --test/chunk_info.c * From 207fc96a6b44303d955ebf72ff3ea6c504208c0a Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Fri, 26 Jul 2024 17:14:01 -0500 Subject: [PATCH 3/9] Fix a doxygen syntax error. --- src/H5Dpublic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index d224f40843e..16125b96966 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -1765,7 +1765,7 @@ H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * * \return \herr_t * - \ \details H5Dwrite_struct_chunk() writes a structured chunk specified by its + * \details H5Dwrite_struct_chunk() writes a structured chunk specified by its * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles * the structured chunk according to the information provided in the * \p chunk_info parameter and using data pointed by \p buf. From 30c7992c89ffc48eac4993f67f2e6c232536663d Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 21:55:45 +0000 Subject: [PATCH 4/9] Committing clang-format changes --- doxygen/examples/H5D_sparse_examples.c | 152 +++++++++++++------------ doxygen/examples/H5P_sparse_examples.c | 37 +++--- src/H5Dpublic.h | 20 ++-- src/H5Oprivate.h | 4 +- src/H5Pdcpl.c | 21 ++-- src/H5Ppublic.h | 10 +- test/sparse_storage.c | 50 ++++---- tools/src/h5stat/h5stat.c | 5 +- 8 files changed, 157 insertions(+), 142 deletions(-) diff --git a/doxygen/examples/H5D_sparse_examples.c b/doxygen/examples/H5D_sparse_examples.c index de6a6f28f19..6969f62c7f3 100644 --- a/doxygen/examples/H5D_sparse_examples.c +++ b/doxygen/examples/H5D_sparse_examples.c @@ -7,13 +7,14 @@ //! int -struct_chunk_cb(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size, void *op_data) +struct_chunk_cb(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size, void *op_data) { // Print out info for each structured chunk printf("offset[0] = %" PRIuHSIZE " offset[1] = %", PRIuHSIZE "\n", offset[0], offset[1]); printf("addr = %" PRIuHADDR " chunk_size = %" PRIuHSIZE "\n", *addr, *chunk_size); - printf("num_sections = %u section_size[0] = %u section_size[1] = %u\n", - chunk_info->num_sections, chunk_info->section_size[0], chunk_info->section_size[1]); + printf("num_sections = %u section_size[0] = %u section_size[1] = %u\n", chunk_info->num_sections, + chunk_info->section_size[0], chunk_info->section_size[1]); return EXIT_SUCCESS; } //! @@ -28,14 +29,14 @@ main(void) __label__ fail_file, fail_space, fail_dcpl, fail_layout; __label__ fail_chunk, fail_dcreate, fail_dwrite, fail_defined; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_dset"; - hid_t fid, lcpl, sid, sid1, did; - hsize_t dim[1] = {50}; - hsize_t chunk_dim[1] = {5}; /* Chunk size */ - int wbuf[50]; /* Write buffer */ + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_dset"; + hid_t fid, lcpl, sid, sid1, did; + hsize_t dim[1] = {50}; + hsize_t chunk_dim[1] = {5}; /* Chunk size */ + int wbuf[50]; /* Write buffer */ - // Create a file + // Create a file if ((fid = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) == H5I_INVALID_HID) { ret_val = EXIT_FAILURE; goto fail_file; @@ -66,7 +67,8 @@ main(void) } // Create an integer dataset with sparse chunk layout - if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == H5I_INVALID_HID) { + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { ret_val = EXIT_FAILURE; goto fail_dcreate; } @@ -95,7 +97,6 @@ main(void) if ((npoints = H5Sget_select_npoints(sid1)) != 5) ret_val = EXIT_FAILURE; - H5Sclose(sid1); fail_defined: @@ -117,11 +118,10 @@ fail_file:; } //! - //! { __label__ fail_file, fail_dset, fail_erase, fail_defined; - hid_t fid, sid, did; + hid_t fid, sid, did; hssize_t npoints; // Open the file @@ -158,11 +158,10 @@ fail_file:; fail_erase: H5Dclose(did); -fail_dset; + fail_dset; H5Fclose(fid); fail_file:; - } //! @@ -172,21 +171,21 @@ fail_file:; __label__ fail_dcreate, fail_direct_write, fail_dread, fail_shyper; __label__ fail_calloc0, fail_calloc1, fail_sencode0, fail_encode1; - hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_direct_dset"; - hsize_t dims[2] = {10, 10}; - hsize_t chunk_dims[2] = {5, 5}; - const hsize_t start[2], count[2], block[2]; - int rbuf[10][10]; - size_t encode_size; - hsize_t wr_offset[2] = {0, 0}; + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + hsize_t dims[2] = {10, 10}; + hsize_t chunk_dims[2] = {5, 5}; + const hsize_t start[2], count[2], block[2]; + int rbuf[10][10]; + size_t encode_size; + hsize_t wr_offset[2] = {0, 0}; H5D_struct_chunk_info_t wr_chk_info; - uint16_t wr_filter_mask[2] = {0, 0}; - size_t wr_section_size[2]; - void *wr_buf[2]; - unsigned char *wr_buf0; - int *wr_buf1; + uint16_t wr_filter_mask[2] = {0, 0}; + size_t wr_section_size[2]; + void *wr_buf[2]; + unsigned char *wr_buf0; + int *wr_buf1; // Open the file if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { @@ -219,14 +218,17 @@ fail_file:; } // Create an integer 2-d dataset with sparse chunk layout - if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == H5I_INVALID_HID) { + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { ret_val = EXIT_FAILURE; goto fail_dcreate; } // Select the 2 x 3 block in chunk index 0 for writing - start[0] = 3; start[1] = 2; - block[0] = 2; block[1] = 3; + start[0] = 3; + start[1] = 2; + block[0] = 2; + block[1] = 3; count[0] = count[1] = 1; if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) { ret_val = EXIT_FAILURE; @@ -260,15 +262,18 @@ fail_file:; } // Set up data into the buffer for section 1 - wr_buf1[0] = 32; wr_buf1[1] = 33; wr_buf1[2] = 34; - wr_buf1[3] = 42; wr_buf1[4] = 43; wr_buf1[5] = 44; - + wr_buf1[0] = 32; + wr_buf1[1] = 33; + wr_buf1[2] = 34; + wr_buf1[3] = 42; + wr_buf1[4] = 43; + wr_buf1[5] = 44; // Set up the buffer for H5D_write_struct_chunk() wr_buf[0] = wr_buf0; wr_buf[1] = wr_buf1; - // Set up chunk info + // Set up chunk info wr_chk_info.type = H5D_SPARSE_CHUNK; wr_chk_info.num_sections = 2; wr_chk_info.filter_mask = wr_filter_mask; @@ -288,17 +293,17 @@ fail_file:; } // Verify elements in rbuf is same as what is written via H5Dwrite_struct_chunk() - if (rbuf[3][2] != wr_buf[1][0] || rbuf[3][3] != wr_buf[1][1] || rbuf[3][4] != wr_buf[1][2] || + if (rbuf[3][2] != wr_buf[1][0] || rbuf[3][3] != wr_buf[1][1] || rbuf[3][4] != wr_buf[1][2] || rbuf[4][2] != wr_buf[1][3] || rbuf[4][3] != wr_buf[1][4] || rbuf[4][4] != wr_buf[1][5]) ret_val = EXIT_FAILURE; fail_direct_write: fail_dread: fail_sencode1: - free(wr_buf1); + free(wr_buf1); fail_calloc1: - free(wr_buf0); + free(wr_buf0); fail_calloc0: fail_sencode0: @@ -317,28 +322,27 @@ fail_file:; H5Fclose(fid); fail_file:; - } //! - + //! { __label__ fail_file, fail_dopen, fail_dwrite, fail_direct_read, fail_dspace; __label__ fail_shyper, fail_calloc0, fail_calloc1, fail_sencode; - hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_direct_dset"; - int wbuf[10][10]; - size_t encode_size; - const hsize_t start[2], count[2], block[2]; - hsize_t rd_offset[2] = {5, 5}; + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + int wbuf[10][10]; + size_t encode_size; + const hsize_t start[2], count[2], block[2]; + hsize_t rd_offset[2] = {5, 5}; H5D_struct_chunk_info_t rd_chk_info; - uint16_t rd_filter_mask[2] = {0, 0}; - size_t rd_section_size[2]; - void *rd_buf[2]; - unsigned char *rd_buf0; - int *rd_buf1; + uint16_t rd_filter_mask[2] = {0, 0}; + size_t rd_section_size[2]; + void *rd_buf[2]; + unsigned char *rd_buf0; + int *rd_buf1; // Open the file if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { @@ -348,7 +352,9 @@ fail_file:; // Open the 2-d dataset with sparse chunk layout if ((did = H5Dopen2(fid, dset_name, H5P_DEFAULT)) < 0) { - ret_val = EXIT_FAILURE; goto fail_dopen; } // Initialize sparse data + ret_val = EXIT_FAILURE; + goto fail_dopen; + } // Initialize sparse data memset(wbuf, 0, sizeof(wbuf)); wbuf[7][6] = 76; @@ -360,15 +366,17 @@ fail_file:; goto fail_dwrite; } - // Retrieve the dataset's dataspace + // Retrieve the dataset's dataspace if ((sid = H5Dget_space(did)) == H5I_INVALID_HID) { ret_val = EXIT_FAILURE; goto fail_dspace; } // Select the 2 x 1 block in chunk index 3 for reading - start[0] = 5; start[1] = 5; - block[0] = 2; block[1] = 1; + start[0] = 5; + start[1] = 5; + block[0] = 2; + block[1] = 1; count[0] = count[1] = 1; if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) { ret_val = EXIT_FAILURE; @@ -415,14 +423,14 @@ fail_file:; ret_val = EXIT_FAILURE; fail_direct_read: - free(rd_buf1); + free(rd_buf1); fail_calloc1: - free(rd_buf0); + free(rd_buf0); fail_calloc0: -fail_shyper; -fail_sencode; + fail_shyper; + fail_sencode; H5Sclose(sid); fail_dspace: @@ -433,17 +441,16 @@ fail_sencode; H5Fclose(fid); fail_file:; - } //! - + //! { __label__ fail_file, fail_dopen; hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_direct_dset"; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; // Open the file if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { @@ -461,24 +468,22 @@ fail_file:; if (H5Dstruct_chunk_iter(did, H5P_DEFAULT, struct_chunk_cb, NULL) < 0) ret_val = EXIT_FAILURE; - H5Dclose(did); fail_dopen: H5Fclose(fid); fail_file:; - } //! - + //! { __label__ fail_file, fail_dopen; - hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_direct_dset"; + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; hsize_t offset[2]; haddr_t addr; hsize_t chunk_size; @@ -532,9 +537,8 @@ fail_file:; H5Fclose(fid); fail_file:; - } //! - + return ret_val; } diff --git a/doxygen/examples/H5P_sparse_examples.c b/doxygen/examples/H5P_sparse_examples.c index 286a82a75ed..6e03f1caa62 100644 --- a/doxygen/examples/H5P_sparse_examples.c +++ b/doxygen/examples/H5P_sparse_examples.c @@ -12,13 +12,13 @@ main(void) //! { - __label__ fail_file, fail_space, fail_dcpl, fail_set_chunk, - __label__ ail_set_layout, fail_set_filter2, fail_dcreate; + __label__ fail_file, fail_space, fail_dcpl, fail_set_chunk, __label__ ail_set_layout, + fail_set_filter2, fail_dcreate; - hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_filter_dset"; - hsize_t dims[2] = {10, 10}; + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_filter_dset"; + hsize_t dims[2] = {10, 10}; hsize_t chunk_dims[2] = {5, 5}; // Open the file @@ -52,13 +52,15 @@ main(void) } // Adds a filter to section 0 - if (H5Pset_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_MANDATORY, 0, NULL) < 0) { + if (H5Pset_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_MANDATORY, 0, NULL) < + 0) { ret_val = EXIT_FAILURE; goto fail_set_filter2; } // Create an integer 2-d dataset with sparse chunk layout - if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == H5I_INVALID_HID) { + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { ret_val = EXIT_FAILURE; goto fail_dcreate; } @@ -82,18 +84,17 @@ main(void) H5Fclose(fid); fail_file:; - } //! - + //! { __label__ fail_file, fail_dopen, fail_dget_plist, fail_pget_filter3; __label__ fail_pget_mfilter2, fail_pget_filter_id3, fail_pget_remove_filter2; hid_t fid, sid, did, dcpl; - char file_name[] = "sparse.h5"; - char dset_name[] = "sparse_filter_dset"; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_filter_dset"; // Open the file if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { @@ -117,13 +118,14 @@ fail_file:; goto fail_pget_filter3; } - if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < - 0) { + if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, + NULL) < 0) { ret_val = EXIT_FAILURE; goto fail_pmod_filter2; } - - if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, NULL, NULL) < 0) { + + if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, + NULL, NULL) < 0) { ret_val = EXIT_FAILURE; goto fail_pget_filter_id3; @@ -156,9 +158,8 @@ fail_file:; H5Fclose(fid); fail_file:; - } //! - + return ret_val; } diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 702cce750c3..34a817fa012 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -52,11 +52,10 @@ typedef enum H5D_layout_t { } H5D_layout_t; //! - /* Types of H5D_STRUCT_CHUNK storage */ /* TBD: H5D_SPARSE_CHUNK is the default storage type for H5D_STRUCT_CHUNK layout when not specified */ -#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ -#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ +#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ +#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ //! /** @@ -1769,12 +1768,12 @@ H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * * \return \herr_t * - * \details H5Dwrite_struct_chunk() writes a structured chunk specified by its - * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles - * the structured chunk according to the information provided in the - * \p chunk_info parameter and using data pointed by \p buf. - * \p buf is an array of pointers to the buffers containing data for each - * section of the structured chunk. + * \details H5Dwrite_struct_chunk() writes a structured chunk specified by its + * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles + * the structured chunk according to the information provided in the + * \p chunk_info parameter and using data pointed by \p buf. + * \p buf is an array of pointers to the buffers containing data for each + * section of the structured chunk. * * Initially, this function will support only sparse chunks of the * fixed-size data. Such chunks have only two sections: one for the @@ -1913,7 +1912,8 @@ H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * * * \since 1.x.x * - * \see H5Dget_struct_chunk_info_by_coord(), H5Dstruct_chunk_iter(), H5Dwrite_struct_chunk(), H5Dread_struct_chunk() + * \see H5Dget_struct_chunk_info_by_coord(), H5Dstruct_chunk_iter(), H5Dwrite_struct_chunk(), + * H5Dread_struct_chunk() * */ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset, diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 24f9a75ff8a..97c25637ed6 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -655,7 +655,7 @@ typedef struct H5O_layout_chunk_t { typedef struct H5O_layout_struct_chunk_t { /* TBD: may add more fields or modify fields for structured chunk */ - unsigned struct_type; /* Structured chunk storage type */ + unsigned struct_type; /* Structured chunk storage type */ /* NOW: H5D_SPARSE_CHUNK, H5D_VL_CHUK */ H5D_chunk_index_t idx_type; /* Type of chunk index */ uint8_t flags; /* Chunk layout flags */ @@ -681,7 +681,7 @@ typedef struct H5O_layout_t { unsigned version; /* Version of message */ const struct H5D_layout_ops_t *ops; /* Pointer to data layout I/O operations */ union { - H5O_layout_chunk_t chunk; /* Information for chunked layout */ + H5O_layout_chunk_t chunk; /* Information for chunked layout */ H5O_layout_struct_chunk_t struct_chunk; /* Information for structured chunk layout */ } u; H5O_storage_t storage; /* Information for storing dataset elements */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index c1fdcb00b35..932902c1ab2 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -161,12 +161,12 @@ #ifdef TBD #define H5D_DEF_LAYOUT_STRUCT_CHUNK \ { \ - H5D_STRUCT_CHUNK, H5O_LAYOUT_VERSION_5, H5D_LOPS_CHUNK, {.struct_chunk = H5D_DEF_LAYOUT_STRUCT_CHUNK_INIT}, \ - H5D_DEF_STORAGE_CHUNK \ - } + H5D_STRUCT_CHUNK, H5O_LAYOUT_VERSION_5, H5D_LOPS_CHUNK, \ + {.struct_chunk = H5D_DEF_LAYOUT_STRUCT_CHUNK_INIT}, H5D_DEF_STORAGE_CHUNK \ + } #endif /* TBD: same as H5D_CHUNKED */ -#define H5D_DEF_LAYOUT_STRUCT_CHUNK \ +#define H5D_DEF_LAYOUT_STRUCT_CHUNK \ { \ H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_CHUNK \ @@ -316,11 +316,11 @@ static const H5O_efl_t H5D_def_efl_g = H5D_CRT_EXT_FILE_LIST_DEF; /* Default static const unsigned H5O_ohdr_min_g = H5D_CRT_MIN_DSET_HDR_SIZE_DEF; /* Default object header minimization */ /* Defaults for each type of layout */ -static const H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; -static const H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; -static const H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; -static const H5O_layout_t H5D_def_layout_struct_chunk_g = H5D_DEF_LAYOUT_STRUCT_CHUNK; -static const H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; +static const H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; +static const H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; +static const H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; +static const H5O_layout_t H5D_def_layout_struct_chunk_g = H5D_DEF_LAYOUT_STRUCT_CHUNK; +static const H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; /*------------------------------------------------------------------------- * Function: H5P__dcrt_reg_prop @@ -1997,7 +1997,6 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) FUNC_LEAVE_API(ret_value) } /* end H5Pset_chunk() */ - /*------------------------------------------------------------------------- * Function: H5Pget_chunk * @@ -3598,7 +3597,7 @@ H5Pset_dset_no_attrs_hint(hid_t dcpl_id, hbool_t minimize) * Purpose: Sets the number of dimensions and the size of each chunk to * the values specified. The dimensionality of the chunk should * match the dimensionality of the dataspace. - * It also sets the structured chunk storage type to FLAG. + * It also sets the structured chunk storage type to FLAG. * * As a side effect, the layout method is changed to * H5D_STRUCT_CHUNK. diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index f4873570049..b6545251477 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6603,7 +6603,7 @@ H5_DLL int H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/); * * The type of structured chunk storage used will be retrieved in * \p flag. The value can be #H5D_SPARSE_CHUNK for storing sparse data - * of any datatype or #H5D_VL_CHUNK for storing dense data of + * of any datatype or #H5D_VL_CHUNK for storing dense data of * variable-length datatype. * * \see H5Pset_struct_chunk(), H5Pget_chunk() @@ -7083,7 +7083,7 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* /** * \ingroup DCPL * - * \brief Sets the size of chunks and the structured chunk storage type + * \brief Sets the size of chunks and the structured chunk storage type * used to store a dataset with structured chunked layout * * \dcpl_id{plist_id} @@ -7107,7 +7107,7 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* * * The parameter \p flag indicates the type of structured chunk storage. * The value can be #H5D_SPARSE_CHUNK for storing sparse data of - * any datatype or #H5D_VL_CHUNK for storing dense data of + * any datatype or #H5D_VL_CHUNK for storing dense data of * variable-length datatype. * * As a side-effect of this function, the layout of the dataset is @@ -7116,7 +7116,7 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* * \note Chunk size cannot exceed the size of a fixed-size dataset. For * example, a dataset consisting of a 5x4 fixed-size array cannot be * defined with 10x10 chunks. - * \note Structured chunk does not have 4GB limit on the chunk size as the + * \note Structured chunk does not have 4GB limit on the chunk size as the * “dense” chunk has due to API limitations. * * \par Example: @@ -7403,7 +7403,7 @@ H5_DLL herr_t H5Pset_shuffle(hid_t plist_id); * the file. * - #H5D_VIRTUAL: Draw raw data from multiple datasets in * different files. - * - #H5D_STRUCT_CHUNK: Compose of two or more sections + * - #H5D_STRUCT_CHUNK: Compose of two or more sections * which together will describe the values * in a chunk. * diff --git a/test/sparse_storage.c b/test/sparse_storage.c index 360c4b73abe..81b0149e030 100644 --- a/test/sparse_storage.c +++ b/test/sparse_storage.c @@ -213,25 +213,25 @@ test_sparse_direct_chunk(hid_t fapl) hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; - int buf[NX][NY]; - size_t encode_size; + int buf[NX][NY]; + size_t encode_size; hsize_t start[2], block[2], count[2]; - hsize_t wr_offset[2] = {0, 0}; + hsize_t wr_offset[2] = {0, 0}; H5D_struct_chunk_info_t wr_chk_info; - uint16_t wr_filter_mask[2] = {0, 0}; + uint16_t wr_filter_mask[2] = {0, 0}; size_t wr_section_size[2]; void *wr_buf[2]; - unsigned char *wr_buf0; - int *wr_buf1; + unsigned char *wr_buf0; + int *wr_buf1; - hsize_t rd_offset[2] = {5, 5}; + hsize_t rd_offset[2] = {5, 5}; H5D_struct_chunk_info_t rd_chk_info; - uint16_t rd_filter_mask[2] = {0, 0}; - size_t rd_section_size[2]; - void *rd_buf[2]; - unsigned char *rd_buf0; - int *rd_buf1; + uint16_t rd_filter_mask[2] = {0, 0}; + size_t rd_section_size[2]; + void *rd_buf[2]; + unsigned char *rd_buf0; + int *rd_buf1; TESTING("APIs for direct chunk I/O on structured chunks"); @@ -262,8 +262,10 @@ test_sparse_direct_chunk(hid_t fapl) if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; - start[0] = 3; start[1] = 2; - block[0] = 2; block[1] = 3; + start[0] = 3; + start[1] = 2; + block[0] = 2; + block[1] = 3; count[0] = count[1] = 1; /* Select the 2x3 block in chunk index 0 for writing */ if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) @@ -284,12 +286,16 @@ test_sparse_direct_chunk(hid_t fapl) FAIL_STACK_ERROR; /* Encode selection into the buffer for section 0 */ - if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) + if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) FAIL_STACK_ERROR; /* Set up data into the bufer for section 1 */ - wr_buf1[0] = 32; wr_buf1[1] = 33; wr_buf1[2] = 34; - wr_buf1[3] = 42; wr_buf1[4] = 43; wr_buf1[5] = 44; + wr_buf1[0] = 32; + wr_buf1[1] = 33; + wr_buf1[2] = 34; + wr_buf1[3] = 42; + wr_buf1[4] = 43; + wr_buf1[5] = 44; /* Set up the buffer for H5D_write_struct_chunk() */ wr_buf[0] = wr_buf0; @@ -324,17 +330,19 @@ test_sparse_direct_chunk(hid_t fapl) if ((sid = H5Dget_space(did)) == H5I_INVALID_HID) FAIL_STACK_ERROR; - + /* Select the 2x1 block in chunk index 3 for reading */ - start[0] = 5; start[1] = 5; - block[0] = 2; block[1] = 1; + start[0] = 5; + start[1] = 5; + block[0] = 2; + block[1] = 1; count[0] = count[1] = 1; if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) FAIL_STACK_ERROR; if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) FAIL_STACK_ERROR; - + rd_section_size[0] = encode_size; rd_section_size[1] = block[0] * block[1] * sizeof(int); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 787a27aed9b..aafc5bc3e9e 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -1281,7 +1281,10 @@ print_dataset_info(const iter_t *iter) printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" - : (u == H5D_CONTIGUOUS ? "CONTIG" : (u == H5D_CHUNKED ? "CHUNKED" : (u == H5D_VIRTUAL ? "VIRTUAL" : "STRUCT_CHUNK")))), + : (u == H5D_CONTIGUOUS + ? "CONTIG" + : (u == H5D_CHUNKED ? "CHUNKED" + : (u == H5D_VIRTUAL ? "VIRTUAL" : "STRUCT_CHUNK")))), iter->dset_layouts[u]); printf("\tNumber of external files : %lu\n", iter->nexternal); From f3e52c1a1b8961bb03dc07b5268c01108bb7b152 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Mon, 12 Aug 2024 18:19:55 -0500 Subject: [PATCH 5/9] Correct spelling and CI failures. --- doxygen/examples/H5D_sparse_examples.c | 2 +- src/H5Pdcpl.c | 4 ++-- test/sparse_storage.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doxygen/examples/H5D_sparse_examples.c b/doxygen/examples/H5D_sparse_examples.c index 6969f62c7f3..9043b7facb7 100644 --- a/doxygen/examples/H5D_sparse_examples.c +++ b/doxygen/examples/H5D_sparse_examples.c @@ -517,7 +517,7 @@ fail_file:; goto fail_get_info; } - // Use the offset just retrieved to obtain structured chunk info by coordindates + // Use the offset just retrieved to obtain structured chunk info by coordinates if (H5Dget_struct_chunk_info_by_coord(did, offset, NULL, &addr, &chunk_size) < 0) { ret_val = EXIT_FAILURE; goto fail_get_info; diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 932902c1ab2..e0c865242c0 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -3607,7 +3607,7 @@ H5Pset_dset_no_attrs_hint(hid_t dcpl_id, hbool_t minimize) *------------------------------------------------------------------------- */ herr_t -H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], unsigned flag) +H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], unsigned H5_ATTR_UNUSED flag) { H5P_genplist_t *plist; /* Property list pointer */ H5O_layout_t chunk_layout; /* Layout information for setting chunk info */ @@ -3637,7 +3637,7 @@ H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], uns chunk_layout.u.chunk.dim[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */ } /* end for */ - /* TBD: should set fields u.struct_chunk e.g. struct_type */ + /* TBD: should set fields u.struct_chunk e.g. struct_type to parameter flag */ /* Get the plist structure */ if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE))) diff --git a/test/sparse_storage.c b/test/sparse_storage.c index 81b0149e030..52ca2badad1 100644 --- a/test/sparse_storage.c +++ b/test/sparse_storage.c @@ -289,7 +289,7 @@ test_sparse_direct_chunk(hid_t fapl) if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) FAIL_STACK_ERROR; - /* Set up data into the bufer for section 1 */ + /* Set up data into the buffer for section 1 */ wr_buf1[0] = 32; wr_buf1[1] = 33; wr_buf1[2] = 34; From 582e5f9bbf02ae1a1bf6028860bf43ad3446e748 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Thu, 15 Aug 2024 01:27:08 -0500 Subject: [PATCH 6/9] --Fix doxygen text for H5Pset/get_struct_chunk() and H5Derase() --Remove H5Pset_layout() from doxygen examples for the new APIs --Rename 3 files --- ...examples.c => H5D_struct_chunk_examples.c} | 12 ------ ...examples.c => H5P_struct_chunk_examples.c} | 6 --- src/H5Dpublic.h | 30 +++++++------- src/H5Ppublic.h | 40 +++++++++---------- test/Makefile.am | 2 +- ...parse_storage.c => struct_chunk_storage.c} | 0 6 files changed, 36 insertions(+), 54 deletions(-) rename doxygen/examples/{H5D_sparse_examples.c => H5D_struct_chunk_examples.c} (97%) rename doxygen/examples/{H5P_sparse_examples.c => H5P_struct_chunk_examples.c} (96%) rename test/{sparse_storage.c => struct_chunk_storage.c} (100%) diff --git a/doxygen/examples/H5D_sparse_examples.c b/doxygen/examples/H5D_struct_chunk_examples.c similarity index 97% rename from doxygen/examples/H5D_sparse_examples.c rename to doxygen/examples/H5D_struct_chunk_examples.c index 9043b7facb7..2d6d8f9908e 100644 --- a/doxygen/examples/H5D_sparse_examples.c +++ b/doxygen/examples/H5D_struct_chunk_examples.c @@ -54,12 +54,6 @@ main(void) goto fail_dcpl; } - // Set dataset layout to sparse chunk */ - if (H5Pset_layout(dcpl, H5D_STRUCT_CHUNK) < 0) { - ret_val = EXIT_FAILURE; - goto fail_layout; - } - // Set to sparse chunked dataset if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { ret_val = EXIT_FAILURE; @@ -205,12 +199,6 @@ fail_file:; goto fail_dcpl; } - // Set dataset layout to sparse chunk */ - if (H5Pset_layout(dcpl, H5D_STRUCT_CHUNK) < 0) { - ret_val = EXIT_FAILURE; - goto fail_set_layout; - } - // Set to sparse chunked dataset if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { ret_val = EXIT_FAILURE; diff --git a/doxygen/examples/H5P_sparse_examples.c b/doxygen/examples/H5P_struct_chunk_examples.c similarity index 96% rename from doxygen/examples/H5P_sparse_examples.c rename to doxygen/examples/H5P_struct_chunk_examples.c index 6e03f1caa62..99612602e89 100644 --- a/doxygen/examples/H5P_sparse_examples.c +++ b/doxygen/examples/H5P_struct_chunk_examples.c @@ -39,12 +39,6 @@ main(void) goto fail_dcpl; } - // Set dataset layout to sparse chunk */ - if (H5Pset_layout(dcpl, H5D_SPARSE_CHUNK) < 0) { - ret_val = EXIT_FAILURE; - goto fail_set_layout; - } - // Set to sparse chunked dataset if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { ret_val = EXIT_FAILURE; diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 34a817fa012..4ed202a40c7 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -53,9 +53,8 @@ typedef enum H5D_layout_t { //! /* Types of H5D_STRUCT_CHUNK storage */ -/* TBD: H5D_SPARSE_CHUNK is the default storage type for H5D_STRUCT_CHUNK layout when not specified */ -#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ -#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ +#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ +#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ //! /** @@ -1710,7 +1709,7 @@ H5_DLL herr_t H5Dclose(hid_t dset_id); * as all elements are defined for non-sparse datasets. * * \par Example: - * \snippet H5D_sparse_examples.c get_defined + * \snippet H5D_struct_chunk_examples.c get_defined * * \since 1.x.x * @@ -1734,15 +1733,16 @@ H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, * causing them to no longer be defined. The part of the dataset to erase is defined - * by \p file_space_id. After this operation, reading from these elements - * will return fill values, and the elements will no longer be included in the - * selection returned by H5Dget_defined(). + * by \p file_space_id and the data transfer properties are defined by \p dxpl_id. + * After this operation, reading from these elements will return fill values, + * and the elements will no longer be included in the selection returned by + * H5Dget_defined(). * - * This function is only useful for datasets with layout TBD:H5D_SPARSE_CHUNK. + * This function is only useful for datasets with #H5D_STRUCT_CHUNK layout. * For other layouts this function will return an error. * * \par Example: - * \snippet H5D_sparse_examples.c erase + * \snippet H5D_struct_chunk_examples.c erase * * \since 1.x.x * @@ -1792,7 +1792,7 @@ H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * \note TBD: need to verify the above notes info are true for structured chunk. * * \par Example: - * \snippet H5D_sparse_examples.c direct_chunk_write + * \snippet H5D_struct_chunk_examples.c direct_chunk_write * * \since 1.x.x * @@ -1845,7 +1845,7 @@ H5_DLL herr_t H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * \note TBD: need to verify the above notes info are true for structured chunk. * * \par Example: - * \snippet H5D_sparse_examples.c direct_chunk_read + * \snippet H5D_struct_chunk_examples.c direct_chunk_read * * \since 1.x.x * @@ -1908,7 +1908,7 @@ H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t * * structured chunk. * * \par Example - * \snippet H5D_sparse_examples.c direct_chunk_get_info + * \snippet H5D_struct_chunk_examples.c direct_chunk_get_info * * \since 1.x.x * @@ -1956,7 +1956,7 @@ H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t c * \note TBD: need to verify the above notes info are true for structured chunk. * * \par Example - * \snippet H5D_sparse_examples.c direct_chunk_get_info + * \snippet H5D_struct_chunk_examples.c direct_chunk_get_info * * \since 1.x.x * @@ -1991,9 +1991,9 @@ H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *of * * \par Example: * Print the info for each structured chunk - * \snippet H5D_sparse_examples.c H5Dstruct_chunk_iter_cb + * \snippet H5D_struct_chunk_examples.c H5Dstruct_chunk_iter_cb * Iterate over all the structured chunks in the dataset. - * \snippet H5D_sparse_examples.c direct_chunk_iter + * \snippet H5D_struct_chunk_examples.c direct_chunk_iter * * \since 1.x.x * diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index b6545251477..8cdab2b39ff 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -2197,7 +2197,7 @@ H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *f * H5Zget_filter_info(). * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter2 + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 * * \see H5Pset_filter2(), H5Pget_filter_by_id3() * @@ -2326,7 +2326,7 @@ H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsig * lists. * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter2 + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 * * \see H5Pset_filter2(), H5Pget_filter3() * @@ -2388,7 +2388,7 @@ H5_DLL int H5Pget_nfilters(hid_t plist_id); * for the section \p section_number, including zero (0) if there are none. * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter * * \see H5Pset_filter2, H5Pget_filter3 * @@ -2491,7 +2491,7 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int * values as described in the discussion of H5Pset_filter2(). * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter2 + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 * * \see H5Premove_filter2(), H5Pset_filter2() * @@ -2635,7 +2635,7 @@ H5_DLL herr_t H5Premove_filter(hid_t plist_id, H5Z_filter_t filter); * * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter2 + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 * * \see H5Pmodify_filter2(), H5Pset_filter2() * @@ -3368,7 +3368,7 @@ H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int fl * for details.) * * \par Example - * \snippet H5P_sparse_examples.c struct_chunk_filter + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter * * \see \ref_filter_pipe, \ref_group_impls * H5Pget_nfilters2(), H5Pget_filter3(), H5Pmodify_filter2() @@ -7083,22 +7083,21 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* /** * \ingroup DCPL * - * \brief Sets the size of chunks and the structured chunk storage type - * used to store a dataset with structured chunked layout + * \brief Sets structured chunked storage. The storage is used for sparse data of + * any datatype, and for dense data with variable-length datatype * * \dcpl_id{plist_id} - * \param[in] ndims The number of dimensions of each chunk + * \param[in] ndims The number of chunk dimensions * \param[in] dim An array defining the size, in dataset elements, of * each chunk - * \param[in] flag Flag that indicates the structured chunk storage type + * \param[in] flag Flag that indicates structured chunk storage type * * \return \herr_t - * \details H5Pset_struct_chunk() sets the size of chunks and the structured - * chunk storage type used to store a dataset with structured chunk - * chunked layout. This function is only valid for dataset - * creation property lists. + * \details H5Pset_struct_chunk() sets structured chunk storage layout, chunk sizes + * and a type of structured chunk storage for a dataset. + * This function is only valid for dataset creation property lists. * - * The \p ndims parameter currently must be the same size as the + * The \p ndims parameter must be the same size as the * rank of the dataset. * * The values of the \p dim array define the size of the chunks @@ -7106,12 +7105,13 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* * values is dataset elements. * * The parameter \p flag indicates the type of structured chunk storage. - * The value can be #H5D_SPARSE_CHUNK for storing sparse data of - * any datatype or #H5D_VL_CHUNK for storing dense data of + * The value can be #H5D_SPARSE_CHUNK to store sparse data of + * any datatype or #H5D_VL_CHUNK to store dense data of * variable-length datatype. * - * As a side-effect of this function, the layout of the dataset is - * changed to #H5D_STRUCT_CHUNK, if it is not already so set. + * As a side-effect of this function, the creation property is modified + * to #H5D_STRUCT_CHUNK storage layout, if it was previously set using + * H5Pset_layout() function with any other storage layout type. * * \note Chunk size cannot exceed the size of a fixed-size dataset. For * example, a dataset consisting of a 5x4 fixed-size array cannot be @@ -7120,7 +7120,7 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* * “dense” chunk has due to API limitations. * * \par Example: - * \snippet H5D_sparse_examples.c get_defined + * \snippet H5D_struct_chunk_examples.c get_defined * * \see H5Pget_struct_chunk(), H5Pset_chunk() * diff --git a/test/Makefile.am b/test/Makefile.am index 117d7b87613..5a98a656cba 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -70,7 +70,7 @@ TEST_PROG= testhdf5 \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ unregister cache_logging cork swmr vol timer event_set \ - onion sparse_storage + onion struct_chunk_storage # List programs to be built when testing here # diff --git a/test/sparse_storage.c b/test/struct_chunk_storage.c similarity index 100% rename from test/sparse_storage.c rename to test/struct_chunk_storage.c From ef171f79692795c67a3b7559daa887ba26b56b43 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 06:30:10 +0000 Subject: [PATCH 7/9] Committing clang-format changes --- src/H5Dpublic.h | 8 ++++---- src/H5Ppublic.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 4ed202a40c7..070b0710cf5 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -53,8 +53,8 @@ typedef enum H5D_layout_t { //! /* Types of H5D_STRUCT_CHUNK storage */ -#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ -#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ +#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ +#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ //! /** @@ -1734,8 +1734,8 @@ H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, * causing them to no longer be defined. The part of the dataset to erase is defined * by \p file_space_id and the data transfer properties are defined by \p dxpl_id. - * After this operation, reading from these elements will return fill values, - * and the elements will no longer be included in the selection returned by + * After this operation, reading from these elements will return fill values, + * and the elements will no longer be included in the selection returned by * H5Dget_defined(). * * This function is only useful for datasets with #H5D_STRUCT_CHUNK layout. diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 8cdab2b39ff..cc69802b0e6 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -7110,7 +7110,7 @@ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims* * variable-length datatype. * * As a side-effect of this function, the creation property is modified - * to #H5D_STRUCT_CHUNK storage layout, if it was previously set using + * to #H5D_STRUCT_CHUNK storage layout, if it was previously set using * H5Pset_layout() function with any other storage layout type. * * \note Chunk size cannot exceed the size of a fixed-size dataset. For From a5a1455841455551ec5dc39c85f618566a1e9e71 Mon Sep 17 00:00:00 2001 From: "vchoi-hdfgroup.org" Date: Thu, 15 Aug 2024 16:31:00 -0500 Subject: [PATCH 8/9] Minor fix to the doxygen description to H5Derase(). --- src/H5Dpublic.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 4ed202a40c7..f5a565e9072 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -1738,8 +1738,9 @@ H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * and the elements will no longer be included in the selection returned by * H5Dget_defined(). * - * This function is only useful for datasets with #H5D_STRUCT_CHUNK layout. - * For other layouts this function will return an error. + * This function is only applicable to the datasets with structured chunk layout + * of the H5D_SPARSE_CHUNK type. For other layouts this function will return + * an error. * * \par Example: * \snippet H5D_struct_chunk_examples.c erase From fff8ec1ebde606c26e149d5eeef1534b68eeeee5 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 21:35:20 +0000 Subject: [PATCH 9/9] Committing clang-format changes --- src/H5Dpublic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index c0ab4595b94..b3462f5e020 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -1738,8 +1738,8 @@ H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); * and the elements will no longer be included in the selection returned by * H5Dget_defined(). * - * This function is only applicable to the datasets with structured chunk layout - * of the H5D_SPARSE_CHUNK type. For other layouts this function will return + * This function is only applicable to the datasets with structured chunk layout + * of the H5D_SPARSE_CHUNK type. For other layouts this function will return * an error. * * \par Example:
#H5Z_FLAG_SPARSE_SELECTIONAdds the filter to the filter pipeline for the encoded - * selection section of the sparse chunk. It has the same - * effect as passing 0. The flag will be ignored if the + * Adds the filter to the filter pipeline for the encoded + * selection section of the sparse chunk. It has the same + * effect as passing 0. The flag will be ignored if the * structured chunk is not sparse. *
#H5Z_FLAG_SPARSE_FIXED_DATAAdds the filter to the filter pipeline for section 1 of - * the sparse chunk. It has the same effect as passing 1. + * Adds the filter to the filter pipeline for section 1 of + * the sparse chunk. It has the same effect as passing 1. *
#H5Z_FLAG_SPARSE_VL_DATAAdds the filter to the filter pipeline for section 2 of - * the sparse chunk if data has variable-length datatype. + * Adds the filter to the filter pipeline for section 2 of + * the sparse chunk if data has variable-length datatype. * It has the same effect as passing 2. *