diff --git a/doxygen/examples/H5D_struct_chunk_examples.c b/doxygen/examples/H5D_struct_chunk_examples.c new file mode 100644 index 00000000000..2d6d8f9908e --- /dev/null +++ b/doxygen/examples/H5D_struct_chunk_examples.c @@ -0,0 +1,532 @@ +/* -*- c-file-style: "stroustrup" -*- */ + +#include "hdf5.h" + +#include +#include + +//! +int +struct_chunk_cb(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size, void *op_data) +{ + // Print out info for each structured chunk + printf("offset[0] = %" PRIuHSIZE " offset[1] = %", PRIuHSIZE "\n", offset[0], offset[1]); + printf("addr = %" PRIuHADDR " chunk_size = %" PRIuHSIZE "\n", *addr, *chunk_size); + printf("num_sections = %u section_size[0] = %u section_size[1] = %u\n", chunk_info->num_sections, + chunk_info->section_size[0], chunk_info->section_size[1]); + return EXIT_SUCCESS; +} +//! + +int +main(void) +{ + int ret_val = EXIT_SUCCESS; + + //! + { + __label__ fail_file, fail_space, fail_dcpl, fail_layout; + __label__ fail_chunk, fail_dcreate, fail_dwrite, fail_defined; + + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_dset"; + hid_t fid, lcpl, sid, sid1, did; + hsize_t dim[1] = {50}; + hsize_t chunk_dim[1] = {5}; /* Chunk size */ + int wbuf[50]; /* Write buffer */ + + // Create a file + if ((fid = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Create a 1D dataspace + if ((sid = H5Screate_simple(1, dim, NULL)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_space; + } + + // Create a dataset creation property list + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcpl; + } + + // Set to sparse chunked dataset + if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { + ret_val = EXIT_FAILURE; + goto fail_chunk; + } + + // Create an integer dataset with sparse chunk layout + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcreate; + } + + // Initialize sparse data + memset(wbuf, 0, sizeof(wbuf)); + wbuf[12] = 12; + wbuf[13] = 13; + wbuf[14] = 14; + wbuf[48] = 48; + wbuf[49] = 49; + + // Write sparse data to the dataset + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dwrite; + } + + // Get dataspace with the defined elements in the dataset + if ((sid1 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_defined; + } + + // There are 5 defined elements + if ((npoints = H5Sget_select_npoints(sid1)) != 5) + ret_val = EXIT_FAILURE; + + H5Sclose(sid1); + +fail_defined: +fail_dwrite: + H5Dclose(did); + +fail_dcreate: +fail_layout: +fail_chunk: + H5Pclose(dcpl); + +fail_dcpl: + H5Sclose(sid); + +fail_space: + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_dset, fail_erase, fail_defined; + hid_t fid, sid, did; + hssize_t npoints; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Open the 1-d dataset with sparse chunk layout + if ((did = H5Dopen2(fid, dset_name, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dset; + } + + // Deletes defined elements from the dataset + if (H5Derase(did, H5S_ALL, H5P_DEFAULT) < 0) { + ret_val = EXIT_FAILURE; + goto fail_erase; + } + + // Get dataspace with the defined elements in the dataset + if ((sid = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_defined; + } + + // No defined elements + if ((npoints = H5Sget_select_npoints(sid1)) != 0) + ret_val = EXIT_FAILURE; + + H5Sclose(sid); + +fail_defined: +fail_erase: + H5Dclose(did); + + fail_dset; + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_space, fail_dcpl, fail_set_chunk, fail_set_layout; + __label__ fail_dcreate, fail_direct_write, fail_dread, fail_shyper; + __label__ fail_calloc0, fail_calloc1, fail_sencode0, fail_encode1; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + hsize_t dims[2] = {10, 10}; + hsize_t chunk_dims[2] = {5, 5}; + const hsize_t start[2], count[2], block[2]; + int rbuf[10][10]; + size_t encode_size; + hsize_t wr_offset[2] = {0, 0}; + H5D_struct_chunk_info_t wr_chk_info; + uint16_t wr_filter_mask[2] = {0, 0}; + size_t wr_section_size[2]; + void *wr_buf[2]; + unsigned char *wr_buf0; + int *wr_buf1; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Create a 2D dataspace + if ((sid = H5Screate_simple(2, dim, NULL)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_space; + } + + // Create a dataset creation property list + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcpl; + } + + // Set to sparse chunked dataset + if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { + ret_val = EXIT_FAILURE; + goto fail_set_chunk; + } + + // Create an integer 2-d dataset with sparse chunk layout + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcreate; + } + + // Select the 2 x 3 block in chunk index 0 for writing + start[0] = 3; + start[1] = 2; + block[0] = 2; + block[1] = 3; + count[0] = count[1] = 1; + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) { + ret_val = EXIT_FAILURE; + goto fail_shyper; + } + + // Get the encoded size for the selection + if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) { + ret_val = EXIT_FAILURE; + goto fail_sencode0; + } + + // Set up the section size for section 0 and section 1 + wr_section_size[0] = encode_size; + wr_section_size[1] = block[0] * block[1] * sizeof(int); + + // Allocate buffers for section 0 (encoded selection) and section 1 (data) + if ((wr_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) { + ret_val = EXIT_FAILURE; + goto fail_calloc0; + } + if ((wr_buf1 = (int *)calloc((size_t)1, wr_section_size[1])) == NULL) { + ret_val = EXIT_FAILURE; + goto fail_calloc1; + } + + // Encode selection into the buffer for section 0 + if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) { + ret_val = EXIT_FAILURE; + goto fail_sencode1; + } + + // Set up data into the buffer for section 1 + wr_buf1[0] = 32; + wr_buf1[1] = 33; + wr_buf1[2] = 34; + wr_buf1[3] = 42; + wr_buf1[4] = 43; + wr_buf1[5] = 44; + + // Set up the buffer for H5D_write_struct_chunk() + wr_buf[0] = wr_buf0; + wr_buf[1] = wr_buf1; + + // Set up chunk info + wr_chk_info.type = H5D_SPARSE_CHUNK; + wr_chk_info.num_sections = 2; + wr_chk_info.filter_mask = wr_filter_mask; + wr_chk_info.section_size = wr_section_size; + wr_chk_info.section_orig_size = wr_section_size; + + /* Write the structured chunk at offset [0,0]: chunk index 0 */ + if (H5Dwrite_struct_chunk(did, dxpl, wr_offset, &wr_chk_info, wr_buf) < 0) { + ret_val = EXIT_FAILURE; + goto fail_direct_write; + } + + /* Read the whole dataset */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, rbuf) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dread; + } + + // Verify elements in rbuf is same as what is written via H5Dwrite_struct_chunk() + if (rbuf[3][2] != wr_buf[1][0] || rbuf[3][3] != wr_buf[1][1] || rbuf[3][4] != wr_buf[1][2] || + rbuf[4][2] != wr_buf[1][3] || rbuf[4][3] != wr_buf[1][4] || rbuf[4][4] != wr_buf[1][5]) + ret_val = EXIT_FAILURE; + +fail_direct_write: +fail_dread: +fail_sencode1: + free(wr_buf1); + +fail_calloc1: + free(wr_buf0); + +fail_calloc0: +fail_sencode0: +fail_shyper: + H5Dclose(did); + +fail_dcreate: +fail_set_layout: +fail_set_chunk: + H5Pclose(dcpl); + +fail_dcpl: + H5Sclose(sid); + +fail_space: + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_dopen, fail_dwrite, fail_direct_read, fail_dspace; + __label__ fail_shyper, fail_calloc0, fail_calloc1, fail_sencode; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + int wbuf[10][10]; + size_t encode_size; + const hsize_t start[2], count[2], block[2]; + hsize_t rd_offset[2] = {5, 5}; + H5D_struct_chunk_info_t rd_chk_info; + uint16_t rd_filter_mask[2] = {0, 0}; + size_t rd_section_size[2]; + void *rd_buf[2]; + unsigned char *rd_buf0; + int *rd_buf1; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Open the 2-d dataset with sparse chunk layout + if ((did = H5Dopen2(fid, dset_name, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dopen; + } // Initialize sparse data + + memset(wbuf, 0, sizeof(wbuf)); + wbuf[7][6] = 76; + wbuf[8][6] = 86; + + // Write sparse data to the dataset + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dwrite; + } + + // Retrieve the dataset's dataspace + if ((sid = H5Dget_space(did)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dspace; + } + + // Select the 2 x 1 block in chunk index 3 for reading + start[0] = 5; + start[1] = 5; + block[0] = 2; + block[1] = 1; + count[0] = count[1] = 1; + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) { + ret_val = EXIT_FAILURE; + goto fail_shyper; + } + + // Get the encoded size for the selection + if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) { + ret_val = EXIT_FAILURE; + goto fail_sencode; + } + + // Set up the section size for section 0 and section 1 + rd_section_size[0] = encode_size; + rd_section_size[1] = block[0] * block[1] * sizeof(int); + + // Allocate buffers for section 0 (encoded selection) and section 1 (data) + if ((rd_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) { + ret_val = EXIT_FAILURE; + goto fail_calloc0; + } + if ((rd_buf1 = (int *)calloc((size_t)1, rd_section_size[1])) == NULL) { + ret_val = EXIT_FAILURE; + goto fail_calloc1; + } + + rd_buf[0] = rd_buf0; + rd_buf[1] = rd_buf1; + + rd_chk_info.type = H5D_SPARSE_CHUNK; + rd_chk_info.num_sections = 2; + rd_chk_info.filter_mask = rd_filter_mask; + rd_chk_info.section_size = rd_section_size; + rd_chk_info.section_orig_size = rd_section_size; + + // Read the structured chunk at offset [5,5]: chunk index 3 + if (H5Dread_struct_chunk(did, dxpl, rd_offset, &rd_chk_info, rd_buf) < 0) { + ret_val = EXIT_FAILURE; + goto fail_direct_read; + } + + // Verify elements in rd_buf via H5Dread_struct_chunk() is same as wbuf via H5Dwrite() + if (rd_buf[1][0] != wbuf[7][6] || rd_buf[1][1] != wbuf[8][6]) + ret_val = EXIT_FAILURE; + +fail_direct_read: + free(rd_buf1); + +fail_calloc1: + free(rd_buf0); + +fail_calloc0: + fail_shyper; + fail_sencode; + H5Sclose(sid); + +fail_dspace: +fail_dwrite: + H5Dclose(did); + +fail_dopen: + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_dopen; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Open the 2-d dataset with sparse chunk layout + if ((did = H5Dopen2(fid, dset_name, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dopen; + } + + // Iterate over the structured chunks in the dataset + if (H5Dstruct_chunk_iter(did, H5P_DEFAULT, struct_chunk_cb, NULL) < 0) + ret_val = EXIT_FAILURE; + + H5Dclose(did); + +fail_dopen: + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_dopen; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_direct_dset"; + hsize_t offset[2]; + haddr_t addr; + hsize_t chunk_size; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Open the 2-d dataset with sparse chunk layout + if ((did = H5Dopen2(fid, dset_name, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dopen; + } + + // Get structured chunk info for chunk index 1 which does not exist + if (H5Dget_struct_chunk_info(did, H5S_ALL, 1, offset, NULL, &addr, &chunk_size) < 0) { + ret_val = EXIT_FAILURE; + goto fail_get_info; + } + + if (chunk_size != 0 && addr != HADDR_UNDEF) { + ret_val = EXIT_FAILURE; + goto fail_verify_info; + } + + // Get structured chunk info for chunk index 3 + if (H5Dget_struct_chunk_info(did, H5S_ALL, 3, offset, NULL, NULL, NULL) < 0) { + ret_val = EXIT_FAILURE; + goto fail_get_info; + } + + // Use the offset just retrieved to obtain structured chunk info by coordinates + if (H5Dget_struct_chunk_info_by_coord(did, offset, NULL, &addr, &chunk_size) < 0) { + ret_val = EXIT_FAILURE; + goto fail_get_info; + } + + if (chunk_size == 0 && addr == HADDR_UNDEF) { + ret_val = EXIT_FAILURE; + goto fail_verify_info; + } + +fail_get_info: +fail_verify_info: +fail_get_info: + H5Dclose(did); + +fail_dopen: + H5Fclose(fid); + +fail_file:; + } + //! + + return ret_val; +} diff --git a/doxygen/examples/H5P_struct_chunk_examples.c b/doxygen/examples/H5P_struct_chunk_examples.c new file mode 100644 index 00000000000..99612602e89 --- /dev/null +++ b/doxygen/examples/H5P_struct_chunk_examples.c @@ -0,0 +1,159 @@ +/* -*- c-file-style: "stroustrup" -*- */ + +#include "hdf5.h" + +#include +#include + +int +main(void) +{ + int ret_val = EXIT_SUCCESS; + + //! + { + __label__ fail_file, fail_space, fail_dcpl, fail_set_chunk, __label__ ail_set_layout, + fail_set_filter2, fail_dcreate; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_filter_dset"; + hsize_t dims[2] = {10, 10}; + hsize_t chunk_dims[2] = {5, 5}; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + // Create a 2D dataspace + if ((sid = H5Screate_simple(2, dim, NULL)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_space; + } + + // Create a dataset creation property list + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcpl; + } + + // Set to sparse chunked dataset + if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) { + ret_val = EXIT_FAILURE; + goto fail_set_chunk; + } + + // Adds a filter to section 0 + if (H5Pset_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_MANDATORY, 0, NULL) < + 0) { + ret_val = EXIT_FAILURE; + goto fail_set_filter2; + } + + // Create an integer 2-d dataset with sparse chunk layout + if ((did = H5Dcreate2(fid, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) == + H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_dcreate; + } + + // Verify the number of filters is correct + if ((nfilters = H5Pget_nfilters2(dcpl, H5Z_FLAG_SPARSE_SELECTION)) != 1) + ret_val = EXIT_FAILURE; + + H5Dclose(did); + +fail_dcreate: +fail_set_layout: +fail_set_chunk: +fail_set_filter2: + H5Pclose(dcpl); + +fail_dcpl: + H5Sclose(sid); + +fail_space: + H5Fclose(fid); + +fail_file:; + } + //! + + //! + { + __label__ fail_file, fail_dopen, fail_dget_plist, fail_pget_filter3; + __label__ fail_pget_mfilter2, fail_pget_filter_id3, fail_pget_remove_filter2; + + hid_t fid, sid, did, dcpl; + char file_name[] = "sparse.h5"; + char dset_name[] = "sparse_filter_dset"; + + // Open the file + if ((fid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) == H5I_INVALID_HID) { + ret_val = EXIT_FAILURE; + goto fail_file; + } + + if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dopen; + } + + if ((dcpl = H5Dget_create_plist(did)) < 0) { + ret_val = EXIT_FAILURE; + goto fail_dget_plist; + } + + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, &flags, NULL, NULL, (size_t)0, NULL, NULL); + if (filtn != H5Z_FILTER_DEFLATE && flags != H5Z_FLAG_MANDATORY) { + ret_val = EXIT_FAILURE; + goto fail_pget_filter3; + } + + if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, + NULL) < 0) { + ret_val = EXIT_FAILURE; + goto fail_pmod_filter2; + } + + if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, + NULL, NULL) < 0) { + + ret_val = EXIT_FAILURE; + goto fail_pget_filter_id3; + } + + if (flags != H5Z_FLAG_OPTIONAL) { + ret_val = EXIT_FAILURE; + goto fail_pget_filter_id3; + } + + if (H5Premove_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE) < 0) { + ret_val = EXIT_FAILURE; + goto fail_remove_filter2; + } + + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, NULL, NULL, NULL, (size_t)0, NULL, NULL); + if (filtn != H5Z_FILTER_NONE) + ret_val = EXIT_FAILURE; + +fail_pget_remove_filter2: +fail_pget_filter_id3: +fail_pget_mfilter2: +fail_pget_filter3: + H5Pclose(dcpl); + +fail_dget_plist: + H5Dclose(did); + +fail_dopen: + H5Fclose(fid); + +fail_file:; + } + //! + + return ret_val; +} diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 76727b58a59..f44e4550edf 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -574,6 +574,16 @@ encoding for object names. #H5Pset_dset_no_attrs_hint/#H5Pget_dset_no_attrs_hint Sets/gets the flag to create minimized dataset object headers. + +#H5Pset_struct_chunk +Sets the size of chunks and the structured chunk storage type + used to store a dataset with structured chunked layout. + + +#H5Pget_struct_chunk +Retrieves the size of chunks and the structured chunk storage type + for the raw data of a dataset with structured chunk layout. + //! [dcpl_table] * @@ -905,6 +915,26 @@ encoding for object names. #H5Pset_fletcher32 Sets up use of the Fletcher32 checksum filter. + +#H5Pset_filter2/#H5Pget_filter3 +Adds/gets a filter to/from the filter pipeline for a specified section of structured chunk. + + +#H5Pget_filter_by_id3 +Returns information about a filter in the pipeline for a specified section of structured chunk. + + +#H5Pget_nfilters2 +Returns the number of filters in the pipeline for a specified section of structured chunk. + + +#H5Premove_filter2 +Remove a filter in the filter pipeline for a specified section of structured chunk. + + +#H5Pmodify_filter2 +Modifies a filter in the filter pipeline for a specified section of structured chunk. + //! [ocpl_table] * diff --git a/src/H5D.c b/src/H5D.c index 7673acc0711..574ea404053 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2455,3 +2455,364 @@ H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t op, void *op_dat done: FUNC_LEAVE_API(ret_value) } /* end H5Dchunk_iter() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_defined + * + * Purpose: Retrieves a dataspace with a selection containing all defined + * elements that are also selected in the input argument + * "file_space_id". + * + * Return: Success: For H5D_SPARSE_CHUNK layout, this function returns + * a dataspace ID with a selection containing all + * defined elements that are also selected in + * "file_space_id". + * For other layouts, this function returns a copy of + * "file_space_id". + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_dataset_get_args_t vol_cb_args; /* Arguments to VOL callback */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API(H5I_INVALID_HID) + + /* Check args */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid dataset identifier"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + vol_cb_args.op_type = H5VL_DATASET_GET_DEFINED; + vol_cb_args.args.get_defined.file_space_id = file_space_id; + vol_cb_args.args.get_defined.space_id = H5I_INVALID_HID; + + /* Get the dataspace with defined elements */ + if (H5VL_dataset_get(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, H5I_INVALID_HID, "can't get dataspce with defined elements"); + + /* Set return value */ + ret_value = vol_cb_args.args.get_defined.space_id; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_defined() */ + +/*------------------------------------------------------------------------- + * Function: H5Derase + * + * Purpose: Deletes elements from a dataset causing them to no longer be defined + * This function is only useful for datasets with H5D_SPARSE_CHUNK layout. + * For other layouts this function will return an error. + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_dataset_specific_args_t vol_cb_args; /* Arguments to VOL callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid dataset identifier"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + vol_cb_args.op_type = H5VL_DATASET_ERASE; + vol_cb_args.args.erase.file_space_id = file_space_id; + + /* Deletes elements from a dataset causing them to no longer be defined */ + if (H5VL_dataset_specific(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to erase elements in dataset"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Derase() */ + +/*------------------------------------------------------------------------- + * Function: H5Dwrite_struct_chunk + * + * Purpose: Writes an entire structured chunk to the file directly. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]) +{ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID"); + if (!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL"); + if (!chunk_info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "chunk_info cannot be NULL"); + if (!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf array not provided"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.write_struct_chunk.offset = offset; + dset_opt_args.write_struct_chunk.chunk_info = chunk_info; + dset_opt_args.write_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; + + /* Write chunk */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dwrite_struct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: H5Dread_struct_chunk + * + * Purpose: Reads an entire structured chunk from the file directly. + * + * Return: Non-negative on success/Negative on failure + * + *--------------------------------------------------------------------------- + */ +herr_t +H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + void *buf[] /*out*/) +{ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID"); + if (!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL"); + if (!chunk_info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "chunk_info cannot be NULL"); + if (!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf array cannot be NULL"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.read_struct_chunk.offset = offset; + dset_opt_args.read_struct_chunk.chunk_info = chunk_info; + dset_opt_args.read_struct_chunk.buf = buf; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK; + vol_cb_args.args = &dset_opt_args; + + /* Read the raw chunk */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dread_struct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_struct_chunk_info + * + * Purpose: Retrieves structured chunk information using chunk index + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hid_t fspace_id; IN: File dataspace selection ID + * hsize_t chunk_idx; IN: Chunk index + * hsize_t *offset OUT: Logical position of the chunk's first + * element in the array + * H5D_struct_chunk_info_t *chunk_info OUT: Information about the structured chunk + * haddr_t *addr OUT: Chunk address in the file + * hsize_t *chunk_size OUT: Chunk size in bytes + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset /*out*/, + H5D_struct_chunk_info_t *chunk_info /*out*/, haddr_t *addr /*out*/, + hsize_t *chunk_size /*out*/) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == offset && NULL == chunk_info && NULL == addr && NULL == chunk_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid arguments, must have at least one non-null output argument"); + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + + /* Set up VOL callback arguments */ + dset_opt_args.get_struct_chunk_info_by_idx.fspace_id = fspace_id; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_idx = chunk_idx; + dset_opt_args.get_struct_chunk_info_by_idx.offset = offset; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_idx.addr = addr; + dset_opt_args.get_struct_chunk_info_by_idx.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX; + vol_cb_args.args = &dset_opt_args; + + /* Call private function to get the chunk info given the chunk's index */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by index"); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Dget_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_struct_chunk_info_by_coord + * + * Purpose: Retrieves information about a chunk specified by its logical + * coordinates. + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hsize_t *offset IN: Logical position of the chunk's first + * element in the array + * H5D_struct_chunk_info_t *chunk_info OUT: Information about the structured chunk + * haddr_t *addr OUT: Chunk address in the file + * hsize_t *chunk_size OUT: Chunk size in bytes + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info /*out*/, haddr_t *addr /*out*/, + hsize_t *chunk_size /*out*/) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + if (NULL == chunk_info && NULL == addr && NULL == chunk_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid arguments, must have at least one non-null output argument"); + if (NULL == offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)"); + + /* Set up VOL callback arguments */ + dset_opt_args.get_struct_chunk_info_by_coord.offset = offset; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_info = chunk_info; + dset_opt_args.get_struct_chunk_info_by_coord.addr = addr; + dset_opt_args.get_struct_chunk_info_by_coord.chunk_size = chunk_size; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD; + vol_cb_args.args = &dset_opt_args; + + /* Call private function to get the chunk info given the chunk's index */ + if (H5VL_dataset_optional(vol_obj, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by its logical coordinates"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_struct_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5Dstruct_chunk_iter + * + * Purpose: Iterates over all structured chunks in dataset calling + * the user specified callback function, "cb", and the + * callback's required data, "op_data". + * + * Parameters: + * hid_t dset_id; IN: Dataset ID + * hid_t dxpl_id; IN: Dataset transfer property list ID + * H5D_struct_chunk_iter_op_t cb IN: Callback function provided by user; called for every chunk + * void *op_data IN/OUT: User-defined data passed on to user callback. + * + * Return: Non-negative on success, negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t op, void *op_data) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier"); + if (NULL == op) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid callback to chunk iteration"); + + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID"); + + /* Set up VOL callback arguments */ + dset_opt_args.struct_chunk_iter.op = op; + dset_opt_args.struct_chunk_iter.op_data = op_data; + vol_cb_args.op_type = H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER; + vol_cb_args.args = &dset_opt_args; + + /* Iterate over the chunks */ + if ((ret_value = H5VL_dataset_optional(vol_obj, &vol_cb_args, dxpl_id, H5_REQUEST_NULL)) < 0) + HERROR(H5E_DATASET, H5E_BADITER, "error iterating over dataset chunks"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dstruct_chunk_iter() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index d232024a3f0..beb90012032 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8253,3 +8253,151 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_get_offset_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5D__write_struct_chunk_direct + * + * Purpose: Internal routine to write a structured chunk directly into the file. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + /* TBD: check for H5D_SPARSE_CHUNK */ + /* assert(layout->type == H5D_SPARSE_CHUNK); */ + + /* TBD: set up and call routine to write the structured chunk */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__write_struct_chunk_direct() */ + +/*------------------------------------------------------------------------- + * Function: H5D__read_struct_chunk_direct + * + * Purpose: Internal routine to read a structured chunk directly from the file. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + /* TBD: check for H5D_SPARSE_CHUNK */ + /* assert(dset && H5D_SPARSE_CHUNK == layout->type); */ + assert(offset); + assert(chunk_info); + assert(buf); + + /* TBD: set up and call routine to read the structured chunk */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__read_struct_chunk_direct() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_struct_chunk_info + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk. + * + * TBD: Is the following also true for structured chunk? + * Note: Currently, the domain of the index in this function is of all + * the written chunks, regardless the dataspace. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space, + hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, + hsize_t H5_ATTR_UNUSED *chunk_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + assert(dset); + assert(dset->shared); + assert(space); + + /* TBD: go get structured chunk information using chunk index */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_struct_chunk_info_by_coord + * + * Purpose: Iterate over the structured chunks in the dataset to get the info + * of the desired chunk, given by its offset coordinates. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, + haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + assert(dset); + assert(dset->shared); + assert(offset); + + /* TBD: go get structured chunk information using chunk coordinates */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_struct_chunk_info_by_coord() */ + +/*------------------------------------------------------------------------- + * Function: H5D__struct_chunk_iter + * + * Purpose: Iterate over all the structured chunks in the dataset + * with given callback and the callback's required data. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, + void H5_ATTR_UNUSED *op_data) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args */ + assert(dset); + assert(dset->shared); + + /* TBD: iterate over all the structured chunks in the dataset */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_iter() */ diff --git a/src/H5Dint.c b/src/H5Dint.c index 2b1d0c7e20b..ffb3b38cfce 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -4012,3 +4012,75 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d) FUNC_LEAVE_NOAPI(d->dcpl_id); } /* end H5D_get_dcpl_id() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_defined + * + * Purpose: Returns the dataspace ID with selection of defined elements + * + * Return: Success: ID for dataspace + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +hid_t +H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) +{ + H5S_t *space = NULL; + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_PACKAGE + + /* TBD: + if (dset->shared->layout.type == H5D_SPARSE_CHUNK) + call routine to get defined elements + else + if (NULL == (space = H5S_copy(fspace, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); + */ + + /* FOR NOW: return copy of fspace */ + if (NULL == (space = H5S_copy(fspace, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); + + /* Create an ID */ + if ((ret_value = H5I_register(H5I_DATASPACE, space, true)) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, FAIL, "unable to register dataspace"); + +done: + if (ret_value < 0) + if (space != NULL) + if (H5S_close(space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataspace"); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5D__get_defined() */ + +/*------------------------------------------------------------------------- + * + * Function: H5D__erase + * + * Purpose: Deletes elements from a dataset + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* TBD: + if (dset->shared->layout.type == H5D_SPARSE_CHUNK) + call routine to delete elements + else + return error + */ + /* FOR NOW: just return success */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__erase() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index c6c50c2eef3..0087c35b05e 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -755,6 +755,22 @@ H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_ H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); #endif /* H5D_CHUNK_DEBUG */ +/* Functions that operate on H5D_SPARSE_CHUNK storage */ +H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace); +H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace); +H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); +H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, + hsize_t chunk_idx, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); +H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); +H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data); + /* format convert */ H5_DLL herr_t H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info, H5D_chk_idx_info_t *new_idx_info); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index fbe60f03735..e393da59654 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -47,10 +47,15 @@ typedef enum H5D_layout_t { H5D_CONTIGUOUS = 1, /**< contiguous layout */ H5D_CHUNKED = 2, /**< chunked or tiled layout */ H5D_VIRTUAL = 3, /**< actual data is stored in other datasets */ - H5D_NLAYOUTS = 4 /**< this one must be last! */ + H5D_STRUCT_CHUNK = 4, /**< structured chunk layout */ + H5D_NLAYOUTS = 5 /**< this one must be last! */ } H5D_layout_t; //! +/* Types of H5D_STRUCT_CHUNK storage */ +#define H5D_SPARSE_CHUNK 0 /* To store sparse data of any datatype */ +#define H5D_VL_CHUNK 1 /* To store dense data of variable-length datatype */ + //! /** * Types of chunk index data structures @@ -128,6 +133,22 @@ typedef enum H5D_vds_view_t { } H5D_vds_view_t; //! +//! +/** + * Used by APIs for direct chunk chunk I/O on filtered and non-filtered structured chunks + */ +typedef struct H5D_struct_chunk_info_t { + H5D_layout_t type; /* Type of the structured chunk */ + uint8_t num_sections; /* Number of sections in the structured chunk */ + uint16_t *filter_mask; /* Array of "num_sections" size */ + /* Contains filter mask for each section */ + size_t *section_size; /* Array of "num_sections" size */ + /* Contains the size of each section */ + size_t *section_orig_size; /* Array of "num_sections" size */ + /* Contains the original size of each section */ +} H5D_struct_chunk_info_t; +//! + //! /** * \brief Callback for H5Pset_append_flush() @@ -250,6 +271,29 @@ typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, void *op_data); //! +//! +/** + * \brief Callback for H5Dstruct_chunk_iter() + * + * \param[in] offset Logical position of the chunk's first element in the array + * \param[in] chunk_info Information about the structured chunk + * \param[in] addr Chunk address in the file, taking the user block (if any) into account + * \param[in] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * \param[in,out] op_data User-defined pointer to data required by the callback function + * \return \li Zero (#H5_ITER_CONT) causes the iterator to continue, returning + * zero when all elements have been processed. + * \li A positive value (#H5_ITER_STOP) causes the iterator to + * immediately return that value, indicating short-circuit success. + * \li A negative (#H5_ITER_ERROR) causes the iterator to immediately + * return that value, indicating failure. + * + * \since 1.x.x + * + */ +typedef int (*H5D_struct_chunk_iter_op_t)(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, + haddr_t *addr, hsize_t *chunk_size, void *op_data); +//! + /********************/ /* Public Variables */ /********************/ @@ -1637,6 +1681,329 @@ H5_DLL herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, */ H5_DLL herr_t H5Dclose(hid_t dset_id); +/** + * -------------------------------------------------------------------------- + *\ingroup H5D + * + * \brief Retrieves a dataspace selection containing defined elements + * + * \dset_id to get the selection of defined elements from + * \param[in] file_space_id Dataspace identifier containing the selection of elements to + * be queried if they are defined + * \dxpl_id + * + * \return Returns a dataspace with a selection containing all defined elements + * that are also selected in \p file_space_id if successful; + * otherwise returns #H5I_INVALID_HID. + * + * \details H5Dget_defined() retrieves a dataspace with only the defined + * elements of a (subset of) dataset selected. The dataset is specified + * by its identifier \p dset_id, and data transfer properties are defined + * by \p dxpl_id. The subset of the dataset to search for defined values is + * given by the selection in \p file_space_id. Setting \p file_space_id to + * #H5S_ALL causes this function to return a selection containing all defined + * values in the dataset. + * + * This function is only useful for datasets with layout TBD: H5D_SPARSE_CHUNK. + * For other layouts this function will simply return a copy of \p file_space_id, + * as all elements are defined for non-sparse datasets. + * + * \par Example: + * \snippet H5D_struct_chunk_examples.c get_defined + * + * \since 1.x.x + * + * \see H5Derase() + * + */ +H5_DLL hid_t H5Dget_defined(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); + +/** + * -------------------------------------------------------------------------- + *\ingroup H5D + * + * \brief Deletes elements from a dataset causing them to no longer be defined + * + * \dset_id to erase elements from + * \param[in] file_space_id Dataspace identifier containing the selection of elements to + * be erased + * \dxpl_id + * + * \return \herr_t + * + * \details H5Derase() deletes elements from a dataset, specified by its identifier \p dset_id, + * causing them to no longer be defined. The part of the dataset to erase is defined + * by \p file_space_id and the data transfer properties are defined by \p dxpl_id. + * After this operation, reading from these elements will return fill values, + * and the elements will no longer be included in the selection returned by + * H5Dget_defined(). + * + * This function is only applicable to the datasets with structured chunk layout + * of the H5D_SPARSE_CHUNK type. For other layouts this function will return + * an error. + * + * \par Example: + * \snippet H5D_struct_chunk_examples.c erase + * + * \since 1.x.x + * + * \see H5Dget_defined() + * + */ +H5_DLL herr_t H5Derase(hid_t dset_id, hid_t file_space_id, hid_t dxpl_id); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Writes a structured chunk from a buffer directly to a dataset in a file + * + * \dset_id + * \dxpl_id + * \param[in] offset Logical position of the chunk's first element in the + * dataspace + * \param[in] chunk_info Information about the structured chunk + * \param[in] buf Array of pointers to the sections of the structured + * chunk; the size of the array is equal to the number + * of sections in the structured chunk + * + * \return \herr_t + * + * \details H5Dwrite_struct_chunk() writes a structured chunk specified by its + * logical offset \p offset to dataset \p dset_id. The HDF5 library assembles + * the structured chunk according to the information provided in the + * \p chunk_info parameter and using data pointed by \p buf. + * \p buf is an array of pointers to the buffers containing data for each + * section of the structured chunk. + * + * Initially, this function will support only sparse chunks of the + * fixed-size data. Such chunks have only two sections: one for the + * encoded selection and the second one for data elements. + * + * \attention Exercise caution when using H5Dread_struct_chunk() and + * H5Dwrite_struct_chunk(), as they read and write data chunks directly + * in a file. H5Dwrite_struct_chunk() bypasses hyperslab selection, the + * conversion of data from one datatype to another, and the filter + * pipeline to write the chunk. Developers should have experience + * with these processes before using this function. + * + * \note H5Dread_struct_chunk() and H5Dwrite_struct_chunk() are currently not supported + * with parallel HDF5 and do not support variable-length types. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \par Example: + * \snippet H5D_struct_chunk_examples.c direct_chunk_write + * + * \since 1.x.x + * + * \see H5Dread_struct_chunk() + * + */ +H5_DLL herr_t H5Dwrite_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Reads a structured chunk directly from a dataset in a file into + * the provided buffer. + * + * \dset_id + * \dxpl_id + * \param[in] offset Logical position of the chunk's first element in the + * dataspace + * \param[in,out] chunk_info Information about the structured chunk + * \param[out] buf Array of pointers to the sections of structured chunk; + * the size of the array is equal to the number of + * sections in the structured chunk + * + * \return \herr_t + * + * \details H5Dread_struct_chunk() reads a structured chunk as specified by its logical + * offset \p offset in a chunked dataset \p dset_id and places data into the provided + * buffers pointed by \p buf. Information about the structured chunk is returned via + * the \p chunk_info parameter. \p buf is an array of pointers to the buffers into which + * data for each section of the structured chunk will be read into. It is + * application’s responsibility to allocate buffers of the appropriate size. + * + * Initially, this function will support only sparse chunks of the fixed-size data. + * Such chunk has only two sections: one for the encoded selection and the second + * one for data elements. + * + * \attention Exercise caution when using H5Dread_struct_chunk() and + * H5Dwrite_struct_chunk(), as they read and write data chunks directly + * in a file. H5Dwrite_struct_chunk() bypasses hyperslab selection, the + * conversion of data from one datatype to another, and the filter + * pipeline to write the chunk. Developers should have experience + * with these processes before using this function. Please see + * Using the Direct Chunk Write Function for more information. + * + * \note H5Dread_struct_chunk() and H5Dwrite_struct_chunk() are currently not supported + * with parallel HDF5 and do not support variable-length datatypes. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \par Example: + * \snippet H5D_struct_chunk_examples.c direct_chunk_read + * + * \since 1.x.x + * + * \see H5Dwrite_struct_chunk() + * + */ +H5_DLL herr_t H5Dread_struct_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, void *buf[]); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Retrieves structured chunk information using chunk index + * + * \dset_id + * \param[in] fspace_id File dataspace selection identifier + * \param[in] chunk_idx Chunk index + * \param[out] offset Logical position of the chunk's first element in the array + * \param[out] chunk_info Information about the structured chunk + * \param[out] addr Chunk address in the file, taking the user block (if any) into account + * \param[out] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * + * \return \herr_t + * + * \details H5Dget_struct_chunk_info() retrieves the offset coordinates, + * \p offset, structured chunk information, \p chunk_info, + * chunk's address, \p addr, and the size, \p chunk_size, + * for the dataset specified by the identifier \p dset_id and + * the chunk specified by the index, \p chunk_idx. + * The chunk belongs to a set of chunks in the selection + * specified by \p fspace_id. If the queried chunk + * does not exist in the file, the size will be set to 0 and + * address to #HADDR_UNDEF. \c NULL can be passed in for + * any \p out parameters. + * + * \note Prior to HDF5 1.14.4, the reported address did not take the + * user block into account. + * + * \p chunk_idx is the chunk index in the selection. + * The index value may have a value of 0 up + * to the number of chunks stored in the file that + * have a nonempty intersection with the file dataspace. + * + * \note As of 1.10.5, the dataspace intersection is not yet + * supported. Hence, the index is of all the written chunks. + * + * \p fspace_id specifies the file dataspace selection. It is + * intended to take #H5S_ALL to specify the current selection. + * + * \note Please be aware that this function currently does not + * support non-trivial selections; thus \p fspace_id has no + * effect. Also, the implementation does not handle the #H5S_ALL + * macro correctly. As a workaround, an application can get + * the dataspace for the dataset using H5Dget_space() and pass that + * in for \p fspace_id. This will be fixed in a future release. + * + * \par + * \note TBD: need to verify the above notes info are true for + * structured chunk. + * + * \par Example + * \snippet H5D_struct_chunk_examples.c direct_chunk_get_info + * + * \since 1.x.x + * + * \see H5Dget_struct_chunk_info_by_coord(), H5Dstruct_chunk_iter(), H5Dwrite_struct_chunk(), + * H5Dread_struct_chunk() + * + */ +H5_DLL herr_t H5Dget_struct_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chunk_idx, hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Retrieves structured chunk information using its coordinates + * + * \dset_id + * \param[in] offset Logical position of the chunk's first element in the array + * \param[out] chunk_info Information about the structured chunk + * \param[out] addr Chunk address in the file, taking the user block + * (if any) into account + * \param[out] chunk_size Chunk size in bytes; 0 if the chunk does not exist + * + * \return \herr_t + * + * \details H5Dget_struct_chunk_info_by_coord() retrieves the structured + * information \p chunk_info, chunk's address, \p addr and the + * size, \p chunk_size, for the dataset specified by + * the identifier \p dset_id and the chunk specified by its + * coordinates \p offset. + * + * \p offset is a pointer to a one-dimensional array with a size + * equal to the dataset's rank. Each element is the logical + * position of the chunk's first element in a dimension. + * + * If the queried chunk does not exist in the file, the size will + * be set to 0 and address to \c HADDR_UNDEF. The value pointed + * to by \p chunk_info will not be modified. \c NULL can be passed + * in for any \p out parameters. + * + * \note Prior to HDF5 1.14.4, the reported address did not take the + * user block into account. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \par Example + * \snippet H5D_struct_chunk_examples.c direct_chunk_get_info + * + * \since 1.x.x + * + * \see H5Dget_struct_chunk_info(), H5Dstruct_chunk_iter(), H5Dwrite_struct_chunk(), H5Dread_struct_chunk() + * + */ +H5_DLL herr_t H5Dget_struct_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, + H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, + hsize_t *chunk_size); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5D + * + * \brief Iterate over all chunks of a chunked dataset + * + * \dset_id + * \param[in] dxpl_id Identifier of a transfer property list + * \param[in] cb User callback function, called for every chunk. + * \param[in] op_data User-defined pointer to data required by op + * + * \return \herr_t + * + * \details H5Dstruct_chunk_iter iterates over all structured chunks in the dataset, + * calling the user supplied callback \p cb, and the callback's required + * data \p op_data. + * + * \note Prior to HDF5 1.14.4, the address passed to the callback did not take + * the user block into account. + * \par + * \note TBD: need to verify the above notes info are true for structured chunk. + * + * \par Example: + * Print the info for each structured chunk + * \snippet H5D_struct_chunk_examples.c H5Dstruct_chunk_iter_cb + * Iterate over all the structured chunks in the dataset. + * \snippet H5D_struct_chunk_examples.c direct_chunk_iter + * + * \since 1.x.x + * + * \see H5Dwrite_struct_chunk(), H5Dread_struct_chunk() + * + */ +H5_DLL herr_t H5Dstruct_chunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_struct_chunk_iter_op_t cb, + void *op_data); + /** * -------------------------------------------------------------------------- * \ingroup ASYNC diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index a5eebcf3985..74251811174 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -427,6 +427,9 @@ typedef struct H5O_efl_t { */ #define H5O_LAYOUT_VERSION_4 4 +/* This version adds support for structured chunk */ +#define H5O_LAYOUT_VERSION_5 5 + /* The default version of the format. (Earlier versions had bugs) */ #define H5O_LAYOUT_VERSION_DEFAULT H5O_LAYOUT_VERSION_3 @@ -650,12 +653,36 @@ typedef struct H5O_layout_chunk_t { } u; } H5O_layout_chunk_t; +typedef struct H5O_layout_struct_chunk_t { + /* TBD: may add more fields or modify fields for structured chunk */ + unsigned struct_type; /* Structured chunk storage type */ + /* NOW: H5D_SPARSE_CHUNK, H5D_VL_CHUK */ + H5D_chunk_index_t idx_type; /* Type of chunk index */ + uint8_t flags; /* Chunk layout flags */ + unsigned ndims; /* Num dimensions in chunk */ + uint32_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ + unsigned enc_bytes_per_dim; /* Encoded # of bytes for storing each chunk dimension */ + uint32_t size; /* Size of chunk in bytes */ + hsize_t nchunks; /* Number of chunks in dataset */ + hsize_t max_nchunks; /* Max. number of chunks in dataset */ + hsize_t chunks[H5O_LAYOUT_NDIMS]; /* # of chunks in each dataset dimension */ + hsize_t max_chunks[H5O_LAYOUT_NDIMS]; /* # of chunks in each dataset's max. dimension */ + hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of chunks in each dimension */ + hsize_t max_down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of chunks in each max dim */ + union { + H5O_layout_chunk_farray_t farray; /* Information for fixed array index */ + H5O_layout_chunk_earray_t earray; /* Information for extensible array index */ + H5O_layout_chunk_bt2_t btree2; /* Information for v2 B-tree index */ + } u; +} H5O_layout_struct_chunk_t; + typedef struct H5O_layout_t { H5D_layout_t type; /* Type of layout */ unsigned version; /* Version of message */ const struct H5D_layout_ops_t *ops; /* Pointer to data layout I/O operations */ union { - H5O_layout_chunk_t chunk; /* Information for chunked layout */ + H5O_layout_chunk_t chunk; /* Information for chunked layout */ + H5O_layout_struct_chunk_t struct_chunk; /* Information for structured chunk layout */ } u; H5O_storage_t storage; /* Information for storing dataset elements */ } H5O_layout_t; diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 6160d36e38a..e0c865242c0 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -85,6 +85,29 @@ } \ } \ } + +#define H5D_DEF_LAYOUT_STRUCT_CHUNK_INIT \ + { \ + H5D_SPARSE_CHUNK, H5D_CHUNK_IDX_BTREE, (uint8_t)0, (unsigned)0, \ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + (unsigned)0, (uint32_t)0, (hsize_t)0, (hsize_t)0, \ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + { \ + { \ + { \ + (uint8_t)0 \ + } \ + } \ + } \ + } #define H5D_DEF_STORAGE_VIRTUAL_INIT \ { \ {HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ @@ -134,6 +157,20 @@ H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_CHUNK \ } +/* TBD: FOR NOW set to CHUNKED */ +#ifdef TBD +#define H5D_DEF_LAYOUT_STRUCT_CHUNK \ + { \ + H5D_STRUCT_CHUNK, H5O_LAYOUT_VERSION_5, H5D_LOPS_CHUNK, \ + {.struct_chunk = H5D_DEF_LAYOUT_STRUCT_CHUNK_INIT}, H5D_DEF_STORAGE_CHUNK \ + } +#endif +/* TBD: same as H5D_CHUNKED */ +#define H5D_DEF_LAYOUT_STRUCT_CHUNK \ + { \ + H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_DEF_STORAGE_CHUNK \ + } #define H5D_DEF_LAYOUT_VIRTUAL \ { \ H5D_VIRTUAL, H5O_LAYOUT_VERSION_4, H5D_LOPS_VIRTUAL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ @@ -279,10 +316,11 @@ static const H5O_efl_t H5D_def_efl_g = H5D_CRT_EXT_FILE_LIST_DEF; /* Default static const unsigned H5O_ohdr_min_g = H5D_CRT_MIN_DSET_HDR_SIZE_DEF; /* Default object header minimization */ /* Defaults for each type of layout */ -static const H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; -static const H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; -static const H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; -static const H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; +static const H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; +static const H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; +static const H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; +static const H5O_layout_t H5D_def_layout_struct_chunk_g = H5D_DEF_LAYOUT_STRUCT_CHUNK; +static const H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; /*------------------------------------------------------------------------- * Function: H5P__dcrt_reg_prop @@ -1778,6 +1816,7 @@ H5P__set_layout(H5P_genplist_t *plist, const H5O_layout_t *layout) case H5D_CHUNKED: case H5D_VIRTUAL: + case H5D_STRUCT_CHUNK: fill.alloc_time = H5D_ALLOC_TIME_INCR; break; @@ -1840,6 +1879,10 @@ H5Pset_layout(hid_t plist_id, H5D_layout_t layout_type) layout = &H5D_def_layout_chunk_g; break; + case H5D_STRUCT_CHUNK: + layout = &H5D_def_layout_struct_chunk_g; + break; + case H5D_VIRTUAL: layout = &H5D_def_layout_virtual_g; break; @@ -3547,3 +3590,117 @@ H5Pset_dset_no_attrs_hint(hid_t dcpl_id, hbool_t minimize) done: FUNC_LEAVE_API(ret_value) } /* H5Pset_dset_no_attrs_hint() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_struct_chunk + * + * Purpose: Sets the number of dimensions and the size of each chunk to + * the values specified. The dimensionality of the chunk should + * match the dimensionality of the dataspace. + * It also sets the structured chunk storage type to FLAG. + * + * As a side effect, the layout method is changed to + * H5D_STRUCT_CHUNK. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], unsigned H5_ATTR_UNUSED flag) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t chunk_layout; /* Layout information for setting chunk info */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check arguments */ + if (ndims <= 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality must be positive"); + if (ndims > H5S_MAX_RANK) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality is too large"); + if (!dim) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no chunk dimensions specified"); + + /* Verify & initialize property's chunk dims */ + /* TBD: for now, just set to H5D_CHUNKED */ + H5MM_memcpy(&chunk_layout, &H5D_def_layout_chunk_g, sizeof(H5D_def_layout_chunk_g)); + + memset(&chunk_layout.u.chunk.dim, 0, sizeof(chunk_layout.u.chunk.dim)); + for (u = 0; u < (unsigned)ndims; u++) { + if (dim[u] == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be positive"); + /* Didn't check for chunk dimensions < 2^32 */ + /* Didn't check for number of elements in chunk < 4 GB */ + chunk_layout.u.chunk.dim[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */ + } /* end for */ + + /* TBD: should set fields u.struct_chunk e.g. struct_type to parameter flag */ + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Set chunk information in property list */ + chunk_layout.u.chunk.ndims = (unsigned)ndims; + if (H5P__set_layout(plist, &chunk_layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_struct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_struct_chunk + * + * Purpose: Retrieves the chunk size of chunked layout. The chunk + * dimensionality is returned and the chunk size in each + * dimension is returned through the DIM argument. At most + * MAX_NDIMS elements of DIM will be initialized. + * It also retrieves the structured chunk storage type in the + * FLAG argument. + * + * Return: Success: Positive Chunk dimensionality. + * + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5Pget_struct_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/, unsigned *flag) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + int ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Peek at the layout property */ + if (H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout"); + if (H5D_STRUCT_CHUNK != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a chunked storage layout"); + + if (dim) { + unsigned u; /* Local index variable */ + + /* Get the dimension sizes */ + for (u = 0; u < layout.u.struct_chunk.ndims && u < (unsigned)max_ndims; u++) + dim[u] = layout.u.struct_chunk.dim[u]; + } /* end if */ + + if (flag) + *flag = layout.u.struct_chunk.struct_type; + + /* Set the return value */ + ret_value = (int)layout.u.struct_chunk.ndims; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_struct_chunk() */ diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index f9d89ab4156..7f179dbe8de 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -93,6 +93,8 @@ static herr_t H5P__ocrt_pipeline_close(const char *name, size_t size, void *valu /* Local routines */ static herr_t H5P__set_filter(H5P_genplist_t *plist, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/]); +static herr_t H5P__set_filter2(H5P_genplist_t *plist, uint64_t section_number, H5Z_filter_t filter, + uint64_t flags, size_t buf_size, const void *buf); /*********************/ /* Package Variables */ @@ -562,6 +564,231 @@ H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t FUNC_LEAVE_API(ret_value) } /* end H5Pmodify_filter() */ +/*------------------------------------------------------------------------- + * Function: H5Pmodify_filter2 + * + * Purpose: Modifies the specified FILTER in the filter pipeline for a + * specified SECTION_NUMBER of the structured chunk. + * + * The PLIST_ID argument is either a dataset or group creation + * creation property list. + * + * The SECTION_NUMBER argument is the specified section of the + * structured chunk + * + * The FILTER arguments specifies the filter identifier + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF_SIZE is the size in bytes of BUF which points to + * a buffer with auxiliary data for the filter. The values + * will be stored in the dataset object header as part of the + * filter information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. + * If the filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * If this bit is clear and the filter fails then the entire + * I/O operation fails. + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (filter < 0 || filter > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identifier"); + if (flags & ~((unsigned)H5Z_FLAG_DEFMASK)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + if (buf_size > 0 && !buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied"); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + Call new H5P_modify_filter() with buf_size and buf instead of cd_nelmts and cd_values + + /* Modify the filter parameters of the I/O pipeline */ + if (H5P_modify_filter(plist, filter, flags, cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't modify filter"); +#endif + + /* For now, just return SUCCEED */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pmodify_filter2() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_filter2 + * + * Purpose: Adds the specified FILTER and corresponding properties to the + * end of the data or link output filter pipeline + * + * The PLIST_ID argument is a dataset creation or group + * creation property list. + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * The FILTER argument is the filter identifier for the filter + * to be added to the pipeline. + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF argument points to a buffer of BUF_SIZE bytes which + * contains auxiliary data for the filter. The values will be + * stored in the dataset object header as part of the filter + * information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. If the + * filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * + * If this bit is clear and the filter fails then the entire I/O + * operation fails. + * + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (filter > 255) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid section number"); + if (filter < 0 || filter > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identifier"); + if (flags & ~((unsigned)H5Z_FLAG_DEFMASK)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags"); + if (buf_size > 0 && !buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied"); + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Call the private function */ + if (H5P__set_filter2(plist, section_number, filter, flags, buf_size, buf) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "failed to call private function"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_filter2() */ + +/*------------------------------------------------------------------------- + * Function: H5P__set_filter2 + * + * Purpose: Adds the specified FILTER and corresponding properties to the + * end of the data or link output filter pipeline. + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * The FILTER argument is the filter identifier for the filter + * to be added to the pipeline. + * + * The FLAGS argument specifies certain + * general properties of the filter and is documented below. + * + * The BUF argument points to a buffer of BUF_SIZE bytes which + * contains auxiliary data for the filter. The values will be + * stored in the dataset object header as part of the filter + * information. + * + * The FLAGS argument is a bit vector of the following fields: + * + * H5Z_FLAG_OPTIONAL(0x0001) + * If this bit is set then the filter is optional. If the + * filter fails during an H5Dwrite() operation then the filter + * is just excluded from the pipeline for the chunk for which it + * failed; the filter will not participate in the pipeline + * during an H5Dread() of the chunk. + * + * If this bit is clear and the filter fails then the entire I/O + * operation fails. + * + * If this bit is set but encoding is disabled for a filter, + * attempting to write will generate an error. + * + * If the filter is not registered, this function tries to load + * it dynamically during run time. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__set_filter2(H5P_genplist_t H5_ATTR_UNUSED *plist, uint64_t H5_ATTR_UNUSED section_number, + H5Z_filter_t H5_ATTR_UNUSED filter, uint64_t H5_ATTR_UNUSED flags, + size_t H5_ATTR_UNUSED buf_size, const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + +#ifdef TBD + Need to set up new H5O_pline_t message Need to do something similar like H5P__set_filter1() as below : + + /* Check if filter is already available */ + if ((filter_avail = H5Z_filter_avail(filter)) < 0) + + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't check filter availability"); + + /* Get the pipeline property to append to */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Add the filter to the I/O pipeline */ + if (H5Z_append(&pline, filter, flags, cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to add filter to pipeline"); + + /* Put the I/O pipeline information back into the property list */ + if (H5P_poke(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set pipeline"); +#endif + + /* For now: just return SUCCEED */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__set_filter2() */ + /*------------------------------------------------------------------------- * Function: H5Pset_filter * @@ -723,6 +950,57 @@ H5Pget_nfilters(hid_t plist_id) FUNC_LEAVE_API(ret_value) } /* end H5Pget_nfilters */ +/*------------------------------------------------------------------------- + * Function: H5Pget_nfilters2 + * + * Purpose: Returns the number of filters in the data or link + * pipeline depending on whether PLIST_ID is a dataset creation + * or group creation property list. + * In each pipeline the filters are numbered from zero through + * N-1 where N is the value returned by this function. + * During output to the file the filters of a pipeline are applied + * in increasing order (the inverse is true for input). + * + * The SECTION_NUMBER argument specified the section of the + * structured chunk to which the filter is applied. + * + * Return: Success: Number of filters or zero if there are none. + * + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5Pget_nfilters2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number) +{ + H5P_genplist_t *plist; /* Property list */ + int ret_value; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + Need to get the new pline with pline.nused for the specified section + + /* Get the pipeline property to query */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Set return value */ + ret_value = (int)(pline.nused); +#endif + + /* For now just return 0 */ + ret_value = 0; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_nfilters2 */ + /*------------------------------------------------------------------------- * Function: H5Pget_filter2 * @@ -804,6 +1082,105 @@ H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t FUNC_LEAVE_API(ret_value) } /* end H5Pget_filter2() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_filter3 + * + * Purpose: This is the query counterpart of H5Pset_filter2(). + * It retrieves information for a filter in the pipeline for a + * specified section of the structured chunk. + * The information returned is about a particular filter number in + * the filter pipeline depending on whether PLIST_ID is a + * dataset creation or group creation property list. + * + * On input: + * SECTION_NUMBER is the specified section of the structured chunk + * + * IDX should be a value between zero and N-1 as described + * for H5Pget_nfilters(). + * + * BUF_SIZE indicates the size in bytes of the buffer pointed to + * by BUF which is allocated by the caller. On exit, it contains the + * values defined by the filter. + * + * FILTER_CONFIG is a bit field containing encode/decode flags from H5Zpublic.h. + * + * The function will return failure if the filter number is out of range. + * + * Return: Success: Filter identification number. + * Failure: H5Z_FILTER_ERROR (Negative) + * + *------------------------------------------------------------------------- + */ +H5Z_filter_t +H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, + unsigned H5_ATTR_UNUSED idx, uint64_t H5_ATTR_UNUSED *flags /*out*/, + size_t H5_ATTR_UNUSED *buf_size /*in,out*/, void H5_ATTR_UNUSED *buf /*out*/, + size_t H5_ATTR_UNUSED namelen, char H5_ATTR_UNUSED name[] /*out*/, + unsigned H5_ATTR_UNUSED *filter_config /*out*/) +{ + H5Z_filter_t ret_value; /* return value */ + + FUNC_ENTER_API(H5Z_FILTER_ERROR) + +#ifdef TBD + + --Check arguments buf_size and buf instead of cd_nelmts and cd_values + --Obtain new pline info for structured chunk + --Call new H5P__get_filter() routine with buf_size and buf in place of cd_nelemts and cd_values + + /* Check args */ + if (cd_nelmts || cd_values) + { + /* + * It's likely that users forget to initialize this on input, so + * we'll check that it has a reasonable value. The actual number + * is unimportant because the H5O layer will detect when a message + * is too large. + */ + if (cd_nelmts && *cd_nelmts > 256) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, + "probable uninitialized *cd_nelmts argument"); + if (cd_nelmts && *cd_nelmts > 0 && !cd_values) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, "client data values not supplied"); + + /* + * If cd_nelmts is null but cd_values is non-null then just ignore + * cd_values + */ + if (!cd_nelmts) + cd_values = NULL; + } /* end if */ + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_FILTER_ERROR, "can't find object for ID"); + + /* Get the pipeline property to query */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5Z_FILTER_ERROR, "can't get pipeline"); + + /* Check index */ + if (idx >= pline.nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, H5Z_FILTER_ERROR, "filter number is invalid"); + + /* Set pointer to particular filter to query */ + filter = &pline.filter[idx]; + + /* Get filter information */ + if (H5P__get_filter(filter, flags, cd_nelmts, cd_values, namelen, name, filter_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5Z_FILTER_ERROR, "can't get filter info"); + + /* Set return value */ + ret_value = filter->id; +#endif + + /* For new just return 0 */ + ret_value = H5Z_FILTER_NONE; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_filter3() */ + /*------------------------------------------------------------------------- * Function: H5P_get_filter_by_id * @@ -914,6 +1291,88 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* FUNC_LEAVE_API(ret_value) } /* end H5Pget_filter_by_id2() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_filter_by_id3 + * + * Purpose: This is an additional query counterpart of H5Pset_filter2(). + * It retrieves information for a filter specified by its identifier + * in the pipeline for a specified section of the structured chunk. + * The information returned is about about a particular filter + * identifier in the filter pipeline depending on whether + * PLIST_ID is a dataset creation or group creation property list. + * + * On input: + * SECTION_NUMBER is the specified section of the structured chunk + * + * ID is the filter ID to retrieve the parameters for + * + * BUF_SIZE indicates the size in bytes of the buffer pointed to + * by BUF which is allocated by the caller. On exit, it contains + * size of the values defined by the filter. + * + * FILTER_CONFIG is a bit field containing encode/decode flags + * from H5Zpublic.h. + * + * If the filter is not set for the property list, an error will be returned. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filter_t id, + uint64_t H5_ATTR_UNUSED *flags /*out*/, size_t H5_ATTR_UNUSED *buf_size /*in,out*/, + void H5_ATTR_UNUSED *buf /*out*/, size_t H5_ATTR_UNUSED namelen, + char H5_ATTR_UNUSED name[] /*out*/, unsigned H5_ATTR_UNUSED *filter_config /*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range"); + +#ifdef TBD + --Check arguments buf_size and buf instead of cd_nelmts and cd_values-- Call new H5P_get_filter_by_id() + routine with buf_size and buf in place of cd_nelemts and cd_values + + if (cd_nelmts || cd_values) + { + /* + * It's likely that users forget to initialize this on input, so + * we'll check that it has a reasonable value. The actual number + * is unimportant because the H5O layer will detect when a message + * is too large. + */ + if (cd_nelmts && *cd_nelmts > 256) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "probable uninitialized *cd_nelmts argument"); + if (cd_nelmts && *cd_nelmts > 0 && !cd_values) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "client data values not supplied"); + + /* + * If cd_nelmts is null but cd_values is non-null then just ignore + * cd_values + */ + if (!cd_nelmts) + cd_values = NULL; + } /* end if */ + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + + /* Get filter information */ + if (H5P_get_filter_by_id(plist, id, flags, cd_nelmts, cd_values, namelen, name, filter_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get filter info"); +#endif + + /* For now just return SUCCEED */ +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_filter_by_id3() */ + /*------------------------------------------------------------------------- * Function: H5Pall_filters_avail * @@ -1025,6 +1484,56 @@ H5Premove_filter(hid_t plist_id, H5Z_filter_t filter) FUNC_LEAVE_API(ret_value) } /* end H5Premove_filter() */ +/*------------------------------------------------------------------------- + * Function: H5Premove_filter2 + * + * Purpose: Deletes a filter for a specified section of the structured chunk + * from the dataset creation or group creation property list; + * deletes all filters if FILTER is H5Z_FILTER_ALL + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Premove_filter2(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_number, + H5Z_filter_t H5_ATTR_UNUSED filter) +{ + H5P_genplist_t *plist; /* Property list */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE))) + HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); + +#ifdef TBD + + --Obtain new pline info for structured chunk then delete the filter + + /* Get the pipeline property to modify */ + if (H5P_peek(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + + /* Check if there are any filters */ + if (pline.filter) { + /* Delete filter */ + if (H5Z_delete(&pline, filter) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't delete filter"); + + /* Put the I/O pipeline information back into the property list */ + if (H5P_poke(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set pipeline"); + } /* end if */ +#endif + + /* For now, just return SUCCEED */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Premove_filter2() */ + /*------------------------------------------------------------------------- * Function: H5Pset_deflate * diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 273250017a3..28a62535435 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -2069,6 +2069,7 @@ H5_DLL herr_t H5Pget_attr_creation_order(hid_t plist_id, unsigned *crt_order_fla * */ H5_DLL herr_t H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact, unsigned *min_dense); + /** * \ingroup OCPL * @@ -2133,6 +2134,81 @@ H5_DLL herr_t H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact, un H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, char name[], unsigned *filter_config /*out*/); +/** + * \ingroup OCPL + * + * \brief Returns information for a filter in the pipeline for a specified section + * of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] idx Sequence number within the filter pipeline of + * the filter for which information is sought + * \param[out] flags Bit vector specifying certain general properties + * of the filter + * \param[in,out] buf_size Size in bytes of \p buf buffer + * \param[out] buf Buffer with auxiliary data for the filter + * \param[in] namelen Anticipated number of characters in \p name + * \param[out] name Name of the filter + * \param[out] filter_config Bit field, as described in H5Zget_filter_info() + * + * \return Returns a negative value on failure, and the filter identifier + * if successful (see #H5Z_filter_t): + * - #H5Z_FILTER_DEFLATE Data compression filter, + * employing the gzip algorithm + * - #H5Z_FILTER_SHUFFLE Data shuffling filter + * - #H5Z_FILTER_FLETCHER32 Error detection filter, employing the + * Fletcher32 checksum algorithm + * - #H5Z_FILTER_SZIP Data compression filter, employing the + * SZIP algorithm + * - #H5Z_FILTER_NBIT Data compression filter, employing the + * N-bit algorithm + * - #H5Z_FILTER_SCALEOFFSET Data compression filter, employing the + * scale-offset algorithm + * + * \details H5Pget_filter3() returns information about a filter for a specified + * section in the structured chunk. The filter is specified + * by its filter number, in a filter pipeline specified by the property + * list with which it is associated. + * + * \p plist_id must be a dataset or group creation property list. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * \p idx is a value between zero and N-1, as described in + * H5Pget_nfilters(). The function will return a negative value if + * the filter number is out of range. + * + * The structure of the \p flags argument is discussed in + * H5Pset_filter2(). + * + * On input, \p buf_size indicates the size of the buffer pointerd + * to by \p buf, as allocated by the caller; on return, + * \p buf_size contains the size of the values defined by the filter. + * + * If \p name is a pointer to an array of at least \p namelen bytes, + * the filter name will be copied into that array. The name will be + * null terminated if \p namelen is large enough. The filter name + * returned will be the name appearing in the file, the name + * registered for the filter, or an empty string. + * + * \p filter_config is the bit field described in + * H5Zget_filter_info(). + * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 + * + * \see H5Pset_filter2(), H5Pget_filter_by_id3() + * + * \since 1.x.x + * + */ +H5_DLL H5Z_filter_t H5Pget_filter3(hid_t plist_id, uint64_t section_number, unsigned idx, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, + unsigned *filter_config /*out*/); + /** * \ingroup OCPL * @@ -2191,6 +2267,77 @@ H5_DLL H5Z_filter_t H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *f H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, char name[] /*out*/, unsigned *filter_config /*out*/); +/** + * \ingroup OCPL + * + * \brief Returns information for a filter specified by its identifier in the + * pipeline for a specified section of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter_id Filter identifier + * \param[out] flags Bit vector specifying certain general + * properties of the filter + * \param[in,out] buf_size Size in bytes of \p buf buffer + * \param[out] buf Auxiliary data for the filter + * \param[in] namelen Length of filter name and number of + * elements in \p name + * \param[out] name[] Name of filter + * \param[out] filter_config Bit field, as described in + * H5Zget_filter_info() + * + * \return \herr_t + * + * \details H5Pget_filter_by_id3() returns information about a filter + * for a specified section in the structured chunk with the + * filter identifier specified in \p filter_id. + * + * \p plist_id must be a dataset or group creation property list + * and \p filter_id must be in the associated filter pipeline. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p filter_id and \p flags parameters are used in the same + * manner as described in the discussion of H5Pset_filter2(). + * + * Aside from the fact that they are used for output, the + * parameters \p buf_size and \p buf are used in the same + * manner as described in the discussion of H5Pset_filter2(). + * On input, the \p buf_size parameter indicates the size in + * bytes of the buffer pointed to by \p buf which is allocated + * by the calling program; on exit it contains the size of the + * values defined by the filter. + * + * On input, the \p namelen parameter indicates the number of + * characters allocated for the filter name by the calling program + * in the array \p name[]. On exit \p name[] contains the name of the + * filter with one character of the name in each element of the + * array. + * + * \p filter_config is the bit field described in + * H5Zget_filter_info(). + * + * If the filter specified in \p filter_id is not set for the + * property list, an error will be returned and + * H5Pget_filter_by_id3() will fail. + * + * \version 1.8.5 Function extended to work with group creation property + * lists. + * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 + * + * \see H5Pset_filter2(), H5Pget_filter3() + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Pget_filter_by_id3(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter_id, + uint64_t *flags /*out*/, size_t *buf_size /*in,out*/, void *buf /*out*/, + size_t namelen /*in*/, char name[] /*out*/, + unsigned *filter_config /*out*/); + /** * \ingroup OCPL * @@ -2216,6 +2363,40 @@ H5_DLL herr_t H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t filter_id, unsig * */ H5_DLL int H5Pget_nfilters(hid_t plist_id); + +/** + * \ingroup OCPL + * + * \brief Returns the number of filters in the pipeline for a section of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * + * \return Returns the number of filters in the pipeline if successful; + * otherwise returns a negative value. + * + * \details H5Pget_nfilters2() returns the number of filters defined in the + * filter pipeline for the section \p section_number that is + * associated with the property list \p plist_id. + * + * In each pipeline, the filters are numbered from 0 through \Code{N-1}, + * where \c N is the value returned by this function. During output to + * the file, the filters are applied in increasing order; during + * input from the file, they are applied in decreasing order. + * + * H5Pget_nfilters() returns the number of filters in the pipeline + * for the section \p section_number, including zero (0) if there are none. + * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter + * + * \see H5Pset_filter2, H5Pget_filter3 + * + * \since 1.0.0 + * + */ +H5_DLL int H5Pget_nfilters2(hid_t plist_id, uint64_t section_number); + /** * \ingroup OCPL * @@ -2248,6 +2429,7 @@ H5_DLL int H5Pget_nfilters(hid_t plist_id); * */ H5_DLL herr_t H5Pget_obj_track_times(hid_t plist_id, hbool_t *track_times); + /** * \ingroup OCPL * @@ -2277,6 +2459,48 @@ H5_DLL herr_t H5Pget_obj_track_times(hid_t plist_id, hbool_t *track_times); */ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/]); + +/** + * \ingroup OCPL + * + * \brief Modifies a filter in the filter pipeline for a specified section of the + * structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter to be modified + * \param[in] flags Bit vector specifying certain general properties + * of the filter + * \param[in] buf_size Size in bytes of \p buf buffer + * \param[in] buf Auxiliary data for the filter + * + * \return \herr_t + * + * \details H5Pmodify_filter2() modifies the specified \p filter in the + * filter pipeline for a specified section of the structured + * chunk. + * + * \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p plist_id parameter must be a dataset or group creation + * property list. + * + * The \p filter, \p flags \p buf_size, and \p buf + * parameters are used in the same manner and accept the same + * values as described in the discussion of H5Pset_filter2(). + * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 + * + * \see H5Premove_filter2(), H5Pset_filter2() + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Pmodify_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf); + /** * \ingroup OCPL * @@ -2343,6 +2567,83 @@ H5_DLL herr_t H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int * */ H5_DLL herr_t H5Premove_filter(hid_t plist_id, H5Z_filter_t filter); + +/** + * \ingroup OCPL + * + * \brief Removes a filter in the filter pipeline for a specified section + * of the structured chunk + * + * \ocpl_id{plist_id} + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter to be deleted + * + * \return \herr_t + * + * \details H5Premove_filter2() removes the specified \p filter for a specified + * \p section_number from the filter pipeline in the dataset or + * group creation property list \p plist_id. + * + * The \p section_number is an integer specifying a section in the + * structured chunk. + * + * The \p filter parameter specifies the filter to be removed. + * Valid values for use in \p filter are as follows: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FILTER_ALLRemoves all filters from the filter pipeline
#H5Z_FILTER_DEFLATEData compression filter, employing the gzip + * algorithm
#H5Z_FILTER_SHUFFLEData shuffling filter
#H5Z_FILTER_FLETCHER32Error detection filter, employing the Fletcher32 + * checksum algorithm
#H5Z_FILTER_SZIPData compression filter, employing the SZIP + * algorithm
#H5Z_FILTER_NBITData compression filter, employing the N-Bit + * algorithm
#H5Z_FILTER_SCALEOFFSETData compression filter, employing the scale-offset + * algorithm
+ * + * Additionally, user-defined filters can be removed with this + * routine by passing the filter identifier with which they were + * registered with the HDF5 library. + * + * Attempting to remove a filter that is not in the filter + * pipeline is an error. + * + * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter2 + * + * \see H5Pmodify_filter2(), H5Pset_filter2() + * + * \since 1.x.x + * + */ +H5_DLL herr_t H5Premove_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter); + /** * \ingroup OCPL * @@ -2420,93 +2721,375 @@ H5_DLL herr_t H5Pset_attr_creation_order(hid_t plist_id, unsigned crt_order_flag * attributes subsequently falls below \p min_dense, the attributes * are returned to compact storage. * - * If \p max_compact is set to 0 (zero), dense storage is always - * used. \p min_dense must be set to 0 (zero) when \p max_compact - * is 0 (zero). + * If \p max_compact is set to 0 (zero), dense storage is always + * used. \p min_dense must be set to 0 (zero) when \p max_compact + * is 0 (zero). + * + * \p plist_id is a dataset or group creation property list + * identifier. + * + * \since 1.8.0 + * + */ +H5_DLL herr_t H5Pset_attr_phase_change(hid_t plist_id, unsigned max_compact, unsigned min_dense); +/** + * \ingroup DCPL + * + * \brief Sets deflate (GNU gzip) compression method and compression level + * + * \ocpl_id{plist_id} + * \param[in] level Compression level + * + * \return \herr_t + * + * \par_compr_note + * + * \details H5Pset_deflate() sets the deflate compression method and the + * compression level, \p level, for a dataset or group creation + * property list, \p plist_id. + * + * The filter identifier set in the property list is + * #H5Z_FILTER_DEFLATE. + * + * The compression level, \p level, is a value from zero to nine, + * inclusive. A compression level of 0 (zero) indicates no + * compression; compression improves but speed slows progressively + * from levels 1 through 9: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Compression LevelGzip Action
0No compression
1Best compression speed; least compression
2 through 8Compression improves; speed degrades
9Best compression ratio; slowest speed
+ * + * Note that setting the compression level to 0 (zero) does not turn + * off use of the gzip filter; it simply sets the filter to perform + * no compression as it processes the data. + * + * HDF5 relies on GNU gzip for this compression. + * + * \version 1.8.5 Function extended to work with group creation property lists. + * \since 1.0.0 + * + */ +H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); + +/** + * \ingroup OCPL + * + * \brief Adds a filter to the filter pipeline + * + * \ocpl_id{plist_id} + * \param[in] filter Filter identifier for the filter to be added to the + * pipeline + * \param[in] flags Bit vector specifying certain general properties of + * the filter + * \param[in] cd_nelmts Number of elements in \p c_values + * \param[in] c_values Auxiliary data for the filter + * + * \return \herr_t + * + * \details H5Pset_filter() adds the specified \p filter identifier and + * corresponding properties to the end of an output filter + * pipeline. + * + * \p plist_id must be either a dataset creation property list or + * group creation property list identifier. If \p plist_id is a + * dataset creation property list identifier, the filter is added + * to the raw data filter pipeline. + * + * If \p plist_id is a group creation property list identifier, + * the filter is added to the link filter pipeline, which filters + * the fractal heap used to store most of the link metadata in + * certain types of groups. The only predefined filters that can + * be set in a group creation property list are the gzip filter + * (#H5Z_FILTER_DEFLATE) and the Fletcher32 error detection filter + * (#H5Z_FILTER_FLETCHER32). + * + * The array \p cd_values contains \p cd_nelmts unsigned integers + * which are auxiliary data for the filter. The values are typically + * used as parameters to control the filter. In a filter's + * \p set_local method (called from \p H5Dcreate), the values are + * interpreted and possibly modified before they are used to control + * the filter. These, possibly modified values, are then stored in + * the dataset object header as auxiliary data for the filter. + * + * The \p flags argument is a bit vector with the following + * fields specifying certain general properties of the filter: + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FLAG_OPTIONALIf this bit is set then the filter is optional. If the + * filter fails (see below) during an H5Dwrite() operation + * then the filter is just excluded from the pipeline for + * the chunk for which it failed; the filter will not + * participate in the pipeline during an H5Dread() of the + * chunk. This is commonly used for compression filters: + * if the filter result would be larger than the input, + * then the compression filter returns failure and the + * uncompressed data is stored in the file.

+ * This flag should not be set for the Fletcher32 checksum + * filter as it will bypass the checksum filter without + * reporting checksum errors to an application.
#H5Z_FLAG_MANDATORYIf the filter is required, that is, set to mandatory, + * and the filter fails, the library's behavior depends + * on whether the chunk cache is in use: + * \li If the chunk cache is enabled, data chunks will + * be flushed to the file during H5Dclose() and the + * library will return the failure in H5Dclose(). + * \li When the chunk cache is disabled or not big enough, + * or the chunk is being evicted from the cache, the + * failure will happen during H5Dwrite(). + * + * In each case, the library will still write to the file + * all data chunks that were processed by the filter + * before the failure occurred.

+ * For example, assume that an application creates a + * dataset of four chunks, the chunk cache is enabled and + * is big enough to hold all four chunks, and the filter + * fails when it tries to write the fourth chunk. The + * actual flush of the chunks will happen during + * H5Dclose(), not H5Dwrite(). By the time H5Dclose() + * fails, the first three chunks will have been written + * to the file. Even though H5Dclose() fails, all the + * resources will be released and the file can be closed + * properly.

+ * If, however, the filter fails on the second chunk, only + * the first chunk will be written to the file as nothing + * further can be written once the filter fails.
+ * The \p filter parameter specifies the filter to be set. Valid + * pre-defined filter identifiers are as follows: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FILTER_DEFLATEData compression filter, employing the gzip + * algorithm
#H5Z_FILTER_SHUFFLEData shuffling filter
#H5Z_FILTER_FLETCHER32Error detection filter, employing the Fletcher32 + * checksum algorithm
#H5Z_FILTER_SZIPData compression filter, employing the SZIP + * algorithm
#H5Z_FILTER_NBITData compression filter, employing the N-Bit + * algorithm
#H5Z_FILTER_SCALEOFFSETData compression filter, employing the scale-offset + * algorithm
+ * Also see H5Pset_edc_check() and H5Pset_filter_callback(). + * + * \note When a non-empty filter pipeline is used with a group creation + * property list, the group will be created with the new group file + * format. The filters will come into play only when dense storage + * is used (see H5Pset_link_phase_change()) and will be applied to + * the group's fractal heap. The fractal heap will contain most of + * the group's link metadata, including link names. * - * \p plist_id is a dataset or group creation property list - * identifier. + * \note When working with group creation property lists, if you are + * adding a filter that is not in HDF5's set of predefined filters, + * i.e., a user-defined or third-party filter, you must first + * determine that the filter will work for a group. See the + * discussion of the set local and can apply callback functions + * in H5Zregister(). * - * \since 1.8.0 + * \note If multiple filters are set for a property list, they will be + * applied to each chunk of raw data for datasets or each block + * of the fractal heap for groups in the order in which they were + * set. * - */ -H5_DLL herr_t H5Pset_attr_phase_change(hid_t plist_id, unsigned max_compact, unsigned min_dense); -/** - * \ingroup DCPL + * \note Filters can be applied only to chunked datasets; they cannot be + * used with other dataset storage methods, such as contiguous, + * compact, or external datasets. * - * \brief Sets deflate (GNU gzip) compression method and compression level + * \note Dataset elements of variable-length and dataset region + * reference datatypes are stored in separate structures in the + * file called heaps. Filters cannot currently be applied to + * these heaps. * - * \ocpl_id{plist_id} - * \param[in] level Compression level + * \note Filter Behavior in HDF5:
+ * Filters can be inserted into the HDF5 pipeline to perform + * functions such as compression and conversion. As such, they are + * a very flexible aspect of HDF5; for example, a user-defined + * filter could provide encryption for an HDF5 dataset. * - * \return \herr_t + * \note A filter can be declared as either required or optional. + * Required is the default status; optional status must be + * explicitly declared. * - * \par_compr_note + * \note A required filter that fails or is not defined causes an + * entire output operation to fail; if it was applied when the + * data was written, such a filter will cause an input operation + * to fail. * - * \details H5Pset_deflate() sets the deflate compression method and the - * compression level, \p level, for a dataset or group creation - * property list, \p plist_id. + * \note The following table summarizes required filter behavior. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Required FILTER_X not availableFILTER_X available
H5Pset_Will fail.Will succeed.
H5Dwrite with FILTER_X setWill fail.Will succeed; FILTER_X will be applied to + * the data.
H5Dread with FILTER_X setWill fail.Will succeed.
+ * \note An optional filter can be set for an HDF5 dataset even when + * the filter is not available. Such a filter can then be + * applied to the dataset when it becomes available on the + * original system or when the file containing the dataset is + * processed on a system on which it is available. * - * The filter identifier set in the property list is - * #H5Z_FILTER_DEFLATE. + * \note A filter can be declared as optional through the use of the + * #H5Z_FLAG_OPTIONAL flag with H5Pset_filter(). * - * The compression level, \p level, is a value from zero to nine, - * inclusive. A compression level of 0 (zero) indicates no - * compression; compression improves but speed slows progressively - * from levels 1 through 9: + * \note Consider a situation where one is creating files that will + * normally be used only on systems where the optional (and + * fictional) filter FILTER_Z is routinely available. One can + * create those files on system A, which lacks FILTER_Z, create + * chunked datasets in the files with FILTER_Z defined in the + * dataset creation property list, and even write data to those + * datasets. The dataset object header will indicate that FILTER_Z + * has been associated with this dataset. But since system A does + * not have FILTER_Z, dataset chunks will be written without it + * being applied. + * + * \note HDF5 has a mechanism for determining whether chunks are + * actually written with the filters specified in the object + * header, so while the filter remains unavailable, system A will + * be able to read the data. Once the file is moved to system B, + * where FILTER_Z is available, HDF5 will apply FILTER_Z to any + * data rewritten or new data written in these datasets. Dataset + * chunks that have been written on system B will then be + * unreadable on system A; chunks that have not been re-written + * since being written on system A will remain readable on system + * A. All chunks will be readable on system B. * + * \note The following table summarizes optional filter behavior. * - * - * - * - * - * - * - * - * - * - * - * - * * - * - * + * + * + * + * * * - * - * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * * *
Compression LevelGzip Action
0No compression
1Best compression speed; least compression
2 through 8Compression improves; speed degradesFILTER_Z not availableFILTER_Z available
with encode and decode
FILTER_Z available decode only
9Best compression ratio; slowest speedH5Pset_Will succeed.Will succeed.Will succeed.
H5Dread with FILTER_Z setWill succeed if FILTER_Z has not actually
+ * been applied to data.
Will succeed.Will succeed.
H5Dwrite with FILTER_Z setWill succeed;
+ * FILTER_Z will not be applied to the data.
Will succeed;
+ * FILTER_Z will be applied to the data.
Will succeed;
+ * FILTER_Z will not be applied to the data.
+ * \note The above principles apply generally in the use of HDF5 + * optional filters insofar as HDF5 does as much as possible to + * complete an operation when an optional filter is unavailable. + * (The SZIP filter is an exception to this rule; see H5Pset_szip() + * for details.) * - * Note that setting the compression level to 0 (zero) does not turn - * off use of the gzip filter; it simply sets the filter to perform - * no compression as it processes the data. - * - * HDF5 relies on GNU gzip for this compression. + * \see \ref_filter_pipe, \ref_group_impls * - * \version 1.8.5 Function extended to work with group creation property lists. - * \since 1.0.0 + * \version 1.8.5 Function applied to group creation property lists. + * \since 1.6.0 * */ -H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); +H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, + const unsigned int c_values[]); + /** * \ingroup OCPL * - * \brief Adds a filter to the filter pipeline + * \brief Adds a filter to the filter pipeline for a specified section of the structured chunk * * \ocpl_id{plist_id} - * \param[in] filter Filter identifier for the filter to be added to the - * pipeline - * \param[in] flags Bit vector specifying certain general properties of - * the filter - * \param[in] cd_nelmts Number of elements in \p cd_values - * \param[in] cd_values Auxiliary data for the filter - * - * \return \herr_t - * - * \details H5Pset_filter() adds the specified \p filter identifier and - * corresponding properties to the end of an output filter - * pipeline. + * \param[in] section_number An integer specifying section number of the structured chunk + * \param[in] filter Filter identifier for the filter to be added to the + * pipeline + * \param[in] flags Bit vector specifying certain general properties of + * the filter + * \param[in] buf_size Size in bytes of \p buf buffer + * \param[in] buf Buffer with an auxiliary data for the filter + * + * \return \herr_t + * + * \details H5Pset_filter2() adds a filter and corresponding properties to the end + * of an output filter pipeline. This function can be used with + * both current chunked storage and structured chunk storage + * including sparse chunk. It also addresses deficiency of + * H5Pset_filter1() in passing the filter’s data as described below. + * + * Note the following differences with H5Pset_filter1(): + * - This function accepts a new parameter \p section_number that + * specifies the section of the structured chunk to which the filter is applied. + * For chunked storage, there is just one section. + * - Data type for the \p flags parameter is changed to uint64_t to provide + * more flexibility to the VOL connectors that use the function. + * - This function passes the filter’s data by using a void pointer to a buffer + * with auxiliary data for the filter instead of unsigned int c_values[]. * * \p plist_id must be either a dataset creation property list or * group creation property list identifier. If \p plist_id is a @@ -2521,7 +3104,36 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * (#H5Z_FILTER_DEFLATE) and the Fletcher32 error detection filter * (#H5Z_FILTER_FLETCHER32). * - * The array \p cd_values contains \p cd_nelmts unsigned integers + * \p section_number specifies the section number. The value + * is 0 to 255 when native HDF5 file format is used. + * For sparse chunk, the convenience flag can be used to specify + * a section of the structured chunk to be filtered as described below: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
#H5Z_FLAG_SPARSE_SELECTIONAdds the filter to the filter pipeline for the encoded + * selection section of the sparse chunk. It has the same + * effect as passing 0. The flag will be ignored if the + * structured chunk is not sparse. + *
#H5Z_FLAG_SPARSE_FIXED_DATAAdds the filter to the filter pipeline for section 1 of + * the sparse chunk. It has the same effect as passing 1. + *
#H5Z_FLAG_SPARSE_VL_DATAAdds the filter to the filter pipeline for section 2 of + * the sparse chunk if data has variable-length datatype. + * It has the same effect as passing 2. + *
+ * + * \p buf points to \p buf_size bytes of buffer * which are auxiliary data for the filter. The values are typically * used as parameters to control the filter. In a filter's * \p set_local method (called from \p H5Dcreate), the values are @@ -2578,6 +3190,7 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * further can be written once the filter fails. * * + * * The \p filter parameter specifies the filter to be set. Valid * pre-defined filter identifiers are as follows: * @@ -2613,6 +3226,11 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * * * Also see H5Pset_edc_check() and H5Pset_filter_callback(). + * TBD: The behavior is the same for H5Pset_edc_check() but the + * functionality is not implemented for varible-length data. + * + * \note TBD: review/modify/add the info in the following note section + * for structured chunk v.s. dense chunks. * * \note When a non-empty filter pipeline is used with a group creation * property list, the group will be created with the new group file @@ -2633,9 +3251,9 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * of the fractal heap for groups in the order in which they were * set. * - * \note Filters can be applied only to chunked datasets; they cannot be - * used with other dataset storage methods, such as contiguous, - * compact, or external datasets. + * \note Filters can be applied only to chunked datasets; + * they cannot be used with other dataset storage methods, such as + * contiguous, compact, or external datasets. * * \note Dataset elements of variable-length and dataset region * reference datatypes are stored in separate structures in the @@ -2735,7 +3353,7 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * * * H5Dwrite with FILTER_Z set - * Will succeed;
+ * Will succeed;
of the structured chunto k * FILTER_Z will not be applied to the data. * Will succeed;
* FILTER_Z will be applied to the data. @@ -2749,14 +3367,19 @@ H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned level); * (The SZIP filter is an exception to this rule; see H5Pset_szip() * for details.) * + * \par Example + * \snippet H5P_struct_chunk_examples.c struct_chunk_filter + * * \see \ref_filter_pipe, \ref_group_impls + * H5Pget_nfilters2(), H5Pget_filter3(), H5Pmodify_filter2() * * \version 1.8.5 Function applied to group creation property lists. * \since 1.6.0 * */ -H5_DLL herr_t H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[]); +H5_DLL herr_t H5Pset_filter2(hid_t plist_id, uint64_t section_number, H5Z_filter_t filter, uint64_t flags, + size_t buf_size, const void *buf); + /** * \ingroup OCPL * @@ -5964,6 +6587,7 @@ H5_DLL herr_t H5Pfill_value_defined(hid_t plist, H5D_fill_value_t *status); * */ H5_DLL herr_t H5Pget_alloc_time(hid_t plist_id, H5D_alloc_time_t *alloc_time /*out*/); + /** * \ingroup DCPL * @@ -5986,6 +6610,39 @@ H5_DLL herr_t H5Pget_alloc_time(hid_t plist_id, H5D_alloc_time_t *alloc_time /*o * */ H5_DLL int H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/); + +/** + * \ingroup DCPL + * + * \brief Retrieves the size of chunks and the structured chunk storage type + * for the raw data of a dataset with structured chunk layout + * + * \dcpl_id{plist_id} + * \param[in] max_ndims Size of the \p dims array + * \param[out] dim Array to store the chunk dimensions + * \param[out] flag Flag to store the structured chunk storage type + * + * \return Returns chunk dimensionality if successful; + * otherwise returns a negative value. + * + * \details H5Pget_struct_chunk() retrieves the size of chunks and the + * structured chunk storage type for the raw data + * of a structured chunk layout dataset. This function is only valid for + * dataset creation property lists. At most, \p max_ndims elements + * of \p dim will be initialized. + * + * The type of structured chunk storage used will be retrieved in + * \p flag. The value can be #H5D_SPARSE_CHUNK for storing sparse data + * of any datatype or #H5D_VL_CHUNK for storing dense data of + * variable-length datatype. + * + * \see H5Pset_struct_chunk(), H5Pget_chunk() + * + * \since 1.x.x + * + */ +H5_DLL int H5Pget_struct_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/, unsigned *flag /*out*/); + /** * * \ingroup DCPL @@ -6408,6 +7065,7 @@ H5_DLL hid_t H5Pget_virtual_vspace(hid_t dcpl_id, size_t index); * */ H5_DLL herr_t H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time); + /** * \ingroup DCPL * @@ -6451,6 +7109,56 @@ H5_DLL herr_t H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time); * */ H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]); + +/** + * \ingroup DCPL + * + * \brief Sets structured chunked storage. The storage is used for sparse data of + * any datatype, and for dense data with variable-length datatype + * + * \dcpl_id{plist_id} + * \param[in] ndims The number of chunk dimensions + * \param[in] dim An array defining the size, in dataset elements, of + * each chunk + * \param[in] flag Flag that indicates structured chunk storage type + * + * \return \herr_t + * \details H5Pset_struct_chunk() sets structured chunk storage layout, chunk sizes + * and a type of structured chunk storage for a dataset. + * This function is only valid for dataset creation property lists. + * + * The \p ndims parameter must be the same size as the + * rank of the dataset. + * + * The values of the \p dim array define the size of the chunks + * to store the dataset's raw data. The unit of measure for \p dim + * values is dataset elements. + * + * The parameter \p flag indicates the type of structured chunk storage. + * The value can be #H5D_SPARSE_CHUNK to store sparse data of + * any datatype or #H5D_VL_CHUNK to store dense data of + * variable-length datatype. + * + * As a side-effect of this function, the creation property is modified + * to #H5D_STRUCT_CHUNK storage layout, if it was previously set using + * H5Pset_layout() function with any other storage layout type. + * + * \note Chunk size cannot exceed the size of a fixed-size dataset. For + * example, a dataset consisting of a 5x4 fixed-size array cannot be + * defined with 10x10 chunks. + * \note Structured chunk does not have 4GB limit on the chunk size as the + * “dense” chunk has due to API limitations. + * + * \par Example: + * \snippet H5D_struct_chunk_examples.c get_defined + * + * \see H5Pget_struct_chunk(), H5Pset_chunk() + * + * \since x.x.x + * + */ +H5_DLL herr_t H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], unsigned flag); + /** * \ingroup DCPL * @@ -6725,6 +7433,9 @@ H5_DLL herr_t H5Pset_shuffle(hid_t plist_id); * the file. * - #H5D_VIRTUAL: Draw raw data from multiple datasets in * different files. + * - #H5D_STRUCT_CHUNK: Compose of two or more sections + * which together will describe the values + * in a chunk. * * Note that a compact storage layout may affect writing data to * the dataset with parallel applications. See the note in diff --git a/src/H5VLconnector.h b/src/H5VLconnector.h index 19e5e11999c..9e9a22dd474 100644 --- a/src/H5VLconnector.h +++ b/src/H5VLconnector.h @@ -221,7 +221,8 @@ typedef enum H5VL_dataset_get_t { H5VL_DATASET_GET_SPACE, /* dataspace */ H5VL_DATASET_GET_SPACE_STATUS, /* space status */ H5VL_DATASET_GET_STORAGE_SIZE, /* storage size */ - H5VL_DATASET_GET_TYPE /* datatype */ + H5VL_DATASET_GET_TYPE, /* datatype */ + H5VL_DATASET_GET_DEFINED /* dataspace selection of defined elements */ } H5VL_dataset_get_t; /* Parameters for dataset 'get' operations */ @@ -259,6 +260,13 @@ typedef struct H5VL_dataset_get_args_t { struct { hid_t type_id; /* Datatype ID (OUT) */ } get_type; + + /* H5VL_DATASET_GET_DEFINED */ + struct { + hid_t file_space_id; /* file dataspace selection to be queried (IN) */ + hid_t space_id; /* dataspace selection of defined elements (OUT) */ + } get_defined; + } args; } H5VL_dataset_get_args_t; @@ -266,7 +274,8 @@ typedef struct H5VL_dataset_get_args_t { typedef enum H5VL_dataset_specific_t { H5VL_DATASET_SET_EXTENT, /* H5Dset_extent */ H5VL_DATASET_FLUSH, /* H5Dflush */ - H5VL_DATASET_REFRESH /* H5Drefresh */ + H5VL_DATASET_REFRESH, /* H5Drefresh */ + H5VL_DATASET_ERASE, /* H5Derase */ } H5VL_dataset_specific_t; /* Parameters for dataset 'specific' operations */ @@ -289,6 +298,11 @@ typedef struct H5VL_dataset_specific_args_t { struct { hid_t dset_id; /* Dataset ID (IN) */ } refresh; + + /* H5VL_DATASET_ERASE */ + struct { + hid_t file_space_id; /* file dataspace selection to be erased (IN) */ + } erase; } args; } H5VL_dataset_specific_args_t; diff --git a/src/H5VLnative.h b/src/H5VLnative.h index f0a185b22d6..19e3196647f 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -72,17 +72,27 @@ typedef union H5VL_native_attr_optional_args_t { /* NOTE: If new values are added here, the H5VL__native_introspect_opt_query * routine must be updated. */ -#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ -#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ -#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ -#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ -#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ -#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ -#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ -#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ +/* TBD: the existing H5VL_NATIVE_DATASET_CHUNK_ITER is not added to + * H5VL__native_introspect_opt_query, therefore I didn't add + * H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER to H5VL__native_introspect_opt_query either. + * Check for sure later. + */ +#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ +#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ +#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ +#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ +#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ +#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ +#define H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK 11 /* H5Dread_struct_chunk */ +#define H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK 12 /* H5Dwrite_struct_chunk */ +#define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX 13 /* H5Dget_struct_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD 14 /* H5Dget_struct_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER 15 /* H5Dstruct_chunk_iter */ /* NOTE: If values over 1023 are added, the H5VL_RESERVED_NATIVE_OPTIONAL macro * must be updated. */ @@ -102,6 +112,21 @@ typedef struct H5VL_native_dataset_chunk_write_t { const void *buf; } H5VL_native_dataset_chunk_write_t; +/* TBD: can we use the same struct for both read and write operations? */ +/* Parameters for native connector's dataset 'read struct chunk' operation */ +typedef struct H5VL_native_dataset_read_struct_chunk_t { + const hsize_t *offset; + H5D_struct_chunk_info_t *chunk_info; + void **buf; +} H5VL_native_dataset_read_struct_chunk_t; + +/* Parameters for native connector's dataset 'write struct chunk' operation */ +typedef struct H5VL_native_dataset_write_struct_chunk_t { + const hsize_t *offset; + H5D_struct_chunk_info_t *chunk_info; + void **buf; +} H5VL_native_dataset_write_struct_chunk_t; + /* Parameters for native connector's dataset 'get vlen buf size' operation */ typedef struct H5VL_native_dataset_get_vlen_buf_size_t { hid_t type_id; @@ -139,6 +164,24 @@ typedef struct H5VL_native_dataset_get_chunk_info_by_coord_t { hsize_t *size; /* Size of chunk in file (OUT) */ } H5VL_native_dataset_get_chunk_info_by_coord_t; +/* Parameters for native connector's dataset 'get struct chunk info by idx' operation */ +typedef struct H5VL_native_dataset_get_struct_chunk_info_by_idx_t { + hid_t fspace_id; /* File dataspace selection */ + hsize_t chunk_idx; /* Chunk index */ + hsize_t *offset; /* Logical position of the chunk's first element in the array (OUT) */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ +} H5VL_native_dataset_get_struct_chunk_info_by_idx_t; + +/* Parameters for native connector's dataset 'get struct chunk info by coord' operation */ +typedef struct H5VL_native_dataset_get_struct_chunk_info_by_coord_t { + const hsize_t *offset; /* Logical position of the chunk's first element in the array */ + H5D_struct_chunk_info_t *chunk_info; /* Information about the structured chunk (OUT) */ + haddr_t *addr; /* Chunk address in the file (OUT) */ + hsize_t *chunk_size; /* Chunk size in bytes (OUT) */ +} H5VL_native_dataset_get_struct_chunk_info_by_coord_t; + /* Parameters for native connector's dataset 'optional' operations */ typedef union H5VL_native_dataset_optional_args_t { /* H5VL_NATIVE_DATASET_FORMAT_CONVERT */ @@ -181,6 +224,24 @@ typedef union H5VL_native_dataset_optional_args_t { void *op_data; /* Context to pass to iteration callback */ } chunk_iter; + /* H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK */ + H5VL_native_dataset_write_struct_chunk_t write_struct_chunk; + + /* H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK */ + H5VL_native_dataset_read_struct_chunk_t read_struct_chunk; + + /* H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX */ + H5VL_native_dataset_get_struct_chunk_info_by_idx_t get_struct_chunk_info_by_idx; + + /* H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD */ + H5VL_native_dataset_get_struct_chunk_info_by_coord_t get_struct_chunk_info_by_coord; + + /* H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER */ + struct { + H5D_struct_chunk_iter_op_t op; /* Structured chunk iteration callback */ + void *op_data; /* Context to pass to iteration callback */ + } struct_chunk_iter; + } H5VL_native_dataset_optional_args_t; /* Values for native VOL connector file optional VOL operations */ diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index ced4569d8d1..7ecdbaf85eb 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -205,7 +205,7 @@ H5VL__native_dataset_io_setup(size_t count, void *obj[], hid_t mem_type_id[], hi * Purpose: Frees memory allocated by H5VL__native_dataset_io_setup() * * Return: SUCCEED/FAIL - * + *------------------------------------------------------------------------- */ static herr_t @@ -497,6 +497,28 @@ H5VL__native_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t H5_ATTR break; } + /* H5Dget_defined */ + case H5VL_DATASET_GET_DEFINED: { + const H5S_t *fspace = NULL; + + assert(dset->shared); + assert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if (args->args.get_defined.file_space_id == H5S_ALL) + fspace = dset->shared->space; + else /* otherwise, use the given space ID */ + if (NULL == (fspace = (const H5S_t *)H5I_object_verify(args->args.get_defined.file_space_id, + H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + /* Call private function */ + if ((args->args.get_defined.space_id = H5D__get_defined(dset, fspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace of defined elements"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't get this type of information from dataset"); } /* end switch */ @@ -547,6 +569,20 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid break; } + /* H5Derase */ + case H5VL_DATASET_ERASE: { + const H5S_t *fspace = NULL; + + if (NULL == + (fspace = (const H5S_t *)H5I_object_verify(args->args.erase.file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + if (H5D__erase(dset, fspace) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to erase dataset elements"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid specific operation"); } /* end switch */ @@ -631,7 +667,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ case H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE: { H5VL_native_dataset_get_chunk_storage_size_t *gcss_args = &opt_args->get_chunk_storage_size; - /* Make sure the dataset is chunked */ + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dget_chunk_storage_storage() is supposed to work for both dense and sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -657,7 +694,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ if (NULL == (space = (const H5S_t *)H5I_object_verify(gnc_args->space_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); - /* Make sure the dataset is chunked */ + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dget_num_chunks() is supposed to work for both dense and sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -722,6 +760,9 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + + /* Make sure the dataset is chunked or sparse chunk */ + /* TBD: H5Dread_chunk() is supposed to work for sparse chunk */ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -747,6 +788,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); @@ -798,6 +840,125 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ break; } + /* H5Dread_struct_chunk */ + case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: { + H5VL_native_dataset_read_struct_chunk_t *read_struct_chunk_args = &opt_args->read_struct_chunk; + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + + /* Check arguments */ + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + /* TBD: check for H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). + */ + if (H5D__chunk_get_offset_copy(dset, read_struct_chunk_args->offset, offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + + /* Read the structured chunk */ + if (H5D__read_struct_chunk_direct(dset, offset_copy, read_struct_chunk_args->chunk_info, + read_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); + + break; + } + + /* H5Dwrite_struct_chunk */ + case H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK: { + H5VL_native_dataset_write_struct_chunk_t *write_struct_chunk_args = &opt_args->write_struct_chunk; + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + + /* Check arguments */ + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); + /* TBD: check for H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). + */ + if (H5D__chunk_get_offset_copy(dset, write_struct_chunk_args->offset, offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + + /* Write the structured chunk */ + if (H5D__write_struct_chunk_direct(dset, offset_copy, write_struct_chunk_args->chunk_info, + write_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); + + break; + } + + /* H5Dget_struct_chunk_info */ + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX: { + H5VL_native_dataset_get_struct_chunk_info_by_idx_t *gcibi_args = + &opt_args->get_struct_chunk_info_by_idx; + const H5S_t *space; + + assert(dset->shared); + assert(dset->shared->space); + + /* TBD: is this true for structured chunk */ + /* When default dataspace is given, use the dataset's dataspace */ + if (gcibi_args->fspace_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if (NULL == (space = (const H5S_t *)H5I_object_verify(gcibi_args->fspace_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID"); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if (H5D__get_struct_chunk_info(dset, space, gcibi_args->chunk_idx, gcibi_args->offset, + gcibi_args->chunk_info, gcibi_args->addr, + gcibi_args->chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get struct chunk info by index"); + + break; + } + + /* H5Dget_struct_chunk_info_by_coord */ + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD: { + H5VL_native_dataset_get_struct_chunk_info_by_coord_t *gcibc_args = + &opt_args->get_struct_chunk_info_by_coord; + + assert(dset->shared); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if (H5D__get_struct_chunk_info_by_coord(dset, gcibc_args->offset, gcibc_args->chunk_info, + gcibc_args->addr, gcibc_args->chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't get struct chunk info by its logical coordinates"); + + break; + } + + /* H5Dstruct_chunk_iter */ + case H5VL_NATIVE_DATASET_STRUCT_CHUNK_ITER: { + /* Sanity check */ + assert(dset->shared); + + /* TBD: Make sure the dataset is H5D_SPARSE_CHUNK */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a sparse chunk dataset"); + + /* Call private function */ + if ((ret_value = H5D__struct_chunk_iter(dset, opt_args->struct_chunk_iter.op, + opt_args->struct_chunk_iter.op_data)) < 0) + HERROR(H5E_DATASET, H5E_BADITER, "struct chunk iteration failed"); + + break; + } + default: HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid optional operation"); } /* end switch */ diff --git a/src/H5VLnative_introspect.c b/src/H5VLnative_introspect.c index 4c1cb53c0db..d6b8dfb8384 100644 --- a/src/H5VLnative_introspect.c +++ b/src/H5VLnative_introspect.c @@ -117,16 +117,20 @@ H5VL__native_introspect_opt_query(void H5_ATTR_UNUSED *obj, H5VL_subclass_t subc case H5VL_NATIVE_DATASET_GET_NUM_CHUNKS: case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX: case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD: + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_IDX: + case H5VL_NATIVE_DATASET_GET_STRUCT_CHUNK_INFO_BY_COORD: case H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE: case H5VL_NATIVE_DATASET_GET_OFFSET: *flags |= H5VL_OPT_QUERY_QUERY_METADATA; break; case H5VL_NATIVE_DATASET_CHUNK_READ: + case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: *flags |= H5VL_OPT_QUERY_READ_DATA; break; case H5VL_NATIVE_DATASET_CHUNK_WRITE: + case H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK: *flags |= H5VL_OPT_QUERY_WRITE_DATA; break; diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index bd557fc8630..65d470817f1 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -263,6 +263,12 @@ typedef enum H5Z_cb_return_t { typedef H5Z_cb_return_t (*H5Z_filter_func_t)(H5Z_filter_t filter, void *buf, size_t buf_size, void *op_data); //! +/* TBD: */ +/* Convenience flag used by H5Pset_filter2() to specify a section of the sparse chunk to be filtered */ +#define H5Z_FLAG_SPARSE_SELECTION 0 +#define H5Z_FLAG_SPARSE_FIXED_DATA 1 +#define H5Z_FLAG_SPARSE_VL_DATA 2 + #ifdef __cplusplus extern "C" { #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8c9b3073b9d..bd4058e0953 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -386,6 +386,7 @@ set (H5_TESTS page_buffer dtypes dsets + sparse_storage select_io_dset chunk_info # compression lib link cmpd_dset diff --git a/test/Makefile.am b/test/Makefile.am index b15e36d3f91..03bbefb55f5 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -69,7 +69,8 @@ TEST_PROG= testhdf5 \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ - unregister cache_logging cork swmr vol timer event_set onion + unregister cache_logging cork swmr vol timer event_set \ + onion struct_chunk_storage # List programs to be built when testing here # diff --git a/test/struct_chunk_storage.c b/test/struct_chunk_storage.c new file mode 100644 index 00000000000..52ca2badad1 --- /dev/null +++ b/test/struct_chunk_storage.c @@ -0,0 +1,1118 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Tests sparse storage based on the concept of structured chunk + */ +#define H5D_FRIEND /*suppress error about including H5Dpkg */ +#define H5D_TESTING +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5FD_TESTING + +#define H5Z_FRIEND /*suppress error about including H5Zpkg */ + +#include "testhdf5.h" +#include "H5srcdir.h" + +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Iprivate.h" +#include "H5Pprivate.h" + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" /* File access */ + +#define H5S_FRIEND /*suppress error about including H5Spkg */ +#include "H5Spkg.h" /* Dataspace */ + +#define H5T_FRIEND /*suppress error about including H5Tpkg */ +#include "H5Tpkg.h" /* Datatype */ + +#define H5A_FRIEND /*suppress error about including H5Apkg */ +#include "H5Apkg.h" /* Attributes */ + +/* Use in version bound test */ +#define H5O_FRIEND /*suppress error about including H5Opkg */ +#include "H5Opkg.h" /* Object headers */ + +#include "H5Dpkg.h" +#include "H5FDpkg.h" +#include "H5VMprivate.h" +#include "H5Zpkg.h" + +static const char *FILENAME[] = {"sparse", /* 0 */ + "sparse_data", /* 1 */ + "sparse_direct_chunk", /* 2 */ + "sparse_query_direct_chunk", /* 3 */ + "sparse_filter", /* 4 */ + "sparse_dense_api", /* 5 */ + NULL}; +#define FILENAME_BUF_SIZE 1024 + +#define SPARSE_DSET "sparse_dset" + +#define RANK 2 +#define NX 10 +#define NY 10 +#define CHUNK_NX 5 +#define CHUNK_NY 5 + +/* Size of a chunk */ +#define CHK_SIZE (CHUNK_NX * CHUNK_NY * sizeof(int)) + +/*------------------------------------------------------------------------- + * Function: test_sparse_data + * + * Purpose: Verify APIs for handling sparse data: + * --H5Dget_defined() + * --H5Derase() + * + * Return: Success: 0 + * Failure: -1 + *------------------------------------------------------------------------- + */ +static herr_t +test_sparse_data(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t sid1 = H5I_INVALID_HID; + hid_t sid2 = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hsize_t dim[1] = {50}; /* 1-d dataspace */ + hsize_t chunk_dim[1] = {5}; /* Chunk size */ + int wbuf[50]; /* Write buffer */ + + TESTING("APIs for handling sparse data"); + + /* Create a file */ + h5_fixname(FILENAME[1], fapl, filename, sizeof filename); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(1, dim, NULL)) < 0) + FAIL_STACK_ERROR; + + /* Create property list for compact dataset creation */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_STRUCT_CHUNK) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Write sparse data to the dataset */ + memset(wbuf, 0, sizeof(wbuf)); + + /* Initialize and write sparse data to the dataset */ + wbuf[1] = 1; + wbuf[12] = 12; + wbuf[13] = 13; + wbuf[14] = 14; + wbuf[22] = 22; + wbuf[23] = 23; + wbuf[24] = 24; + wbuf[48] = 48; + wbuf[49] = 49; + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + /* Get defined elements */ + /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ + if ((sid1 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* TBD: Verify defined elements in sid1 are as expected */ + + /* Erase all defined elements */ + /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ + if (H5Derase(did, sid1, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + + /* Call H5Dget_defined() again after H5Derase() */ + if ((sid2 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* TBD: Verify nothing is defined in sid2 */ + + if (H5Sclose(sid1) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid2) < 0) + FAIL_STACK_ERROR; + + /* Closing */ + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(sid); + H5Sclose(sid1); + H5Sclose(sid2); + H5Pclose(dcpl); + H5Dclose(did); + H5Fclose(fid); + } + H5E_END_TRY + + return FAIL; +} /* end test_sparse_data() */ + +/*------------------------------------------------------------------------- + * Function: test_sparse_direct_chunk + * + * Purpose: Verify APIs for direct chunk I/O on structured chunk: + * --H5Dwrite_struct_chunk() + * --H5Dread_struct_chunk() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static herr_t +test_sparse_direct_chunk(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; + + int buf[NX][NY]; + size_t encode_size; + hsize_t start[2], block[2], count[2]; + + hsize_t wr_offset[2] = {0, 0}; + H5D_struct_chunk_info_t wr_chk_info; + uint16_t wr_filter_mask[2] = {0, 0}; + size_t wr_section_size[2]; + void *wr_buf[2]; + unsigned char *wr_buf0; + int *wr_buf1; + + hsize_t rd_offset[2] = {5, 5}; + H5D_struct_chunk_info_t rd_chk_info; + uint16_t rd_filter_mask[2] = {0, 0}; + size_t rd_section_size[2]; + void *rd_buf[2]; + unsigned char *rd_buf0; + int *rd_buf1; + + TESTING("APIs for direct chunk I/O on structured chunks"); + + /* Create a file */ + h5_fixname(FILENAME[2], fapl, filename, sizeof filename); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* + * Create the data space with unlimited dimensions. + */ + if ((sid = H5Screate_simple(RANK, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* + * Create a new dataset within the file using dcpl + */ + if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + start[0] = 3; + start[1] = 2; + block[0] = 2; + block[1] = 3; + count[0] = count[1] = 1; + /* Select the 2x3 block in chunk index 0 for writing */ + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) + FAIL_STACK_ERROR; + + /* Get the encoded size for the selection */ + if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + + /* Set up section size for section 0 and section 1 */ + wr_section_size[0] = encode_size; + wr_section_size[1] = block[0] * block[1] * sizeof(int); + + /* Allocate buffers for section 0 (encoded selection) and section 1 (data) */ + if ((wr_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) + FAIL_STACK_ERROR; + if ((wr_buf1 = (int *)calloc((size_t)1, wr_section_size[1])) == NULL) + FAIL_STACK_ERROR; + + /* Encode selection into the buffer for section 0 */ + if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + + /* Set up data into the buffer for section 1 */ + wr_buf1[0] = 32; + wr_buf1[1] = 33; + wr_buf1[2] = 34; + wr_buf1[3] = 42; + wr_buf1[4] = 43; + wr_buf1[5] = 44; + + /* Set up the buffer for H5D_write_struct_chunk() */ + wr_buf[0] = wr_buf0; + wr_buf[1] = wr_buf1; + + wr_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + wr_chk_info.num_sections = 2; + wr_chk_info.filter_mask = wr_filter_mask; + wr_chk_info.section_size = wr_section_size; + wr_chk_info.section_orig_size = wr_section_size; + + /* Write the structured chunk at offset [0,0]: chunk index 0 */ + if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, wr_offset, &wr_chk_info, wr_buf) < 0) + FAIL_STACK_ERROR; + + /* Read the whole dataset */ + if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + FAIL_STACK_ERROR; + /* TBD: Verify buf read has data as in wr_buf1[] at location wr_buf0[] */ + + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + + if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + + if ((sid = H5Dget_space(did)) == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Select the 2x1 block in chunk index 3 for reading */ + start[0] = 5; + start[1] = 5; + block[0] = 2; + block[1] = 1; + count[0] = count[1] = 1; + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) + FAIL_STACK_ERROR; + + if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + + rd_section_size[0] = encode_size; + rd_section_size[1] = block[0] * block[1] * sizeof(int); + + /* Allocate buffers for section 0 (encoded selection) and section 1 (data) */ + if ((rd_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) + FAIL_STACK_ERROR; + if ((rd_buf1 = (int *)calloc((size_t)1, rd_section_size[1])) == NULL) + FAIL_STACK_ERROR; + + rd_buf[0] = rd_buf0; + rd_buf[1] = rd_buf1; + + rd_chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + rd_chk_info.num_sections = 2; + rd_chk_info.filter_mask = rd_filter_mask; + rd_chk_info.section_size = rd_section_size; + rd_chk_info.section_orig_size = rd_section_size; + + /* Read the structured chunk at offset [5,5] */ + if (H5Dread_struct_chunk(did, H5P_DEFAULT, rd_offset, &rd_chk_info, rd_buf) < 0) + FAIL_STACK_ERROR; + /* Verify rd_chk_info and rd_buf are the same as wr_chk_info and wr_buf */ + + /* + * Close/release resources. + */ + if (H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if (H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + H5_FAILED(); + return 1; + +} /* test_sparse_direct_chunk() */ + +/*------------------------------------------------------------------------- + * Function: verify_get_struct_chunk_info (helper function) + * + * Purpose: Verifies that H5Dget_struct_chunk_info returns correct + * values for a chunk. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +verify_get_struct_chunk_info(hid_t did, hid_t sid, hsize_t chk_index, + const hsize_t H5_ATTR_UNUSED *exp_offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], + hsize_t H5_ATTR_UNUSED exp_chk_size) +{ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + H5D_struct_chunk_info_t out_chunk_info[50]; + + /* Get info of the chunk specified by chk_index */ + if (H5Dget_struct_chunk_info(did, sid, chk_index, out_offset, out_chunk_info, &out_addr, &out_chk_size) < + 0) + TEST_ERROR; + +#ifdef TBD + + /* Verify info from H5Dget_struct_chunk_info() with expected chunk info */ + + if (out_offset[0] != exp_offset[0]) + FAIL_PUTS_ERROR("unexpected offset[0]"); + if (out_offset[1] != exp_offset[1]) + FAIL_PUTS_ERROR("unexpected offset[1]"); + + Compare out_chunk_info with exp_chunk_info + + if (HADDR_UNDEF == out_addr) FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); + + if (out_chk_size != exp_chk_size) + FAIL_PUTS_ERROR("unexpected chunk size"); + +#endif + + /* For now, just return SUCCEED */ + + return SUCCEED; + +error: + return FAIL; +} /* verify_get_struct_chunk_info() */ + +/*------------------------------------------------------------------------- + * + * Function: verify_get_struct_chunk_info_by_coord (helper function) + * + * Purpose: Verifies that H5Dget_struct_chunk_info_by_coord returns correct + * values for a chunk. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +verify_get_struct_chunk_info_by_coord(hid_t did, hsize_t *offset, + H5D_struct_chunk_info_t H5_ATTR_UNUSED exp_chunk_info[], + hsize_t H5_ATTR_UNUSED exp_chk_size) +{ + hsize_t out_chk_size = 0; /* Size of an allocated/written chunk */ + haddr_t out_addr = 0; /* Address of an allocated/written chunk */ + H5D_struct_chunk_info_t out_chunk_info[50]; + + /* Get info of the chunk at logical coordinates specified by offset */ + if (H5Dget_struct_chunk_info_by_coord(did, offset, out_chunk_info, &out_addr, &out_chk_size) < 0) + TEST_ERROR; + +#ifdef TBD + { + /* Verify info from H5Dget_struct_chunk_info_by_coord() with expected chunk info */ + + if (HADDR_UNDEF == out_addr) + FAIL_PUTS_ERROR("address cannot be HADDR_UNDEF"); + + Compare out_chunk_info with exp_chunk_info + + if (out_chk_size != exp_chk_size) FAIL_PUTS_ERROR("unexpected chunk size"); + } +#endif + + /* For now, just return SUCCEED */ + + return SUCCEED; + +error: + return FAIL; +} /* verify_get_struct_chunk_info_by_coord() */ + +typedef struct struct_chunk_iter_info_t { + hsize_t offset[2]; + H5D_struct_chunk_info_t *chunk_info; + haddr_t addr; + hsize_t chunk_size; +} struct_chunk_iter_info_t; + +typedef struct struct_chunk_iter_udata_t { + struct_chunk_iter_info_t *struct_chunk_info; + int last_index; +} struct_chunk_iter_udata_t; + +static int +iter_cb_struct(const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size, + void *op_data) +{ + struct_chunk_iter_udata_t *cidata = (struct_chunk_iter_udata_t *)op_data; + int idx = cidata->last_index + 1; + + cidata->struct_chunk_info[idx].offset[0] = offset[0]; + cidata->struct_chunk_info[idx].offset[1] = offset[1]; + cidata->struct_chunk_info[idx].chunk_info = chunk_info; + cidata->struct_chunk_info[idx].addr = *addr; + cidata->struct_chunk_info[idx].chunk_size = *chunk_size; + + cidata->last_index++; + + return H5_ITER_CONT; +} /* iter_cb_struct() */ + +/*------------------------------------------------------------------------- + * Function: test_sparse_direct_chunk_query + * + * Purpose: Verify APIs for direct chunk I/O query on structured chunk: + * --H5Dget_struct_chunk_info() + * --H5Dget_struct_chunk_info_by_coord() + * --H5Dstruct_chunk_iter() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_sparse_direct_chunk_query(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + + struct_chunk_iter_info_t chunk_infos[2]; /* Chunk infos filled up by iterator */ + struct_chunk_iter_udata_t udata; /* udata for iteration */ + H5D_struct_chunk_info_t chk_info; + + uint16_t filter_mask[2] = {0, 0}; + hsize_t offset[2] = {0, 0}; + size_t section_size[2] = {32, 48}; + void *write_buf[2]; + hsize_t in0[4] = {3, 2, 4, 4}; /* Encoded coordinates: [3,2] - [4,4] */ + hsize_t in1[6] = {66, 69, 72, 96, 99, 102}; /* Data: 66,69,72,96,99,102 */ + + TESTING("APIs for direct chunk I/O query on structured chunk"); + + /* Create the file */ + h5_fixname(FILENAME[3], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if (did < 0) + TEST_ERROR; + + write_buf[0] = in0; + write_buf[1] = in1; + + chk_info.type = 4; /* should be H5D_SPARSE_CHUNK */ + chk_info.num_sections = 2; + chk_info.filter_mask = filter_mask; + chk_info.section_size = section_size; + chk_info.section_orig_size = section_size; + + /* Write the structured chunk at offset */ + if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) + FAIL_STACK_ERROR; + + /* Verify info of the first and only chunk via H5Dget_struct_chunk_info() */ + if (verify_get_struct_chunk_info(did, H5S_ALL, 0, offset, &chk_info, CHK_SIZE) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_struct_chunk_info failed\n"); + + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + /* Write the structured chunk at offset */ + if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) + FAIL_STACK_ERROR; + + /* Verify info of the chunk at offset [CHUNK_NX,CHUNK_NY] via H5Dget_struct_chunk_info_by_coord() */ + if (verify_get_struct_chunk_info_by_coord(did, offset, &chk_info, CHK_SIZE) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_struct_chunk_info_by_coord failed\n"); + + /* For now, H5Dstruct_chunk_iter() just returns SUCCEED without actual iteration */ + udata.struct_chunk_info = chunk_infos; + udata.last_index = -1; + if (H5Dstruct_chunk_iter(did, H5P_DEFAULT, &iter_cb_struct, &udata) < 0) + TEST_ERROR; + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_sparse_direct_chunk_query() */ + +/*------------------------------------------------------------------------- + * + * Function: test_sparse_filter() + * + * Purpose: Verify the following APIs for structured chunk filtering: + * --H5Pset_filter2() + * --H5Pget_nfilters2() + * --H5Pget_filters3() + * --H5Pget_filter_by_id3() + * --H5Premove_filter2() + * --H5Pmodify_filter2() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_sparse_filter(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int nfilters; + H5Z_filter_t filtn; + uint64_t flags; + + TESTING("APIs for structured chunk filtering"); + + /* Create the file */ + h5_fixname(FILENAME[4], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + if (H5Pset_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_MANDATORY, 0, NULL) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + /* TBD: verify nfilters is correct; for now it is 0 */ + nfilters = H5Pget_nfilters2(dcpl, H5Z_FLAG_SPARSE_SELECTION); + if (nfilters) + TEST_ERROR; + + if (did < 0) + TEST_ERROR; + + if (H5Dclose(did) < 0) + TEST_ERROR; + + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + + if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if ((dcpl = H5Dget_create_plist(did)) < 0) + FAIL_STACK_ERROR; + + /* TBD: verify that filtn is H5Z_FILTER_DEFLATE and flags is H5Z_FLAG_MANDATORY */ + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, &flags, NULL, NULL, (size_t)0, NULL, NULL); + + /* For now: just verify it's 0 */ + if (filtn != H5Z_FILTER_NONE) + TEST_ERROR; + + /* TBD: Modify the filter's flags to optional */ + if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < + 0) + FAIL_STACK_ERROR; + + /* TBD: verify that flags is H5Z_FLAG_OPTIONAL */ + if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, NULL, + NULL) < 0) + FAIL_STACK_ERROR; + + /* TBD: Remove the filter */ + if (H5Premove_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE) < 0) + FAIL_STACK_ERROR; + + /* TBD: verify that filtn is H5Z_FILTER_NONE */ + filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, NULL, NULL, NULL, (size_t)0, NULL, NULL); + + /* For now: just verify it's 0 */ + if (filtn != H5Z_FILTER_NONE) + TEST_ERROR; + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_sparse_filter() */ + +typedef struct chunk_iter_info_t { + hsize_t offset[2]; + unsigned filter_mask; + haddr_t addr; + hsize_t size; +} chunk_iter_info_t; + +typedef struct chunk_iter_udata_t { + chunk_iter_info_t *chunk_info; + int last_index; +} chunk_iter_udata_t; + +static int +iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) +{ + chunk_iter_udata_t *cidata = (chunk_iter_udata_t *)op_data; + int idx = cidata->last_index + 1; + + cidata->chunk_info[idx].offset[0] = offset[0]; + cidata->chunk_info[idx].offset[1] = offset[1]; + cidata->chunk_info[idx].filter_mask = filter_mask; + cidata->chunk_info[idx].addr = addr; + cidata->chunk_info[idx].size = size; + + cidata->last_index++; + + return H5_ITER_CONT; +} /* iter_cb() */ + +/*------------------------------------------------------------------------- + * Function: test_dense_chunk_api_on_sparse() + * + * Purpose: Verify the following dense chunk APIs will fail for + * H5D_SPARSE_CHUNK layout: + * --H5Dwrite_chunk() + * --H5Dget_chunk_info() + * --H5Dget_chunk_info_by_coord() + * --H5Dchunk_iter() + * Verify the following dense chunk APIs will succeed for + * H5D_SPARSE_CHUNK layout: + * --H5Dread_chunk() + * --H5Dget_chunk_storage_size() + * --H5Dget_num_chunks() + * + * Return: # of errors + * + *------------------------------------------------------------------------- + */ +static int +test_dense_chunk_api_on_sparse(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hid_t dcpl = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + chunk_iter_info_t chunk_infos[2]; + chunk_iter_udata_t udata; + hsize_t nchunks = 0; + hsize_t chunk_nbytes; + hsize_t offset[2] = {0, 0}; + int direct_buf[CHUNK_NX][CHUNK_NY]; + haddr_t addr = 0; + uint32_t filters = 0; + + TESTING("APIs for direct chunk I/O: dense chunk functions on sparse layout"); + + /* Create the file */ + h5_fixname(FILENAME[3], fapl, filename, sizeof filename); + + /* Create a new file. */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* Enable chunking */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* TBD: need to set to H5D_SPARSE_CHUNK */ + if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) + FAIL_STACK_ERROR; + + /* The layout is set to H5D_CHUNKED as a side-effect */ + if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* Create a new dataset using dcpl creation properties */ + did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if (did < 0) + TEST_ERROR; + + H5E_BEGIN_TRY + { + H5Dwrite_chunk(did, H5P_DEFAULT, 0, offset, CHK_SIZE, direct_buf); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + H5Dget_chunk_info(did, H5S_ALL, 0, NULL, NULL, &addr, NULL); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + H5Dget_chunk_info_by_coord(did, offset, NULL, &addr, NULL); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5E_BEGIN_TRY + { + udata.chunk_info = chunk_infos; + udata.last_index = -1; + H5Dchunk_iter(did, H5P_DEFAULT, &iter_cb, &udata); + } + H5E_END_TRY + /* TBD: set return status and verify that it should fail */ + + H5Dread_chunk(did, H5P_DEFAULT, offset, &filters, direct_buf); + /* TBD: should succeed */ + + H5Dget_num_chunks(did, sid, &nchunks); + /* TBD: should succeed */ + + H5Dget_chunk_storage_size(did, offset, &chunk_nbytes); + /* TBD: should succeed */ + + /* Release resource */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Remove the test file */ + HDremove(filename); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; +} /* test_dense_chunk_api_on_sparse() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests for sparse data + * Tests are copied and modified from: + * --test/dsets.c + * --test/direct_chunk.c + * --test/chunk_info.c + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + char filename[FILENAME_BUF_SIZE]; + hid_t file, grp, fapl, fapl2; + hid_t fcpl = H5I_INVALID_HID, fcpl2 = H5I_INVALID_HID; + unsigned new_format; + unsigned paged; + unsigned minimized_ohdr; + int mdc_nelmts; + size_t rdcc_nelmts; + size_t rdcc_nbytes; + double rdcc_w0; + int nerrors = 0; + const char *driver_name; + bool contig_addr_vfd; /* Whether VFD used has a contiguous address space */ + bool driver_is_default_compatible; + + /* Don't run this test using certain file drivers */ + driver_name = h5_get_test_driver_name(); + + /* Current VFD that does not support contiguous address space */ + contig_addr_vfd = (bool)(strcmp(driver_name, "split") != 0 && strcmp(driver_name, "multi") != 0); + + /* Set the random # seed */ + srand((unsigned)time(NULL)); + + /* Testing setup */ + h5_test_init(); + fapl = h5_fileaccess(); + + if (h5_driver_is_default_vfd_compatible(fapl, &driver_is_default_compatible) < 0) + TEST_ERROR; + + /* Turn off the chunk cache, so all the chunks are immediately written to disk */ + if (H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) + goto error; + rdcc_nbytes = 0; + if (H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) + goto error; + + /* Copy the file access property list */ + if ((fapl2 = H5Pcopy(fapl)) < 0) + TEST_ERROR; + + /* Set the "use the latest version of the format" bounds for creating objects in the file */ + if (H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if ((fcpl2 = H5Pcopy(fcpl)) < 0) + TEST_ERROR; + + /* Set file space strategy to paged aggregation and persisting free-space */ + if (H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, true, (hsize_t)1) < 0) + TEST_ERROR; + + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + /* Test with paged aggregation enabled or not */ + for (paged = false; paged <= true; paged++) { + + /* Temporary: skip testing for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ + if (!contig_addr_vfd && paged) + continue; + + for (minimized_ohdr = false; minimized_ohdr <= true; minimized_ohdr++) { + + /* Test with old & new format groups */ + /* Use new_format for NOW */ + for (new_format = true; new_format <= true; new_format++) { + hid_t my_fapl, my_fcpl; + + /* Set the FAPL for the type of format */ + if (new_format) { + my_fapl = fapl2; + if (paged) { + my_fcpl = fcpl2; + puts("\nTesting with new file format and paged aggregation"); + } + else { + my_fcpl = fcpl; + puts("\nTesting with new file format and non-paged aggregation"); + } + } /* end if */ + else { + my_fapl = fapl; + if (paged) { + my_fcpl = fcpl2; + puts("Testing with old file format and paged aggregation:"); + } + else { + my_fcpl = fcpl; + puts("Testing with old file format and non-paged aggregation:"); + } + } /* end else */ + + /* Create the file for this test */ + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, my_fcpl, my_fapl)) < 0) + goto error; + + if (true == minimized_ohdr) { + if (0 > H5Fset_dset_no_attrs_hint(file, true)) + goto error; + puts("(minimized dataset object headers with file setting)"); + } + + /* Cause the library to emit initial messages */ + if ((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) + goto error; + if (H5Gclose(grp) < 0) + goto error; + + /* Create its own testfile */ + nerrors += (test_sparse_data(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_direct_chunk(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_direct_chunk_query(my_fapl) < 0 ? 1 : 0); + nerrors += (test_sparse_filter(my_fapl) < 0 ? 1 : 0); + nerrors += (test_dense_chunk_api_on_sparse(my_fapl) < 0 ? 1 : 0); + + if (H5Fclose(file) < 0) + goto error; + } /* end for new_format */ + } /* end for minimized_ohdr */ + } /* end for paged */ + + /* Close property lists */ + if (H5Pclose(fapl2) < 0) + TEST_ERROR; + if (H5Pclose(fcpl) < 0) + TEST_ERROR; + if (H5Pclose(fcpl2) < 0) + TEST_ERROR; + + /* Verify symbol table messages are cached */ + nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0); + + if (nerrors) + goto error; + printf("All sparse storage tests passed.\n"); + + h5_cleanup(FILENAME, fapl); + + exit(EXIT_SUCCESS); + +error: + nerrors = MAX(1, nerrors); + printf("***** %d SPARSE STORAGE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); + exit(EXIT_FAILURE); +} /* end main() */ diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 71c507801ee..aafc5bc3e9e 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -1281,7 +1281,10 @@ print_dataset_info(const iter_t *iter) printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" - : (u == H5D_CONTIGUOUS ? "CONTIG" : (u == H5D_CHUNKED ? "CHUNKED" : "VIRTUAL"))), + : (u == H5D_CONTIGUOUS + ? "CONTIG" + : (u == H5D_CHUNKED ? "CHUNKED" + : (u == H5D_VIRTUAL ? "VIRTUAL" : "STRUCT_CHUNK")))), iter->dset_layouts[u]); printf("\tNumber of external files : %lu\n", iter->nexternal); diff --git a/tools/test/h5stat/expected/h5stat_dims1.ddl b/tools/test/h5stat/expected/h5stat_dims1.ddl index 07b2900545b..43f3f7dbfd7 100644 --- a/tools/test/h5stat/expected/h5stat_dims1.ddl +++ b/tools/test/h5stat/expected/h5stat_dims1.ddl @@ -32,6 +32,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 23 Dataset layout counts[CHUNKED]: 0 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_dims2.ddl b/tools/test/h5stat/expected/h5stat_dims2.ddl index dbccd05bb42..90924a7d7d5 100644 --- a/tools/test/h5stat/expected/h5stat_dims2.ddl +++ b/tools/test/h5stat/expected/h5stat_dims2.ddl @@ -23,6 +23,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 23 Dataset layout counts[CHUNKED]: 0 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_filters-cache.ddl b/tools/test/h5stat/expected/h5stat_filters-cache.ddl index 7383f0b31ff..1dc68e375e9 100644 --- a/tools/test/h5stat/expected/h5stat_filters-cache.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-cache.ddl @@ -57,6 +57,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 2 Dataset layout counts[CHUNKED]: 12 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 2 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_filters-d.ddl b/tools/test/h5stat/expected/h5stat_filters-d.ddl index eee7e1845d4..af98ee27f39 100644 --- a/tools/test/h5stat/expected/h5stat_filters-d.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-d.ddl @@ -19,6 +19,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 2 Dataset layout counts[CHUNKED]: 12 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 2 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_filters-dT.ddl b/tools/test/h5stat/expected/h5stat_filters-dT.ddl index e513b3a5f28..b49426cab4a 100644 --- a/tools/test/h5stat/expected/h5stat_filters-dT.ddl +++ b/tools/test/h5stat/expected/h5stat_filters-dT.ddl @@ -19,6 +19,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 2 Dataset layout counts[CHUNKED]: 12 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 2 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_filters.ddl b/tools/test/h5stat/expected/h5stat_filters.ddl index 7383f0b31ff..1dc68e375e9 100644 --- a/tools/test/h5stat/expected/h5stat_filters.ddl +++ b/tools/test/h5stat/expected/h5stat_filters.ddl @@ -57,6 +57,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 2 Dataset layout counts[CHUNKED]: 12 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 2 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_idx.ddl b/tools/test/h5stat/expected/h5stat_idx.ddl index 1b6ae0c90ae..99563bab249 100644 --- a/tools/test/h5stat/expected/h5stat_idx.ddl +++ b/tools/test/h5stat/expected/h5stat_idx.ddl @@ -57,6 +57,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 0 Dataset layout counts[CHUNKED]: 2 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_links2.ddl b/tools/test/h5stat/expected/h5stat_links2.ddl index 9fc82cd850d..0502f39fd4d 100644 --- a/tools/test/h5stat/expected/h5stat_links2.ddl +++ b/tools/test/h5stat/expected/h5stat_links2.ddl @@ -65,6 +65,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 23 Dataset layout counts[CHUNKED]: 0 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_newgrat.ddl b/tools/test/h5stat/expected/h5stat_newgrat.ddl index 130fe2ff9e6..9c75f2bdb51 100644 --- a/tools/test/h5stat/expected/h5stat_newgrat.ddl +++ b/tools/test/h5stat/expected/h5stat_newgrat.ddl @@ -55,6 +55,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 1 Dataset layout counts[CHUNKED]: 0 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_numattrs2.ddl b/tools/test/h5stat/expected/h5stat_numattrs2.ddl index 638781b1e5c..0565889fe35 100644 --- a/tools/test/h5stat/expected/h5stat_numattrs2.ddl +++ b/tools/test/h5stat/expected/h5stat_numattrs2.ddl @@ -66,6 +66,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 23 Dataset layout counts[CHUNKED]: 0 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: diff --git a/tools/test/h5stat/expected/h5stat_tsohm.ddl b/tools/test/h5stat/expected/h5stat_tsohm.ddl index 9369950c999..8c0b2a8c923 100644 --- a/tools/test/h5stat/expected/h5stat_tsohm.ddl +++ b/tools/test/h5stat/expected/h5stat_tsohm.ddl @@ -54,6 +54,7 @@ Dataset layout information: Dataset layout counts[CONTIG]: 0 Dataset layout counts[CHUNKED]: 3 Dataset layout counts[VIRTUAL]: 0 + Dataset layout counts[STRUCT_CHUNK]: 0 Number of external files : 0 Dataset filters information: Number of datasets with: