Update plugin versions, zlib-ng header, release note (#5266)

CMakeFilters.cmake

@@ -85,7 +85,6 @@ endif ()
 if(NOT DEFINED ZLIBNG_PACKAGE_NAME)
   set(ZLIBNG_PACKAGE_NAME "zlib-ng")
 endif ()
-option (HDF5_ENABLE_Z_LIB_SUPPORT "Enable Zlib Filters" ON)
 if (HDF5_ENABLE_Z_LIB_SUPPORT)
   if (NOT H5_ZLIB_HEADER)
     if (NOT ZLIB_USE_EXTERNAL)
@@ -149,7 +148,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT)
     message (VERBOSE "Filter HDF5_ZLIB is ON")
   else ()
     set (HDF5_ENABLE_Z_LIB_SUPPORT OFF CACHE BOOL "" FORCE)
-    message (FATAL_ERROR " ZLib support in HDF5 was enabled but not found")
+    message (WARNING " ZLib support in HDF5 was enabled but not found")
   endif ()
   message (VERBOSE "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
 endif ()
@@ -161,7 +160,6 @@ set(H5_SZIP_FOUND FALSE)
 if(NOT DEFINED LIBAEC_PACKAGE_NAME)
   set(LIBAEC_PACKAGE_NAME "libaec")
 endif ()
-option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" ON)
 if (HDF5_ENABLE_SZIP_SUPPORT)
   option (HDF5_ENABLE_SZIP_ENCODING "Use SZip Encoding" ON)
   if (NOT SZIP_USE_EXTERNAL)
@@ -208,6 +206,6 @@ if (HDF5_ENABLE_SZIP_SUPPORT)
     endif ()
   else ()
     set (HDF5_ENABLE_SZIP_SUPPORT OFF CACHE BOOL "" FORCE)
-    message (FATAL_ERROR "SZIP support in HDF5 was enabled but not found")
+    message (WARNING "SZIP support in HDF5 was enabled but not found")
   endif ()
 endif ()

CMakeLists.txt

@@ -910,8 +910,8 @@ include (UserMacros.cmake)
 #-----------------------------------------------------------------------------
 # Include filter (zlib, szip, etc.) macros
 #-----------------------------------------------------------------------------
-option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" OFF)
-option (HDF5_ENABLE_Z_LIB_SUPPORT "Enable Zlib Filters" OFF)
+option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" ON)
+option (HDF5_ENABLE_Z_LIB_SUPPORT "Enable Zlib Filters" ON)
 if (HDF5_ENABLE_Z_LIB_SUPPORT OR HDF5_ENABLE_SZIP_SUPPORT)
   include (CMakeFilters.cmake)
 endif ()

CMakePresets.json

@@ -45,13 +45,13 @@
       "name": "ci-base-plugins",
       "hidden": true,
       "cacheVariables": {
-        "BSHUF_TGZ_NAME": {"type": "STRING", "value": "bitshuffle-0.5.1.tar.gz"},
+        "BSHUF_TGZ_NAME": {"type": "STRING", "value": "bitshuffle-0.5.2.tar.gz"},
         "BSHUF_PACKAGE_NAME": {"type": "STRING", "value": "bshuf"},
         "BLOSC_TGZ_NAME": {"type": "STRING", "value": "c-blosc-1.21.6.tar.gz"},
         "BLOSC_PACKAGE_NAME": {"type": "STRING", "value": "blosc"},
         "BLOSC_ZLIB_TGZ_NAME": {"type": "STRING", "value": "zlib-1.3.tar.gz"},
         "BLOSC_ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"},
-        "BLOSC2_TGZ_NAME": {"type": "STRING", "value": "c-blosc2-2.15.1.tar.gz"},
+        "BLOSC2_TGZ_NAME": {"type": "STRING", "value": "c-blosc2-2.15.2.tar.gz"},
         "BLOSC2_PACKAGE_NAME": {"type": "STRING", "value": "blosc2"},
         "BLOSC2_ZLIB_TGZ_NAME": {"type": "STRING", "value": "zlib-1.3.tar.gz"},
         "BLOSC2_ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"},

config/cmake/cacheinit.cmake

@@ -79,15 +79,15 @@ set (KWSYS_USE_LOCALCONTENT OFF CACHE BOOL "Use local file for KWSYS FetchConten
 # filter plugin options
 ########################
 
-set (PLUGIN_TGZ_ORIGPATH "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshot" CACHE STRING "Use PLUGINS from original location" FORCE)
+set (PLUGIN_TGZ_ORIGPATH "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshot-1.14" CACHE STRING "Use PLUGINS from original location" FORCE)
 set (PLUGIN_TGZ_NAME "hdf5_plugins-1.14.tar.gz" CACHE STRING "Use PLUGINS from compressed file" FORCE)
 set (PLUGIN_USE_LOCALCONTENT ON CACHE BOOL "Use local file for PLUGIN FetchContent" FORCE)
 set (PLUGIN_PACKAGE_NAME "pl" CACHE STRING "Name of PLUGIN package" FORCE)
 set (PLUGIN_GIT_URL "https://github.com/HDFGroup/hdf5_plugins.git" CACHE STRING "Use plugins from HDF Group repository" FORCE)
 set (PLUGIN_GIT_BRANCH "master" CACHE STRING "" FORCE)
 set (H5PL_VERS_MAJOR "1" CACHE STRING "Major version of hdf5 package for PLUGIN package" FORCE)
 set (H5PL_VERS_MINOR "14" CACHE STRING "Minor version of hdf5 package for PLUGIN package" FORCE)
-set (H5PL_VERS_RELEASE "5" CACHE STRING "Release version of hdf5 package for PLUGIN package" FORCE)
+set (H5PL_VERS_RELEASE "6" CACHE STRING "Release version of hdf5 package for PLUGIN package" FORCE)
 
 #############
 # bitshuffle
@@ -97,7 +97,7 @@ set (BSHUF_GIT_URL "https://github.com/kiyo-masui/bitshuffle.git" CACHE STRING "
 set (BSHUF_GIT_BRANCH "master" CACHE STRING "" FORCE)
 
 set (BSHUF_TGZ_ORIGPATH "https://github.com/kiyo-masui/bitshuffle/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE)
-set (BSHUF_TGZ_NAME "bitshuffle-0.5.1.tar.gz" CACHE STRING "Use BSHUF from compressed file" FORCE)
+set (BSHUF_TGZ_NAME "bitshuffle-0.5.2.tar.gz" CACHE STRING "Use BSHUF from compressed file" FORCE)
 
 set (BSHUF_PACKAGE_NAME "bshuf" CACHE STRING "Name of BSHUF package" FORCE)
 
@@ -129,7 +129,7 @@ set (BLOSC2_GIT_URL "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Use BL
 set (BLOSC2_GIT_BRANCH "main" CACHE STRING "" FORCE)
 
 set (BLOSC2_TGZ_ORIGPATH "https://github.com/Blosc/c-blosc2/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE)
-set (BLOSC2_TGZ_NAME "c-blosc2-2.15.1.tar.gz" CACHE STRING "Use BLOSC2 from compressed file" FORCE)
+set (BLOSC2_TGZ_NAME "c-blosc2-2.15.2.tar.gz" CACHE STRING "Use BLOSC2 from compressed file" FORCE)
 
 set (BLOSC2_PACKAGE_NAME "blosc2" CACHE STRING "Name of BLOSC2 package" FORCE)
 

release_docs/INSTALL_CMake.txt

@@ -879,7 +879,7 @@ if (BUILD_TESTING)
 ---------------- External Library Options ---------------------
 HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building (NO GIT TGZ)"       "NO"
 HDF5_ENABLE_PLUGIN_SUPPORT  "Enable PLUGIN Filters"                              OFF
-HDF5_ENABLE_SZIP_SUPPORT    "Use SZip Filter"                                    OFF
+HDF5_ENABLE_SZIP_SUPPORT    "Use SZip Filter"                                    ON
 HDF5_ENABLE_Z_LIB_SUPPORT   "Enable Zlib Filters"                                ON
 
 if (HDF5_USE_ZLIB_NG)

release_docs/RELEASE.txt

@@ -47,7 +47,13 @@ New Features
 
     Configuration:
     -------------
-    -
+    - The documentation in the source folder, doc, has been converted to doxygen files.
+
+    - Added support for native zlib-ng compression.
+
+      Changed the zlib-ng CMake logic to prefer the native zlib-ng library. Added
+      #ifdef around the compression function calls. Added including the correct
+      header file with the same #ifdef.
 
 
     Library:
@@ -120,7 +126,28 @@ Bug Fixes since HDF5-1.14.5 release
 
     Configuration
     -------------
-    -
+    - Use pre-installed libaec compression library
+
+      The CMake logic for finding the libaec compression library has been
+      modified for a system-installed version of the library. Two options
+      must be set;
+          HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO
+          <LIB_PKG_NAME>_USE_EXTERNAL:BOOL=OFF
+      where <LIB_PKG_NAME> is one of ZLIB, ZLIBNG, SZIP, PLUGIN.
+      Note that HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO disables building all plugins
+      and external libraries in-line with the HDF5 library.
+
+      In addition, the <LIB_PKG_NAME>_ROOT environment variables must be set,
+      where <LIB_PKG_NAME> is one of ZLIB, ZLIBNG, SZIP, libaec, PLUGIN.
+      Note that libaec is the expected name for using the libaec library in place of original szip.
+
+      See INSTALL_CMake.txt for more detailed information.
+
+    - Removed the module search find_package for szip library
+
+      There is not a szip module file to use, so the find_package only uses
+      find_package in config mode. The choice then is to either build szip, with libaec,
+      inline or find a system installed szip library, built with CMake.
 
 
     Tools

src/CMakeLists.txt

@@ -711,8 +711,10 @@ set (H5Z_SOURCES
     ${HDF5_SRC_DIR}/H5Ztrans.c
 )
 if (H5_ZLIB_HEADER)
-  SET_PROPERTY(SOURCE ${HDF5_SRC_DIR}/H5Zdeflate.c PROPERTY
-      COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
+  message(VERBOSE "H5_ZLIB_HEADER for library=${H5_ZLIB_HEADER}")
+  set_source_files_properties(${HDF5_SRC_DIR}/H5Zdeflate.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
 endif ()
 
 

src/H5Zdeflate.c

@@ -74,8 +74,12 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
 
     if (flags & H5Z_FLAG_REVERSE) {
         /* Input; uncompress */
-        z_stream z_strm;             /* zlib parameters */
-        size_t   nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+        zng_stream z_strm; /* zlib parameters */
+#else
+        z_stream z_strm; /* zlib parameters */
+#endif
+        size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */
 
         /* Allocate space for the compressed buffer */
         if (NULL == (outbuf = H5MM_malloc(nalloc)))
@@ -89,21 +93,34 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         H5_CHECKED_ASSIGN(z_strm.avail_out, unsigned, nalloc, size_t);
 
         /* Initialize the uncompression routines */
+#if defined(H5_HAVE_ZLIBNG_H)
+        if (Z_OK != zng_inflateInit(&z_strm))
+            HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
+#else
         if (Z_OK != inflateInit(&z_strm))
             HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
+#endif
 
         /* Loop to uncompress the buffer */
         do {
             /* Uncompress some data */
+#if defined(H5_HAVE_ZLIBNG_H)
+            status = zng_inflate(&z_strm, Z_SYNC_FLUSH);
+#else
             status = inflate(&z_strm, Z_SYNC_FLUSH);
+#endif
 
             /* Check if we are done uncompressing data */
             if (Z_STREAM_END == status)
                 break; /*done*/
 
             /* Check for error */
             if (Z_OK != status) {
+#if defined(H5_HAVE_ZLIBNG_H)
+                (void)zng_inflateEnd(&z_strm);
+#else
                 (void)inflateEnd(&z_strm);
+#endif
                 HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
             }
             else {
@@ -114,7 +131,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
                     /* Allocate a buffer twice as big */
                     nalloc *= 2;
                     if (NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) {
+#if defined(H5_HAVE_ZLIBNG_H)
+                        (void)zng_inflateEnd(&z_strm);
+#else
                         (void)inflateEnd(&z_strm);
+#endif
                         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
                                     "memory allocation failed for deflate uncompression");
                     } /* end if */
@@ -137,7 +158,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         ret_value = z_strm.total_out;
 
         /* Finish uncompressing the stream */
+#if defined(H5_HAVE_ZLIBNG_H)
+        (void)zng_inflateEnd(&z_strm);
+#else
         (void)inflateEnd(&z_strm);
+#endif
     } /* end if */
     else {
         /*
@@ -147,9 +172,13 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
          */
         const Bytef *z_src = (const Bytef *)(*buf);
         Bytef       *z_dst; /*destination buffer		*/
-        uLongf       z_dst_nbytes = (uLongf)compressBound(nbytes);
-        uLong        z_src_nbytes = (uLong)nbytes;
-        int          aggression; /* Compression aggression setting */
+#if defined(H5_HAVE_ZLIBNG_H)
+        uLongf z_dst_nbytes = (uLongf)zng_compressBound(nbytes);
+#else
+        uLongf z_dst_nbytes = (uLongf)compressBound(nbytes);
+#endif
+        uLong z_src_nbytes = (uLong)nbytes;
+        int   aggression; /* Compression aggression setting */
 
         /* Set the compression aggression level */
         H5_CHECKED_ASSIGN(aggression, int, cd_values[0], unsigned);
@@ -160,7 +189,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[]
         z_dst = (Bytef *)outbuf;
 
         /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+        status = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
         status = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
         /* Check for various zlib errors */
         if (Z_BUF_ERROR == status)

test/CMakeLists.txt

@@ -477,6 +477,12 @@ endforeach ()
 ##############################################################################
 ######### Also special handling of link libs #############
 #-- Adding test for chunk_info
+if (H5_ZLIB_HEADER)
+  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/chunk_info.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
+endif ()
 add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c)
 target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}")
 target_compile_definitions(chunk_info PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")
@@ -498,6 +504,13 @@ if (HDF5_ENABLE_FORMATTERS)
 endif ()
 
 #-- Adding test for direct_chunk
+if (H5_ZLIB_HEADER)
+  message(VERBOSE "H5_ZLIB_HEADER for direct_chunk=${H5_ZLIB_HEADER}")
+
+  set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/direct_chunk.c
+               PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
+  )
+endif ()
 add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c)
 target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}")
 target_compile_definitions(direct_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}")

test/chunk_info.c

@@ -40,7 +40,12 @@
 
 #include "testhdf5.h"
 #ifdef H5_HAVE_FILTER_DEFLATE
-#include "zlib.h"
+#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
+#define H5_ZLIB_HEADER "zlib.h"
+#endif
+#if defined(H5_ZLIB_HEADER)
+#include H5_ZLIB_HEADER /* "zlib.h" */
+#endif
 #endif
 
 /* Test file names, using H5F_libver_t as indices */
@@ -534,7 +539,11 @@ test_get_chunk_info_highest_v18(hid_t fapl)
     z_dst = (Bytef *)inbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Set the chunk size to the compressed chunk size */
     chunk_size = (hsize_t)z_dst_nbytes;

test/direct_chunk.c

@@ -12,12 +12,14 @@
 
 #include "h5test.h"
 
+#ifdef H5_HAVE_FILTER_DEFLATE
 #if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER)
 #define H5_ZLIB_HEADER "zlib.h"
 #endif
 #if defined(H5_ZLIB_HEADER)
 #include H5_ZLIB_HEADER /* "zlib.h" */
 #endif
+#endif
 
 #define FILE_NAME "direct_chunk.h5"
 
@@ -205,7 +207,11 @@ test_direct_chunk_write(hid_t file)
     z_dst  = (Bytef *)outbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Check for various zlib errors */
     if (Z_BUF_ERROR == ret) {
@@ -284,7 +290,11 @@ test_direct_chunk_write(hid_t file)
     z_dst  = (Bytef *)outbuf;
 
     /* Perform compression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+    ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#else
     ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+#endif
 
     /* Check for various zlib errors */
     if (Z_BUF_ERROR == ret) {
@@ -1522,8 +1532,12 @@ test_direct_chunk_read_no_cache(hid_t file)
             if (filter_mask != 0)
                 goto error;
 
-            /* Perform decompression from the source to the destination buffer */
+                /* Perform decompression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+            ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#else
             ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#endif
 
             /* Check for various zlib errors */
             if (Z_BUF_ERROR == ret) {
@@ -1713,8 +1727,12 @@ test_direct_chunk_read_cache(hid_t file, bool flush)
             if (filter_mask != 0)
                 goto error;
 
-            /* Perform decompression from the source to the destination buffer */
+                /* Perform decompression from the source to the destination buffer */
+#if defined(H5_HAVE_ZLIBNG_H)
+            ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#else
             ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes);
+#endif
 
             /* Check for various zlib errors */
             if (Z_BUF_ERROR == ret) {

tools/test/perform/CMakeLists.txt

@@ -112,6 +112,12 @@ endif ()
 set (zip_perf_SOURCES
     ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/zip_perf.c
 )
+if (H5_ZLIB_HEADER)
+  message (VERBOSE "H5_ZLIB_HEADER for zip_perf=${H5_ZLIB_HEADER}")
+  if (HDF5_USE_ZLIB_NG)
+    add_compile_definitions(H5_HAVE_ZLIBNG_H=1 H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
+  endif ()
+endif ()
 add_executable (zip_perf ${zip_perf_SOURCES})
 target_include_directories (zip_perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
 if (NOT BUILD_SHARED_LIBS)