Fix dataset creation failure on Ubuntu 24 (#131)

* Remove vol-tests submodule * Show HSDS log on CI failure * Fix warnings * Add Ubuntu 22/24 to CI * Fix bad length in snprintf
HDFGroup · Dec 18, 2024 · 82f89c3 · 82f89c3
1 parent 56703e8
commit 82f89c3
Show file tree

Hide file tree

Showing 12 changed files with 97 additions and 214 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -27,13 +27,18 @@ env:
   BUCKET_NAME: hsdstest
 
 jobs:
-  build_and_test_with_autotools:
+  build_and_test:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-22.04, ubuntu-latest]
+        build_system: ["autotools", "cmake"]
         python-version: ["3.10"]
-        hdf5-branch: ["hdf5_1_14"]  
+        hdf5-branch: ["hdf5_1_14", "develop"]
+        exclude:
+          # hdf5 2.0.0+ does not support autotools
+          - hdf5-branch: "develop"
+            build_system: "autotools"
 
     runs-on: ${{matrix.os}}
     steps:
@@ -50,6 +55,7 @@ jobs:
           sudo apt-get install libyajl-dev
 
       - name: Get Autotools Dependencies
+        if: matrix.build_system == 'autotools'
         run: |
           sudo apt update
           sudo apt install automake autoconf libtool libtool-bin
@@ -59,7 +65,25 @@ jobs:
         with:
           path: ${{github.workspace}}/vol-rest
 
-      - name: Autotools Configure + Build HDF5        
+      - name: CMake Configure + Build HDF5
+        if: matrix.build_system == 'cmake'
+        run: |
+          mkdir ${{github.workspace}}/hdf5/build
+          cd ./build
+          cmake \
+          -DHDF5_BUILD_HL_LIB=ON \
+          -DBUILD_SHARED_LIBS=ON \
+          -DHDF5_TEST_API=ON \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install \
+          ..
+          make 
+          make install
+        shell: bash
+        working-directory: ${{github.workspace}}/hdf5
+
+      - name: Autotools Configure + Build HDF5  
+        if: matrix.build_system == 'autotools'      
         run: |
           ./autogen.sh
           ./configure --prefix=${{github.workspace}}/hdf5install \
@@ -70,21 +94,44 @@ jobs:
         shell: bash
         working-directory: ${{github.workspace}}/hdf5
 
+      - name: CMake Configure REST VOL
+        if: matrix.build_system == 'cmake'
+        run: |
+          mkdir ./build
+          cd ./build
+          CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "Unix Makefiles" -DHDF5_ROOT=${{github.workspace}}/hdf5install \
+          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/vol-rest/install \
+          ..
+        shell: bash
+        working-directory: ${{github.workspace}}/vol-rest
+
       - name: Autotools Configure REST VOL
+        if: matrix.build_system == 'autotools'
         run: |
           ./autogen.sh
           mkdir ${{github.workspace}}/vol-rest/install
-          CFLAGS="-D_POSIX_C_SOURCE=200809L" ./configure --prefix=${{github.workspace}}/vol-rest/install --with-hdf5=${{github.workspace}}/hdf5install
+          CFLAGS="-D_POSIX_C_SOURCE=200809L" ./configure \
+          --prefix=${{github.workspace}}/vol-rest/install \
+          --with-hdf5=${{github.workspace}}/hdf5install
         shell: bash
         working-directory: ${{github.workspace}}/vol-rest
 
-      - name: Build + Install REST VOL
+      - name: Build + Install REST VOL (Autotools)
+        if: matrix.build_system == 'autotools'
         run: |
-          make
+          make -j
           make install
         shell: bash
         working-directory: ${{github.workspace}}/vol-rest/
 
+      - name: Build + Install REST VOL (CMake)
+        if: matrix.build_system == 'cmake'
+        run: |
+          make -j
+          make install
+        shell: bash
+        working-directory: ${{github.workspace}}/vol-rest/build/
+
       - uses: actions/checkout@v4
         with:
           repository: HDFGroup/hsds
@@ -114,19 +161,12 @@ jobs:
           cd ${{github.workspace}}/hsds
           pytest
 
-      - name: Install valgrind
-        run: |
-          sudo apt update
-          sudo apt install valgrind
-        working-directory: ${{ github.workspace }}
-
       # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
       - name: Fix requests version
         run: |
           pip install requests==2.31.0
 
       - name: Start HSDS
-        if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
         run: | 
               cd ${{github.workspace}}/hsds
               mkdir ${{github.workspace}}/hsdsdata &&
@@ -140,149 +180,34 @@ jobs:
         working-directory: ${{github.workspace}}/hsds
 
       - name: Test HSDS
-        if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
         run: |
               python tests/integ/setup_test.py
         working-directory: ${{github.workspace}}/hsds
 
-      - name: Test REST VOL
+      - name: Test REST VOL (Autotools)
+        if: matrix.build_system == 'autotools'
         working-directory: ${{github.workspace}}/vol-rest/
+        env:
+          HDF5_PLUGIN_PATH: ${{github.workspace}}/vol-rest/install/lib
         run: |
-              HDF5_PLUGIN_PATH=${{github.workspace}}/vol-rest/install/lib HDF5_VOL_CONNECTOR=REST ./test/test_rest_vol
-
-  build_and_test_with_cmake:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.10"]
-        hdf5-branch: ["hdf5_1_14", "develop"]  
-
-    runs-on: ${{matrix.os}}
-    steps:
-      - uses: actions/checkout@v4
-        with: 
-          repository: HDFGroup/hdf5
-          ref: ${{matrix.hdf5-branch}}
-          path: ${{github.workspace}}/hdf5
-
-      - name: Get REST VOL dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libcurl4-openssl-dev
-          sudo apt-get install libyajl-dev
-              
-      - name: Get REST VOL
-        uses: actions/checkout@v4
-        with:
-          path: ${{github.workspace}}/vol-rest
+               ./test/test_rest_vol
 
-      - name: CMake Configure + Build HDF5        
-        run: |
-          mkdir ${{github.workspace}}/hdf5/build
-          cd ./build
-          cmake \
-          -DHDF5_BUILD_HL_LIB=ON \
-          -DBUILD_SHARED_LIBS=ON -DHDF5_ENABLE_SZIP_SUPPORT=OFF \
-          -DHDF5_TEST_API=ON \
-          -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF \
-          -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHDF5_ENABLE_THREADSAFE=OFF \
-          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install \
-          ..
-          make 
-          make install
-        shell: bash
-        working-directory: ${{github.workspace}}/hdf5
-
-      - name: CMake Configure REST VOL
-        run: |
-          mkdir ./build
-          cd ./build
-          CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "Unix Makefiles" -DHDF5_ROOT=${{github.workspace}}/hdf5install \
-          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/vol-rest/install \
-          ..
-        shell: bash
-        working-directory: ${{github.workspace}}/vol-rest
-
-      - name: Build + Install REST VOL
-        run: |
-          make
-          make install
-        shell: bash
-        working-directory: ${{github.workspace}}/vol-rest/build
-
-      - uses: actions/checkout@v4
-        with:
-          repository: HDFGroup/hsds
-          path: ${{github.workspace}}/hsds
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install HSDS dependencies
-        shell: bash
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Install HSDS package
-        shell: bash
-        run: |
-          cd ${{github.workspace}}/hsds
-          pip install -e .
-          
-      - name: Run HSDS unit tests
-        shell: bash
-        run: |
-          cd ${{github.workspace}}/hsds
-          pytest
-
-      - name: Install valgrind
-        run: |
-          sudo apt update
-          sudo apt install valgrind
-        working-directory: ${{ github.workspace }}
-
-        # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
-      - name: Fix requests version
+      - name: Test REST VOL (CMake)
+        if: matrix.build_system == 'cmake'
+        working-directory: ${{github.workspace}}/vol-rest/build/
+        env:
+          HDF5_PLUGIN_PATH: ${{github.workspace}}/vol-rest/install/lib
         run: |
-          pip install requests==2.31.0
+              ctest -R "test_rest_vol" -VV
 
-      - name: Start HSDS
-        if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
-        run: | 
-            cd ${{github.workspace}}/hsds
-            mkdir ${{github.workspace}}/hsdsdata &&
-            mkdir ${{github.workspace}}/hsdsdata/hsdstest &&
-            cp admin/config/groups.default admin/config/groups.txt &&
-            cp admin/config/passwd.default admin/config/passwd.txt &&
-            cp admin/config/groups.default admin/config/groups.txt &&
-            cp admin/config/passwd.default admin/config/passwd.txt
-            ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 &
-            sleep 10
+      - name: Show HSDS Logs on Fail
+        if: ${{failure()}}
         working-directory: ${{github.workspace}}/hsds
-
-      - name: Test HSDS
-        if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
-        run: |
-              python tests/integ/setup_test.py
-        working-directory: ${{github.workspace}}/hsds
-
-      - name: Set HDF5 Plugin path
-        run: |
-              echo "HDF5_PLUGIN_PATH=${{github.workspace}}/vol-rest/build/bin/" >> $GITHUB_ENV
-              echo "HDF5_VOL_CONNECTOR=REST" >> $GITHUB_ENV
-
-      - name: Test REST VOL
-        working-directory: ${{github.workspace}}/vol-rest/build/
         run: |
-              valgrind --leak-check=full -s ctest -R "test_rest_vol" -VV
+          cat hs.log
 
 #      TODO: Attribute, dataset, link, and testhdf5 tests currently fail
 #      - name: Test REST VOL with API
 #        run: |
-#              valgrind --leak-check=full -s ctest -R "vol-rest" -VV
+#              ctest -R "vol-rest" -VV
 #        working-directory: ${{github.workspace}}/hdf5/build/
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +0,0 @@
-[submodule "test/vol-tests"]
-	path = test/vol-tests
-	url = https://github.com/HDFGroup/vol-tests

diff --git a/build_vol_autotools.sh b/build_vol_autotools.sh
@@ -163,12 +163,6 @@ if [ "$NPROCS" -eq "0" ]; then
     fi
 fi
 
-# Ensure that the HDF5 and VOL tests submodules get checked out
-if [ -z "$(ls -A ${SCRIPT_DIR}/${HDF5_DIR})" ]; then
-    git submodule init
-    git submodule update
-fi
-
 # If the user hasn't already, first build HDF5
 if [ "$build_hdf5" = true ]; then
     echo "*****************"

diff --git a/build_vol_cmake.sh b/build_vol_cmake.sh
@@ -175,12 +175,6 @@ if [ "$NPROCS" -eq "0" ]; then
     fi
 fi
 
-# Ensure that the vol-tests submodule gets checked out
-if [ -z "$(ls -A ${SCRIPT_DIR}/test/vol-tests)" ]; then
-    git submodule init
-    git submodule update
-fi
-
 # Build the REST VOL connector against HDF5.
 echo "*******************************************"
 echo "* Building REST VOL connector and test suite *"
@@ -206,20 +200,4 @@ fi
 
 echo "REST VOL built"
 
-# Clean out the old CMake cache
-rm -f "${BUILD_DIR}/CMakeCache.txt"
-
-# Configure vol-tests
-
-mkdir -p "${BUILD_DIR}/tests/vol-tests"
-cd "${BUILD_DIR}/tests/vol-tests"
-
-CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "${CMAKE_GENERATOR}"  "-DHDF5_DIR=${HDF5_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" "${CONNECTOR_DEBUG_OPT}" "${CURL_DEBUG_OPT}" "${MEM_TRACK_OPT}" "${THREAD_SAFE_OPT}" "${SCRIPT_DIR}/test/vol-tests"
-
-echo "Build files generated for vol-tests"
-
-make || exit 1
-
-echo "VOL tests built"
-
 exit 0