Merge branch 'OSGeo:master' into master

a0x8o · Feb 13, 2024 · 1c912b6 · 1c912b6
2 parents f30dd4b + d7aed6e
commit 1c912b6
Show file tree

Hide file tree

Showing 176 changed files with 4,925 additions and 2,943 deletions.
diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml
@@ -43,10 +43,7 @@ jobs:
             ${{ runner.os }}-${{ env.cache-name }}
     - name: Install dependency
       run: |
-        # This currently fails as of https://lists.osgeo.org/pipermail/ubuntu/2023-October/002046.html
-        # sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
-        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6B827C12C2D425E227EDCA75089EBE08314DF160
-        sudo add-apt-repository -y http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu/
+        sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
         sudo apt-get update
         sudo apt-get install -y -q bison libaec-dev libjpeg-dev libgif-dev liblzma-dev libzstd-dev libgeos-dev git \
            libcurl4-gnutls-dev libproj-dev libxml2-dev  libxerces-c-dev libnetcdf-dev netcdf-bin \
@@ -554,20 +551,22 @@ jobs:
       run: cat $GITHUB_WORKSPACE/build/gdal.pc
 
   build-mac:
-    runs-on: macOS-latest
+    # M1 MacOS runner: https://github.blog/changelog/2024-01-30-github-actions-introducing-the-new-m1-macos-runner-available-to-open-source/
+    runs-on: macOS-14
     env:
       # Disable Arrow/Parquet because the VM provides libraries in /usr/local/lib/
       # that cause Illegal instruction error when running tests. I suspect the
       # Arrow/Parquet libraries to be built with AVX2 support but the VM worker
       # doesn't support it.
       # Disable use of libkml since boost headers cannot be found
-      CMAKE_OPTIONS: -DCFITSIO_ROOT=/usr/local/opt/cfitsio  -DPoppler_ROOT=/usr/local/opt/poppler -DPROJ_ROOT=/usr/local/opt/proj -DSPATIALITE_ROOT=/usr/local/opt/libspatialite -DPostgreSQL_ROOT=/usr/local/opt/libpq -DEXPAT_ROOT=/usr/local/opt/expat -DXercesC_ROOT=/usr/local/opt/xerces-c -DSQLite3_ROOT=/usr/local/opt/sqlite -DOpenSSL_ROOT=/usr/local/opt/openssl -DPNG_ROOT=/usr/local/opt/libpng -DJPEG_ROOT=/usr/local/opt/jpeg-turbo -DEXPECTED_JPEG_LIB_VERSION=80 -DOpenJPEG_ROOT=/usr/local/opt/openjpeg -DCURL_ROOT=/usr/local/opt/curl -DGDAL_USE_ARROW=OFF -DGDAL_USE_PARQUET=OFF -DGDAL_USE_LIBKML=OFF
+      # Disable ODBC since the postinstall checks don't manage to find sql.h
+      CMAKE_OPTIONS: -DCFITSIO_ROOT=/opt/homebrew/Cellar/cfitsio  -DPoppler_ROOT=/opt/homebrew/Cellar/poppler -DPROJ_ROOT=/opt/homebrew/Cellar/proj -DSPATIALITE_ROOT=/opt/homebrew/Cellar/libspatialite -DPostgreSQL_ROOT=/opt/homebrew/Cellar/libpq -DEXPAT_ROOT=/opt/homebrew/Cellar/expat -DXercesC_ROOT=/opt/homebrew/Cellar/xerces-c -DSQLite3_ROOT=/opt/homebrew/Cellar/sqlite -DOpenSSL_ROOT=/opt/homebrew/Cellar/openssl -DPNG_ROOT=/opt/homebrew/Cellar/libpng -DJPEG_ROOT=/opt/homebrew/Cellar/jpeg-turbo -DEXPECTED_JPEG_LIB_VERSION=80 -DOpenJPEG_ROOT=/opt/homebrew/Cellar/openjpeg -DCURL_ROOT=/opt/homebrew/Cellar/curl -DGDAL_USE_LIBKML=OFF -DGDAL_USE_ODBC=OFF
       cache-name: cmake-macos
     steps:
     - name: Setup xcode
       uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
       with:
-        xcode-version: 14.2
+        xcode-version: 14.3
     - name: Checkout GDAL
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: Setup cache
@@ -583,18 +582,17 @@ jobs:
         cmake --version
     - name: Install dependency
       run: |
-        rm -rf /usr/local/lib/node_modules # conflicts with node@18 from brew
+        rm -rf /opt/homebrew/lib/node_modules # conflicts with node@18 from brew
         brew update
         brew outdated
         brew unlink python
         brew install --overwrite [email protected] [email protected]
         brew install postgresql || brew link postgresql
         brew install pkg-config freexl libxml2 libspatialite geos proj libgeotiff openjpeg giflib libaec postgis poppler doxygen unixodbc jpeg-turbo aom jpeg-xl libheif libarchive libkml boost
         brew install ccache swig
+        brew link --force sqlite
         # gdal is automatically installed as a dependency for postgis
         brew uninstall --ignore-dependencies gdal
-        # Remove broken Arrow as it causes CMake detection issue
-        rm -rf /usr/local/lib/cmake/Arrow
     - name: Configure ccache
       run: |
           echo CCACHE_BASEDIR=$PWD >> ${GITHUB_ENV}
@@ -605,7 +603,7 @@ jobs:
     - name: Configure
       run: |
         mkdir -p $GITHUB_WORKSPACE/build
-        export PKG_CONFIG_PATH=/usr/local/opt/libxml2/lib/pkgconfig
+        export PKG_CONFIG_PATH=/opt/homebrew/Cellar/libxml2/lib/pkgconfig
         # FIXME: remove -DBUILD_PYTHON_BINDINGS=OFF. Python tests fail with "ModuleNotFoundError: No module named '_gdal'" with macos-12
         cmake -Werror=dev -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal ${CMAKE_OPTIONS} -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror -DUSE_CCACHE=ON -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DWERROR_DEV_FLAG="-Werror=dev" -DBUILD_PYTHON_BINDINGS=OFF
         #$(cat $GITHUB_WORKSPACE/build/python_executable.txt) -m pip install --user -r $GITHUB_WORKSPACE/autotest/requirements.txt
@@ -630,7 +628,7 @@ jobs:
         ./autotest/postinstall/test_gdal-config.sh $GITHUB_WORKSPACE/install-gdal
     - name: Test post-install usage (with CMake)
       run: |
-        ./autotest/postinstall/test_cmake.sh $GITHUB_WORKSPACE/install-gdal
+        ./autotest/postinstall/test_cmake.sh $GITHUB_WORKSPACE/install-gdal --dynamic --disable-odbc
     - name: ccache statistics
       run: ccache -s
 

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -1,8 +1,12 @@
 name: "CodeQL"
 
 on:
-  push:
-  pull_request:
+    push:
+        paths-ignore:
+            - 'doc/**'
+    pull_request:
+        paths-ignore:
+            - 'doc/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
@@ -41,7 +45,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2
+      uses: github/codeql-action/init@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -144,6 +148,6 @@ jobs:
         key: ${{ steps.restore-cache.outputs.cache-primary-key }}
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2
+      uses: github/codeql-action/analyze@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
       with:
         category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -71,6 +71,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2
+        uses: github/codeql-action/upload-sarif@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
         with:
           sarif_file: results.sarif
diff --git a/apps/gdalbuildvrt_lib.cpp b/apps/gdalbuildvrt_lib.cpp
@@ -42,8 +42,9 @@
 
 #include <algorithm>
 #include <memory>
-#include <vector>
 #include <set>
+#include <string>
+#include <vector>
 
 #include "commonutils.h"
 #include "cpl_conv.h"

diff --git a/apps/gdalmdimtranslate_bin.cpp b/apps/gdalmdimtranslate_bin.cpp
@@ -44,6 +44,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg = nullptr)
             "                         [-if <format>]... [-of <format>]\n"
             "                         [-co <NAME>=<VALUE>]...\n"
             "                         [-array <array_spec>]...\n"
+            "                         [-arrayoption <NAME>=<VALUE>]...\n"
             "                         [-group <group_spec>]...\n"
             "                         [-subset <subset_spec>]...\n"
             "                         [-scaleaxes <scaleaxes_spec>]\n"

diff --git a/apps/gdalmdimtranslate_lib.cpp b/apps/gdalmdimtranslate_lib.cpp
@@ -45,6 +45,7 @@ struct GDALMultiDimTranslateOptions
     std::string osFormat{};
     CPLStringList aosCreateOptions{};
     std::vector<std::string> aosArraySpec{};
+    CPLStringList aosArrayOptions{};
     std::vector<std::string> aosSubset{};
     std::vector<std::string> aosScaleFactor{};
     std::vector<std::string> aosGroup{};
@@ -1293,7 +1294,8 @@ static bool CopyGroup(
         }
     }
 
-    auto arrayNames = poSrcGroup->GetMDArrayNames();
+    auto arrayNames =
+        poSrcGroup->GetMDArrayNames(psOptions->aosArrayOptions.List());
     for (const auto &name : arrayNames)
     {
         if (!TranslateArray(oDimRemapper, nullptr, name, poSrcRootGroup,
@@ -1542,7 +1544,8 @@ CopyToNonMultiDimensionalDriver(GDALDriver *poDriver, const char *pszDest,
     }
     else
     {
-        auto srcArrayNames = poRG->GetMDArrayNames();
+        auto srcArrayNames = poRG->GetMDArrayNames(
+            psOptions ? psOptions->aosArrayOptions.List() : nullptr);
         for (const auto &srcArrayName : srcArrayNames)
         {
             auto tmpArray = poRG->OpenMDArray(srcArrayName);
@@ -1725,7 +1728,8 @@ GDALMultiDimTranslate(const char *pszDest, GDALDatasetH hDstDS, int nSrcCount,
     GDALDataset *poTmpSrcDS = poSrcDS;
     if (psOptions &&
         (!psOptions->aosArraySpec.empty() || !psOptions->aosGroup.empty() ||
-         !psOptions->aosSubset.empty() || !psOptions->aosScaleFactor.empty()))
+         !psOptions->aosSubset.empty() || !psOptions->aosScaleFactor.empty() ||
+         !psOptions->aosArrayOptions.empty()))
     {
         poTmpDS.reset(VRTDataset::CreateMultiDimensional("", nullptr, nullptr));
         CPLAssert(poTmpDS);
@@ -1832,7 +1836,11 @@ GDALMultiDimTranslateOptions *GDALMultiDimTranslateOptionsNew(
             ++i;
             psOptions->aosArraySpec.push_back(papszArgv[i]);
         }
-
+        else if (i < argc - 1 && EQUAL(papszArgv[i], "-arrayoption"))
+        {
+            ++i;
+            psOptions->aosArrayOptions.AddString(papszArgv[i]);
+        }
         else if (i < argc - 1 && EQUAL(papszArgv[i], "-group"))
         {
             ++i;

diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp
@@ -57,6 +57,7 @@
 #include "cpl_threadsafe_queue.hpp"
 
 #include <atomic>
+#include <cmath>
 #include <limits>
 #include <fstream>
 #include <string>
@@ -4889,4 +4890,137 @@ TEST_F(test_cpl, VSIGetCanonicalFilename)
     VSIUnlink(osLC.c_str());
 }
 
+TEST_F(test_cpl, CPLStrtod)
+{
+    {
+        const char *pszVal = "5";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 5.0);
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+
+    {
+        const char *pszVal = "5 foo";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 5.0);
+        EXPECT_EQ(pszEnd, pszVal + 1);
+    }
+
+    {
+        const char *pszVal = "foo";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 0.0);
+        EXPECT_EQ(pszEnd, pszVal);
+    }
+
+    {
+        const char *pszVal = "-inf";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  -std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "-Inf";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  -std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "-INF";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  -std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "-Infinity";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  -std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "-1.#INF";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  -std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+
+    {
+        const char *pszVal = "inf";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "Inf";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "INF";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "Infinity";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "1.#INF";
+        char *pszEnd = nullptr;
+        EXPECT_EQ(CPLStrtod(pszVal, &pszEnd),
+                  std::numeric_limits<double>::infinity());
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+
+    {
+        const char *pszVal = "-1.#QNAN";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "-1.#IND";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "1.#QNAN";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "1.#SNAN";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "NaN";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+    {
+        const char *pszVal = "nan";
+        char *pszEnd = nullptr;
+        EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd)));
+        EXPECT_EQ(pszEnd, pszVal + strlen(pszVal));
+    }
+}
+
 }  // namespace
diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp
@@ -2969,7 +2969,7 @@ TEST_F(test_gdal, GDALDatasetReportError)
     CPLPushErrorHandler(CPLQuietErrorHandler);
     poSrcDS->ReportError("%foo", CE_Warning, CPLE_AppDefined, "bar");
     CPLPopErrorHandler();
-    EXPECT_STREQ(CPLGetLastErrorMsg(), "bar");
+    EXPECT_STREQ(CPLGetLastErrorMsg(), "%foo: bar");
 
     CPLPushErrorHandler(CPLQuietErrorHandler);
     poSrcDS->ReportError(

diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp
@@ -425,6 +425,11 @@ TEST_F(test_ogr, geometry_get_point)
     }
 }
 
+TEST_F(test_ogr, OGR_G_CreateGeometry_unknown)
+{
+    EXPECT_EQ(OGR_G_CreateGeometry(wkbUnknown), nullptr);
+}
+
 TEST_F(test_ogr, style_manager)
 {
     OGRStyleMgrH hSM = OGR_SM_Create(nullptr);

diff --git a/autotest/gcore/bmp_read.py b/autotest/gcore/bmp_read.py
@@ -26,10 +26,13 @@
 # Boston, MA 02111-1307, USA.
 ###############################################################################
 
-
 import gdaltest
 import pytest
 
+from osgeo import gdal
+
+pytestmark = pytest.mark.require_driver("BMP")
+
 init_list = [
     ("1bit.bmp", 200),
     ("4bit_pal.bmp", 2587),
@@ -43,12 +46,19 @@
     init_list,
     ids=[tup[0].split(".")[0] for tup in init_list],
 )
-@pytest.mark.require_driver("BMP")
 def test_bmp_open(filename, checksum):
     ut = gdaltest.GDALTest("BMP", filename, 1, checksum)
     ut.testOpen()
 
 
+def test_bmp_read_more_than_4GB():
+
+    ds = gdal.Open("/vsisparse/data/bmp/huge_sparse.xml")
+    assert ds.RasterXSize == 65536
+    assert ds.RasterYSize == 65536
+    assert ds.GetRasterBand(1).ReadRaster(65535, 65535, 1, 1) == b"\0"
+
+
 def test_bmp_online_1():
 
     gdaltest.download_or_skip(

diff --git a/autotest/gcore/data/bmp/huge_header.bmp.bin b/autotest/gcore/data/bmp/huge_header.bmp.bin
diff --git a/autotest/gcore/data/bmp/huge_sparse.xml b/autotest/gcore/data/bmp/huge_sparse.xml
@@ -0,0 +1,9 @@
+<VSISparseFile>
+  <Length>12884901942</Length>
+  <SubfileRegion>
+    <Filename relative="1">huge_header.bmp.bin</Filename>
+    <DestinationOffset>0</DestinationOffset>
+    <SourceOffset>0</SourceOffset>
+    <RegionLength>128</RegionLength>
+  </SubfileRegion>
+</VSISparseFile>