diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 436e145f36f6..c6c73bb69187 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -29,7 +29,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index c2fd53ccad34..8111b4b4f575 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -141,12 +141,12 @@ jobs: - name: Install pdfium run: | - wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ + wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && sudo chown -R root:root install \ && sudo mv install/lib/* /usr/lib/ \ && sudo mv install/include/* /usr/include/ \ - && sudo rm -rf install-ubuntu2004-rev5952.tar.gz install \ + && sudo rm -rf install-ubuntu2004-rev6309.tar.gz install \ && sudo apt-get update -y \ && sudo apt-get install -y --fix-missing --no-install-recommends liblcms2-dev - name: Configure ccache @@ -409,7 +409,7 @@ jobs: shell: pwsh run: | echo "JAVA_HOME=$env:JAVA_HOME_11_X64" >> %GITHUB_ENV% - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@392cf345b1784333caa1a1185081c71e6ffd61bc # v3.0.2 with: activate-environment: gdalenv miniforge-variant: Mambaforge @@ -417,8 +417,6 @@ jobs: use-mamba: true auto-update-conda: true use-only-tar-bz2: false - - run: | - cmake --version - name: Install dependency shell: bash -l {0} run: | @@ -429,12 +427,16 @@ jobs: cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \ sqlite tiledb zstd cryptopp cgal doxygen librttopo libkml openssl xz \ openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \ - arrow-cpp pyarrow libaec + arrow-cpp pyarrow libaec cmake + - name: Check CMake version + shell: bash -l {0} + run: | + cmake --version - name: Install pdfium shell: bash -l {0} run: | - curl -LOs https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-win10-vs2019-x64-rev5952.zip - unzip install-win10-vs2019-x64-rev5952.zip + curl -LOs https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-win10-vs2019-x64-rev6309.zip + unzip install-win10-vs2019-x64-rev6309.zip mv install install-pdfium - name: Remove conflicting libraries @@ -452,7 +454,7 @@ jobs: # Build PDF driver as plugin due to the PDFium build including libopenjp2 symbols which would conflict with external libopenjp2 run: | mkdir -p $GITHUB_WORKSPACE/build - cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA_PREFIX}" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON + cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config Release -j 2 @@ -504,7 +506,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@392cf345b1784333caa1a1185081c71e6ffd61bc # v3.0.2 with: activate-environment: gdalenv miniforge-variant: Mambaforge @@ -512,17 +514,19 @@ jobs: use-mamba: true auto-update-conda: true use-only-tar-bz2: false - - run: | - cmake --version - name: Install dependency shell: bash -l {0} run: | - conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml + conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml cmake + - name: Check CMake version + shell: bash -l {0} + run: | + cmake --version - name: Configure shell: bash -l {0} run: | mkdir -p $GITHUB_WORKSPACE/build - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 @@ -530,12 +534,12 @@ jobs: shell: bash -l {0} run: | rm -f build/CMakeCache.txt - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Configure with even less dependencies, and disabling all optional drivers shell: bash -l {0} run: | rm -f build/CMakeCache.txt - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 @@ -649,7 +653,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@392cf345b1784333caa1a1185081c71e6ffd61bc # v3.0.2 with: activate-environment: gdalenv python-version: 3.9 diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 91dddf6bd13b..c0a53bca8c6b 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -83,7 +83,7 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 doxygen: runs-on: ubuntu-latest diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index af089a34730a..daa0604025ba 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0 + uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -61,6 +61,7 @@ jobs: - name: Install dependencies run: | + sudo apt-get update sudo apt-get install -y ccache cmake g++ swig python3-numpy libproj-dev libqhull-dev sudo apt-get install -y \ libblosc-dev \ @@ -148,6 +149,6 @@ jobs: key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0 + uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index e0d044655ccf..ddcfa46891e9 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: true matrix: - platform: ['ubuntu-latest','windows-latest','macos-latest'] + platform: ['ubuntu-latest','windows-latest','macos-latest','macos-14'] env: GHA_CI_PLATFORM: ${{ matrix.platform }} @@ -47,7 +47,7 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-${{ steps.get-date.outputs.today }}-conda-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@392cf345b1784333caa1a1185081c71e6ffd61bc # v3.0.2 with: #miniforge-variant: Mambaforge miniforge-version: latest @@ -72,7 +72,7 @@ jobs: source ../ci/travis/conda/compile.sh working-directory: ./gdal-feedstock - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: ${{ matrix.platform }}-conda-package path: ./gdal-feedstock/packages/ diff --git a/.github/workflows/delete_untagged_containers.yml b/.github/workflows/delete_untagged_containers.yml index b6321b784857..4510b0c21ca7 100644 --- a/.github/workflows/delete_untagged_containers.yml +++ b/.github/workflows/delete_untagged_containers.yml @@ -14,6 +14,7 @@ jobs: delete-untagged-containers: name: Delete all containers from gdal-deps without tags runs-on: ubuntu-latest + if: github.repository == 'OSGeo/gdal' steps: - name: Delete all containers from gdal-deps without tags uses: Chizkiyahu/delete-untagged-ghcr-action@bbbab219998078a91c9b283dac9389b825894603 # v3.2.0 diff --git a/.github/workflows/doc_build.yml b/.github/workflows/doc_build.yml index 05dbf587d7d7..0b93863643c9 100644 --- a/.github/workflows/doc_build.yml +++ b/.github/workflows/doc_build.yml @@ -81,15 +81,15 @@ jobs: # run: | # make spelling # working-directory: ./doc - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: PDF path: doc/build/latex/gdal.pdf - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: HTML path: doc/build/html/* - #- uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + #- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 # with: # name: Misspelled # path: doc/build/spelling/output.txt diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 22d6dbbcf307..7f855b968121 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -322,14 +322,14 @@ jobs: docker push ${CONTAINER_NAME_FULL} - name: Upload coverage artifacts - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: ${{ matrix.id == 'coverage' }} with: name: coverage_index.html path: build-${{ matrix.id }}/coverage_html/index.html - name: Upload coverage artifacts - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: ${{ matrix.id == 'coverage' }} with: name: HTML diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1dc6e6d7e747..28e3d5e5bf23 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@392cf345b1784333caa1a1185081c71e6ffd61bc # v3.0.2 with: channels: conda-forge auto-update-conda: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5c8033037594..71e2979ebde0 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -63,7 +63,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: SARIF file path: results.sarif @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0 + uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: sarif_file: results.sarif diff --git a/.github/workflows/ubuntu_20.04/Dockerfile.ci b/.github/workflows/ubuntu_20.04/Dockerfile.ci index 54798996984c..58b223bcb2b3 100644 --- a/.github/workflows/ubuntu_20.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_20.04/Dockerfile.ci @@ -182,12 +182,12 @@ RUN mkdir geos \ && rm -rf geos # Install pdfium -RUN wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ +RUN wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && chown -R root:root install \ && mv install/lib/* /usr/lib/ \ && mv install/include/* /usr/include/ \ - && rm -rf install-ubuntu2004-rev5952.tar.gz install + && rm -rf install-ubuntu2004-rev6309.tar.gz install # HANA: client side # Install hdbsql tool diff --git a/.github/workflows/ubuntu_22.04/Dockerfile.ci b/.github/workflows/ubuntu_22.04/Dockerfile.ci index d257da558c10..a983e3aceb49 100644 --- a/.github/workflows/ubuntu_22.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_22.04/Dockerfile.ci @@ -112,6 +112,13 @@ RUN curl -L -O https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_A && rm -rf FileGDB_API-64gcc51 \ && ldconfig +# Oracle : client side (proprietary software) +RUN curl -L -O https://download.oracle.com/otn_software/linux/instantclient/199000/instantclient-basic-linux.x64-19.9.0.0.0dbru.zip \ + && curl -L -O https://download.oracle.com/otn_software/linux/instantclient/199000/instantclient-sdk-linux.x64-19.9.0.0.0dbru.zip \ + && unzip instantclient-basic-linux.x64-19.9.0.0.0dbru.zip -d /opt \ + && unzip instantclient-sdk-linux.x64-19.9.0.0.0dbru.zip -d opt \ + && apt-get install -y libaio1 + COPY requirements.txt /tmp/ RUN python3 -m pip install -U -r /tmp/requirements.txt diff --git a/.github/workflows/ubuntu_22.04/build.sh b/.github/workflows/ubuntu_22.04/build.sh index b797888baf72..79057fbf42ba 100755 --- a/.github/workflows/ubuntu_22.04/build.sh +++ b/.github/workflows/ubuntu_22.04/build.sh @@ -2,11 +2,15 @@ set -e +LD_LIBRARY_PATH="/opt/instantclient_19_9:/opt/instantclient_19_9/lib:${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH + cmake ${GDAL_SOURCE_DIR:=..} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_FLAGS="-Werror" \ -DCMAKE_CXX_FLAGS="-Werror" \ -DUSE_CCACHE=ON \ + -DOracle_ROOT=/opt/instantclient_19_9 \ -DGDAL_USE_GEOTIFF_INTERNAL:BOOL=ON \ -DGDAL_USE_TIFF_INTERNAL:BOOL=ON diff --git a/.github/workflows/ubuntu_22.04/test.sh b/.github/workflows/ubuntu_22.04/test.sh index 5addcb8694eb..9af1dd53f7b7 100755 --- a/.github/workflows/ubuntu_22.04/test.sh +++ b/.github/workflows/ubuntu_22.04/test.sh @@ -4,6 +4,9 @@ set -e . ../scripts/setdevenv.sh +LD_LIBRARY_PATH="/opt/instantclient_19_9:/opt/instantclient_19_9/lib:${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH + export PYTEST="python3 -m pytest -vv -p no:sugar --color=no" # Run C++ tests diff --git a/apps/gdalbuildvrt_bin.cpp b/apps/gdalbuildvrt_bin.cpp index 84feff19088d..0437ca0f3335 100644 --- a/apps/gdalbuildvrt_bin.cpp +++ b/apps/gdalbuildvrt_bin.cpp @@ -56,6 +56,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg) " [-srcnodata \"[ ]...\"] [-vrtnodata " "\"[ ]...\"\n" " [-ignore_srcmaskband]\n" + " [-nodata_max_mask_threshold ]\n" " [-a_srs ]\n" " [-r " "{nearest|bilinear|cubic|cubicspline|lanczos|average|mode}]\n" diff --git a/apps/gdalbuildvrt_lib.cpp b/apps/gdalbuildvrt_lib.cpp index b73d5efea156..b07f416cf5a9 100644 --- a/apps/gdalbuildvrt_lib.cpp +++ b/apps/gdalbuildvrt_lib.cpp @@ -250,6 +250,8 @@ class VRTBuilder char *pszResampling = nullptr; char **papszOpenOptions = nullptr; bool bUseSrcMaskBand = true; + bool bNoDataFromMask = false; + double dfMaskValueThreshold = 0; /* Internal variables */ char *pszProjectionRef = nullptr; @@ -285,6 +287,7 @@ class VRTBuilder int bAllowProjectionDifference, int bAddAlpha, int bHideNoData, int nSubdataset, const char *pszSrcNoData, const char *pszVRTNoData, bool bUseSrcMaskBand, + bool bNoDataFromMask, double dfMaskValueThreshold, const char *pszOutputSRS, const char *pszResampling, const char *const *papszOpenOptionsIn); @@ -306,7 +309,8 @@ VRTBuilder::VRTBuilder( double maxYIn, int bSeparateIn, int bAllowProjectionDifferenceIn, int bAddAlphaIn, int bHideNoDataIn, int nSubdatasetIn, const char *pszSrcNoDataIn, const char *pszVRTNoDataIn, - bool bUseSrcMaskBandIn, const char *pszOutputSRSIn, + bool bUseSrcMaskBandIn, bool bNoDataFromMaskIn, + double dfMaskValueThresholdIn, const char *pszOutputSRSIn, const char *pszResamplingIn, const char *const *papszOpenOptionsIn) : bStrict(bStrictIn) { @@ -366,6 +370,8 @@ VRTBuilder::VRTBuilder( pszOutputSRS = (pszOutputSRSIn) ? CPLStrdup(pszOutputSRSIn) : nullptr; pszResampling = (pszResamplingIn) ? CPLStrdup(pszResamplingIn) : nullptr; bUseSrcMaskBand = bUseSrcMaskBandIn; + bNoDataFromMask = bNoDataFromMaskIn; + dfMaskValueThreshold = dfMaskValueThresholdIn; } /************************************************************************/ @@ -564,11 +570,14 @@ std::string VRTBuilder::AnalyseRaster(GDALDatasetH hDS, double ds_minY = ds_maxY + GDALGetRasterYSize(hDS) * padfGeoTransform[GEOTRSFRM_NS_RES]; - const int _nBands = GDALGetRasterCount(hDS); + int _nBands = GDALGetRasterCount(hDS); if (_nBands == 0) { return "Dataset has no bands"; } + if (bNoDataFromMask && + poDS->GetRasterBand(_nBands)->GetColorInterpretation() == GCI_AlphaBand) + _nBands--; GDALRasterBand *poFirstBand = poDS->GetRasterBand(1); poFirstBand->GetBlockSize(&psDatasetProperties->nBlockXSize, @@ -1299,8 +1308,20 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) static_cast(hVRTBand); VRTSimpleSource *poSimpleSource; - if (bAllowSrcNoData && - psDatasetProperties->abHasNoData[nSelBand - 1]) + if (bNoDataFromMask) + { + auto poNoDataFromMaskSource = new VRTNoDataFromMaskSource(); + poSimpleSource = poNoDataFromMaskSource; + poNoDataFromMaskSource->SetParameters( + (nVRTNoDataCount > 0) + ? ((j < nVRTNoDataCount) + ? padfVRTNoData[j] + : padfVRTNoData[nVRTNoDataCount - 1]) + : 0, + dfMaskValueThreshold); + } + else if (bAllowSrcNoData && + psDatasetProperties->abHasNoData[nSelBand - 1]) { auto poComplexSource = new VRTComplexSource(); poSimpleSource = poComplexSource; @@ -1748,6 +1769,8 @@ struct GDALBuildVRTOptions char *pszResampling; char **papszOpenOptions; bool bUseSrcMaskBand; + bool bNoDataFromMask; + double dfMaskValueThreshold; /*! allow or suppress progress monitor and other non-error output */ int bQuiet; @@ -1926,7 +1949,8 @@ GDALDatasetH GDALBuildVRT(const char *pszDest, int nSrcCount, psOptions->bSeparate, psOptions->bAllowProjectionDifference, psOptions->bAddAlpha, psOptions->bHideNoData, psOptions->nSubdataset, psOptions->pszSrcNoData, psOptions->pszVRTNoData, - psOptions->bUseSrcMaskBand, psOptions->pszOutputSRS, + psOptions->bUseSrcMaskBand, psOptions->bNoDataFromMask, + psOptions->dfMaskValueThreshold, psOptions->pszOutputSRS, psOptions->pszResampling, psOptions->papszOpenOptions); GDALDatasetH hDstDS = static_cast( @@ -1997,6 +2021,8 @@ GDALBuildVRTOptionsNew(char **papszArgv, psOptions->pfnProgress = GDALDummyProgress; psOptions->pProgressData = nullptr; psOptions->bUseSrcMaskBand = true; + psOptions->bNoDataFromMask = false; + psOptions->dfMaskValueThreshold = 0; psOptions->bStrict = false; /* -------------------------------------------------------------------- */ @@ -2183,6 +2209,12 @@ GDALBuildVRTOptionsNew(char **papszArgv, { psOptions->bUseSrcMaskBand = false; } + else if (EQUAL(papszArgv[iArg], "-nodata_max_mask_threshold") && + iArg + 1 < argc) + { + psOptions->bNoDataFromMask = true; + psOptions->dfMaskValueThreshold = CPLAtofM(papszArgv[++iArg]); + } else if (papszArgv[iArg][0] == '-') { CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", diff --git a/apps/gdalinfo_lib.cpp b/apps/gdalinfo_lib.cpp index d72cbbe036ab..577fc617b962 100644 --- a/apps/gdalinfo_lib.cpp +++ b/apps/gdalinfo_lib.cpp @@ -290,30 +290,31 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) GDALGetDriverShortName(hDriver), GDALGetDriverLongName(hDriver)); } - // The list of files of a raster FileGDB is not super useful and potentially - // super long, so omit it, unless the -json mode is enabled - char **papszFileList = - (!bJson && EQUAL(GDALGetDriverShortName(hDriver), "OpenFileGDB")) - ? nullptr - : GDALGetFileList(hDataset); - - if (papszFileList == nullptr || *papszFileList == nullptr) + if (psOptions->bShowFileList) { - if (bJson) + // The list of files of a raster FileGDB is not super useful and potentially + // super long, so omit it, unless the -json mode is enabled + char **papszFileList = + (!bJson && EQUAL(GDALGetDriverShortName(hDriver), "OpenFileGDB")) + ? nullptr + : GDALGetFileList(hDataset); + + if (!papszFileList || *papszFileList == nullptr) { - json_object *poFiles = json_object_new_array(); - json_object_object_add(poJsonObject, "files", poFiles); + if (bJson) + { + json_object *poFiles = json_object_new_array(); + json_object_object_add(poJsonObject, "files", poFiles); + } + else + { + Concat(osStr, psOptions->bStdoutOutput, + "Files: none associated\n"); + } } else { - Concat(osStr, psOptions->bStdoutOutput, "Files: none associated\n"); - } - } - else - { - if (bJson) - { - if (psOptions->bShowFileList) + if (bJson) { json_object *poFiles = json_object_new_array(); @@ -327,20 +328,17 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) json_object_object_add(poJsonObject, "files", poFiles); } - } - else - { - Concat(osStr, psOptions->bStdoutOutput, "Files: %s\n", - papszFileList[0]); - if (psOptions->bShowFileList) + else { + Concat(osStr, psOptions->bStdoutOutput, "Files: %s\n", + papszFileList[0]); for (int i = 1; papszFileList[i] != nullptr; i++) Concat(osStr, psOptions->bStdoutOutput, " %s\n", papszFileList[i]); } } + CSLDestroy(papszFileList); } - CSLDestroy(papszFileList); if (bJson) { diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp index dd1f56814fb3..25a5929bf5d2 100644 --- a/apps/gdaltindex_bin.cpp +++ b/apps/gdaltindex_bin.cpp @@ -51,7 +51,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg) " [-skip_different_projection] [-t_srs ]\n" " [-src_srs_name field_name] [-src_srs_format " "{AUTO|WKT|EPSG|PROJ}]\n" - " [-lyr_name ]\n" + " [-lyr_name ] [-lco =]...\n" " [-gti_filename ]\n" " [-tr ] [-te " "]\n" diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 59a6bbe124eb..79770cd2030b 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -75,6 +75,7 @@ struct GDALTileIndexOptions std::string osFormat{}; std::string osIndexLayerName{}; std::string osLocationField = "location"; + CPLStringList aosLCO{}; std::string osTargetSRS{}; bool bWriteAbsolutePath = false; bool bSkipDifferentProjection = false; @@ -504,9 +505,9 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, oSRS = *poSrcSRS; } - poLayer = poTileIndexDS->CreateLayer(osLayerName.c_str(), - oSRS.IsEmpty() ? nullptr : &oSRS, - wkbPolygon, nullptr); + poLayer = poTileIndexDS->CreateLayer( + osLayerName.c_str(), oSRS.IsEmpty() ? nullptr : &oSRS, wkbPolygon, + psOptions->aosLCO.List()); if (!poLayer) return nullptr; @@ -1193,6 +1194,11 @@ GDALTileIndexOptionsNew(char **papszArgv, CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); psOptions->osLocationField = papszArgv[++iArg]; } + else if (strcmp(papszArgv[iArg], "-lco") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->aosLCO.AddString(papszArgv[++iArg]); + } else if (strcmp(papszArgv[iArg], "-t_srs") == 0) { CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 8d924e07bd8d..ed17f2290f74 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -2705,8 +2705,8 @@ static GDALDatasetH GDALWarpDirect(const char *pszDest, GDALDatasetH hDstDS, // Check if transformation is inversible { - double dfX = GDALGetRasterXSize(hDstDS) / 2; - double dfY = GDALGetRasterYSize(hDstDS) / 2; + double dfX = GDALGetRasterXSize(hDstDS) / 2.0; + double dfY = GDALGetRasterYSize(hDstDS) / 2.0; double dfZ = 0; int bSuccess = false; const auto nErrorCounterBefore = CPLGetErrorCounter(); @@ -4818,16 +4818,56 @@ static void RemoveZeroWidthSlivers(OGRGeometry *poGeom) const OGRwkbGeometryType eType = wkbFlatten(poGeom->getGeometryType()); if (eType == wkbMultiPolygon) { - for (auto poSubGeom : *(poGeom->toMultiPolygon())) + auto poMP = poGeom->toMultiPolygon(); + int nNumGeometries = poMP->getNumGeometries(); + for (int i = 0; i < nNumGeometries; /* incremented in loop */) { - RemoveZeroWidthSlivers(poSubGeom); + auto poPoly = poMP->getGeometryRef(i); + RemoveZeroWidthSlivers(poPoly); + if (poPoly->IsEmpty()) + { + CPLDebug("WARP", + "RemoveZeroWidthSlivers: removing empty polygon"); + poMP->removeGeometry(i, /* bDelete = */ true); + --nNumGeometries; + } + else + { + ++i; + } } } else if (eType == wkbPolygon) { - for (auto poSubGeom : *(poGeom->toPolygon())) + auto poPoly = poGeom->toPolygon(); + if (auto poExteriorRing = poPoly->getExteriorRing()) { - RemoveZeroWidthSlivers(poSubGeom); + RemoveZeroWidthSlivers(poExteriorRing); + if (poExteriorRing->getNumPoints() < 4) + { + poPoly->empty(); + return; + } + } + int nNumInteriorRings = poPoly->getNumInteriorRings(); + for (int i = 0; i < nNumInteriorRings; /* incremented in loop */) + { + auto poRing = poPoly->getInteriorRing(i); + RemoveZeroWidthSlivers(poRing); + if (poRing->getNumPoints() < 4) + { + CPLDebug( + "WARP", + "RemoveZeroWidthSlivers: removing empty interior ring"); + constexpr int OFFSET_EXTERIOR_RING = 1; + poPoly->removeRing(i + OFFSET_EXTERIOR_RING, + /* bDelete = */ true); + --nNumInteriorRings; + } + else + { + ++i; + } } } else if (eType == wkbLineString) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 4a74a870a731..a3c404c53133 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -4324,6 +4324,21 @@ SetupTargetLayer::Setup(OGRLayer *poSrcLayer, const char *pszNewLayerName, bPreserveFID = false; } } + // Detect scenario of converting from GPX to a format like GPKG + // Cf https://github.com/OSGeo/gdal/issues/9225 + else if (!bPreserveFID && !m_bUnsetFid && !bAppend && + m_poSrcDS->GetDriver() && + EQUAL(m_poSrcDS->GetDriver()->GetDescription(), "GPX") && + pszDestCreationOptions && + (strstr(pszDestCreationOptions, "='FID'") != nullptr || + strstr(pszDestCreationOptions, "=\"FID\"") != nullptr) && + CSLFetchNameValue(m_papszLCO, "FID") == nullptr) + { + CPLDebug("GDALVectorTranslate", + "Forcing -preserve_fid because source is GPX and layers " + "have FID cross references"); + bPreserveFID = true; + } // Detect scenario of converting GML2 with fid attribute to GPKG else if (EQUAL(m_poDstDS->GetDriver()->GetDescription(), "GPKG") && CSLFetchNameValue(m_papszLCO, "FID") == nullptr) diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index f2ab0ed538cc..5c655d4b0cb4 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -3128,6 +3128,7 @@ TEST_F(test_cpl, cpl_minixml) CPLXMLNode *psElt = CPLCreateXMLElementAndValue(psRoot, "Elt", "value"); CPLAddXMLAttributeAndValue(psElt, "attr1", "val1"); CPLAddXMLAttributeAndValue(psElt, "attr2", "val2"); + EXPECT_GE(CPLXMLNodeGetRAMUsageEstimate(psRoot), 0); char *str = CPLSerializeXMLTree(psRoot); CPLDestroyXMLNode(psRoot); ASSERT_STREQ( diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index a750d6ab50fe..69493680ff2f 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -768,15 +768,30 @@ def test_vsicurl_planetary_computer_url_signing(server): ) with webserver.install_http_handler(handler): - with gdaltest.config_option( - "VSICURL_PC_SAS_SIGN_HREF_URL", - "http://localhost:%d/pc_sas_sign_href?href=" % server.port, - ): - statres = gdal.VSIStatL( - "/vsicurl?pc_url_signing=yes&url=http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" - % server.port + try: + gdal.SetPathSpecificOption( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing" + % server.port, + "VSICURL_PC_URL_SIGNING", + "YES", + ) + + with gdaltest.config_option( + "VSICURL_PC_SAS_SIGN_HREF_URL", + "http://localhost:%d/pc_sas_sign_href?href=" % server.port, + ): + statres = gdal.VSIStatL( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" + % server.port + ) + assert statres.size == 3 + finally: + gdal.SetPathSpecificOption( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing" + % server.port, + "VSICURL_PC_URL_SIGNING", + None, ) - assert statres.size == 3 # Check that signing request is done since it has expired gdal.VSICurlClearCache() @@ -788,14 +803,14 @@ def test_vsicurl_planetary_computer_url_signing(server): % server.port, 200, {}, - '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin?my_token"}' + '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin?my_token2"}' % server.port, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, - {"Content-Length": "3"}, + {"Content-Length": "4"}, ) with webserver.install_http_handler(handler): @@ -807,7 +822,7 @@ def test_vsicurl_planetary_computer_url_signing(server): "/vsicurl?pc_url_signing=yes&url=http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" % server.port ) - assert statres.size == 3 + assert statres.size == 4 # Check that signing request is not needed gdal.VSICurlClearCache() @@ -815,7 +830,7 @@ def test_vsicurl_planetary_computer_url_signing(server): handler = webserver.SequentialHandler() handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, {"Content-Length": "3"}, ) @@ -838,12 +853,12 @@ def test_vsicurl_planetary_computer_url_signing(server): % server.port, 200, {}, - '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing2.bin?my_token2"}' + '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing2.bin?my_token3"}' % server.port, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing2.bin?my_token2", + "/test_vsicurl_planetary_computer_url_signing2.bin?my_token3", 200, {"Content-Length": "4"}, ) @@ -866,14 +881,14 @@ def test_vsicurl_planetary_computer_url_signing(server): handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, {"Content-Length": "3"}, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing2.bin?my_token2", + "/test_vsicurl_planetary_computer_url_signing2.bin?my_token3", 200, {"Content-Length": "4"}, ) diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index 6e1353c11eeb..0ba955c2f316 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -937,8 +937,9 @@ def test_vsifile_opendir(basepath): for i in range(4): entry = gdal.GetNextDirEntry(d) assert entry - if entry.name == "test": - entries_found.append(entry.name) + name = entry.name.replace("\\", "/") + if name == "test": + entries_found.append(name) assert (entry.mode & 32768) != 0 assert entry.modeKnown assert entry.size == 3 @@ -946,14 +947,14 @@ def test_vsifile_opendir(basepath): assert entry.mtime != 0 assert entry.mtimeKnown assert not entry.extra - elif entry.name == "subdir": - entries_found.append(entry.name) + elif name == "subdir": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2": - entries_found.append(entry.name) + elif name == "subdir/subdir2": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2/test2": - entries_found.append(entry.name) + elif name == "subdir/subdir2/test2": + entries_found.append(name) assert (entry.mode & 32768) != 0 else: assert False, entry.name @@ -970,19 +971,20 @@ def test_vsifile_opendir(basepath): for i in range(4): entry = gdal.GetNextDirEntry(d) assert entry - if entry.name == "test": - entries_found.append(entry.name) + name = entry.name.replace("\\", "/") + if name == "test": + entries_found.append(name) assert (entry.mode & 32768) != 0 if os.name == "posix" and basepath == "tmp/": assert entry.size == 0 - elif entry.name == "subdir": - entries_found.append(entry.name) + elif name == "subdir": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2": - entries_found.append(entry.name) + elif name == "subdir/subdir2": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2/test2": - entries_found.append(entry.name) + elif name == "subdir/subdir2/test2": + entries_found.append(name) assert (entry.mode & 32768) != 0 if os.name == "posix" and basepath == "tmp/": assert entry.size == 0 @@ -1009,7 +1011,10 @@ def test_vsifile_opendir(basepath): # Depth 1 files = set( - [l_entry.name for l_entry in gdal.listdir(basepath + "/vsifile_opendir", 1)] + [ + l_entry.name.replace("\\", "/") + for l_entry in gdal.listdir(basepath + "/vsifile_opendir", 1) + ] ) assert files == set(["test", "subdir", "subdir/subdir2"]) @@ -1030,18 +1035,19 @@ def test_vsifile_opendir(basepath): entry = gdal.GetNextDirEntry(d) assert entry.name == "subdir" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2" + sep = "\\" if "\\" in entry.name else "/" + assert entry.name.replace("\\", "/") == "subdir/subdir2" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2/test2" + assert entry.name.replace("\\", "/") == "subdir/subdir2/test2" entry = gdal.GetNextDirEntry(d) assert not entry gdal.CloseDir(d) - d = gdal.OpenDir(basepath + "/vsifile_opendir", -1, ["PREFIX=subdir/sub"]) + d = gdal.OpenDir(basepath + "/vsifile_opendir", -1, ["PREFIX=subdir" + sep + "sub"]) entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2" + assert entry.name.replace("\\", "/") == "subdir/subdir2" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2/test2" + assert entry.name.replace("\\", "/") == "subdir/subdir2/test2" entry = gdal.GetNextDirEntry(d) assert not entry gdal.CloseDir(d) diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index 106869ab86cc..cacec03b5b45 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -3734,17 +3734,58 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp prefix_path = "\\\\?\\" + tmp_path_str - f = gdal.VSIFOpenL(prefix_path + "\\testsync.txt", "wb") - assert f - gdal.VSIFCloseL(f) - - # S3 to local: S3 file is newer + # S3 to local gdal.VSICurlClearCache() handler = webserver.SequentialHandler() - handler.add( "GET", - "/out/testsync.txt", + "/bucket/", + 200, + {}, + """ + + + + false + + subdir/ + 2037-01-01T00:00:01.000Z + 0 + + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/", + 200, + {}, + """ + + + + false + + subdir/ + 2037-01-01T00:00:01.000Z + 0 + + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/subdir/testsync.txt", 206, { "Content-Length": "3", @@ -3755,15 +3796,35 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp ) handler.add( "GET", - "/out/testsync.txt", + "/bucket/?delimiter=%2F&prefix=subdir%2F", + 200, + {}, + """ + + subdir/ + + false + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/subdir/testsync.txt", 200, {"Content-Length": "3", "Last-Modified": "Mon, 01 Jan 2037 00:00:01 GMT"}, "foo", ) with webserver.install_http_handler(handler): - assert gdal.Sync("/vsis3/out/testsync.txt", prefix_path, options=options) + assert gdal.Sync("/vsis3/bucket/", prefix_path, options=options) - # Local to S3: S3 file is newer + assert gdal.VSIStatL(prefix_path + "\\subdir\\testsync.txt") is not None + + # Local to S3 gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add("GET", "/out/", 404) @@ -3779,14 +3840,12 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp false - testsync.txt - 2037-01-01T00:00:01.000Z - 3 """, ) - handler.add("PUT", "/out/testsync.txt", 200) + handler.add("PUT", "/out/subdir/", 200) + handler.add("PUT", "/out/subdir/testsync.txt", 200) with webserver.install_http_handler(handler): assert gdal.Sync(prefix_path + "\\", "/vsis3/out/", options=options) diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data new file mode 100644 index 000000000000..000d0eb8d63d --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data @@ -0,0 +1,195 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:24 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Tue, 11 Feb 2025 05:49:25 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 7564 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 171838 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "Information about the Blue Marble Next Generation (2004) data (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "Information about the Blue Marble Next Generation (2004) data (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/mapml", + "title" : "Information about the Blue Marble Next Generation (2004) data (as MapML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=mapml" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "Information about the Blue Marble Next Generation (2004) data (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/dggs-list", + "title" : "Discrete Global Grid Systems for Blue Marble Next Generation (2004)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/dggs" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "application/json", + "title" : "Schema (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "text/plain", + "title" : "Schema (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "text/html", + "title" : "Schema (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "application/json", + "title" : "Queryables (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "text/plain", + "title" : "Queryables (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "text/html", + "title" : "Queryables (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage", + "type" : "image/png", + "title" : "Blue Marble Next Generation (2004) (as PNG; Note: requesting large extent may result in generalized data)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage?f=png" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage", + "type" : "image/tiff; application=geotiff", + "title" : "Blue Marble Next Generation (2004) (as GeoTIFF; Note: requesting large extent may result in generalized data)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage?f=tif" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage-domainset", + "type" : "application/json", + "title" : "Blue Marble Next Generation (2004) (domain set of the coverage for this collection)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage/domainset?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage-rangetype", + "type" : "application/json", + "title" : "Blue Marble Next Generation (2004) (range type of the coverage for this collection)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage/rangetype?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/png", + "title" : "Default map (as PNG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.png" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/jpeg", + "title" : "Default map (as JPG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.jpg" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/tif", + "title" : "Default map (as GeoTIFF)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.tif" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets available for this dataset (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets available for this dataset (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets available for this dataset (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=html" + }, + { + "rel" : "styles", + "type" : "text/html", + "title" : "Styles for Blue Marble Next Generation (2004) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=html" + }, + { + "rel" : "styles", + "type" : "application/json", + "title" : "Styles for Blue Marble Next Generation (2004) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=json" + }, + { + "rel" : "styles", + "type" : "text/plain", + "title" : "Styles for Blue Marble Next Generation (2004) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=econ" + } + ], + "title" : "Blue Marble Next Generation (2004)", + "extent" : { + "spatial" : { + "bbox" : [ [ -180, -90, 180, 90 ] ], + "grid" : [ + { + "cellsCount" : 131072, + "resolution" : 0.0027465820312 + }, + { + "cellsCount" : 65536, + "resolution" : 0.0027465820312 + } + ] + }, + "temporal" : { + "interval" : [ [ "2004-01", "2004-12" ] ], + "grid" : { + "cellsCount" : 12, + "resolution" : "P1M" + } + } + }, + "crs" : [ + "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "http://www.opengis.net/def/crs/EPSG/0/4326", + "http://www.opengis.net/def/crs/EPSG/0/3857", + "http://www.opengis.net/def/crs/EPSG/0/3395" + ], + "storageCrs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "id" : "blueMarble", + "dataType" : "map", + "attribution" : "NASA Earth Observatory", + "minScaleDenominator" : 1091957.5469310893677, + "minCellSize" : 0.0027465820312 +} diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data new file mode 100644 index 000000000000..0760475e8302 Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data new file mode 100644 index 000000000000..ff0efadec872 Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data new file mode 100644 index 000000000000..70944fa0c06f Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data new file mode 100644 index 000000000000..d044f8585f0a --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data @@ -0,0 +1,138 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:25 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Thu, 13 Feb 2025 04:01:39 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 5775 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 5505 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3395", + "dataType" : "map", + "tileMatrixSetLimits" : [ + { "tileMatrix" : "0", "minTileRow" : 0, "maxTileRow" : 0, "minTileCol" : 0, "maxTileCol" : 0 }, + { "tileMatrix" : "1", "minTileRow" : 0, "maxTileRow" : 1, "minTileCol" : 0, "maxTileCol" : 1 }, + { "tileMatrix" : "2", "minTileRow" : 0, "maxTileRow" : 3, "minTileCol" : 0, "maxTileCol" : 3 }, + { "tileMatrix" : "3", "minTileRow" : 0, "maxTileRow" : 7, "minTileCol" : 0, "maxTileCol" : 7 }, + { "tileMatrix" : "4", "minTileRow" : 0, "maxTileRow" : 15, "minTileCol" : 0, "maxTileCol" : 15 }, + { "tileMatrix" : "5", "minTileRow" : 0, "maxTileRow" : 31, "minTileCol" : 0, "maxTileCol" : 31 }, + { "tileMatrix" : "6", "minTileRow" : 0, "maxTileRow" : 63, "minTileCol" : 0, "maxTileCol" : 63 }, + { "tileMatrix" : "7", "minTileRow" : 0, "maxTileRow" : 127, "minTileCol" : 0, "maxTileCol" : 127 }, + { "tileMatrix" : "8", "minTileRow" : 0, "maxTileRow" : 255, "minTileCol" : 0, "maxTileCol" : 255 }, + { "tileMatrix" : "9", "minTileRow" : 0, "maxTileRow" : 511, "minTileCol" : 0, "maxTileCol" : 511 } + ], + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "The JSON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "The ECON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "The HTML representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?=html" + }, + { + "rel" : "alternate", + "type" : "application/json+tile", + "title" : "The TileJSON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=tilejson" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldMercatorWGS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldMercatorWGS84Quad" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/geodata", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble" + }, + { + "rel" : "item", + "type" : "application/vnd.gnosis-map-tile", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as GNOSIS Map Tiles)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.gmt", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/png", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as PNG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.png", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/jpeg", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as JPG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.jpg", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/tiff; application=geotiff", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as GeoTIFF)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.tif", + "templated" : true + } + ], + "layers" : [ + { + "id" : "blueMarble", + "dataType" : "map", + "minScaleDenominator" : 1091957.5469310893677, + "minCellSize" : 305.748113140705, + "maxTileMatrix" : "9", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/geodata", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble" + } + ], + "propertiesSchema" : { + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "type" : "object", + "properties" : { "a" : { + "title" : "Alpha channel", + "type" : "integer", + "x-ogc-property-seq" : 4 + }, "b" : { + "title" : "Blue channel", + "type" : "integer", + "x-ogc-property-seq" : 3 + }, "g" : { + "title" : "Green channel", + "type" : "integer", + "x-ogc-property-seq" : 2 + }, "r" : { + "title" : "Red channel", + "type" : "integer", + "x-ogc-property-seq" : 1 + } } + } + } + ], + "centerPoint" : { + "coordinates" : [ 0, 0 ], + "tileMatrix" : "4", + "scaleDenominator" : 34942641.501794859767, + "cellSize" : 9783.9396205025605, + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + } +} diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data new file mode 100644 index 000000000000..1694f8e8e181 --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data @@ -0,0 +1,433 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:25 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Thu, 13 Feb 2025 04:01:39 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 20038 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 5505 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "The JSON representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "The ECON representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "The HTML representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (default style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (default style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (default style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (evi style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (evi style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (evi style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (nir style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (nir style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (nir style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (scl style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (scl style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (scl style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (ndvi style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (ndvi style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (ndvi style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (evi2 style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (evi2 style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (evi2 style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=html" + } + ], + "tilesets" : [ + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/CDB1GlobalGrid", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/4326", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "CDB1GlobalGridTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/CDB1GlobalGrid" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "CDB1GlobalGrid map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "CDB1GlobalGrid map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "CDB1GlobalGrid map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GlobalCRS84Pixel", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GlobalCRS84PixelTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GlobalCRS84Pixel" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GlobalCRS84Scale", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GlobalCRS84ScaleTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GlobalCRS84Scale" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GlobalCRS84Scale map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GlobalCRS84Scale map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GlobalCRS84Scale map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GNOSISGlobalGrid", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/4326", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GNOSISGlobalGridTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GNOSISGlobalGrid" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GoogleCRS84Quad", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GoogleCRS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GoogleCRS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GoogleCRS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GoogleCRS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GoogleCRS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/ISEA9R", + "crs" : "http://www.opengis.net/def/crs/OGC/0/153456", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "ISEA9RTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/ISEA9R" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "ISEA9R map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "ISEA9R map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "ISEA9R map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3857", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WebMercatorQuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WebMercatorQuad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WebMercatorQuad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WebMercatorQuad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WebMercatorQuad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldCRS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldCRS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WorldCRS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WorldCRS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WorldCRS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3395", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldMercatorWGS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldMercatorWGS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=tilejson" + } + ] + } + ] +} diff --git a/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf b/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf new file mode 100644 index 000000000000..5c942a9c5c31 Binary files /dev/null and b/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf differ diff --git a/autotest/gdrivers/ecw.py b/autotest/gdrivers/ecw.py index 388e31663c75..008057c1c11e 100755 --- a/autotest/gdrivers/ecw.py +++ b/autotest/gdrivers/ecw.py @@ -2210,6 +2210,66 @@ def test_ecw_read_uint32_jpeg2000(): ) +############################################################################### +# Test unsupported XML SRS + + +def test_jp2ecw_unsupported_srs_for_gmljp2(tmp_vsimem): + + if gdaltest.jp2ecw_drv is None or not has_write_support(): + pytest.skip("ECW write support not available") + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate(filename, "data/byte.tif", outputSRS=wkt, format="JP2ECW") + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() + + ############################################################################### diff --git a/autotest/gdrivers/jp2kak.py b/autotest/gdrivers/jp2kak.py index 209032e4e7c6..c8f14d7aa6eb 100755 --- a/autotest/gdrivers/jp2kak.py +++ b/autotest/gdrivers/jp2kak.py @@ -33,7 +33,7 @@ import gdaltest import pytest -from osgeo import gdal +from osgeo import gdal, osr pytestmark = pytest.mark.require_driver("JP2KAK") @@ -953,3 +953,60 @@ def find_xml_node(ar, element_name, only_attributes=False): if found is not None: return found return None + + +############################################################################### +# Test unsupported XML SRS + + +def test_jp2kak_unsupported_srs_for_gmljp2(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate(filename, "data/byte.tif", outputSRS=wkt, format="JP2KAK") + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() diff --git a/autotest/gdrivers/jp2openjpeg.py b/autotest/gdrivers/jp2openjpeg.py index e38952725803..feaddd440910 100755 --- a/autotest/gdrivers/jp2openjpeg.py +++ b/autotest/gdrivers/jp2openjpeg.py @@ -3850,3 +3850,76 @@ def test_jp2openjpeg_vrt_protocol(): webserver.server_stop(webserver_process, webserver_port) gdal.VSICurlClearCache() + + +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/9236 + + +def test_jp2openjpeg_limit_resolution_count_from_image_size(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + assert gdal.Translate(filename, "data/byte.tif", width=1024, height=7) + + # Check number of resolutions + ret = gdal.GetJPEG2000StructureAsString(filename, ["ALL=YES"]) + assert '2' in ret + + +############################################################################### +# Test unsupported XML SRS + + +def test_jp2openjpeg_unsupported_srs_for_gmljp2(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate( + filename, "data/byte.tif", outputSRS=wkt, format="JP2OpenJPEG" + ) + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() diff --git a/autotest/gdrivers/ogcapi.py b/autotest/gdrivers/ogcapi.py index d078ca0bb81d..f12a239ec7ed 100644 --- a/autotest/gdrivers/ogcapi.py +++ b/autotest/gdrivers/ogcapi.py @@ -438,3 +438,42 @@ def test_ogc_api_raster_tiles(): assert ds.GetRasterBand(1).ReadBlock( ds.RasterXSize // 2 // 256, ds.RasterYSize // 2 // 256 ) + + +@pytest.mark.parametrize( + "image_format,raster_count,statistics", + ( + ("AUTO", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("PNG", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("PNG_PREFERRED", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("JPEG", 3, [0.0, 255.0, 83.83631896972656, 30.486497283147653]), + ("JPEG_PREFERRED", 3, [0.0, 255.0, 83.83631896972656, 30.486497283147653]), + ("GEOTIFF", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ), +) +@pytest.mark.require_driver("WMS") +def test_ogc_api_raster_tiles_format(image_format, raster_count, statistics): + + ds = gdal.OpenEx( + f"OGCAPI:http://127.0.0.1:{gdaltest.webserver_port}/fakeogcapi/collections/blueMarble", + gdal.OF_RASTER, + open_options=[ + "API=TILES", + "CACHE=NO", + "TILEMATRIXSET=WorldMercatorWGS84Quad", + f"IMAGE_FORMAT={image_format}", + ], + ) + + assert ds is not None + + assert ds.RasterCount == raster_count + assert ds.RasterXSize == 131072 + assert ds.GetRasterBand(1).GetOverviewCount() == 9 + + # For some reason these tests fail on Github on Ubuntu 20.04 gcc and Ubuntu 20.04 coverage, but + # pass on all other builds. + # assert ds.RasterYSize == 1586181 + # assert ds.GetRasterBand(1).GetStatistics(True, True) == statistics + + del ds diff --git a/autotest/gdrivers/pdf.py b/autotest/gdrivers/pdf.py index e9a38ab8e89f..7db4786de764 100755 --- a/autotest/gdrivers/pdf.py +++ b/autotest/gdrivers/pdf.py @@ -1364,6 +1364,48 @@ def test_pdf_layers(poppler_or_pdfium): assert not (cs4 == cs1 or cs4 == cs2), "did not get expected checksum" +############################################################################### +# Check layers support + + +def test_pdf_layers_with_same_name_on_different_pages(poppler_or_pdfium): + + ds = gdal.Open("data/pdf/layer_with_same_name_on_different_pages.pdf") + layers = ds.GetMetadata_List("LAYERS") + assert layers == [ + "LAYER_00_NAME=Map_Frame (page 1)", + "LAYER_01_NAME=Map_Frame.Map (page 1)", + "LAYER_02_NAME=Map_Frame.Map.States (page 1)", + "LAYER_03_NAME=Map_Frame (page 2)", + "LAYER_04_NAME=Map_Frame.Map (page 2)", + "LAYER_05_NAME=Map_Frame.Map.States (page 2)", + "LAYER_06_NAME=Map_Frame (page 3)", + "LAYER_07_NAME=Map_Frame.Map (page 3)", + "LAYER_08_NAME=Map_Frame.Map.States (page 3)", + "LAYER_09_NAME=Map_Frame (page 4)", + "LAYER_10_NAME=Map_Frame.Map (page 4)", + "LAYER_11_NAME=Map_Frame.Map.States (page 4)", + ] + + for page in (1, 2, 3, 4): + ds = gdal.Open( + f"PDF:{page}:data/pdf/layer_with_same_name_on_different_pages.pdf" + ) + layers = ds.GetMetadata_List("LAYERS") + assert layers == [ + "LAYER_00_NAME=Map_Frame", + "LAYER_01_NAME=Map_Frame.Map", + "LAYER_02_NAME=Map_Frame.Map.States", + ] + cs_all = ds.GetRasterBand(1).Checksum() + + ds = gdal.OpenEx( + f"PDF:{page}:data/pdf/layer_with_same_name_on_different_pages.pdf", + open_options=["LAYERS_OFF=Map_Frame.Map.States"], + ) + assert ds.GetRasterBand(1).Checksum() != cs_all + + ############################################################################### # Test MARGIN, EXTRA_STREAM, EXTRA_LAYER_NAME and EXTRA_IMAGES options diff --git a/autotest/gdrivers/vrtderived.py b/autotest/gdrivers/vrtderived.py index e02b4c459f37..b5fd42965881 100755 --- a/autotest/gdrivers/vrtderived.py +++ b/autotest/gdrivers/vrtderived.py @@ -1023,6 +1023,51 @@ def identity(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize _validate(xml) +############################################################################### + + +@pytest.mark.parametrize("dtype", range(1, gdal.GDT_TypeCount)) +def test_vrt_derived_dtype(tmp_vsimem, dtype): + pytest.importorskip("numpy") + + input_fname = tmp_vsimem / "input.tif" + + nx = 1 + ny = 1 + + with gdal.GetDriverByName("GTiff").Create( + input_fname, nx, ny, 1, eType=gdal.GDT_Int8 + ) as input_ds: + input_ds.GetRasterBand(1).Fill(1) + gt = input_ds.GetGeoTransform() + + vrt_xml = f""" + + {', '.join([str(x) for x in gt])} + + Python + identity + + + + {input_fname} + 1 + + + + """ + + with gdal.config_option("GDAL_VRT_ENABLE_PYTHON", "YES"): + with gdal.Open(vrt_xml) as vrt_ds: + arr = vrt_ds.ReadAsArray() + if dtype not in {gdal.GDT_CInt16, gdal.GDT_CInt32}: + assert arr[0, 0] == 1 + assert vrt_ds.GetRasterBand(1).DataType == dtype + + ############################################################################### # Cleanup. diff --git a/autotest/ogr/data/csv/header_with_line_break.csv b/autotest/ogr/data/csv/header_with_line_break.csv new file mode 100644 index 000000000000..e56f470e181b --- /dev/null +++ b/autotest/ogr/data/csv/header_with_line_break.csv @@ -0,0 +1,6 @@ +Column one,Column two,"Column with a +line break",Column three,"Another +line break",Column four,Column five +1,2,3,4,5,6,7 +4,5,6,7,8,9,10 +7,8,9,10,11,12,13 diff --git a/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles b/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles new file mode 100644 index 000000000000..44057c2036e3 Binary files /dev/null and b/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles differ diff --git a/autotest/ogr/ogr_csv.py b/autotest/ogr/ogr_csv.py index b8efff32ca28..9008bec734ba 100755 --- a/autotest/ogr/ogr_csv.py +++ b/autotest/ogr/ogr_csv.py @@ -2988,6 +2988,37 @@ def test_ogr_csv_getextent3d(tmp_vsimem): ############################################################################### +def test_ogr_csv_read_header_with_line_break(): + + ds = ogr.Open("data/csv/header_with_line_break.csv") + lyr = ds.GetLayer(0) + lyr_defn = lyr.GetLayerDefn() + assert [ + lyr_defn.GetFieldDefn(i).GetName() for i in range(lyr_defn.GetFieldCount()) + ] == [ + "Column one", + "Column two", + "Column with a\nline break", + "Column three", + "Another\nline break", + "Column four", + "Column five", + ] + f = lyr.GetNextFeature() + assert [f.GetField(i) for i in range(lyr_defn.GetFieldCount())] == [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + ] + + +############################################################################### + + if __name__ == "__main__": gdal.UseExceptions() if len(sys.argv) != 2: diff --git a/autotest/ogr/ogr_fgdb.py b/autotest/ogr/ogr_fgdb.py index da1a1ea8dd3e..f0c483500dcd 100755 --- a/autotest/ogr/ogr_fgdb.py +++ b/autotest/ogr/ogr_fgdb.py @@ -2875,7 +2875,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "my forward path label" assert rel.GetBackwardPathLabel() == "my backward path label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -2887,7 +2887,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -2901,7 +2901,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["destination_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -2913,7 +2913,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -2925,7 +2925,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_many_to_many") assert rel is not None @@ -2939,7 +2939,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["dest_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index 6d73b0106f0c..6d58c77795dd 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -2295,6 +2295,30 @@ def test_ogr_gpkg_22(tmp_vsimem): assert f.GetFID() == 1234567890123 +############################################################################### +# Test creating a feature with FID 0 + + +def test_ogr_gpkg_create_with_fid_0(tmp_vsimem): + + fname = tmp_vsimem / "test_ogr_gpkg_create_with_fid_0.gpkg" + + ds = gdaltest.gpkg_dr.CreateDataSource(fname) + lyr = ds.CreateLayer("test") + + feat = ogr.Feature(lyr.GetLayerDefn()) + feat.SetFID(0) + lyr.CreateFeature(feat) + feat = None + + ds = None + + ds = ogr.Open(fname) + lyr = ds.GetLayerByName("test") + f = lyr.GetNextFeature() + assert f.GetFID() == 0 + + ############################################################################### # Test not nullable fields @@ -7008,6 +7032,13 @@ def test_ogr_gpkg_relations(tmp_vsimem, tmp_path): assert rel.GetRightMappingTableFields() == ["related_id"] assert rel.GetRelatedTableType() == "attributes" + # ensure that the mapping table, which is present in gpkgext_relations but + # NOT gpkg_contents can be opened as a layer + lyr = ds.GetLayer("my_mapping_table") + assert lyr is not None + assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "base_id" + assert lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "related_id" + lyr = ds.GetLayer("a") lyr.Rename("a_renamed") lyr.AlterFieldDefn( @@ -7086,6 +7117,33 @@ def test_ogr_gpkg_relations(tmp_vsimem, tmp_path): assert rel.GetRightMappingTableFields() == ["related_id"] assert rel.GetRelatedTableType() == "features" + # a one-to-many relationship defined using foreign key constraints + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + ds.ExecuteSQL( + "CREATE TABLE test_relation_a(artistid INTEGER PRIMARY KEY, artistname TEXT)" + ) + ds.ExecuteSQL( + "CREATE TABLE test_relation_b(trackid INTEGER, trackname TEXT, trackartist INTEGER, FOREIGN KEY(trackartist) REFERENCES test_relation_a(artistid))" + ) + ds = None + + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + assert ds.GetRelationshipNames() == [ + "custom_type", + "test_relation_a_test_relation_b", + ] + assert ds.GetRelationship("custom_type") is not None + rel = ds.GetRelationship("test_relation_a_test_relation_b") + assert rel is not None + assert rel.GetName() == "test_relation_a_test_relation_b" + assert rel.GetLeftTableName() == "test_relation_a" + assert rel.GetRightTableName() == "test_relation_b" + assert rel.GetCardinality() == gdal.GRC_ONE_TO_MANY + assert rel.GetType() == gdal.GRT_ASSOCIATION + assert rel.GetLeftTableFields() == ["artistid"] + assert rel.GetRightTableFields() == ["trackartist"] + assert rel.GetRelatedTableType() == "features" + ds = None @@ -7260,6 +7318,31 @@ def get_query_row_count(query): == 1 ) + # validate mapping table was created + assert get_query_row_count("SELECT * FROM 'origin_table_dest_table'") == 0 + # validate mapping table is present in gpkg_contents + assert ( + get_query_row_count( + "SELECT * FROM gpkg_contents WHERE table_name='origin_table_dest_table' AND data_type='attributes'" + ) + == 1 + ) + # force delete from gpkg_contents, and then ensure that we CAN successfully + # load layers which are present ONLY in gpkgext_relations but NOT + # gpkg_contents (i.e. datasources which follow the Related Tables specification + # exactly) + ds.ExecuteSQL( + "DELETE FROM gpkg_contents WHERE table_name='origin_table_dest_table'" + ) + + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + # ensure that the mapping table, which is present in gpkgext_relations but + # NOT gpkg_contents can be opened as a layer + lyr = ds.GetLayer("origin_table_dest_table") + assert lyr is not None + assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "base_id" + assert lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "related_id" + lyr = ds.CreateLayer("origin_table2", geom_type=ogr.wkbNone) fld_defn = ogr.FieldDefn("o_pkey", ogr.OFTInteger) assert lyr.CreateField(fld_defn) == ogr.OGRERR_NONE diff --git a/autotest/ogr/ogr_gpx.py b/autotest/ogr/ogr_gpx.py index f9bff84c1dc7..3af9c0a10a8c 100755 --- a/autotest/ogr/ogr_gpx.py +++ b/autotest/ogr/ogr_gpx.py @@ -572,3 +572,22 @@ def test_ogr_gpx_N_MAX_LINKS(): lyr = ds.GetLayerByName("track_points") f = lyr.GetNextFeature() assert f["link3_href"] is None + + +############################################################################### +# Test preservation of FID when converting to GPKG +# (https://github.com/OSGeo/gdal/issues/9225) + + +@pytest.mark.require_driver("GPKG") +def test_ogr_gpx_convert_to_gpkg(tmp_vsimem): + + outfilename = str(tmp_vsimem / "out.gpkg") + gdal.VectorTranslate(outfilename, "data/gpx/test.gpx") + + ds = ogr.Open(outfilename) + lyr = ds.GetLayer("tracks") + f = lyr.GetNextFeature() + assert f.GetFID() == 0 + f = lyr.GetNextFeature() + assert f.GetFID() == 1 diff --git a/autotest/ogr/ogr_ods.py b/autotest/ogr/ogr_ods.py index 6977cd3fa980..16fccb20fbd3 100755 --- a/autotest/ogr/ogr_ods.py +++ b/autotest/ogr/ogr_ods.py @@ -59,6 +59,8 @@ def ogr_ods_check(ds): assert lyr.TestCapability("foo") == 0 + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 + lyr = ds.GetLayer(6) assert lyr.GetName() == "Feuille7", "bad layer name" @@ -363,6 +365,7 @@ def test_ogr_ods_8(): drv = ogr.GetDriverByName("ODS") ds = drv.CreateDataSource("/vsimem/ogr_ods_8.ods") lyr = ds.CreateLayer("foo") + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 lyr.CreateField(ogr.FieldDefn("Field1", ogr.OFTInteger64)) f = ogr.Feature(lyr.GetLayerDefn()) f.SetField(0, 1) diff --git a/autotest/ogr/ogr_openfilegdb.py b/autotest/ogr/ogr_openfilegdb.py index 901e10ae3990..c9c2ad519b6c 100755 --- a/autotest/ogr/ogr_openfilegdb.py +++ b/autotest/ogr/ogr_openfilegdb.py @@ -2386,7 +2386,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "my forward path label" assert rel.GetBackwardPathLabel() == "my backward path label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -2398,7 +2398,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -2412,7 +2412,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["destination_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -2424,7 +2424,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -2436,7 +2436,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_many_to_many") assert rel is not None @@ -2450,7 +2450,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["dest_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_openfilegdb_write.py b/autotest/ogr/ogr_openfilegdb_write.py index 7e105bfe8823..690a127879d4 100755 --- a/autotest/ogr/ogr_openfilegdb_write.py +++ b/autotest/ogr/ogr_openfilegdb_write.py @@ -2833,7 +2833,7 @@ def test_ogr_openfilegdb_write_relationships(): assert retrieved_rel.GetRightTableFields() == ["dest_pkey"] assert retrieved_rel.GetForwardPathLabel() == "fwd label" assert retrieved_rel.GetBackwardPathLabel() == "backward label" - assert retrieved_rel.GetRelatedTableType() == "feature" + assert retrieved_rel.GetRelatedTableType() == "features" items_lyr = ds.GetLayerByName("GDB_Items") f = items_lyr.GetFeature(8) @@ -3182,7 +3182,7 @@ def test_ogr_openfilegdb_write_relationships(): assert retrieved_rel.GetRightTableFields() == ["dest_pkey"] assert retrieved_rel.GetForwardPathLabel() == "my new fwd label" assert retrieved_rel.GetBackwardPathLabel() == "my new backward label" - assert retrieved_rel.GetRelatedTableType() == "feature" + assert retrieved_rel.GetRelatedTableType() == "features" # change relationship tables lyr = ds.CreateLayer("new_origin_table", geom_type=ogr.wkbNone) diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index 42e2a9c3106b..40c7d5c8a3dc 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -307,12 +307,6 @@ def pg_quote_with_E(pg_autotest_ds): return True -@pytest.fixture(scope="module") -def pg_retrieve_fid(pg_version): - - return pg_version >= (8, 2) - - ############################################################################### # Create table from data/poly.shp @@ -2023,7 +2017,7 @@ def test_ogr_pg_39(pg_ds): @only_with_postgis -def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): +def test_ogr_pg_39_bis(pg_ds, pg_has_postgis): schema = current_schema(pg_ds) @@ -2038,10 +2032,6 @@ def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): # Create a view pg_ds.ExecuteSQL("CREATE VIEW testview AS SELECT * FROM inherited") - if pg_postgis_version[0] < 2: - pg_ds.ExecuteSQL( - f"INSERT INTO geometry_columns VALUES ( '', '{schema}', 'testview', 'wkb_geometry', 2, -1, 'POINT') " - ) pg_ds.ExecuteSQL( "INSERT INTO inherited (col1, wkb_geometry) VALUES ( 'a', GeomFromEWKT('POINT (0 1)') )" ) @@ -2069,10 +2059,7 @@ def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): ), ("bad geometry %s" % feat.GetGeometryRef().ExportToWkt()) # Test another geometry column - if pg_postgis_version[0] < 2: - pg_ds.ExecuteSQL( - f"INSERT INTO geometry_columns VALUES ( '', '{schema}', 'testview', 'point25D', 3, -1, 'POINT') " - ) + pg_ds.ExecuteSQL( "UPDATE inherited SET \"point25D\" = GeomFromEWKT('POINT (0 1 2)') " ) @@ -2796,12 +2783,9 @@ def test_ogr_pg_53_bis(tmp_path, pg_ds): @only_with_postgis -def test_ogr_pg_54(pg_ds, pg_postgis_version): +def test_ogr_pg_54(pg_ds): - if pg_postgis_version[0] >= 2: - sql_lyr = pg_ds.ExecuteSQL("SELECT ST_AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") - else: - sql_lyr = pg_ds.ExecuteSQL("SELECT AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") + sql_lyr = pg_ds.ExecuteSQL("SELECT ST_AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") feat = sql_lyr.GetNextFeature() pg_ds.ReleaseResultSet(sql_lyr) @@ -3538,7 +3522,7 @@ def test_ogr_pg_70bis(pg_ds, pg_postgis_schema): @only_with_postgis -def test_ogr_pg_71(pg_ds, pg_postgis_version): +def test_ogr_pg_71(pg_ds): curve_lyr = pg_ds.CreateLayer("test_curve") curve_lyr2 = pg_ds.CreateLayer( @@ -3574,25 +3558,7 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): "GEOMETRYCOLLECTION (CIRCULARSTRING (0 1,2 3,4 5),COMPOUNDCURVE ((0 1,2 3,4 5)),CURVEPOLYGON ((0 0,0 1,1 1,1 0,0 0)),MULTICURVE ((0 0,1 1)),MULTISURFACE (((0 0,0 10,10 10,10 0,0 0))))", ]: - # would cause PostGIS 1.X to crash - if pg_postgis_version[0] < 2 and wkt == "CURVEPOLYGON EMPTY": - continue - # Parsing error of WKT by PostGIS 1.X - if ( - pg_postgis_version[0] < 2 - and "MULTICURVE" in wkt - and "CIRCULARSTRING" in wkt - ): - continue - postgis_in_wkt = wkt - while True: - z_pos = postgis_in_wkt.find("Z ") - # PostGIS 1.X doesn't like Z in WKT - if pg_postgis_version[0] < 2 and z_pos >= 0: - postgis_in_wkt = postgis_in_wkt[0:z_pos] + postgis_in_wkt[z_pos + 2 :] - else: - break # Test parsing PostGIS WKB lyr = pg_ds.ExecuteSQL("SELECT ST_GeomFromText('%s')" % postgis_in_wkt) @@ -3604,18 +3570,11 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): pg_ds.ReleaseResultSet(lyr) expected_wkt = wkt - if pg_postgis_version[0] < 2 and "EMPTY" in wkt: - expected_wkt = "GEOMETRYCOLLECTION EMPTY" assert out_wkt == expected_wkt # Test parsing PostGIS WKT - if pg_postgis_version[0] >= 2: - fct = "ST_AsText" - else: - fct = "AsEWKT" - lyr = pg_ds.ExecuteSQL( - "SELECT %s(ST_GeomFromText('%s'))" % (fct, postgis_in_wkt) + "SELECT ST_AsText(ST_GeomFromText('%s'))" % (postgis_in_wkt) ) f = lyr.GetNextFeature() g = f.GetGeometryRef() @@ -3625,8 +3584,6 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): pg_ds.ReleaseResultSet(lyr) expected_wkt = wkt - if pg_postgis_version[0] < 2 and "EMPTY" in wkt: - expected_wkt = "GEOMETRYCOLLECTION EMPTY" assert out_wkt == expected_wkt g = ogr.CreateGeometryFromWkt(wkt) @@ -3643,13 +3600,9 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): assert ret == 0, wkt fid = f.GetFID() - # AsEWKT() in PostGIS 1.X does not like CIRCULARSTRING EMPTY - if pg_postgis_version[0] < 2 and "CIRCULARSTRING" in wkt and "EMPTY" in wkt: - continue - lyr = pg_ds.ExecuteSQL( - "SELECT %s(wkb_geometry) FROM %s WHERE ogc_fid = %d" - % (fct, active_lyr.GetName(), fid) + "SELECT ST_AsText(wkb_geometry) FROM %s WHERE ogc_fid = %d" + % (active_lyr.GetName(), fid) ) f = lyr.GetNextFeature() g = f.GetGeometryRef() @@ -4165,14 +4118,14 @@ def ogr_pg_76_get_transaction_state(ds): ) -def test_ogr_pg_76(pg_ds, pg_postgis_version, use_postgis): +def test_ogr_pg_76(pg_ds, use_postgis): assert pg_ds.TestCapability(ogr.ODsCTransactions) == 1 level = int(pg_ds.GetMetadataItem("nSoftTransactionLevel", "_DEBUG_")) assert level == 0 - if use_postgis and pg_postgis_version[0] >= 2: + if use_postgis: pg_ds.StartTransaction() lyr = pg_ds.CreateLayer("will_not_be_created", options=["OVERWRITE=YES"]) lyr.CreateField(ogr.FieldDefn("foo", ogr.OFTString)) @@ -4505,10 +4458,7 @@ def test_ogr_pg_77(pg_ds, tmp_path): @only_with_postgis -def test_ogr_pg_78(pg_ds, pg_postgis_version): - - if pg_postgis_version[0] < 2: - pytest.skip("Requires PostGIS >= 2.0") +def test_ogr_pg_78(pg_ds): pg_ds.ExecuteSQL("CREATE TABLE ogr_pg_78 (ID INTEGER PRIMARY KEY)") pg_ds.ExecuteSQL("ALTER TABLE ogr_pg_78 ADD COLUMN my_geom GEOMETRY") @@ -4802,7 +4752,7 @@ def ogr_pg_83_ids(param): ], ids=ogr_pg_83_ids, ) -def test_ogr_pg_83(pg_ds, pg_postgis_version, geom_type, options, wkt, expected_wkt): +def test_ogr_pg_83(pg_ds, geom_type, options, wkt, expected_wkt): lyr = pg_ds.CreateLayer("ogr_pg_83", geom_type=geom_type, options=options) f = ogr.Feature(lyr.GetLayerDefn()) @@ -4821,12 +4771,8 @@ def test_ogr_pg_83(pg_ds, pg_postgis_version, geom_type, options, wkt, expected_ if "GEOM_TYPE=geography" in options: return - # Cannot do AddGeometryColumn( 'GEOMETRYM', 3 ) with PostGIS 2, and doesn't accept inserting a XYM geometry - if ( - pg_postgis_version[0] >= 2 - and geom_type == ogr.wkbUnknown - and options == ["DIM=XYM"] - ): + # Cannot do AddGeometryColumn( 'GEOMETRYM', 3 ) with PostGIS >= 2, and doesn't accept inserting a XYM geometry + if geom_type == ogr.wkbUnknown and options == ["DIM=XYM"]: return lyr = pg_ds.CreateLayer( diff --git a/autotest/ogr/ogr_pgeo.py b/autotest/ogr/ogr_pgeo.py index e9e147445335..66f4a3f39aed 100755 --- a/autotest/ogr/ogr_pgeo.py +++ b/autotest/ogr/ogr_pgeo.py @@ -910,7 +910,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "forward label" assert rel.GetBackwardPathLabel() == "backward label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -922,7 +922,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -936,7 +936,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["rel_pk"] assert rel.GetRightTableFields() == ["primary_key"] assert rel.GetRightMappingTableFields() == ["rel_primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -948,7 +948,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -962,7 +962,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == None assert rel.GetRightTableFields() == ["primary_key"] assert rel.GetRightMappingTableFields() == None - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_pmtiles.py b/autotest/ogr/ogr_pmtiles.py index 0757e95a9d59..81361f82def5 100755 --- a/autotest/ogr/ogr_pmtiles.py +++ b/autotest/ogr/ogr_pmtiles.py @@ -617,3 +617,15 @@ def test_ogr_pmtiles_read_corrupted_min_zoom_larger_than_30(): ds = ogr.Open(tmpfilename) assert gdal.GetLastErrorMsg() == "Clamping max_zoom from 255 to 30" assert ds.GetMetadataItem("ZOOM_LEVEL") == "30" + + +############################################################################### + + +def test_ogr_pmtiles_read_with_many_directories(): + + ds = gdal.OpenEx( + "data/pmtiles/subset7_truncated.pmtiles", open_options=["ZOOM_LEVEL=0"] + ) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() != 0 diff --git a/autotest/ogr/ogr_xlsx.py b/autotest/ogr/ogr_xlsx.py index 39971b128d96..164bdd835d29 100755 --- a/autotest/ogr/ogr_xlsx.py +++ b/autotest/ogr/ogr_xlsx.py @@ -59,6 +59,8 @@ def ogr_xlsx_check(ds): assert lyr.TestCapability("foo") == 0 + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 + lyr = ds.GetLayer(6) assert lyr.GetName() == "Feuille7", "bad layer name" @@ -277,6 +279,7 @@ def test_ogr_xlsx_8(): ds = ogr.GetDriverByName("XLSX").CreateDataSource("/vsimem/ogr_xlsx_8.xlsx") lyr = ds.CreateLayer("foo") + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 for i in range(30): lyr.CreateField(ogr.FieldDefn("Field%d" % (i + 1))) f = ogr.Feature(lyr.GetLayerDefn()) diff --git a/autotest/osr/osr_xml.py b/autotest/osr/osr_xml.py index f804aec67ded..41ef4d1377bf 100755 --- a/autotest/osr/osr_xml.py +++ b/autotest/osr/osr_xml.py @@ -31,6 +31,7 @@ import re import gdaltest +import pytest from osgeo import osr @@ -200,3 +201,58 @@ def test_osr_xml_2(): expected = re.sub(r' gml:id="[^"]*"', "", expected, 0) assert got == expected + + +############################################################################### +# Test the osr.SpatialReference.ExportToXML() function. +# + + +def test_osr_xml_export_failure(): + + srs = osr.SpatialReference() + srs.ImportFromWkt( + """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + ) + + with pytest.raises( + Exception, match="Unhandled projection method Albers_Conic_Equal_Area" + ): + srs.ExportToXML() diff --git a/autotest/pyscripts/gdal2tiles/test_logger.py b/autotest/pyscripts/gdal2tiles/test_logger.py new file mode 100644 index 000000000000..de9cd087b1bd --- /dev/null +++ b/autotest/pyscripts/gdal2tiles/test_logger.py @@ -0,0 +1,72 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: gdal2tiles.py testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import pytest + +from osgeo import gdal +from osgeo_utils import gdal2tiles + + +def test_gdal2tiles_logger(): + + if gdal.GetDriverByName("PNG") is None: + pytest.skip("PNG driver is missing") + + gdal2tiles.main( + argv=[ + "gdal2tiles", + "--verbose", + "-z", + "13-14", + "../../gcore/data/byte.tif", + "/vsimem/gdal2tiles", + ] + ) + + assert set(gdal.ReadDirRecursive("/vsimem/gdal2tiles")) == set( + [ + "13/", + "13/1418/", + "13/1418/4916.png", + "13/1419/", + "13/1419/4916.png", + "14/", + "14/2837/", + "14/2837/9833.png", + "14/2838/", + "14/2838/9833.png", + "googlemaps.html", + "leaflet.html", + "openlayers.html", + "tilemapresource.xml", + ] + ) + gdal.RmdirRecursive("/vsimem/gdal2tiles") diff --git a/autotest/pyscripts/gdal2tiles/test_vsimem.py b/autotest/pyscripts/gdal2tiles/test_vsimem.py index c6c61ce946d3..340e0238f7c4 100644 --- a/autotest/pyscripts/gdal2tiles/test_vsimem.py +++ b/autotest/pyscripts/gdal2tiles/test_vsimem.py @@ -40,7 +40,9 @@ def test_gdal2tiles_vsimem(): if gdal.GetDriverByName("PNG") is None: pytest.skip("PNG driver is missing") - gdal2tiles.main(argv=["-q", "../../gcore/data/byte.tif", "/vsimem/gdal2tiles"]) + gdal2tiles.main( + argv=["gdal2tiles", "-q", "../../gcore/data/byte.tif", "/vsimem/gdal2tiles"] + ) assert set(gdal.ReadDirRecursive("/vsimem/gdal2tiles")) == set( [ diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index c49c816f67fc..94f449d1e282 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -748,3 +748,158 @@ def test_gdalbuildvrt_lib_stable_average(): vrt_gt = vrt_ds.GetGeoTransform() assert vrt_gt == gt + + +############################################################################### + + +def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgba(tmp_vsimem): + + ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 4) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + # Test remapping of second valid pixel at 0 to 1 + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x00") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x02\x02") + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x03\x03") + ds.GetRasterBand(4).WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.GetRasterBand(4).SetColorInterpretation(gdal.GCI_AlphaBand) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128, VRTNodata=0) + assert vrt_ds.RasterCount == 3 + assert vrt_ds.GetRasterBand(1).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x00\x01" + assert vrt_ds.GetRasterBand(2).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x00\x02" + assert vrt_ds.GetRasterBand(3).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x00\x03" + + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (0, 1) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128.5, VRTNodata=0) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x00\x01" + + # VRTNodata=255, test remapping of 255 to 254 + ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 2) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\xFF") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128, VRTNodata=255) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\xFF\xFE" + + +############################################################################### + + +def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgb_mask(tmp_vsimem): + + # UInt16, VRTNodata=0 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 3, gdal.GDT_UInt16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 1, 0)) + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 2, 2)) + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 3, 2)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=0) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (0, 1) + + assert struct.unpack( + "B" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Byte) + ) == (0, 1) + + # UInt16, VRTNodata=65535 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_UInt16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 1, 65535)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=65535 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (65535, 65534) + + # Int16, VRTNodata=-32768 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Int16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("h" * 2, 1, -32768)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=-32768 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (-32768, -32767) + + # Int16, VRTNodata=32767 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Int16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("h" * 2, 1, 32767)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=32767 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (32767, 32766) + + # Float32, VRTNodata=0 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Float32) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("f" * 2, 1, 0)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=0) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "f" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Float32) + ) == pytest.approx((0.0, 0.001)) + + # Float32, VRTNodata=1 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 3, 1, 1, gdal.GDT_Float32) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, struct.pack("f" * 3, 0, 1, 2)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 3, 1, b"\x00\xFF\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=1) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "f" * 3, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Float32) + ) == pytest.approx((1.0, 1.001, 2.0)) diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index 5edd1027b92f..1ca76128ea5f 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -286,10 +286,12 @@ def test_gdaltindex_lib_gti_non_xml(tmp_path, four_tiles): colorInterpretation="gray", mask=True, metadataOptions={"foo": "bar"}, + layerCreationOptions=["FID=my_fid"], ) ds = ogr.Open(index_filename) lyr = ds.GetLayer(0) + assert lyr.GetFIDColumn() == "my_fid" assert lyr.GetMetadataItem("RESX") == "60" assert lyr.GetMetadataItem("RESY") == "60" assert lyr.GetMetadataItem("MINX") == "0" diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 07ccacfc5c09..0585b04cd271 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -31,6 +31,7 @@ ############################################################################### import collections +import json import shutil import struct @@ -3004,6 +3005,94 @@ def test_gdalwarp_lib_cutline_zero_width_sliver(tmp_vsimem): assert ds is not None +############################################################################### +# Test cutline with zero-width sliver + + +def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_polygon(tmp_vsimem): + + geojson = { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [-101.43346, 36.91886], + [-101.43337, 36.91864], + [-101.43332, 36.91865], + [-101.43342, 36.91888], + [-101.43346, 36.91886], + ] + ], + # The below polygon has a zero-width sliver + [ + [ + [-101.4311, 36.91909], + [-101.43106, 36.91913], + [-101.43111, 36.91908], + [-101.4311, 36.91909], + ] + ], + ], + } + gdal.FileFromMemBuffer(tmp_vsimem / "cutline.geojson", json.dumps(geojson)) + src_ds = gdal.GetDriverByName("MEM").Create("", 100, 100) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + src_ds.SetSpatialRef(srs) + src_ds.SetGeoTransform([-101.5, 0.1, 0, 37, 0, -0.1]) + ds = gdal.Warp( + "", src_ds, format="MEM", cutlineDSName=tmp_vsimem / "cutline.geojson" + ) + assert ds is not None + + +############################################################################### +# Test cutline with zero-width sliver + + +def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_inner_ring(tmp_vsimem): + + geojson = { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [-101.5, 37], + [-101.4, 37], + [-101.4, 36.9], + [-101.5, 36.9], + [-101.5, 37], + ], + # The below ring has a zero-width sliver + [ + [-101.4311, 36.91909], + [-101.43106, 36.91913], + [-101.43111, 36.91908], + [-101.4311, 36.91909], + ], + # This one is OK + [ + [-101.49, 36.95], + [-101.48, 36.95], + [-101.48, 36.94], + [-101.49, 36.94], + [-101.49, 36.95], + ], + ] + ], + } + gdal.FileFromMemBuffer(tmp_vsimem / "cutline.geojson", json.dumps(geojson)) + src_ds = gdal.GetDriverByName("MEM").Create("", 100, 100) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + src_ds.SetSpatialRef(srs) + src_ds.SetGeoTransform([-101.6, 0.1, 0, 37.1, 0, -0.1]) + ds = gdal.Warp( + "", src_ds, format="MEM", cutlineDSName=tmp_vsimem / "cutline.geojson" + ) + assert ds is not None + + ############################################################################### # Test support for propagating coordinate epoch diff --git a/autotest/utilities/test_ogrinfo_lib.py b/autotest/utilities/test_ogrinfo_lib.py index 1fdc10130dce..0cdcbc214fe3 100755 --- a/autotest/utilities/test_ogrinfo_lib.py +++ b/autotest/utilities/test_ogrinfo_lib.py @@ -637,7 +637,7 @@ def test_ogrinfo_lib_relationships(): ret = gdal.VectorInfo(ds) expected = """Relationship: composite_many_to_many Type: Composite - Related table type: feature + Related table type: features Cardinality: ManyToMany Left table name: table6 Right table name: table7 @@ -667,7 +667,7 @@ def test_ogrinfo_lib_json_relationships(): # print(ret["relationships"]["composite_many_to_many"]) assert ret["relationships"]["composite_many_to_many"] == { "type": "Composite", - "related_table_type": "feature", + "related_table_type": "features", "cardinality": "ManyToMany", "left_table_name": "table6", "right_table_name": "table7", diff --git a/ci/travis/conda/compile.sh b/ci/travis/conda/compile.sh index c200601633d0..dcb73e80a665 100755 --- a/ci/travis/conda/compile.sh +++ b/ci/travis/conda/compile.sh @@ -5,18 +5,24 @@ mkdir -p packages CONDA_PLAT="" if grep -q "windows" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="win" + ARCH="64" fi if grep -q "ubuntu" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="linux" + ARCH="64" fi -if grep -q "macos" <<< "$GHA_CI_PLATFORM"; then +if grep -q "macos-14" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="osx" + ARCH="arm64" +elif grep -q "macos" <<< "$GHA_CI_PLATFORM"; then + CONDA_PLAT="osx" + ARCH="64" fi -conda build recipe --clobber-file recipe/recipe_clobber.yaml --output-folder packages -m ".ci_support/${CONDA_PLAT}_64_.yaml" -conda create -y -n test -c ./packages/${CONDA_PLAT}-64 python libgdal gdal +conda build recipe --clobber-file recipe/recipe_clobber.yaml --output-folder packages -m ".ci_support/${CONDA_PLAT}_${ARCH}_.yaml" +conda create -y -n test -c ./packages/${CONDA_PLAT}-${ARCH} python libgdal gdal conda deactivate conda activate test diff --git a/doc/source/about_no_title.rst b/doc/source/about_no_title.rst index 4b9d5f68523e..4c6634af523f 100644 --- a/doc/source/about_no_title.rst +++ b/doc/source/about_no_title.rst @@ -1,13 +1,13 @@ .. include:: ./substitutions.rst -GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source :ref:`license` by the `Open Source Geospatial Foundation`_. As a library, it presents a single raster abstract data model and single vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line utilities for data translation and processing. The `NEWS`_ page describes the January 2024 GDAL/OGR 3.8.3 release. +GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source :ref:`license` by the `Open Source Geospatial Foundation`_. As a library, it presents a single raster abstract data model and single vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line utilities for data translation and processing. The `NEWS`_ page describes the February 2024 GDAL/OGR 3.8.4 release. .. image:: ../images/OSGeo_project.png :alt: OSGeo project :target: `Open Source Geospatial Foundation`_ .. _`Open Source Geospatial Foundation`: http://www.osgeo.org/ -.. _`NEWS`: https://github.com/OSGeo/gdal/blob/v3.8.3/NEWS.md +.. _`NEWS`: https://github.com/OSGeo/gdal/blob/v3.8.4/NEWS.md See :ref:`software_using_gdal` diff --git a/doc/source/api/python/osgeo.gdal.rst b/doc/source/api/python/osgeo.gdal.rst index f344667e2470..a3ac74b7b754 100644 --- a/doc/source/api/python/osgeo.gdal.rst +++ b/doc/source/api/python/osgeo.gdal.rst @@ -5,3 +5,4 @@ osgeo.gdal module :members: :undoc-members: :show-inheritance: + :exclude-members: thisown diff --git a/doc/source/development/dev_documentation.rst b/doc/source/development/dev_documentation.rst index 6946670ea065..e23d8965d7bf 100644 --- a/doc/source/development/dev_documentation.rst +++ b/doc/source/development/dev_documentation.rst @@ -30,6 +30,26 @@ Once installed, running ``sphinx-autobuild -b html source build`` from the ``doc and serve it on a local web server at ``http://127.0.0.1:8000``. The pages served will be automatically refreshed as changes are made to underlying ``rst`` documentation files. +Python API documentation +------------------------ + +Sphinx uses the `autodoc `_ extension +to generate documentation for the Python API from Python function docstrings. +To be correctly parsed by ``autodoc``, docstrings should follow the `numpydoc Style guide `_. +Docstrings may be found in two locations. If the function was defined in Python +(i.e., using a ``%pythoncode`` SWIG directive), then the docstring must be +placed within the function definition. If the function is defined in C++ only, +then the docstring should be placed in a separate file +containing only docstrings (located in :source_file:`swig/include/python/docs`). +Sphinx loads the Python bindings when generating documentation, so for it to see any changes +the following steps must be completed: + +- rebuild the Python bindings from the build directory (``cmake --build . --target python_binding``) +- make the updated Python bindings visible to Python, either by installing them, or by running ``scripts/setdevenv.sh`` + from the build directory +- update the timestamp of the ``rst`` files associated with the page where the documentation appears (e.g., ``touch doc/source/api/python/osgeo.ogr.rst``) + + .. _rst_style: Sphinx RST Style guide diff --git a/doc/source/download.rst b/doc/source/download.rst index 686f2e0f72e8..8583de31abc6 100644 --- a/doc/source/download.rst +++ b/doc/source/download.rst @@ -13,15 +13,21 @@ Download Current Release ------------------------------------------------------------------------------ +* **2024-02-18** `gdal-3.8.4.tar.gz`_ `3.8.4 Release Notes`_ (`3.8.4 md5`_) + +.. _`3.8.4 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.4/NEWS.md +.. _`gdal-3.8.4.tar.gz`: https://github.com/OSGeo/gdal/releases/download/v3.8.4/gdal-3.8.4.tar.gz +.. _`3.8.4 md5`: https://github.com/OSGeo/gdal/releases/download/v3.8.4/gdal-3.8.4.tar.gz.md5 + +Past Releases +------------------------------------------------------------------------------ + * **2024-01-08** `gdal-3.8.3.tar.gz`_ `3.8.3 Release Notes`_ (`3.8.3 md5`_) .. _`3.8.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.3/NEWS.md .. _`gdal-3.8.3.tar.gz`: https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz .. _`3.8.3 md5`: https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz.md5 -Past Releases ------------------------------------------------------------------------------- - * **2023-12-20** `gdal-3.8.2.tar.gz`_ `3.8.2 Release Notes`_ (`3.8.2 md5`_) .. _`3.8.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.2/NEWS.md diff --git a/doc/source/drivers/raster/jp2ecw.rst b/doc/source/drivers/raster/jp2ecw.rst index 006748fa97e3..122fcba198d2 100644 --- a/doc/source/drivers/raster/jp2ecw.rst +++ b/doc/source/drivers/raster/jp2ecw.rst @@ -280,6 +280,15 @@ JP2ECW driver also arranges JP2 codestream to allow optimal access to power of two overviews. This is controlled with the creation option LEVELS." +Create support +-------------- + +While the driver advertizes the Create() capability, contrary to most other +drivers that implement it, the implementation of RasterIO() and WriteBlock() +in the JP2ECW driver does not support arbitrary random writing. +Data must be written in the dataset from top to bottom, whole line(s) at a +time. + Configuration Options --------------------- diff --git a/doc/source/drivers/raster/ogcapi.rst b/doc/source/drivers/raster/ogcapi.rst index 09d84ae05136..04204af49f7e 100644 --- a/doc/source/drivers/raster/ogcapi.rst +++ b/doc/source/drivers/raster/ogcapi.rst @@ -81,16 +81,18 @@ The following open options are available: GeoJSON items otherwise. - .. oo:: IMAGE_FORMAT - :choices: AUTO, PNG, PNG_PREFERRED, JPEG, JPEG_PREFERRED + :choices: AUTO, PNG, PNG_PREFERRED, JPEG, JPEG_PREFERRED, GEOTIFF :default: AUTO Which format to use for pixel acquisition, for tiles or map API. - Defaults to AUTO, which means - that PNG will be used if available, and fallback to JPEG otherwise. - If specifying PNG or JPEG, they must be available, otherwise the driver will - return an error. If specifying the one of the PNG_PREFERRED or JPEG_PREFERRED - value, the specified format will be used if available, and the driver will - fallback to the other format otherwise. + AUTO - This is the default and specifies that PNG images will be checked first, + then JPEG and then any additional formats the server supports. + PNG_PREFERRED - Same as AUTO + JPEG_PREFERRED - Similar to AUTO, but the order is JPEG, PNG and then any additional + formats the server supports + JPEG - Use only JPEG images. If none are available then the driver will return an error + PNG - Use only PNG images. If none are available then the driver will return an error + GEOTIFF - Use only GEOTIFF images. If none are available then the driver will return an error - .. oo:: VECTOR_FORMAT :choices: AUTO, GEOJSON, GEOJSON_PREFERRED, MVT, MVT_PREFERRED diff --git a/doc/source/drivers/raster/pdf.rst b/doc/source/drivers/raster/pdf.rst index 3825c56e3fc2..bdc949b6550b 100644 --- a/doc/source/drivers/raster/pdf.rst +++ b/doc/source/drivers/raster/pdf.rst @@ -721,9 +721,15 @@ Only GDAL builds against static builds of PDFium have been tested. Building PDFium can be challenging, and particular builds must be used to work properly with GDAL. -With GDAL >= 3.8 +With GDAL >= 3.9 ++++++++++++++++ +The scripts in the ``__ +repository must be used to build a patched version of PDFium. + +With GDAL = 3.8 ++++++++++++++++ + The scripts in the ``__ repository must be used to build a patched version of PDFium. diff --git a/doc/source/drivers/raster/usgsdem.rst b/doc/source/drivers/raster/usgsdem.rst index 0559038d7e60..7157b2598588 100644 --- a/doc/source/drivers/raster/usgsdem.rst +++ b/doc/source/drivers/raster/usgsdem.rst @@ -10,7 +10,7 @@ USGSDEM -- USGS ASCII DEM (and CDED) GDAL includes support for reading USGS ASCII DEM files. This is the traditional format used by USGS before being replaced by SDTS, and is -the format used for CDED DEM data products from the Canada. Most popular +the format used for CDED DEM data products from Canada. Most popular variations on USGS DEM files should be supported, including correct recognition of coordinate system, and georeferenced positioning. diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index 8b3333304e81..49476c354b1f 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -17,6 +17,11 @@ potentially applied as well as various kinds of metadata altered or added. VRT descriptions of datasets can be saved in an XML format normally given the extension .vrt. +Note .vrt files starting with + +- open with :ref:`ogrinfo`, etc. +- open with :ref:`gdalinfo`, etc. + The VRT format can also describe :ref:`gdal_vrttut_warped` and :ref:`gdal_vrttut_pansharpen` @@ -81,9 +86,13 @@ The following creations options are supported: .vrt Format ----------- -A `XML schema of the GDAL VRT format `_ +A `XML schema of the GDAL VRT format `_ is available. +Note, .vrt files starting with +- open with ogrinfo, etc. +- open with gdalinfo, etc. + Virtual files stored on disk are kept in an XML format with the following elements. @@ -185,7 +194,7 @@ The attributes for VRTRasterBand are: - **blockYSize** (optional, GDAL >= 3.3): block height. If not specified, defaults to the minimum of the raster height and 128. -This element may have Metadata, ColorInterp, NoDataValue, HideNoDataValue, ColorTable, GDALRasterAttributeTable, Description and MaskBand subelements as well as the various kinds of source elements such as SimpleSource, ComplexSource, AveragedSource, KernelFilteredSource and ArraySource. A raster band may have many "sources" indicating where the actual raster data should be fetched from, and how it should be mapped into the raster bands pixel space. +This element may have Metadata, ColorInterp, NoDataValue, HideNoDataValue, ColorTable, GDALRasterAttributeTable, Description and MaskBand subelements as well as the various kinds of source elements such as SimpleSource, ComplexSource, AveragedSource, NoDataFromMaskSource, KernelFilteredSource and ArraySource. A raster band may have many "sources" indicating where the actual raster data should be fetched from, and how it should be mapped into the raster bands pixel space. The allowed subelements for VRTRasterBand are : @@ -304,6 +313,8 @@ The allowed subelements for VRTRasterBand are : - **AveragedSource**: The AveragedSource is derived from the SimpleSource and shares the same properties except that it uses an averaging resampling instead of a nearest neighbour algorithm as in SimpleSource, when the size of the destination rectangle is not the same as the size of the source rectangle. Note: a more general mechanism to specify resampling algorithms can be used. See above paragraph about the 'resampling' attribute. +- **NoDataFromMaskSource**: (GDAL >= 3.9) The NoDataFromMaskSource is derived from the SimpleSource and shares the same properties except that it replaces the value of the source with the value of the NODATA child element when the value of the mask band of the source is less or equal to the MaskValueThreshold child element. + - **ComplexSource**: The ComplexSource_ is derived from the SimpleSource (so it shares the SourceFilename, SourceBand, SrcRect and DstRect elements), but it provides support to rescale and offset the range of the source values. Certain regions of the source can be masked by specifying the NODATA value, or starting with GDAL 3.3, with the true element. - **KernelFilteredSource**: The KernelFilteredSource_ is a pixel source derived from the Simple Source (so it shares the SourceFilename, SourceBand, SrcRect and DstRect elements, but it also passes the data through a simple filtering kernel specified with the Kernel element. @@ -498,6 +509,25 @@ For example, a Gaussian blur: +NoDataFromMaskSource +~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.9 + +The NoDataFromMaskSource is derived from the SimpleSource and shares the same properties except that it replaces the value of the source with the value of the NODATA child element when the value of the mask band of the source is less or equal to the MaskValueThreshold child element. +An optional RemappedValue element can be set to specify the value onto which valid pixels whose value is the one of NODATA should be remapped to. When RemappedValue is not explicitly specified, for Byte bands, if NODATA=255, it is implicitly set to 254, otherwise it is set to NODATA+1. + +.. code-block:: xml + + + in.tif + 1 + 128 + 0 + 1 + + + ArraySource ~~~~~~~~~~~ @@ -726,13 +756,13 @@ Except if (from top priority to lesser priority) : ------------------------------- So far we have described how to derive new virtual datasets from existing -files supports by GDAL. However, it is also common to need to utilize +files supported by GDAL. However, it is also common to need to utilize raw binary raster files for which the regular layout of the data is known but for which no format specific driver exists. This can be accomplished by writing a .vrt file describing the raw file. For example, the following .vrt describes a raw raster file containing -floating point complex pixels in a file called l2p3hhsso.img. The +floating point complex pixels in a file called *l2p3hhsso.img*. The image data starts from the first byte (ImageOffset=0). The byte offset between pixels is 8 (PixelOffset=8), the size of a CFloat32. The byte offset from the start of one line to the start of the next is 9376 bytes diff --git a/doc/source/drivers/raster/xyz.rst b/doc/source/drivers/raster/xyz.rst index 0cb9c409b0a9..66c0d7d4c45d 100644 --- a/doc/source/drivers/raster/xyz.rst +++ b/doc/source/drivers/raster/xyz.rst @@ -14,7 +14,8 @@ the documentation of the :ref:`gdal_grid` utility). Those datasets are ASCII files with (at least) 3 columns, each line containing the X and Y coordinates of the center of the cell and the -value of the cell. +value of the cell. (Note the XYZ driver only uses the first band of +the dataset. I.e., columns beyond the third are ignored.) The spacing between each cell must be constant. diff --git a/doc/source/drivers/vector/gml.rst b/doc/source/drivers/vector/gml.rst index 4e8080845b1d..89eb17d83eb2 100644 --- a/doc/source/drivers/vector/gml.rst +++ b/doc/source/drivers/vector/gml.rst @@ -791,7 +791,7 @@ Syntax of .gfs files -------------------- A XML Schema for .gfs files can be found at -https://raw.githubusercontent.com/OSGeo/gdal/master/data/gfs.xsd . +https://raw.githubusercontent.com/OSGeo/gdal/master/ogr/ogrsf_frmts/gml/data/gfs.xsd . Let's consider the following test.gml file : diff --git a/doc/source/drivers/vector/gpkg.rst b/doc/source/drivers/vector/gpkg.rst index 8f9425c041cb..798192145868 100644 --- a/doc/source/drivers/vector/gpkg.rst +++ b/doc/source/drivers/vector/gpkg.rst @@ -168,15 +168,20 @@ Relationships .. versionadded:: 3.6 -Relationship retrieval is supported, respecting the OGC GeoPackage Related Tables Extension. -If the Related Tables Extension is not in use then relationships will be reported for tables -which utilize FOREIGN KEY constraints. +Many-to-many relationship retrieval is supported, respecting the OGC GeoPackage Related Tables Extension. +One-to-many relationships will also be reported for tables which utilize FOREIGN KEY constraints. Relationship creation, deletion and updating is supported since GDAL 3.7. Relationships can only be updated to change their base or related table fields, or the relationship related table type. It is not permissible to change the base or related table itself, or the mapping table details. If this is desired then a new relationship should be created instead. +Note that when a many-to-many relationship is created in a GeoPackage, GDAL will always +insert the mapping table into the gpkg_contents table. Formally this is not required +by the Related Tables Extension (instead, the table should only be listed in gpkgext_relations), +however failing to list the mapping table in gpkg_contents prevents it from being usable +in some other applications (e.g. ESRI software). + Dataset open options -------------------- diff --git a/doc/source/drivers/vector/pg.rst b/doc/source/drivers/vector/pg.rst index 055958a22572..490e1caa9708 100644 --- a/doc/source/drivers/vector/pg.rst +++ b/doc/source/drivers/vector/pg.rst @@ -19,6 +19,8 @@ instead use the :ref:`PostgreSQL SQL Dump driver `. You can find additional information on the driver in the :ref:`Advanced OGR PostgreSQL driver Information ` page. +Starting with GDAL 3.9, only PostgreSQL >= 9 and PostGIS >= 2 are supported. + Driver capabilities ------------------- @@ -81,7 +83,7 @@ and named views will be treated as layers. The driver also supports the `geography `__ -column type introduced in PostGIS 1.5. +column type. The driver also supports reading and writing the following non-linear geometry types :CIRCULARSTRING, COMPOUNDCURVE, @@ -228,7 +230,7 @@ Layer Creation Options :choices: geometry, geography, BYTEA, OID The GEOM_TYPE layer creation option can be set to one - of "geometry", "geography" (PostGIS >= 1.5), "BYTEA" or "OID" to + of "geometry", "geography", "BYTEA" or "OID" to force the type of geometry used for a table. For a PostGIS database, "geometry" is the default value. PostGIS "geography" assumes a geographic SRS (before PostGIS 2.2, it was even required to be EPSG:4326), but the @@ -266,7 +268,7 @@ Layer Creation Options :choices: 2, 3, XYM, XYZM Control the dimension of the layer. Important - to set to 2 for 2D layers with PostGIS 1.0+ as it has constraints on + to set to 2 for 2D layers as it has constraints on the geometry dimension during loading. - .. lco:: GEOMETRY_NAME diff --git a/doc/source/drivers/vector/vrt.rst b/doc/source/drivers/vector/vrt.rst index 93a2555d69ff..b1de74ac1050 100644 --- a/doc/source/drivers/vector/vrt.rst +++ b/doc/source/drivers/vector/vrt.rst @@ -544,3 +544,7 @@ Other Notes that is possible. For instance if the source is an RDBMS. You can turn off that feature by setting the *useSpatialSubquery* attribute of the GeometryField element to FALSE. +- .vrt files starting with + - open with ogrinfo, etc. + - open with gdalinfo, etc. + diff --git a/doc/source/index.rst b/doc/source/index.rst index 1f37d01ff378..31f6f8829ba7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -4,7 +4,10 @@ GDAL .. include:: ./about_no_title.rst -This documentation is also available as a `PDF file `_. +This documentation is also available as a `PDF file `_, +and `a .ZIP of individual HTML pages `_ for offline browsing. (The .ZIP also includes that .PDF.) + + .. toctree:: :maxdepth: 2 diff --git a/doc/source/programs/gdal2xyz.rst b/doc/source/programs/gdal2xyz.rst index a3dc4738e959..027990c56c5d 100644 --- a/doc/source/programs/gdal2xyz.rst +++ b/doc/source/programs/gdal2xyz.rst @@ -22,14 +22,13 @@ Synopsis [-skipnodata] [-csv] [-srcnodata ] [-dstnodata ] - [] + Description ----------- The :program:`gdal2xyz` utility can be used to translate a raster file into xyz format. -`gdal2xyz` can be used as an alternative to `gdal_translate of=xyz`, but supporting other options, -for example: +`gdal2xyz` can be used as an alternative to `gdal_translate of=xyz`. Features include: * Select more then one band * Skip or replace nodata value @@ -101,5 +100,7 @@ Examples gdal2xyz -b 1 -b 2 -dstnodata 0 input.tif output.txt -To create a text file in `xyz` format from the input file `input.tif`, including the first and second bands, -while replacing the dataset nodata values with zeros. +To create a text file in `xyz` format from the input file `input.tif`. +The first columns, x and y, are the coordinates of the centers of each cell. +The remaining colums represent the first and second bands. +We also replace the dataset nodata values with zeros. diff --git a/doc/source/programs/gdal_grid.rst b/doc/source/programs/gdal_grid.rst index 91ac2c90683d..806905379d87 100644 --- a/doc/source/programs/gdal_grid.rst +++ b/doc/source/programs/gdal_grid.rst @@ -48,6 +48,9 @@ computer. .. include:: options/ot.rst +If not set then a default type is used, which might not be supported +by the relevant driver, causing a error. + .. include:: options/of.rst .. option:: -txe @@ -444,15 +447,27 @@ content: -This description specifies so called 2.5D geometry with three coordinates X, Y -and Z. Z value will be used for interpolation. Now you can use *dem.vrt* -with all OGR programs (start with :ref:`ogrinfo` to test that everything works -fine). The datasource will contain a single layer called *"dem"* filled -with point features constructed from values in CSV file. Using this technique -you can handle CSV files with more than three columns, switch columns, etc. +This description specifies so called 2.5D geometry with three coordinates +X, Y and Z. The Z value will be used for interpolation. Now you can +use *dem.vrt* with all OGR programs (start with :ref:`ogrinfo` to test that +everything works fine). The datasource will contain a single layer called +*"dem"* filled with point features constructed from values in the CSV file. +Using this technique you can handle CSV files with more than three +columns, switch columns, etc. OK, now the final step: + +.. code-block:: -If your CSV file does not contain column headers then it can be handled in the -following way: + gdal_grid dem.vrt demv.tif + +Or, if we do not wish to use a VRT file: + +.. code-block:: + + gdal_grid -l dem -oo X_POSSIBLE_NAMES=Easting \ + -oo Y_POSSIBLE_NAMES=Northing -zfield Elevation dem.csv dem.tif + +If your CSV file does not contain column headers then it can be handled +in the VRT file in the following way: .. code-block:: xml @@ -461,6 +476,10 @@ following way: The :ref:`vector.csv` description page contains details on CSV format supported by GDAL/OGR. +Creating multiband files: not directly possible with gdal_grid. +One might use gdal_grid multiple times to create one band per file, +and then use gdalbuildvrt -separate + gdal_translate to combine the one-band-files into a single one. + C API ----- @@ -475,7 +494,8 @@ Values to interpolate will be read from Z value of geometry record. :: - gdal_grid -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff + gdal_grid -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 \ + -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff The next command does the same thing as the previous one, but reads values to interpolate from the attribute field specified with **-zfield** option @@ -485,5 +505,7 @@ The :config:`GDAL_NUM_THREADS` is also set to parallelize the computation. :: - gdal_grid -zfield "Elevation" -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff --config GDAL_NUM_THREADS ALL_CPUS + gdal_grid -zfield "Elevation" -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 \ + -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt \ + dem.tiff --config GDAL_NUM_THREADS ALL_CPUS diff --git a/doc/source/programs/gdal_rasterize.rst b/doc/source/programs/gdal_rasterize.rst index 71060b79ba11..32121529f2b1 100644 --- a/doc/source/programs/gdal_rasterize.rst +++ b/doc/source/programs/gdal_rasterize.rst @@ -6,7 +6,7 @@ gdal_rasterize .. only:: html - Burns vector geometries into a raster. + Burns vector geometries into a raster .. Index:: gdal_rasterize @@ -202,7 +202,7 @@ raster data is only supported since GDAL 2.1.0. The GDAL supported output file. Must support update mode access. This file will be created (or overwritten if it already exists). -The program create a new target raster image when any of the :option:`-of`, +The program creates a new target raster image when any of the :option:`-of`, :option:`-a_nodata`, :option:`-init`, :option:`-a_srs`, :option:`-co`, :option:`-te`, :option:`-tr`, :option:`-tap`, :option:`-ts`, or :option:`-ot` options are used. The resolution or size must be specified using the :option:`-tr` or :option:`-ts` option for all new @@ -216,13 +216,13 @@ This utility is also callable from C with :cpp:func:`GDALRasterize`. .. versionadded:: 2.1 -Example -------- +Examples +-------- The following would burn all polygons from mask.shp into the RGB TIFF file work.tif with the color red (RGB = 255,0,0). -:: +.. code-block:: gdal_rasterize -b 1 -b 2 -b 3 -burn 255 -burn 0 -burn 0 -l mask mask.shp work.tif @@ -230,7 +230,7 @@ file work.tif with the color red (RGB = 255,0,0). The following would burn all "class A" buildings into the output elevation file, pulling the top elevation from the ROOF_H attribute. -:: +.. code-block:: gdal_rasterize -a ROOF_H -where "class='A'" -l footprints footprints.shp city_dem.tif @@ -238,6 +238,6 @@ The following would burn all polygons from footprint.shp into a new 1000x1000 rgb TIFF as the color red. Note that :option:`-b` is not used; the order of the :option:`-burn` options determines the bands of the output raster. -:: +.. code-block:: gdal_rasterize -burn 255 -burn 0 -burn 0 -ot Byte -ts 1000 1000 -l footprints footprints.shp mask.tif diff --git a/doc/source/programs/gdal_translate.rst b/doc/source/programs/gdal_translate.rst index e55d4e43f681..8c13b82d5e56 100644 --- a/doc/source/programs/gdal_translate.rst +++ b/doc/source/programs/gdal_translate.rst @@ -443,7 +443,8 @@ To create a JPEG-compressed TIFF with internal mask from a RGBA dataset :: - gdal_translate rgba.tif withmask.tif -b 1 -b 2 -b 3 -mask 4 -co COMPRESS=JPEG -co PHOTOMETRIC=YCBCR --config GDAL_TIFF_INTERNAL_MASK YES + gdal_translate rgba.tif withmask.tif -b 1 -b 2 -b 3 -mask 4 -co COMPRESS=JPEG \ + -co PHOTOMETRIC=YCBCR --config GDAL_TIFF_INTERNAL_MASK YES To create a RGBA dataset from a RGB dataset with a mask diff --git a/doc/source/programs/gdalbuildvrt.rst b/doc/source/programs/gdalbuildvrt.rst index 253e1bcc462a..f3bf3eec0a11 100644 --- a/doc/source/programs/gdalbuildvrt.rst +++ b/doc/source/programs/gdalbuildvrt.rst @@ -24,6 +24,7 @@ Synopsis [-addalpha] [-hidenodata] [-srcnodata "[ ]..."] [-vrtnodata "[ ]..." [-ignore_srcmaskband] + [-nodata_max_mask_threshold ] [-a_srs ] [-r {nearest|bilinear|cubic|cubicspline|lanczos|average|mode}] [-oo =]... @@ -146,6 +147,14 @@ changed in later versions. not be taken into account, and in case of overlapping between sources, the last one will override previous ones in areas of overlap. +.. option:: -nodata_max_mask_threshold + + .. versionadded:: 3.9 + + Insert a source, which replaces the value of the source + with the value of :option:`-vrtnodata` (or 0 if not specified) when the value + of the mask band of the source is less or equal to the threshold. + .. option:: -b Select an input to be processed. Bands are numbered from 1. diff --git a/doc/source/programs/gdallocationinfo.rst b/doc/source/programs/gdallocationinfo.rst index 8814cced6f75..5f8d9df82970 100644 --- a/doc/source/programs/gdallocationinfo.rst +++ b/doc/source/programs/gdallocationinfo.rst @@ -94,15 +94,15 @@ pixel. Currently it reports: - The location of the pixel in pixel/line space. - The result of a LocationInfo metadata query against the datasource. - This is implement for VRT files which will report the + This is implemented for VRT files which will report the file(s) used to satisfy requests for that pixel, and by the - :ref:`raster.mbtiles` driver + :ref:`raster.mbtiles` driver. - The raster pixel value of that pixel for all or a subset of the bands. - The unscaled pixel value if a Scale and/or Offset apply to the band. The pixel selected is requested by x/y coordinate on the command line, or read from stdin. More than one coordinate pair can be supplied when reading -coordinates from stdin. By default pixel/line coordinates are expected. +coordinates from stdin. By default integer pixel/line coordinates are expected. However with use of the :option:`-geoloc`, :option:`-wgs84`, or :option:`-l_srs` switches it is possible to specify the location in other coordinate systems. diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index f54cf6d28e73..15cd905f4188 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -21,7 +21,7 @@ Synopsis [-f ] [-tileindex ] [-write_absolute_path] [-skip_different_projection] [-t_srs ] [-src_srs_name ] [-src_srs_format {AUTO|WKT|EPSG|PROJ}] - [-lyr_name ] + [-lyr_name ] [-lco =]... [-gti_filename ] [-tr ] [-te ] [-ot ] [-bandcount ] [-nodata [,...]] @@ -130,6 +130,12 @@ tileindex, or as input for the :ref:`GTI ` driver. Layer name to create/append to in the output tile index file. +.. option:: -lco = + + .. versionadded:: 3.9 + + Layer creation option (format specific) + .. option:: The name of the output file to create/append to. The default dataset will diff --git a/doc/source/programs/gdaltransform.rst b/doc/source/programs/gdaltransform.rst index 6411c2d6ea43..38242760900a 100644 --- a/doc/source/programs/gdaltransform.rst +++ b/doc/source/programs/gdaltransform.rst @@ -6,7 +6,7 @@ gdaltransform .. only:: html - Transforms coordinates. + Transforms coordinates .. Index:: gdaltransform @@ -16,7 +16,7 @@ Synopsis .. code-block:: gdaltransform [--help] [--help-general] - [-i] [-s_srs ] [-t_srs ] [-to >NAME>=]... + [-i] [-s_srs ] [-t_srs ] [-to =]... [-s_coord_epoch ] [-t_coord_epoch ] [-ct ] [-order ] [-tps] [-rpc] [-geoloc] [-gcp [elevation]]... [-output_xy] @@ -116,13 +116,15 @@ projection,including GCP-based transformations. .. option:: - File with source projection definition or GCP's. If - not given, source projection is read from the command-line :option:`-s_srs` - or :option:`-gcp` parameters + Raster dataset with source projection definition or GCPs. If + not given, source projection/GCPs are read from the command-line :option:`-s_srs` + or :option:`-gcp` parameters. + + Note that only the SRS and/or GCPs of this input file is taken into account, and not its pixel content. .. option:: - File with destination projection definition. + Raster dataset with destination projection definition. Coordinates are read as pairs, triples (for 3D,) or (since GDAL 3.0.0,) quadruplets (for X,Y,Z,time) of numbers per line from standard @@ -144,7 +146,7 @@ Reprojection Example Simple reprojection from one projected coordinate system to another: -:: +.. code-block:: bash gdaltransform -s_srs EPSG:28992 -t_srs EPSG:31370 177502 311865 @@ -152,9 +154,9 @@ Simple reprojection from one projected coordinate system to another: Produces the following output in meters in the "Belge 1972 / Belgian Lambert 72" projection: -:: +.. code-block:: bash - 244510.77404604 166154.532871342 -1046.79270555763 + 244296.724777415 165937.350217148 0 Image RPC Example +++++++++++++++++ @@ -165,14 +167,14 @@ used, the transformation is from output georeferenced (WGS84) coordinates back to image coordinates. -:: +.. code-block:: bash gdaltransform -i -rpc 06OCT20025052-P2AS-005553965230_01_P001.TIF 125.67206 39.85307 50 Produces this output measured in pixels and lines on the image: -:: +.. code-block:: bash 3499.49282422381 2910.83892848414 50 @@ -182,7 +184,7 @@ X,Y,Z,time transform 15-term time-dependent Helmert coordinate transformation from ITRF2000 to ITRF93 for a coordinate at epoch 2000.0 -:: +.. code-block:: bash gdaltransform -ct "+proj=pipeline +step +proj=unitconvert +xy_in=deg \ +xy_out=rad +step +proj=cart +step +proj=helmert +convention=position_vector \ @@ -194,6 +196,6 @@ for a coordinate at epoch 2000.0 Produces this output measured in longitude degrees, latitude degrees and ellipsoid height in metre: -:: +.. code-block:: bash 2.0000005420366 49.0000003766711 -0.0222802283242345 diff --git a/doc/source/programs/gdalwarp.rst b/doc/source/programs/gdalwarp.rst index 463113ede342..1da8b4f5c52a 100644 --- a/doc/source/programs/gdalwarp.rst +++ b/doc/source/programs/gdalwarp.rst @@ -71,7 +71,7 @@ with control information. Blue, Green, Red, NearInfraRed in an output dataset with bands ordered as Red, Green, Blue. - :: + .. code-block:: bash gdalwarp in_bgrn.tif out_rgb.tif -b 3 -b 2 -b 1 -overwrite @@ -84,7 +84,7 @@ with control information. is only useful when updating an existing dataset, e.g to warp one band at at time. - :: + .. code-block:: bash gdal_create -if in_red.tif -bands 3 out_rgb.tif gdalwarp in_red.tif out_rgb.tif -srcband 1 -dstband 1 @@ -579,13 +579,13 @@ less than 100% then you know things are IO bound. Otherwise they are CPU bound. The ``--debug`` option may also provide useful information. For instance, after running the following: -.. code-block:: +.. code-block:: bash gdalwarp --debug on abc.tif def.tif a message like the following will be output: -:: +.. code-block:: GDAL: 224 block reads on 32 block band 1 of utm.tif @@ -594,7 +594,7 @@ that 224 block reads were done, implying that lots of data was having to be re-read, presumably because of a limited IO cache. You will also see messages like: -:: +.. code-block:: GDAL: GDALWarpKernel()::GWKNearestNoMasksByte() Src=0,0,512x512 Dst=0,0,512x512 @@ -648,7 +648,7 @@ Examples - Basic transformation: -:: +.. code-block:: bash gdalwarp -t_srs EPSG:4326 input.tif output.tif @@ -657,7 +657,7 @@ Examples control points mapping the corners to lat/long could be warped to a UTM projection with a command like this: -:: +.. code-block:: bash gdalwarp -t_srs '+proj=utm +zone=11 +datum=WGS84' -overwrite raw_spot.tif utm11.tif @@ -667,19 +667,21 @@ Examples .. versionadded:: 2.2 -:: +.. code-block:: bash - gdalwarp -overwrite HDF4_SDS:ASTER_L1B:"pg-PR1B0000-2002031402_100_001":2 pg-PR1B0000-2002031402_100_001_2.tif + gdalwarp -overwrite HDF4_SDS:ASTER_L1B:"pg-PR1B0000-2002031402_100_001":2 \ + pg-PR1B0000-2002031402_100_001_2.tif - To apply a cutline on a un-georeferenced image and clip from pixel (220,60) to pixel (1160,690): -:: +.. code-block:: bash - gdalwarp -overwrite -to SRC_METHOD=NO_GEOTRANSFORM -to DST_METHOD=NO_GEOTRANSFORM -te 220 60 1160 690 -cutline cutline.csv in.png out.tif + gdalwarp -overwrite -to SRC_METHOD=NO_GEOTRANSFORM -to DST_METHOD=NO_GEOTRANSFORM \ + -te 220 60 1160 690 -cutline cutline.csv in.png out.tif where cutline.csv content is like: -:: +.. code-block:: id,WKT 1,"POLYGON((....))" @@ -688,7 +690,7 @@ where cutline.csv content is like: .. versionadded:: 2.2 -:: +.. code-block:: bash gdalwarp -overwrite in_dem.tif out_dem.tif -s_srs EPSG:4326+5773 -t_srs EPSG:4979 diff --git a/doc/source/programs/ogrlineref.rst b/doc/source/programs/ogrlineref.rst index 744fd4219bf0..b3969b1d1875 100644 --- a/doc/source/programs/ogrlineref.rst +++ b/doc/source/programs/ogrlineref.rst @@ -6,7 +6,7 @@ ogrlineref .. only:: html - Create linear reference and provide some calculations using it. + Create linear reference and provide some calculations using it .. Index:: ogrlineref @@ -42,8 +42,8 @@ The :program:`ogrlineref` program can be used to: - return the portion of the path according to the "linear referenced" begin and end distances -The :program:`ogrlineref` creates a linear reference - a file containing -a segments of special length (e.g. 1 km in reference units) and gets coordinates, +The :program:`ogrlineref` utility creates a linear reference - a file containing +segments of a certain length (e.g. 1 km in reference units.) The user can get coordinates, linear referenced distances or sublines (subpaths) from this file. The utility does not require the ``M`` or ``Z`` components in the geometry. The results can be stored in any OGR supported format. diff --git a/doc/source/software_using_gdal.rst b/doc/source/software_using_gdal.rst index 7a2150ae827f..f25df05bdb7a 100644 --- a/doc/source/software_using_gdal.rst +++ b/doc/source/software_using_gdal.rst @@ -72,6 +72,7 @@ Proprietary license / Other - `3D DEM Viewer `_ from MS MacroSystem. - `Cadcorp SIS: `_ A Windows GIS with a GDAL and OGR plugins. +- `Carmenta Engine `_ (previously known as SpatialAce): A GIS Rapid Application Development environment - `CARTO `_ A cloud mapping platform to analyze and visualize geospatial data. - `Cartographica `_ Macintosh GIS package. - `CatchmentSIM `_ A Windows terrain analysis model for hydrologic applications. @@ -113,7 +114,6 @@ Proprietary license / Other - `SkylineGlobe `_ The Skyline suite of interactive applications allows you to build, view, query and analyze customized, virtual 3D landscapes. - `SpacEyes3D `_ 3D visualization software for cartographic data. - `Spatial Manager `_ A product suite designed designed to manage spatial data in a simple, fast and inexpensive way. Uses GDAL to import/export data. -- `Carmenta Engine `_ (previously known as SpatialAce): A GIS Rapid Application Development environment - `TacitView `_ An imagery visualization and exploitation package for military intelligence. - `TatukGIS `_ Desktop GIS mapping and data editing application. - `Team Awareness Kit `_ Suite of georeferenced imagery and situational awareness tools developed for military planning and execution, now available for civilian use. diff --git a/doc/source/sponsors/index.rst b/doc/source/sponsors/index.rst index bd17ad8a0f44..059f558eea7d 100644 --- a/doc/source/sponsors/index.rst +++ b/doc/source/sponsors/index.rst @@ -131,6 +131,10 @@ the health of the project: `Regrid `__ + .. container:: horizontal-logo + + `Satelligence `__ + .. container:: horizontal-logo `Space Intelligence `__ diff --git a/doc/source/user/sql_sqlite_dialect.rst b/doc/source/user/sql_sqlite_dialect.rst index c482622667bb..f1c6d8fd8259 100644 --- a/doc/source/user/sql_sqlite_dialect.rst +++ b/doc/source/user/sql_sqlite_dialect.rst @@ -343,11 +343,11 @@ returns: :: OGRFeature(SELECT):0 - POINT (2.342878767069653 48.85661793020374) + POINT (2.34287687375113 48.856622357411) .. code-block:: shell - ogrinfo cities.csv -dialect sqlite -sql "SELECT *, ogr_geocode(city, 'country') AS country, ST_Centroid(ogr_geocode(city)) FROM cities" + ogrinfo cities.csv -dialect sqlite -sql "SELECT *, ogr_geocode(city, 'country_code') AS country_code, ST_Centroid(ogr_geocode(city)) FROM cities" returns: @@ -357,57 +357,30 @@ returns: :: OGRFeature(SELECT):0 - id (Real) = 1 - city (String) = Paris - country (String) = France métropolitaine - POINT (2.342878767069653 48.85661793020374) + city (String) = Paris + country_code (String) = fr + POINT (2.34287687375113 48.856622357411) OGRFeature(SELECT):1 - id (Real) = 2 - city (String) = London - country (String) = United Kingdom - POINT (-0.109369427546499 51.500506667319407) + city (String) = London + country_code (String) = gb + POINT (-0.109415723431508 51.5004964757441) OGRFeature(SELECT):2 - id (Real) = 3 - city (String) = Rennes - country (String) = France métropolitaine - POINT (-1.68185153381778 48.111663929761093) + city (String) = Rennes + country_code (String) = fr + POINT (-1.68185479486048 48.1116771631195) OGRFeature(SELECT):3 - id (Real) = 4 - city (String) = Strasbourg - country (String) = France métropolitaine - POINT (7.767762859150757 48.571233274141846) + city (String) = New York + country_code (String) = us + POINT (-73.9388908443975 40.6632061220125) OGRFeature(SELECT):4 - id (Real) = 5 - city (String) = New York - country (String) = United States of America - POINT (-73.938140243499049 40.663799577449979) - - OGRFeature(SELECT):5 - id (Real) = 6 - city (String) = Berlin - country (String) = Deutschland - POINT (13.402306623451983 52.501470321410636) - - OGRFeature(SELECT):6 - id (Real) = 7 - city (String) = Beijing - POINT (116.391195 39.9064702) - - OGRFeature(SELECT):7 - id (Real) = 8 - city (String) = Brasilia - country (String) = Brasil - POINT (-52.830435216371839 -10.828214867369699) - - OGRFeature(SELECT):8 - id (Real) = 9 - city (String) = Moscow - country (String) = Российская Федерация - POINT (37.367988106866868 55.556208255649558) + city (String) = Beijing + country_code (String) = cn + POINT (116.3912972 39.9057136) + .. highlight:: sql diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst index 1e36919d9922..dbd62e56ea5c 100644 --- a/doc/source/user/virtual_file_systems.rst +++ b/doc/source/user/virtual_file_systems.rst @@ -390,7 +390,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following - proxy=value - proxyauth=value - proxyuserpwd=value -- pc_url_signing=yes/no: whether to use the URL signing mechanism of Microsoft Planetary Computer (https://planetarycomputer.microsoft.com/docs/concepts/sas/). (GDAL >= 3.5.2) +- pc_url_signing=yes/no: whether to use the URL signing mechanism of Microsoft Planetary Computer (https://planetarycomputer.microsoft.com/docs/concepts/sas/). (GDAL >= 3.5.2). Note that starting with GDAL 3.9, this may also be set with the path-specific option ( cf :cpp:func:`VSISetPathSpecificOption`) ``VSICURL_PC_URL_SIGNING`` set to ``YES``. - pc_collection=name: name of the collection of the dataset for Planetary Computer URL signing. Only used when pc_url_signing=yes. (GDAL >= 3.5.2) Partial downloads (requires the HTTP server to support random reading) are done with a 16 KB granularity by default. Starting with GDAL 2.3, the chunk size can be configured with the :config:`CPL_VSIL_CURL_CHUNK_SIZE` configuration option, with a value in bytes. If the driver detects sequential reading, it will progressively increase the chunk size up to 128 times :config:`CPL_VSIL_CURL_CHUNK_SIZE` (so 2 MB by default) to improve download performance. diff --git a/docker/README.md b/docker/README.md index 1d8362c4eaea..0e22b44ab15c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -98,11 +98,11 @@ If you are getting a ``: arena 0 background thread creation failed (1) # Images of releases -Tagged images of recent past releases are available. The last ones (at time of writing) are for GDAL 3.8.3 and PROJ 9.3.1, for linux/amd64 and linux/arm64: -* ghcr.io/osgeo/gdal:alpine-small-3.8.3 -* ghcr.io/osgeo/gdal:alpine-normal-3.8.3 -* ghcr.io/osgeo/gdal:ubuntu-small-3.8.3 -* ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 +Tagged images of recent past releases are available. The last ones (at time of writing) are for GDAL 3.8.4 and PROJ 9.3.1, for linux/amd64 and linux/arm64: +* ghcr.io/osgeo/gdal:alpine-small-3.8.4 +* ghcr.io/osgeo/gdal:alpine-normal-3.8.4 +* ghcr.io/osgeo/gdal:ubuntu-small-3.8.4 +* ghcr.io/osgeo/gdal:ubuntu-full-3.8.4 ## Multi-arch Images diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile index ee94d2ad288f..21ce77bc742f 100644 --- a/docker/ubuntu-full/Dockerfile +++ b/docker/ubuntu-full/Dockerfile @@ -197,12 +197,12 @@ RUN . /buildscripts/bh-set-envvars.sh \ ARG WITH_PDFIUM=yes RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$" ; then ( \ - wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ + wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && chown -R root:root install \ && mv install/lib/* /usr/lib/ \ && mv install/include/* /usr/include/ \ - && rm -rf install-ubuntu2004-rev5952.tar.gz install \ + && rm -rf install-ubuntu2004-rev6309.tar.gz install \ && apt-get update -y \ && apt-get install -y --fix-missing --no-install-recommends liblcms2-dev${APT_ARCH_SUFFIX} \ && rm -rf /var/lib/apt/lists/* \ @@ -243,7 +243,8 @@ RUN . /buildscripts/bh-set-envvars.sh \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-acero-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt-get update -y \ && apt-get install -y --fix-missing --no-install-recommends rsync ccache \ diff --git a/frmts/ecw/ecwcreatecopy.cpp b/frmts/ecw/ecwcreatecopy.cpp index ed0ac5b0f266..cb9b8738b066 100644 --- a/frmts/ecw/ecwcreatecopy.cpp +++ b/frmts/ecw/ecwcreatecopy.cpp @@ -1032,10 +1032,33 @@ CPLErr GDALECWCompressor::Initialize( const char *pszGMLJP2V2Def = CSLFetchNameValue(papszOptions, "GMLJP2V2_DEF"); if (pszGMLJP2V2Def != nullptr) + { WriteJP2Box(oJP2MD.CreateGMLJP2V2(nXSize, nYSize, pszGMLJP2V2Def, poSrcDS)); + } else - WriteJP2Box(oJP2MD.CreateGMLJP2(nXSize, nYSize)); + { + if (!poSRS || poSRS->IsEmpty() || + GDALJP2Metadata::IsSRSCompatible(poSRS)) + { + WriteJP2Box(oJP2MD.CreateGMLJP2(nXSize, nYSize)); + } + else if (CSLFetchNameValue(papszOptions, "GMLJP2")) + { + CPLError(CE_Warning, CPLE_AppDefined, + "GMLJP2 box was explicitly required but " + "cannot be written due " + "to lack of georeferencing and/or unsupported " + "georeferencing " + "for GMLJP2"); + } + else + { + CPLDebug( + "JP2ECW", + "Cannot write GMLJP2 box due to unsupported SRS"); + } + } } if (CPLFetchBool(papszOptions, "GeoJP2", true)) WriteJP2Box(oJP2MD.CreateJP2GeoTIFF()); diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 60021b170eca..c05bb12213fb 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -5066,6 +5066,44 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, } } +#ifdef HAVE_JXL + if (l_nCompression == COMPRESSION_JXL) + { + // Reflects tif_jxl's GetJXLDataType() + if (eType != GDT_Byte && eType != GDT_UInt16 && eType != GDT_Float32) + { + ReportError(pszFilename, CE_Failure, CPLE_NotSupported, + "Data type %s not supported for JXL compression. Only " + "Byte, UInt16, Float32 are supported", + GDALGetDataTypeName(eType)); + return nullptr; + } + const struct + { + GDALDataType eDT; + int nBitsPerSample; + } asSupportedDTBitsPerSample[] = { + {GDT_Byte, 8}, + {GDT_UInt16, 16}, + {GDT_Float32, 32}, + }; + for (const auto &sSupportedDTBitsPerSample : asSupportedDTBitsPerSample) + { + if (eType == sSupportedDTBitsPerSample.eDT && + l_nBitsPerSample != sSupportedDTBitsPerSample.nBitsPerSample) + { + ReportError( + pszFilename, CE_Failure, CPLE_NotSupported, + "Bits per sample=%d not supported for JXL compression. " + "Only %d is supported for %s data type.", + l_nBitsPerSample, sSupportedDTBitsPerSample.nBitsPerSample, + GDALGetDataTypeName(eType)); + return nullptr; + } + } + } +#endif + int nPredictor = PREDICTOR_NONE; pszValue = CSLFetchNameValue(papszParamList, "PREDICTOR"); if (pszValue != nullptr) diff --git a/frmts/hfa/hfadataset.h b/frmts/hfa/hfadataset.h index 018cd18befc4..0c49a98d3660 100644 --- a/frmts/hfa/hfadataset.h +++ b/frmts/hfa/hfadataset.h @@ -53,7 +53,7 @@ class HFADataset final : public GDALPamDataset { friend class HFARasterBand; - HFAHandle hHFA; + HFAHandle hHFA = nullptr; bool bMetadataDirty = false; diff --git a/frmts/hfa/hfaopen.cpp b/frmts/hfa/hfaopen.cpp index e9890058096c..f8ee8fecdd2c 100644 --- a/frmts/hfa/hfaopen.cpp +++ b/frmts/hfa/hfaopen.cpp @@ -3859,9 +3859,9 @@ CPLErr HFARenameReferences(HFAHandle hHFA, const char *pszNewBase, // Update the filename. if (strncmp(osFileName, pszOldBase, strlen(pszOldBase)) == 0) { - CPLString osNew = pszNewBase; - osNew += osFileName.c_str() + strlen(pszOldBase); - osFileName = osNew; + std::string osNew = pszNewBase; + osNew += (osFileName.c_str() + strlen(pszOldBase)); + osFileName = std::move(osNew); } apoNodeList[iNode]->SetStringField("dependent.string", osFileName); diff --git a/frmts/jp2kak/jp2kakdataset.cpp b/frmts/jp2kak/jp2kakdataset.cpp index 287fea1931b1..4738b11c69d5 100644 --- a/frmts/jp2kak/jp2kakdataset.cpp +++ b/frmts/jp2kak/jp2kakdataset.cpp @@ -2701,7 +2701,7 @@ static GDALDataset *JP2KAKCreateCopy(const char *pszFilename, { const char *pszGMLJP2V2Def = CSLFetchNameValue(papszOptions, "GMLJP2V2_DEF"); - GDALJP2Box *poBox; + GDALJP2Box *poBox = nullptr; if (pszGMLJP2V2Def != nullptr) { poBox = oJP2MD.CreateGMLJP2V2(nXSize, nYSize, pszGMLJP2V2Def, @@ -2709,18 +2709,42 @@ static GDALDataset *JP2KAKCreateCopy(const char *pszFilename, } else { - poBox = oJP2MD.CreateGMLJP2(nXSize, nYSize); + const OGRSpatialReference *poSRS = + poSrcDS->GetGCPCount() > 0 ? poSrcDS->GetGCPSpatialRef() + : poSrcDS->GetSpatialRef(); + if (!poSRS || poSRS->IsEmpty() || + GDALJP2Metadata::IsSRSCompatible(poSRS)) + { + poBox = oJP2MD.CreateGMLJP2(nXSize, nYSize); + } + else if (CSLFetchNameValue(papszOptions, "GMLJP2")) + { + CPLError(CE_Warning, CPLE_AppDefined, + "GMLJP2 box was explicitly required but cannot be " + "written due " + "to lack of georeferencing and/or unsupported " + "georeferencing " + "for GMLJP2"); + } + else + { + CPLDebug("JP2KAK", + "Cannot write GMLJP2 box due to unsupported SRS"); + } } - try + if (poBox) { - JP2KAKWriteBox(&family, poBox); - } - catch (...) - { - CPLDebug("JP2KAK", "JP2KAKWriteBox) - caught exception."); - oCodeStream.destroy(); - delete poBox; - return nullptr; + try + { + JP2KAKWriteBox(&family, poBox); + } + catch (...) + { + CPLDebug("JP2KAK", "JP2KAKWriteBox) - caught exception."); + oCodeStream.destroy(); + delete poBox; + return nullptr; + } } } if (CPLFetchBool(papszOptions, "GeoJP2", true)) diff --git a/frmts/jp2lura/jp2luradataset.cpp b/frmts/jp2lura/jp2luradataset.cpp index c3382e3811d7..6af9b66dcf7a 100644 --- a/frmts/jp2lura/jp2luradataset.cpp +++ b/frmts/jp2lura/jp2luradataset.cpp @@ -713,10 +713,19 @@ GDALDataset *JP2LuraDataset::CreateCopy(const char *pszFilename, { bGeoreferencingCompatOfGeoJP2 = true; oJP2MD.SetGeoTransform(adfGeoTransform); + + if (poSRS && !poSRS->IsEmpty()) + { + bGeoreferencingCompatOfGMLJP2 = + GDALJP2Metadata::IsSRSCompatible(poSRS); + if (!bGeoreferencingCompatOfGMLJP2) + { + CPLDebug( + "JP2LURA", + "Cannot write GMLJP2 box due to unsupported SRS"); + } + } } - bGeoreferencingCompatOfGMLJP2 = - poSRS != nullptr && !poSRS->IsEmpty() && - poSrcDS->GetGeoTransform(adfGeoTransform) == CE_None; } if (poSrcDS->GetMetadata("RPC") != nullptr) { diff --git a/frmts/mrf/mrf_band.cpp b/frmts/mrf/mrf_band.cpp index 97783fa3fb6a..d3eade863d2f 100644 --- a/frmts/mrf/mrf_band.cpp +++ b/frmts/mrf/mrf_band.cpp @@ -379,10 +379,20 @@ static void *ZstdCompBlock(buf_mgr &src, size_t extrasize, int c_level, dst = dbuff.data(); } - size_t val = - ZSTD_compressCCtx(cctx, dst, size, src.buffer, src.size, c_level); + // Use the streaming interface, it's faster and better + // See discussion at https://github.com/facebook/zstd/issues/3729 + ZSTD_outBuffer output = {dst, size, 0}; + ZSTD_inBuffer input = {src.buffer, src.size, 0}; + // Set level + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, c_level); + // First, pass a continue flag, otherwise it will compress in one go + size_t val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_continue); + // If it worked, pass the end flag to flush the buffer + if (val == 0) + val = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); if (ZSTD_isError(val)) return nullptr; + val = output.pos; // If we didn't need the buffer, packed data is already in the user buffer if (dbuff.empty()) diff --git a/frmts/ogcapi/gdalogcapidataset.cpp b/frmts/ogcapi/gdalogcapidataset.cpp index 60434f426ec0..46d91ca463a5 100644 --- a/frmts/ogcapi/gdalogcapidataset.cpp +++ b/frmts/ogcapi/gdalogcapidataset.cpp @@ -76,6 +76,7 @@ class OGCAPIDataset final : public GDALDataset double m_adfGeoTransform[6]; OGRSpatialReference m_oSRS{}; + CPLString m_osTileData{}; // Classic OGC API features /items access std::unique_ptr m_poOAPIFDS{}; @@ -92,6 +93,8 @@ class OGCAPIDataset final : public GDALDataset CPLString BuildURL(const std::string &href) const; void SetRootURLFromURL(const std::string &osURL); + int FigureBands(const std::string &osContentType, + const CPLString &osImageURL); bool InitFromFile(GDALOpenInfo *poOpenInfo); bool InitFromURL(GDALOpenInfo *poOpenInfo); @@ -110,6 +113,12 @@ class OGCAPIDataset final : public GDALDataset ", " MEDIA_TYPE_JSON, CPLStringList *paosHeaders = nullptr); + std::unique_ptr + OpenTile(const CPLString &osURLPattern, int nMatrix, int nColumn, int nRow, + bool &bEmptyContent, unsigned int nOpenTileFlags = 0, + const CPLString &osPrefix = {}, + const char *const *papszOpenOptions = nullptr); + bool InitWithMapAPI(GDALOpenInfo *poOpenInfo, const CPLJSONObject &oCollection, double dfXMin, double dfYMin, double dfXMax, double dfYMax); @@ -235,7 +244,6 @@ class OGCAPITiledLayer final false; // prevent recursion in EstablishFields() OGCAPITiledLayerFeatureDefn *m_poFeatureDefn = nullptr; OGREnvelope m_sEnvelope{}; - CPLString m_osTileData{}; std::unique_ptr m_poUnderlyingDS{}; OGRLayer *m_poUnderlyingLayer = nullptr; int m_nCurY = 0; @@ -587,6 +595,53 @@ bool OGCAPIDataset::DownloadJSon(const CPLString &osURL, CPLJSONDocument &oDoc, return oDoc.LoadMemory(osResult); } +/************************************************************************/ +/* OpenTile() */ +/************************************************************************/ + +std::unique_ptr +OGCAPIDataset::OpenTile(const CPLString &osURLPattern, int nMatrix, int nColumn, + int nRow, bool &bEmptyContent, + unsigned int nOpenTileFlags, const CPLString &osPrefix, + const char *const *papszOpenTileOptions) +{ + CPLString osURL(osURLPattern); + osURL.replaceAll("{tileMatrix}", CPLSPrintf("%d", nMatrix)); + osURL.replaceAll("{tileCol}", CPLSPrintf("%d", nColumn)); + osURL.replaceAll("{tileRow}", CPLSPrintf("%d", nRow)); + + CPLString osContentType; + if (!this->Download(osURL, nullptr, nullptr, m_osTileData, osContentType, + true, nullptr)) + { + return nullptr; + } + + bEmptyContent = m_osTileData.empty(); + if (bEmptyContent) + return nullptr; + + CPLString osTempFile; + osTempFile.Printf("/vsimem/ogcapi/%p", this); + VSIFCloseL(VSIFileFromMemBuffer(osTempFile.c_str(), + reinterpret_cast(&m_osTileData[0]), + m_osTileData.size(), false)); + + GDALDataset *result = nullptr; + + if (osPrefix.empty()) + result = GDALDataset::Open(osTempFile.c_str(), nOpenTileFlags, nullptr, + papszOpenTileOptions); + else + result = + GDALDataset::Open((osPrefix + ":" + osTempFile).c_str(), + nOpenTileFlags, nullptr, papszOpenTileOptions); + + VSIUnlink(osTempFile); + + return std::unique_ptr(result); +} + /************************************************************************/ /* Identify() */ /************************************************************************/ @@ -628,6 +683,37 @@ void OGCAPIDataset::SetRootURLFromURL(const std::string &osURL) m_osRootURL.assign(pszStr, pszPtr - pszStr); } +/************************************************************************/ +/* FigureBands() */ +/************************************************************************/ + +int OGCAPIDataset::FigureBands(const std::string &osContentType, + const CPLString &osImageURL) +{ + int result = 0; + + if (osContentType == "image/png") + { + result = 4; + } + else if (osContentType == "image/jpeg") + { + result = 3; + } + else + { + // Since we don't know the format download a tile and find out + bool bEmptyContent = false; + std::unique_ptr dataset = + OpenTile(osImageURL, 0, 0, 0, bEmptyContent, GDAL_OF_RASTER); + + // Return the bands from the image, if we didn't get an image then assume 3. + result = dataset ? (int)dataset->GetBands().size() : 3; + } + + return result; +} + /************************************************************************/ /* InitFromFile() */ /************************************************************************/ @@ -1004,21 +1090,64 @@ bool OGCAPIDataset::InitFromURL(GDALOpenInfo *poOpenInfo) /* SelectImageURL() */ /************************************************************************/ -static const CPLString SelectImageURL(const char *const *papszOptionOptions, - const CPLString &osPNG_URL, - const CPLString &osJPEG_URL) +static const std::pair +SelectImageURL(const char *const *papszOptionOptions, + std::map &oMapItemUrls) { - const char *pszFormat = + // Map IMAGE_FORMATS to their content types. Would be nice if this was + // globally defined someplace + const std::map> + oFormatContentTypeMap = { + {"AUTO", + {"image/png", "image/jpeg", "image/tiff; application=geotiff"}}, + {"PNG_PREFERRED", + {"image/png", "image/jpeg", "image/tiff; application=geotiff"}}, + {"JPEG_PREFERRED", + {"image/jpeg", "image/png", "image/tiff; application=geotiff"}}, + {"PNG", {"image/png"}}, + {"JPEG", {"image/jpeg"}}, + {"GEOTIFF", {"image/tiff; application=geotiff"}}}; + + // Get the IMAGE_FORMAT + const std::string osFormat = CSLFetchNameValueDef(papszOptionOptions, "IMAGE_FORMAT", "AUTO"); - if (EQUAL(pszFormat, "AUTO") || EQUAL(pszFormat, "PNG_PREFERRED")) - return !osPNG_URL.empty() ? osPNG_URL : osJPEG_URL; - else if (EQUAL(pszFormat, "PNG")) - return osPNG_URL; - else if (EQUAL(pszFormat, "JPEG")) - return osJPEG_URL; - else if (EQUAL(pszFormat, "JPEG_PREFERRED")) - return !osJPEG_URL.empty() ? osJPEG_URL : osPNG_URL; - return CPLString(); + + // Get a list of content types we will search for in priority order based on IMAGE_FORMAT + auto iterFormat = oFormatContentTypeMap.find(osFormat); + if (iterFormat == oFormatContentTypeMap.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unknown IMAGE_FORMAT specified: %s", osFormat.c_str()); + return std::pair(); + } + std::vector oContentTypes = iterFormat->second; + + // For "special" IMAGE_FORMATS we will also accept additional content types + // specified by the server. Note that this will likely result in having + // some content types duplicated in the vector but that is fine. + if (osFormat == "AUTO" || osFormat == "PNG_PREFERRED" || + osFormat == "JPEG_PREFERRED") + { + std::transform(oMapItemUrls.begin(), oMapItemUrls.end(), + std::back_inserter(oContentTypes), + [](const auto &pair) -> const std::string & + { return pair.first; }); + } + + // Loop over each content type - return the first one we find + for (auto &oContentType : oContentTypes) + { + auto iterContentType = oMapItemUrls.find(oContentType); + if (iterContentType != oMapItemUrls.end()) + { + return *iterContentType; + } + } + + CPLError(CE_Failure, CPLE_AppDefined, + "Server does not support specified IMAGE_FORMAT: %s", + osFormat.c_str()); + return std::pair(); } /************************************************************************/ @@ -1052,35 +1181,40 @@ bool OGCAPIDataset::InitWithMapAPI(GDALOpenInfo *poOpenInfo, double dfYMin, double dfXMax, double dfYMax) { auto oLinks = oRoot["links"].ToArray(); - CPLString osPNG_URL; - CPLString osJPEG_URL; + + // Key - mime type, Value url + std::map oMapItemUrls; for (const auto &oLink : oLinks) { if (oLink["rel"].ToString() == "http://www.opengis.net/def/rel/ogc/1.0/map" && - oLink["type"].ToString() == "image/png") + oLink["type"].IsValid()) { - osPNG_URL = BuildURL(oLink["href"].ToString()); + oMapItemUrls[oLink["type"].ToString()] = + BuildURL(oLink["href"].ToString()); } - else if (oLink["rel"].ToString() == - "http://www.opengis.net/def/rel/ogc/1.0/map" && - oLink["type"].ToString() == "image/jpeg") + else { - osJPEG_URL = BuildURL(oLink["href"].ToString()); + // For lack of additional information assume we are getting some bytes + oMapItemUrls["application/octet-stream"] = + BuildURL(oLink["href"].ToString()); } } - CPLString osImageURL = - SelectImageURL(poOpenInfo->papszOpenOptions, osPNG_URL, osJPEG_URL); + const std::pair oContentUrlPair = + SelectImageURL(poOpenInfo->papszOpenOptions, oMapItemUrls); + const std::string osContentType = oContentUrlPair.first; + const std::string osImageURL = oContentUrlPair.second; + if (osImageURL.empty()) { CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find link to PNG or JPEG images"); + "Cannot find link to tileset items"); return false; } - const int l_nBands = ((osImageURL == osPNG_URL) ? 4 : 3); + int l_nBands = FigureBands(osContentType, osImageURL); int nOverviewCount = 0; int nLargestDim = std::max(nRasterXSize, nRasterYSize); while (nLargestDim > 256) @@ -1098,7 +1232,7 @@ bool OGCAPIDataset::InitWithMapAPI(GDALOpenInfo *poOpenInfo, CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "MAX_CONNECTIONS", CPLGetConfigOption("GDAL_MAX_CONNECTIONS", "5"))); CPLString osWMS_XML; - char *pszEscapedURL = CPLEscapeString(osImageURL, -1, CPLES_XML); + char *pszEscapedURL = CPLEscapeString(osImageURL.c_str(), -1, CPLES_XML); osWMS_XML.Printf("" " " " %s" @@ -1631,12 +1765,14 @@ bool OGCAPIDataset::InitWithTilesAPI(GDALOpenInfo *poOpenInfo, CPLError(CE_Failure, CPLE_AppDefined, "Missing links for tileset"); return false; } - CPLString osPNG_URL; - CPLString osJPEG_URL; + + // Key - mime type, Value url + std::map oMapItemUrls; CPLString osMVT_URL; CPLString osGEOJSON_URL; CPLString osTilingSchemeURL; bool bTilingSchemeURLJson = false; + for (const auto &oLink : oLinks) { const auto osRel = oLink.GetString("rel"); @@ -1657,13 +1793,15 @@ bool OGCAPIDataset::InitWithTilesAPI(GDALOpenInfo *poOpenInfo, } else if (bIsMap) { - if (osRel == "item" && osType == "image/png") + if (osRel == "item" && !osType.empty()) { - osPNG_URL = BuildURL(oLink["href"].ToString()); + oMapItemUrls[osType] = BuildURL(oLink["href"].ToString()); } - else if (osRel == "item" && osType == "image/jpeg") + else if (osRel == "item") { - osJPEG_URL = BuildURL(oLink["href"].ToString()); + // For lack of additional information assume we are getting some bytes + oMapItemUrls["application/octet-stream"] = + BuildURL(oLink["href"].ToString()); } } else @@ -1719,8 +1857,11 @@ bool OGCAPIDataset::InitWithTilesAPI(GDALOpenInfo *poOpenInfo, } } - const CPLString osRasterURL = - SelectImageURL(poOpenInfo->papszOpenOptions, osPNG_URL, osJPEG_URL); + const std::pair oContentUrlPair = + SelectImageURL(poOpenInfo->papszOpenOptions, oMapItemUrls); + const std::string osContentType = oContentUrlPair.first; + const std::string osRasterURL = oContentUrlPair.second; + const CPLString osVectorURL = SelectVectorFormatURL( poOpenInfo->papszOpenOptions, osMVT_URL, osGEOJSON_URL); if (osRasterURL.empty() && osVectorURL.empty()) @@ -1877,7 +2018,8 @@ bool OGCAPIDataset::InitWithTilesAPI(GDALOpenInfo *poOpenInfo, CPLGetConfigOption("GDAL_WMS_MAX_CONNECTIONS", "5"))); const char *pszTileMatrix = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "TILEMATRIX"); - const int l_nBands = ((osRasterURL == osPNG_URL) ? 4 : 3); + + int l_nBands = FigureBands(osContentType, osRasterURL); for (const auto &tileMatrix : tms->tileMatrixList()) { @@ -2331,37 +2473,17 @@ void OGCAPITiledLayer::ResetReading() GDALDataset *OGCAPITiledLayer::OpenTile(int nX, int nY, bool &bEmptyContent) { - bEmptyContent = false; - CPLString osURL(m_osTileURL); - int nCoalesce = GetCoalesceFactorForRow(nY); if (nCoalesce <= 0) return nullptr; nX = (nX / nCoalesce) * nCoalesce; - osURL.replaceAll("{tileCol}", CPLSPrintf("%d", nX)); - osURL.replaceAll("{tileRow}", CPLSPrintf("%d", nY)); - - CPLString osContentType; - if (!m_poDS->Download(osURL, nullptr, nullptr, m_osTileData, osContentType, - true, nullptr)) - { - return nullptr; - } - bEmptyContent = m_osTileData.empty(); - if (bEmptyContent) - return nullptr; - - CPLString osTempFile; - osTempFile.Printf("/vsimem/ogcapi/%p", this); - VSIFCloseL(VSIFileFromMemBuffer(osTempFile.c_str(), - reinterpret_cast(&m_osTileData[0]), - m_osTileData.size(), false)); + const char *const *papszOpenOptions = nullptr; + CPLString poPrefix; + CPLStringList aosOpenOptions; - GDALDataset *poTileDS; if (m_bIsMVT) { - CPLStringList aosOpenOptions; const double dfOriX = m_bInvertAxis ? m_oTileMatrix.mTopLeftY : m_oTileMatrix.mTopLeftX; const double dfOriY = @@ -2382,16 +2504,16 @@ GDALDataset *OGCAPITiledLayer::OpenTile(int nX, int nY, bool &bEmptyContent) "@GEOREF_TILEDIMY", CPLSPrintf("%.18g", m_oTileMatrix.mResY * m_oTileMatrix.mTileWidth)); - poTileDS = - GDALDataset::Open(("MVT:" + osTempFile).c_str(), GDAL_OF_VECTOR, - nullptr, aosOpenOptions.List()); - } - else - { - poTileDS = GDALDataset::Open(osTempFile.c_str(), GDAL_OF_VECTOR); + + papszOpenOptions = aosOpenOptions.List(); + poPrefix = "MVT"; } - VSIUnlink(osTempFile); - return poTileDS; + + std::unique_ptr dataset = m_poDS->OpenTile( + m_osTileURL, stoi(m_oTileMatrix.mId), nX, nY, bEmptyContent, + GDAL_OF_VECTOR, poPrefix, papszOpenOptions); + + return dataset.release(); } /************************************************************************/ diff --git a/frmts/opjlike/jp2opjlikedataset.cpp b/frmts/opjlike/jp2opjlikedataset.cpp index b857d78cd44f..b803c2846397 100644 --- a/frmts/opjlike/jp2opjlikedataset.cpp +++ b/frmts/opjlike/jp2opjlikedataset.cpp @@ -2177,9 +2177,11 @@ GDALDataset *JP2OPJLikeDataset::CreateCopy( } const int nMaxTileDim = std::max(nBlockXSize, nBlockYSize); + const int nMinTileDim = std::min(nBlockXSize, nBlockYSize); int nNumResolutions = 1; /* Pickup a reasonable value compatible with PROFILE_1 requirements */ - while ((nMaxTileDim >> (nNumResolutions - 1)) > 128) + while ((nMaxTileDim >> (nNumResolutions - 1)) > 128 && + (nMinTileDim >> nNumResolutions) > 0) nNumResolutions++; int nMinProfile1Resolutions = nNumResolutions; const char *pszResolutions = @@ -2188,6 +2190,7 @@ GDALDataset *JP2OPJLikeDataset::CreateCopy( { nNumResolutions = atoi(pszResolutions); if (nNumResolutions <= 0 || nNumResolutions >= 32 || + (nMinTileDim >> nNumResolutions) == 0 || (nMaxTileDim >> nNumResolutions) == 0) { CPLError(CE_Warning, CPLE_NotSupported, @@ -2458,7 +2461,7 @@ GDALDataset *JP2OPJLikeDataset::CreateCopy( else { const OGRSpatialReference *poSRS = poSrcDS->GetSpatialRef(); - if (poSRS != nullptr) + if (poSRS) { bGeoreferencingCompatOfGeoJP2 = TRUE; oJP2MD.SetSpatialRef(poSRS); @@ -2468,10 +2471,18 @@ GDALDataset *JP2OPJLikeDataset::CreateCopy( { bGeoreferencingCompatOfGeoJP2 = TRUE; oJP2MD.SetGeoTransform(adfGeoTransform); + if (poSRS && !poSRS->IsEmpty()) + { + bGeoreferencingCompatOfGMLJP2 = + GDALJP2Metadata::IsSRSCompatible(poSRS); + if (!bGeoreferencingCompatOfGMLJP2) + { + CPLDebug( + CODEC::debugId(), + "Cannot write GMLJP2 box due to unsupported SRS"); + } + } } - bGeoreferencingCompatOfGMLJP2 = - poSRS != nullptr && !poSRS->IsEmpty() && - poSrcDS->GetGeoTransform(adfGeoTransform) == CE_None; } if (poSrcDS->GetMetadata("RPC") != nullptr) { diff --git a/frmts/pdf/gdal_pdf.h b/frmts/pdf/gdal_pdf.h index 9ea6db31d457..a567ecb3d124 100644 --- a/frmts/pdf/gdal_pdf.h +++ b/frmts/pdf/gdal_pdf.h @@ -267,24 +267,32 @@ class PDFDataset final : public GDALPamDataset #endif GDALPDFObject *m_poCatalogObject = nullptr; GDALPDFObject *GetCatalog(); + GDALPDFArray *GetPagesKids(); #if defined(HAVE_POPPLER) || defined(HAVE_PDFIUM) - void AddLayer(const char *pszLayerName); + void AddLayer(const std::string &osName, int iPage); + void CreateLayerList(); + std::string + BuildPostfixedLayerNameAndAddLayer(const std::string &osName, + const std::pair &oOCGRef, + int iPageOfInterest, int nPageCount); #endif #if defined(HAVE_POPPLER) - void ExploreLayersPoppler(GDALPDFArray *poArray, CPLString osTopLayer, + void ExploreLayersPoppler(GDALPDFArray *poArray, int iPageOfInterest, + int nPageCount, CPLString osTopLayer, int nRecLevel, int &nVisited, bool &bStop); - void FindLayersPoppler(); + void FindLayersPoppler(int iPageOfInterest); void TurnLayersOnOffPoppler(); std::vector> m_oLayerOCGListPoppler{}; #endif #ifdef HAVE_PDFIUM - void ExploreLayersPdfium(GDALPDFArray *poArray, int nRecLevel, + void ExploreLayersPdfium(GDALPDFArray *poArray, int iPageOfInterest, + int nPageCount, int nRecLevel, CPLString osTopLayer = ""); - void FindLayersPdfium(); + void FindLayersPdfium(int iPageOfInterest); void PDFiumRenderPageBitmap(FPDF_BITMAP bitmap, FPDF_PAGE page, int start_x, int start_y, int size_x, int size_y, const char *pszRenderingOptions); @@ -306,7 +314,18 @@ class PDFDataset final : public GDALPamDataset m_oMapOCGNumGenToVisibilityStatePdfium{}; #endif - CPLStringList m_osLayerList{}; + // Map OCGs identified by their (number, generation) to the list of pages + // where they are referenced from. + std::map, std::vector> m_oMapOCGNumGenToPages{}; + + struct LayerStruct + { + std::string osName{}; + int nInsertIdx = 0; + int iPage = 0; + }; + std::vector m_oLayerNameSet{}; + CPLStringList m_aosLayerNames{}; struct LayerWithRef { @@ -326,6 +345,8 @@ class PDFDataset final : public GDALPamDataset const char *pszLayerName); void FindLayersGeneric(GDALPDFDictionary *poPageDict); + void MapOCGsToPages(); + bool m_bUseOCG = false; static const char *GetOption(char **papszOpenOptions, diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index 616097b2076b..be30415cb4ad 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -1566,18 +1566,17 @@ class GDALPDFiumRenderDeviceDriver : public RenderDeviceDriverIface return m_poParent->GetBackDrop(); } - virtual bool SetDIBits(const RetainPtr &pBitmap, - uint32_t color, const FX_RECT &src_rect, - int dest_left, int dest_top, + virtual bool SetDIBits(RetainPtr bitmap, uint32_t color, + const FX_RECT &src_rect, int dest_left, int dest_top, BlendMode blend_type) override { if (!bEnableBitmap && !bTemporaryEnableVectorForTextStroking) return true; - return m_poParent->SetDIBits(pBitmap, color, src_rect, dest_left, - dest_top, blend_type); + return m_poParent->SetDIBits(std::move(bitmap), color, src_rect, + dest_left, dest_top, blend_type); } - virtual bool StretchDIBits(const RetainPtr &pBitmap, + virtual bool StretchDIBits(RetainPtr bitmap, uint32_t color, int dest_left, int dest_top, int dest_width, int dest_height, const FX_RECT *pClipRect, @@ -1586,21 +1585,20 @@ class GDALPDFiumRenderDeviceDriver : public RenderDeviceDriverIface { if (!bEnableBitmap && !bTemporaryEnableVectorForTextStroking) return true; - return m_poParent->StretchDIBits(pBitmap, color, dest_left, dest_top, - dest_width, dest_height, pClipRect, - options, blend_type); + return m_poParent->StretchDIBits(std::move(bitmap), color, dest_left, + dest_top, dest_width, dest_height, + pClipRect, options, blend_type); } - virtual bool StartDIBits(const RetainPtr &pBitmap, - int bitmap_alpha, uint32_t color, - const CFX_Matrix &matrix, + virtual bool StartDIBits(RetainPtr bitmap, float alpha, + uint32_t color, const CFX_Matrix &matrix, const FXDIB_ResampleOptions &options, std::unique_ptr *handle, BlendMode blend_type) override { if (!bEnableBitmap && !bTemporaryEnableVectorForTextStroking) return true; - return m_poParent->StartDIBits(pBitmap, bitmap_alpha, color, matrix, + return m_poParent->StartDIBits(std::move(bitmap), alpha, color, matrix, options, handle, blend_type); } @@ -1655,21 +1653,21 @@ class GDALPDFiumRenderDeviceDriver : public RenderDeviceDriverIface return m_poParent->MultiplyAlpha(alpha); } - bool MultiplyAlpha(const RetainPtr &mask) override + bool MultiplyAlphaMask(RetainPtr mask) override { - return m_poParent->MultiplyAlpha(mask); + return m_poParent->MultiplyAlphaMask(std::move(mask)); } #if defined(_SKIA_SUPPORT_) - virtual bool SetBitsWithMask(const RetainPtr &pBitmap, - const RetainPtr &pMask, int left, - int top, int bitmap_alpha, + virtual bool SetBitsWithMask(RetainPtr bitmap, + RetainPtr mask, int left, + int top, float alpha, BlendMode blend_type) override { if (!bEnableBitmap && !bTemporaryEnableVectorForTextStroking) return true; - return m_poParent->SetBitsWithMask(pBitmap, pMask, left, top, - bitmap_alpha, blend_type); + return m_poParent->SetBitsWithMask(bitmap, mask, left, top, alpha, + blend_type); } virtual void SetGroupKnockout(bool group_knockout) override { @@ -3561,31 +3559,106 @@ void PDFDataset::ParseInfo(GDALPDFObject *poInfoObj) #if defined(HAVE_POPPLER) || defined(HAVE_PDFIUM) /************************************************************************/ -/* AddLayer() */ +/* AddLayer() */ /************************************************************************/ -void PDFDataset::AddLayer(const char *pszLayerName) +void PDFDataset::AddLayer(const std::string &osName, int iPage) { - int nNewIndex = m_osLayerList.size() /*/ 2*/; + LayerStruct layerStruct; + layerStruct.osName = osName; + layerStruct.nInsertIdx = static_cast(m_oLayerNameSet.size()); + layerStruct.iPage = iPage; + m_oLayerNameSet.emplace_back(std::move(layerStruct)); +} + +/************************************************************************/ +/* CreateLayerList() */ +/************************************************************************/ + +void PDFDataset::CreateLayerList() +{ + // Sort layers by prioritizing page number and then insertion index + std::sort(m_oLayerNameSet.begin(), m_oLayerNameSet.end(), + [](const LayerStruct &a, const LayerStruct &b) + { + if (a.iPage < b.iPage) + return true; + if (a.iPage > b.iPage) + return false; + return a.nInsertIdx < b.nInsertIdx; + }); - if (nNewIndex == 100) + if (m_oLayerNameSet.size() >= 100) + { + for (const auto &oLayerStruct : m_oLayerNameSet) + { + m_aosLayerNames.AddNameValue( + CPLSPrintf("LAYER_%03d_NAME", m_aosLayerNames.size()), + oLayerStruct.osName.c_str()); + } + } + else + { + for (const auto &oLayerStruct : m_oLayerNameSet) + { + m_aosLayerNames.AddNameValue( + CPLSPrintf("LAYER_%02d_NAME", m_aosLayerNames.size()), + oLayerStruct.osName.c_str()); + } + } +} + +/************************************************************************/ +/* BuildPostfixedLayerNameAndAddLayer() */ +/************************************************************************/ + +/** Append a suffix with the page number(s) to the provided layer name, if + * it makes sense (that is if it is a multiple page PDF and we haven't selected + * a specific name). And also call AddLayer() on it if successful. + * If may return an empty string if the layer isn't used by the page of interest + */ +std::string PDFDataset::BuildPostfixedLayerNameAndAddLayer( + const std::string &osName, const std::pair &oOCGRef, + int iPageOfInterest, int nPageCount) +{ + std::string osPostfixedName = osName; + int iLayerPage = 0; + if (nPageCount > 1 && !m_oMapOCGNumGenToPages.empty()) { - CPLStringList osNewLayerList; - for (int i = 0; i < 100; i++) + const auto oIterToPages = m_oMapOCGNumGenToPages.find(oOCGRef); + if (oIterToPages != m_oMapOCGNumGenToPages.end()) { - osNewLayerList.AddNameValue(CPLSPrintf("LAYER_%03d_NAME", i), - m_osLayerList[/*2 * */ i] + - strlen("LAYER_00_NAME=")); + const auto &anPages = oIterToPages->second; + if (iPageOfInterest > 0) + { + if (std::find(anPages.begin(), anPages.end(), + iPageOfInterest) == anPages.end()) + { + return std::string(); + } + } + else if (anPages.size() == 1) + { + iLayerPage = anPages.front(); + osPostfixedName += CPLSPrintf(" (page %d)", anPages.front()); + } + else + { + osPostfixedName += " (pages "; + for (size_t j = 0; j < anPages.size(); ++j) + { + if (j > 0) + osPostfixedName += ", "; + osPostfixedName += CPLSPrintf("%d", anPages[j]); + } + osPostfixedName += ')'; + } } - m_osLayerList = std::move(osNewLayerList); } - char szFormatName[64]; - snprintf(szFormatName, sizeof(szFormatName), "LAYER_%%0%dd_NAME", - nNewIndex >= 100 ? 3 : 2); + AddLayer(osPostfixedName, iLayerPage); - m_osLayerList.AddNameValue(CPLSPrintf(szFormatName, nNewIndex), - pszLayerName); + return osPostfixedName; } #endif // defined(HAVE_POPPLER) || defined(HAVE_PDFIUM) @@ -3597,6 +3670,7 @@ void PDFDataset::AddLayer(const char *pszLayerName) /************************************************************************/ void PDFDataset::ExploreLayersPoppler(GDALPDFArray *poArray, + int iPageOfInterest, int nPageCount, CPLString osTopLayer, int nRecLevel, int &nVisited, bool &bStop) { @@ -3630,13 +3704,13 @@ void PDFDataset::ExploreLayersPoppler(GDALPDFArray *poArray, } else osTopLayer = std::move(osName); - AddLayer(osTopLayer.c_str()); + AddLayer(osTopLayer, 0); m_oLayerOCGListPoppler.push_back(std::pair(osTopLayer, nullptr)); } else if (poObj->GetType() == PDFObjectType_Array) { - ExploreLayersPoppler(poObj->GetArray(), osCurLayer, nRecLevel + 1, - nVisited, bStop); + ExploreLayersPoppler(poObj->GetArray(), iPageOfInterest, nPageCount, + osCurLayer, nRecLevel + 1, nVisited, bStop); if (bStop) return; osCurLayer = ""; @@ -3667,10 +3741,17 @@ void PDFDataset::ExploreLayersPoppler(GDALPDFArray *poArray, OptionalContentGroup *ocg = optContentConfig->findOcgByRef(r); if (ocg) { - AddLayer(osCurLayer.c_str()); + const auto oRefPair = std::pair(poObj->GetRefNum().toInt(), + poObj->GetRefGen()); + const std::string osPostfixedName = + BuildPostfixedLayerNameAndAddLayer( + osCurLayer, oRefPair, iPageOfInterest, nPageCount); + if (osPostfixedName.empty()) + continue; + m_oLayerOCGListPoppler.push_back( - std::make_pair(osCurLayer, ocg)); - m_aoLayerWithRef.emplace_back(osCurLayer.c_str(), + std::make_pair(osPostfixedName, ocg)); + m_aoLayerWithRef.emplace_back(osPostfixedName.c_str(), poObj->GetRefNum(), r.gen); } } @@ -3682,8 +3763,13 @@ void PDFDataset::ExploreLayersPoppler(GDALPDFArray *poArray, /* FindLayersPoppler() */ /************************************************************************/ -void PDFDataset::FindLayersPoppler() +void PDFDataset::FindLayersPoppler(int iPageOfInterest) { + int nPageCount = 0; + const auto poPages = GetPagesKids(); + if (poPages) + nPageCount = poPages->GetLength(); + OCGs *optContentConfig = m_poDocPoppler->getOptContentConfig(); if (optContentConfig == nullptr || !optContentConfig->isOk()) return; @@ -3694,7 +3780,8 @@ void PDFDataset::FindLayersPoppler() GDALPDFArray *poArray = GDALPDFCreateArray(array); int nVisited = 0; bool bStop = false; - ExploreLayersPoppler(poArray, CPLString(), 0, nVisited, bStop); + ExploreLayersPoppler(poArray, iPageOfInterest, nPageCount, CPLString(), + 0, nVisited, bStop); delete poArray; } else @@ -3706,14 +3793,15 @@ void PDFDataset::FindLayersPoppler() { const char *pszLayerName = (const char *)ocg->getName()->c_str(); - AddLayer(pszLayerName); + AddLayer(pszLayerName, 0); m_oLayerOCGListPoppler.push_back( std::make_pair(CPLString(pszLayerName), ocg)); } } } - m_oMDMD_PDF.SetMetadata(m_osLayerList.List(), "LAYERS"); + CreateLayerList(); + m_oMDMD_PDF.SetMetadata(m_aosLayerNames.List(), "LAYERS"); } /************************************************************************/ @@ -3897,14 +3985,15 @@ void PDFDataset::TurnLayersOnOffPoppler() /* ExploreLayersPdfium() */ /************************************************************************/ -void PDFDataset::ExploreLayersPdfium(GDALPDFArray *poArray, int nRecLevel, +void PDFDataset::ExploreLayersPdfium(GDALPDFArray *poArray, int iPageOfInterest, + int nPageCount, int nRecLevel, CPLString osTopLayer) { if (nRecLevel == 16) return; - int nLength = poArray->GetLength(); - CPLString osCurLayer; + const int nLength = poArray->GetLength(); + std::string osCurLayer; for (int i = 0; i < nLength; i++) { GDALPDFObject *poObj = poArray->Get(i); @@ -3912,18 +4001,20 @@ void PDFDataset::ExploreLayersPdfium(GDALPDFArray *poArray, int nRecLevel, continue; if (i == 0 && poObj->GetType() == PDFObjectType_String) { - CPLString osName = PDFSanitizeLayerName(poObj->GetString().c_str()); + const std::string osName = + PDFSanitizeLayerName(poObj->GetString().c_str()); if (!osTopLayer.empty()) - osTopLayer = osTopLayer + "." + osName; + osTopLayer = std::string(osTopLayer).append(".").append(osName); else osTopLayer = osName; - AddLayer(osTopLayer.c_str()); + AddLayer(osTopLayer, 0); m_oMapLayerNameToOCGNumGenPdfium[osTopLayer] = std::pair(-1, -1); } else if (poObj->GetType() == PDFObjectType_Array) { - ExploreLayersPdfium(poObj->GetArray(), nRecLevel + 1, osCurLayer); - osCurLayer = ""; + ExploreLayersPdfium(poObj->GetArray(), iPageOfInterest, nPageCount, + nRecLevel + 1, osCurLayer); + osCurLayer.clear(); } else if (poObj->GetType() == PDFObjectType_Dictionary) { @@ -3931,24 +4022,29 @@ void PDFDataset::ExploreLayersPdfium(GDALPDFArray *poArray, int nRecLevel, GDALPDFObject *poName = poDict->Get("Name"); if (poName != nullptr && poName->GetType() == PDFObjectType_String) { - CPLString osName = + const std::string osName = PDFSanitizeLayerName(poName->GetString().c_str()); // coverity[copy_paste_error] if (!osTopLayer.empty()) { - osCurLayer = osTopLayer; - osCurLayer += '.'; - osCurLayer += std::move(osName); + osCurLayer = + std::string(osTopLayer).append(".").append(osName); } else - osCurLayer = std::move(osName); + osCurLayer = osName; // CPLDebug("PDF", "Layer %s", osCurLayer.c_str()); - AddLayer(osCurLayer.c_str()); - m_aoLayerWithRef.emplace_back(osCurLayer, poObj->GetRefNum(), - poObj->GetRefGen()); - m_oMapLayerNameToOCGNumGenPdfium[osCurLayer] = + const auto oRefPair = std::pair(poObj->GetRefNum().toInt(), poObj->GetRefGen()); + const std::string osPostfixedName = + BuildPostfixedLayerNameAndAddLayer( + osCurLayer, oRefPair, iPageOfInterest, nPageCount); + if (osPostfixedName.empty()) + continue; + + m_aoLayerWithRef.emplace_back( + osPostfixedName, poObj->GetRefNum(), poObj->GetRefGen()); + m_oMapLayerNameToOCGNumGenPdfium[osPostfixedName] = oRefPair; } } } @@ -3958,8 +4054,13 @@ void PDFDataset::ExploreLayersPdfium(GDALPDFArray *poArray, int nRecLevel, /* FindLayersPdfium() */ /************************************************************************/ -void PDFDataset::FindLayersPdfium() +void PDFDataset::FindLayersPdfium(int iPageOfInterest) { + int nPageCount = 0; + const auto poPages = GetPagesKids(); + if (poPages) + nPageCount = poPages->GetLength(); + GDALPDFObject *poCatalog = GetCatalog(); if (poCatalog == nullptr || poCatalog->GetType() != PDFObjectType_Dictionary) @@ -3967,7 +4068,8 @@ void PDFDataset::FindLayersPdfium() GDALPDFObject *poOrder = poCatalog->LookupObject("OCProperties.D.Order"); if (poOrder != nullptr && poOrder->GetType() == PDFObjectType_Array) { - ExploreLayersPdfium(poOrder->GetArray(), 0); + ExploreLayersPdfium(poOrder->GetArray(), iPageOfInterest, nPageCount, + 0); } #if 0 else @@ -3989,7 +4091,8 @@ void PDFDataset::FindLayersPdfium() } #endif - m_oMDMD_PDF.SetMetadata(m_osLayerList.List(), "LAYERS"); + CreateLayerList(); + m_oMDMD_PDF.SetMetadata(m_aosLayerNames.List(), "LAYERS"); } /************************************************************************/ @@ -4181,6 +4284,84 @@ PDFDataset::VisibilityState PDFDataset::GetVisibilityStateForOGCPdfium(int nNum, #endif /* HAVE_PDFIUM */ +/************************************************************************/ +/* GetPagesKids() */ +/************************************************************************/ + +GDALPDFArray *PDFDataset::GetPagesKids() +{ + const auto poCatalog = GetCatalog(); + if (!poCatalog || poCatalog->GetType() != PDFObjectType_Dictionary) + { + return nullptr; + } + const auto poKids = poCatalog->LookupObject("Pages.Kids"); + if (!poKids || poKids->GetType() != PDFObjectType_Array) + { + return nullptr; + } + return poKids->GetArray(); +} + +/************************************************************************/ +/* MapOCGsToPages() */ +/************************************************************************/ + +void PDFDataset::MapOCGsToPages() +{ + const auto poKidsArray = GetPagesKids(); + if (!poKidsArray) + { + return; + } + const int nKidsArrayLenght = poKidsArray->GetLength(); + for (int iPage = 0; iPage < nKidsArrayLenght; ++iPage) + { + const auto poPage = poKidsArray->Get(iPage); + if (poPage && poPage->GetType() == PDFObjectType_Dictionary) + { + const auto poXObject = poPage->LookupObject("Resources.XObject"); + if (poXObject && poXObject->GetType() == PDFObjectType_Dictionary) + { + for (const auto &oNameObjectPair : + poXObject->GetDictionary()->GetValues()) + { + const auto poProperties = + oNameObjectPair.second->LookupObject( + "Resources.Properties"); + if (poProperties && + poProperties->GetType() == PDFObjectType_Dictionary) + { + const auto &oMap = + poProperties->GetDictionary()->GetValues(); + for (const auto &[osKey, poObj] : oMap) + { + if (poObj->GetRefNum().toBool() && + poObj->GetType() == PDFObjectType_Dictionary) + { + GDALPDFObject *poType = + poObj->GetDictionary()->Get("Type"); + GDALPDFObject *poName = + poObj->GetDictionary()->Get("Name"); + if (poType && + poType->GetType() == PDFObjectType_Name && + poType->GetName() == "OCG" && poName && + poName->GetType() == PDFObjectType_String) + { + m_oMapOCGNumGenToPages + [std::pair(poObj->GetRefNum().toInt(), + poObj->GetRefGen())] + .push_back(iPage + 1); + } + } + } + } + } + } + } + } +} + /************************************************************************/ /* FindLayerOCG() */ /************************************************************************/ @@ -5268,6 +5449,8 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) CPLFree(pszNeatLineWkt); } + poDS->MapOCGsToPages(); + #ifdef HAVE_POPPLER if (bUseLib.test(PDFLIB_POPPLER)) { @@ -5299,7 +5482,8 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) } /* Find layers */ - poDS->FindLayersPoppler(); + poDS->FindLayersPoppler( + (bOpenSubdataset || bOpenSubdatasetImage) ? iPage : 0); /* Turn user specified layers on or off */ poDS->TurnLayersOnOffPoppler(); @@ -5371,7 +5555,8 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) delete poRoot; /* Find layers */ - poDS->FindLayersPdfium(); + poDS->FindLayersPdfium((bOpenSubdataset || bOpenSubdatasetImage) ? iPage + : 0); /* Turn user specified layers on or off */ poDS->TurnLayersOnOffPdfium(); diff --git a/frmts/rasterlite/rasterlitedataset.cpp b/frmts/rasterlite/rasterlitedataset.cpp index 322ea55f8180..691d675a1d6b 100644 --- a/frmts/rasterlite/rasterlitedataset.cpp +++ b/frmts/rasterlite/rasterlitedataset.cpp @@ -1096,7 +1096,7 @@ GDALDataset *RasterliteDataset::Open(GDALOpenInfo *poOpenInfo) if (osTableName.empty()) { int nCountSubdataset = 0; - int nLayers = OGR_DS_GetLayerCount(hDS); + const int nLayers = OGR_DS_GetLayerCount(hDS); /* -------------------------------------------------------------------- */ /* Add raster layers as subdatasets */ @@ -1105,14 +1105,15 @@ GDALDataset *RasterliteDataset::Open(GDALOpenInfo *poOpenInfo) for (int i = 0; i < nLayers; i++) { OGRLayerH hLyr = OGR_DS_GetLayer(hDS, i); - const char *pszLayerName = OGR_L_GetName(hLyr); - if (strstr(pszLayerName, "_metadata")) + const std::string osLayerName = OGR_L_GetName(hLyr); + const auto nPosMetadata = osLayerName.find("_metadata"); + if (nPosMetadata != std::string::npos) { - char *pszShortName = CPLStrdup(pszLayerName); - *strstr(pszShortName, "_metadata") = '\0'; + const std::string osShortName = + osLayerName.substr(0, nPosMetadata); - CPLString osRasterTableName = pszShortName; - osRasterTableName += "_rasters"; + const std::string osRasterTableName = + std::string(osShortName).append("_rasters"); if (OGR_DS_GetLayerByName(hDS, osRasterTableName.c_str()) != nullptr) @@ -1120,21 +1121,19 @@ GDALDataset *RasterliteDataset::Open(GDALOpenInfo *poOpenInfo) if (poDS == nullptr) { poDS = new RasterliteDataset(); - osTableName = pszShortName; + osTableName = osShortName; } - CPLString osSubdatasetName; + std::string osSubdatasetName; if (!STARTS_WITH_CI(poOpenInfo->pszFilename, "RASTERLITE:")) osSubdatasetName += "RASTERLITE:"; osSubdatasetName += poOpenInfo->pszFilename; osSubdatasetName += ",table="; - osSubdatasetName += pszShortName; + osSubdatasetName += osShortName; poDS->AddSubDataset(osSubdatasetName.c_str()); nCountSubdataset++; } - - CPLFree(pszShortName); } } diff --git a/frmts/tiledb/tiledbcommon.cpp b/frmts/tiledb/tiledbcommon.cpp index 21f51959d324..5679a6f5b9eb 100644 --- a/frmts/tiledb/tiledbcommon.cpp +++ b/frmts/tiledb/tiledbcommon.cpp @@ -133,9 +133,17 @@ int TileDBDataset::Identify(GDALOpenInfo *poOpenInfo) return TRUE; } + const bool bIsS3OrGS = + STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || + STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); + // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. + if (!bIsS3OrGS && STARTS_WITH(poOpenInfo->pszFilename, "/vsi")) + { + return false; + } + if (poOpenInfo->bIsDirectory || - ((STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/")) && + (bIsS3OrGS && !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif"))) { tiledb::Context ctx; @@ -289,7 +297,7 @@ GDALDataset *TileDBDataset::Open(GDALOpenInfo *poOpenInfo) { if (!poCandidateArray) { - poCandidateArray = poArray; + poCandidateArray = std::move(poArray); } else { diff --git a/frmts/tiledb/tiledbdrivercore.cpp b/frmts/tiledb/tiledbdrivercore.cpp index 607d624b2413..d9ae98c5da1d 100644 --- a/frmts/tiledb/tiledbdrivercore.cpp +++ b/frmts/tiledb/tiledbdrivercore.cpp @@ -51,10 +51,20 @@ static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) return TRUE; } - if (poOpenInfo->bIsDirectory || - ((STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/")) && - !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif"))) + const bool bIsS3OrGS = STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || + STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); + // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. + if (!bIsS3OrGS && STARTS_WITH(poOpenInfo->pszFilename, "/vsi")) + { + return false; + } + + if (poOpenInfo->bIsDirectory) + { + return GDAL_IDENTIFY_UNKNOWN; + } + + if (bIsS3OrGS && !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif")) { return GDAL_IDENTIFY_UNKNOWN; } diff --git a/frmts/vrt/data/gdalvrt.xsd b/frmts/vrt/data/gdalvrt.xsd index da0296e0fa7e..cc9c13cba596 100644 --- a/frmts/vrt/data/gdalvrt.xsd +++ b/frmts/vrt/data/gdalvrt.xsd @@ -187,6 +187,7 @@ + @@ -406,6 +407,21 @@ + + + + + + + + + + + + + + + diff --git a/frmts/vrt/vrtdataset.cpp b/frmts/vrt/vrtdataset.cpp index 69e65fa68ae6..5e99d7437ed6 100644 --- a/frmts/vrt/vrtdataset.cpp +++ b/frmts/vrt/vrtdataset.cpp @@ -328,10 +328,14 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn) { } CPLAssert(psLastChild); // we have at least rasterXSize + bool bHasWarnedAboutRAMUsage = false; + size_t nAccRAMUsage = 0; for (int iBand = 0; iBand < nBands; iBand++) { - CPLXMLNode *psBandTree = static_cast(papoBands[iBand]) - ->SerializeToXML(pszVRTPathIn); + CPLXMLNode *psBandTree = + static_cast(papoBands[iBand]) + ->SerializeToXML(pszVRTPathIn, bHasWarnedAboutRAMUsage, + nAccRAMUsage); if (psBandTree != nullptr) { @@ -345,7 +349,8 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ if (m_poMaskBand) { - CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(pszVRTPathIn); + CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML( + pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage); if (psBandTree != nullptr) { @@ -429,7 +434,7 @@ VRTRasterBand *VRTDataset::InitBand(const char *pszSubclass, int nBand, /* XMLInit() */ /************************************************************************/ -CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) +CPLErr VRTDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn) { if (pszVRTPathIn != nullptr) @@ -438,7 +443,7 @@ CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ /* Check for an SRS node. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psSRSNode = CPLGetXMLNode(psTree, "SRS"); + const CPLXMLNode *psSRSNode = CPLGetXMLNode(psTree, "SRS"); if (psSRSNode) { if (m_poSRS) @@ -475,11 +480,12 @@ CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ /* Check for a GeoTransform node. */ /* -------------------------------------------------------------------- */ - if (strlen(CPLGetXMLValue(psTree, "GeoTransform", "")) > 0) + const char *pszGT = CPLGetXMLValue(psTree, "GeoTransform", ""); + if (strlen(pszGT) > 0) { - const char *pszGT = CPLGetXMLValue(psTree, "GeoTransform", ""); - char **papszTokens = CSLTokenizeStringComplex(pszGT, ",", FALSE, FALSE); - if (CSLCount(papszTokens) != 6) + const CPLStringList aosTokens( + CSLTokenizeStringComplex(pszGT, ",", FALSE, FALSE)); + if (aosTokens.size() != 6) { CPLError(CE_Warning, CPLE_AppDefined, "GeoTransform node does not have expected six values."); @@ -487,19 +493,15 @@ CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) else { for (int iTA = 0; iTA < 6; iTA++) - m_adfGeoTransform[iTA] = CPLAtof(papszTokens[iTA]); + m_adfGeoTransform[iTA] = CPLAtof(aosTokens[iTA]); m_bGeoTransformSet = TRUE; } - - CSLDestroy(papszTokens); } /* -------------------------------------------------------------------- */ /* Check for GCPs. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psGCPList = CPLGetXMLNode(psTree, "GCPList"); - - if (psGCPList != nullptr) + if (const CPLXMLNode *psGCPList = CPLGetXMLNode(psTree, "GCPList")) { GDALDeserializeGCPListFromXML(psGCPList, &m_pasGCPList, &m_nGCPCount, &m_poGCP_SRS); @@ -515,9 +517,9 @@ CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ /* Parse dataset mask band first */ - CPLXMLNode *psMaskBandNode = CPLGetXMLNode(psTree, "MaskBand"); + const CPLXMLNode *psMaskBandNode = CPLGetXMLNode(psTree, "MaskBand"); - CPLXMLNode *psChild = nullptr; + const CPLXMLNode *psChild = nullptr; if (psMaskBandNode) psChild = psMaskBandNode->psChild; else @@ -576,8 +578,7 @@ CPLErr VRTDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) } } - CPLXMLNode *psGroup = CPLGetXMLNode(psTree, "Group"); - if (psGroup) + if (const CPLXMLNode *psGroup = CPLGetXMLNode(psTree, "Group")) { const char *pszName = CPLGetXMLValue(psGroup, "name", nullptr); if (pszName == nullptr || !EQUAL(pszName, "/")) diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index 59dc1ecf91af..6c123027779a 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -124,6 +124,10 @@ class CPL_DLL VRTSource // do nothing /* coverity[uninit_member] */ } + inline operator GByte() const + { + return value; + } }; #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -150,7 +154,7 @@ class CPL_DLL VRTSource int bApproxOK, GDALProgressFunc pfnProgress, void *pProgressData) = 0; - virtual CPLErr XMLInit(CPLXMLNode *psTree, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, std::map &) = 0; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) = 0; @@ -168,17 +172,17 @@ class CPL_DLL VRTSource }; typedef VRTSource *(*VRTSourceParser)( - CPLXMLNode *, const char *, + const CPLXMLNode *, const char *, std::map &oMapSharedSources); VRTSource * -VRTParseCoreSources(CPLXMLNode *psTree, const char *, +VRTParseCoreSources(const CPLXMLNode *psTree, const char *, std::map &oMapSharedSources); VRTSource * -VRTParseFilterSources(CPLXMLNode *psTree, const char *, +VRTParseFilterSources(const CPLXMLNode *psTree, const char *, std::map &oMapSharedSources); VRTSource * -VRTParseArraySource(CPLXMLNode *psTree, const char *, +VRTParseArraySource(const CPLXMLNode *psTree, const char *, std::map &oMapSharedSources); /************************************************************************/ @@ -331,7 +335,7 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset char **papszOptions) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath); - virtual CPLErr XMLInit(CPLXMLNode *, const char *); + virtual CPLErr XMLInit(const CPLXMLNode *, const char *); virtual CPLErr IBuildOverviews(const char *, int, const int *, int, const int *, GDALProgressFunc, void *, @@ -415,7 +419,7 @@ class CPL_DLL VRTWarpedDataset final : public VRTDataset const char *pszDomain = "") override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; - virtual CPLErr XMLInit(CPLXMLNode *, const char *) override; + virtual CPLErr XMLInit(const CPLXMLNode *, const char *) override; virtual CPLErr AddBand(GDALDataType eType, char **papszOptions = nullptr) override; @@ -477,10 +481,10 @@ class VRTPansharpenedDataset final : public VRTDataset virtual CPLErr FlushCache(bool bAtClosing) override; - virtual CPLErr XMLInit(CPLXMLNode *, const char *) override; + virtual CPLErr XMLInit(const CPLXMLNode *, const char *) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; - CPLErr XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, + CPLErr XMLInit(const CPLXMLNode *psTree, const char *pszVRTPath, GDALRasterBandH hPanchroBandIn, int nInputSpectralBandsIn, GDALRasterBandH *pahInputSpectralBandsIn); @@ -563,9 +567,11 @@ class CPL_DLL VRTRasterBand CPL_NON_FINAL : public GDALRasterBand VRTRasterBand(); virtual ~VRTRasterBand(); - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &); - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath); + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage); CPLErr SetNoDataValue(double) override; CPLErr SetNoDataValueAsInt64(int64_t nNoData) override; @@ -703,9 +709,11 @@ class CPL_DLL VRTSourcedRasterBand CPL_NON_FINAL : public VRTRasterBand virtual CPLErr SetMetadataItem(const char *pszName, const char *pszValue, const char *pszDomain = "") override; - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &) override; - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; virtual double GetMinimum(int *pbSuccess = nullptr) override; virtual double GetMaximum(int *pbSuccess = nullptr) override; @@ -804,7 +812,9 @@ class CPL_DLL VRTWarpedRasterBand final : public VRTRasterBand GDALDataType eType = GDT_Unknown); virtual ~VRTWarpedRasterBand(); - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; virtual CPLErr IReadBlock(int, int, void *) override; virtual CPLErr IWriteBlock(int, int, void *) override; @@ -825,7 +835,9 @@ class VRTPansharpenedRasterBand final : public VRTRasterBand GDALDataType eDataType = GDT_Unknown); virtual ~VRTPansharpenedRasterBand(); - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; virtual CPLErr IReadBlock(int, int, void *) override; @@ -897,16 +909,18 @@ class CPL_DLL VRTDerivedRasterBand CPL_NON_FINAL : public VRTSourcedRasterBand GDALDerivedPixelFuncWithArgs pfnPixelFunc, const char *pszMetadata); - static std::pair * + static const std::pair * GetPixelFunction(const char *pszFuncNameIn); void SetPixelFunctionName(const char *pszFuncNameIn); void SetSourceTransferType(GDALDataType eDataType); void SetPixelFunctionLanguage(const char *pszLanguage); - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &) override; - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; virtual double GetMinimum(int *pbSuccess = nullptr) override; virtual double GetMaximum(int *pbSuccess = nullptr) override; @@ -945,9 +959,11 @@ class CPL_DLL VRTRawRasterBand CPL_NON_FINAL : public VRTRasterBand GDALDataType eType = GDT_Unknown); virtual ~VRTRawRasterBand(); - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &) override; - virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; virtual CPLErr IRasterIO(GDALRWFlag, int, int, int, int, void *, int, int, GDALDataType, GSpacing nPixelSpace, @@ -994,7 +1010,7 @@ class VRTDriver final : public GDALDriver const char *pszDomain = "") override; VRTSource * - ParseSource(CPLXMLNode *psSrc, const char *pszVRTPath, + ParseSource(const CPLXMLNode *psSrc, const char *pszVRTPath, std::map &oMapSharedSources); void AddSourceParser(const char *pszElementName, VRTSourceParser pfnParser); }; @@ -1075,7 +1091,7 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource double dfYDstRatio); virtual ~VRTSimpleSource(); - virtual CPLErr XMLInit(CPLXMLNode *psTree, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, std::map &) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; @@ -1189,6 +1205,50 @@ class VRTAveragedSource final : public VRTSimpleSource } }; +/************************************************************************/ +/* VRTNoDataFromMaskSource */ +/************************************************************************/ + +class VRTNoDataFromMaskSource final : public VRTSimpleSource +{ + CPL_DISALLOW_COPY_ASSIGN(VRTNoDataFromMaskSource) + + bool m_bNoDataSet = false; + double m_dfNoDataValue = 0; + double m_dfMaskValueThreshold = 0; + bool m_bHasRemappedValue = false; + double m_dfRemappedValue = 0; + + public: + VRTNoDataFromMaskSource(); + virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, int nBufXSize, + int nBufYSize, GDALDataType eBufType, + GSpacing nPixelSpace, GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArgIn, + WorkingState &oWorkingState) override; + + virtual double GetMinimum(int nXSize, int nYSize, int *pbSuccess) override; + virtual double GetMaximum(int nXSize, int nYSize, int *pbSuccess) override; + virtual CPLErr GetHistogram(int nXSize, int nYSize, double dfMin, + double dfMax, int nBuckets, + GUIntBig *panHistogram, int bIncludeOutOfRange, + int bApproxOK, GDALProgressFunc pfnProgress, + void *pProgressData) override; + + void SetParameters(double dfNoDataValue, double dfMaskValueThreshold); + void SetParameters(double dfNoDataValue, double dfMaskValueThreshold, + double dfRemappedValue); + + virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, + std::map &) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + virtual const char *GetType() override + { + return "VRTNoDataFromMaskSource"; + } +}; + /************************************************************************/ /* VRTComplexSource */ /************************************************************************/ @@ -1272,7 +1332,7 @@ class CPL_DLL VRTComplexSource CPL_NON_FINAL : public VRTSimpleSource void *pProgressData) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &) override; virtual const char *GetType() override { @@ -1355,7 +1415,7 @@ class VRTKernelFilteredSource CPL_NON_FINAL : public VRTFilteredSource VRTKernelFilteredSource(); virtual ~VRTKernelFilteredSource(); - virtual CPLErr XMLInit(CPLXMLNode *psTree, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, std::map &) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; @@ -1378,7 +1438,7 @@ class VRTAverageFilteredSource final : public VRTKernelFilteredSource explicit VRTAverageFilteredSource(int nKernelSize); virtual ~VRTAverageFilteredSource(); - virtual CPLErr XMLInit(CPLXMLNode *psTree, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, std::map &) override; virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; }; @@ -1394,7 +1454,7 @@ class VRTFuncSource final : public VRTSource VRTFuncSource(); virtual ~VRTFuncSource(); - virtual CPLErr XMLInit(CPLXMLNode *, const char *, + virtual CPLErr XMLInit(const CPLXMLNode *, const char *, std::map &) override { return CE_Failure; diff --git a/frmts/vrt/vrtderivedrasterband.cpp b/frmts/vrt/vrtderivedrasterband.cpp index e2de0b4fbca7..cf340548966b 100644 --- a/frmts/vrt/vrtderivedrasterband.cpp +++ b/frmts/vrt/vrtderivedrasterband.cpp @@ -50,10 +50,6 @@ using namespace GDALPy; #define GDAL_VRT_ENABLE_PYTHON_DEFAULT "TRUSTED_MODULES" #endif -static std::map> - osMapPixelFunction; - /* Flags for getting buffers */ #define PyBUF_WRITABLE 0x0001 #define PyBUF_FORMAT 0x0004 @@ -88,6 +84,9 @@ static PyObject *GDALCreateNumpyArray(PyObject *pCreateArray, void *pBuffer, case GDT_Byte: pszDataType = "uint8"; break; + case GDT_Int8: + pszDataType = "int8"; + break; case GDT_UInt16: pszDataType = "uint16"; break; @@ -100,6 +99,12 @@ static PyObject *GDALCreateNumpyArray(PyObject *pCreateArray, void *pBuffer, case GDT_Int32: pszDataType = "int32"; break; + case GDT_Int64: + pszDataType = "int64"; + break; + case GDT_UInt64: + pszDataType = "uint64"; + break; case GDT_Float32: pszDataType = "float32"; break; @@ -116,7 +121,8 @@ static PyObject *GDALCreateNumpyArray(PyObject *pCreateArray, void *pBuffer, case GDT_CFloat64: pszDataType = "complex128"; break; - default: + case GDT_Unknown: + case GDT_TypeCount: CPLAssert(FALSE); break; } @@ -221,6 +227,20 @@ void VRTDerivedRasterBand::Cleanup() { } +/************************************************************************/ +/* GetGlobalMapPixelFunction() */ +/************************************************************************/ + +static std::map> & +GetGlobalMapPixelFunction() +{ + static std::map> + gosMapPixelFunction; + return gosMapPixelFunction; +} + /************************************************************************/ /* AddPixelFunction() */ /************************************************************************/ @@ -251,7 +271,7 @@ CPLErr CPL_STDCALL GDALAddDerivedBandPixelFunc( return CE_None; } - osMapPixelFunction[pszName] = { + GetGlobalMapPixelFunction()[pszName] = { [pfnNewFunction](void **papoSources, int nSources, void *pData, int nBufXSize, int nBufYSize, GDALDataType eSrcType, GDALDataType eBufType, int nPixelSpace, int nLineSpace, @@ -289,13 +309,13 @@ CPLErr CPL_STDCALL GDALAddDerivedBandPixelFuncWithArgs( const char *pszName, GDALDerivedPixelFuncWithArgs pfnNewFunction, const char *pszMetadata) { - if (pszName == nullptr || pszName[0] == '\0' || pfnNewFunction == nullptr) + if (!pszName || pszName[0] == '\0' || !pfnNewFunction) { return CE_None; } - osMapPixelFunction[pszName] = {pfnNewFunction, - pszMetadata != nullptr ? pszMetadata : ""}; + GetGlobalMapPixelFunction()[pszName] = {pfnNewFunction, + pszMetadata ? pszMetadata : ""}; return CE_None; } @@ -343,7 +363,7 @@ CPLErr VRTDerivedRasterBand::AddPixelFunction( * @return A derived band pixel function, or NULL if none have been * registered for pszFuncName. */ -std::pair * +const std::pair * VRTDerivedRasterBand::GetPixelFunction(const char *pszFuncNameIn) { if (pszFuncNameIn == nullptr || pszFuncNameIn[0] == '\0') @@ -351,9 +371,10 @@ VRTDerivedRasterBand::GetPixelFunction(const char *pszFuncNameIn) return nullptr; } - auto oIter = osMapPixelFunction.find(pszFuncNameIn); + const auto &oMapPixelFunction = GetGlobalMapPixelFunction(); + const auto oIter = oMapPixelFunction.find(pszFuncNameIn); - if (oIter == osMapPixelFunction.end()) + if (oIter == oMapPixelFunction.end()) return nullptr; return &(oIter->second); @@ -935,7 +956,7 @@ CPLErr VRTDerivedRasterBand::IRasterIO( } /* ---- Get pixel function for band ---- */ - std::pair *poPixelFunc = nullptr; + const std::pair *poPixelFunc = nullptr; std::vector> oAdditionalArgs; if (EQUAL(m_poPrivate->m_osLanguage, "C")) @@ -1396,7 +1417,7 @@ int VRTDerivedRasterBand::IGetDataCoverageStatus( /************************************************************************/ CPLErr VRTDerivedRasterBand::XMLInit( - CPLXMLNode *psTree, const char *pszVRTPath, + const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -1447,10 +1468,11 @@ CPLErr VRTDerivedRasterBand::XMLInit( return CE_Failure; } - CPLXMLNode *psArgs = CPLGetXMLNode(psTree, "PixelFunctionArguments"); + const CPLXMLNode *const psArgs = + CPLGetXMLNode(psTree, "PixelFunctionArguments"); if (psArgs != nullptr) { - for (CPLXMLNode *psIter = psArgs->psChild; psIter != nullptr; + for (const CPLXMLNode *psIter = psArgs->psChild; psIter; psIter = psIter->psNext) { if (psIter->eType == CXT_Attribute) @@ -1487,9 +1509,12 @@ CPLErr VRTDerivedRasterBand::XMLInit( /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTDerivedRasterBand::SerializeToXML(const char *pszVRTPath) +CPLXMLNode *VRTDerivedRasterBand::SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { - CPLXMLNode *psTree = VRTSourcedRasterBand::SerializeToXML(pszVRTPath); + CPLXMLNode *psTree = VRTSourcedRasterBand::SerializeToXML( + pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage); /* -------------------------------------------------------------------- */ /* Set subclass. */ diff --git a/frmts/vrt/vrtdriver.cpp b/frmts/vrt/vrtdriver.cpp index 73a17a1565c1..2883944b4e13 100644 --- a/frmts/vrt/vrtdriver.cpp +++ b/frmts/vrt/vrtdriver.cpp @@ -136,7 +136,7 @@ void VRTDriver::AddSourceParser(const char *pszElementName, /************************************************************************/ VRTSource * -VRTDriver::ParseSource(CPLXMLNode *psSrc, const char *pszVRTPath, +VRTDriver::ParseSource(const CPLXMLNode *psSrc, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -555,6 +555,7 @@ void GDALRegister_VRT() poDriver->AddSourceParser("SimpleSource", VRTParseCoreSources); poDriver->AddSourceParser("ComplexSource", VRTParseCoreSources); poDriver->AddSourceParser("AveragedSource", VRTParseCoreSources); + poDriver->AddSourceParser("NoDataFromMaskSource", VRTParseCoreSources); poDriver->AddSourceParser("KernelFilteredSource", VRTParseFilterSources); poDriver->AddSourceParser("ArraySource", VRTParseArraySource); diff --git a/frmts/vrt/vrtfilters.cpp b/frmts/vrt/vrtfilters.cpp index dfd41b7a1137..3350afeff354 100644 --- a/frmts/vrt/vrtfilters.cpp +++ b/frmts/vrt/vrtfilters.cpp @@ -592,7 +592,7 @@ CPLErr VRTKernelFilteredSource::FilterData(int nXSize, int nYSize, /************************************************************************/ CPLErr VRTKernelFilteredSource::XMLInit( - CPLXMLNode *psTree, const char *pszVRTPath, + const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -705,7 +705,7 @@ CPLXMLNode *VRTKernelFilteredSource::SerializeToXML(const char *pszVRTPath) /************************************************************************/ VRTSource * -VRTParseFilterSources(CPLXMLNode *psChild, const char *pszVRTPath, +VRTParseFilterSources(const CPLXMLNode *psChild, const char *pszVRTPath, std::map &oMapSharedSources) { diff --git a/frmts/vrt/vrtmultidim.cpp b/frmts/vrt/vrtmultidim.cpp index d4305915a126..d4706b081659 100644 --- a/frmts/vrt/vrtmultidim.cpp +++ b/frmts/vrt/vrtmultidim.cpp @@ -2627,7 +2627,7 @@ class VRTArraySource : public VRTSource } CPLErr - XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, + XMLInit(const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) override; CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; }; @@ -2706,7 +2706,7 @@ ParseSingleSourceArray(const CPLXMLNode *psSingleSourceArray, /************************************************************************/ CPLErr VRTArraySource::XMLInit( - CPLXMLNode *psTree, const char *pszVRTPath, + const CPLXMLNode *psTree, const char *pszVRTPath, std::map & /*oMapSharedSources*/) { const auto poArray = ParseArray(psTree, pszVRTPath, "ArraySource"); @@ -2978,7 +2978,7 @@ static std::shared_ptr ParseArray(const CPLXMLNode *psTree, /************************************************************************/ VRTSource * -VRTParseArraySource(CPLXMLNode *psChild, const char *pszVRTPath, +VRTParseArraySource(const CPLXMLNode *psChild, const char *pszVRTPath, std::map &oMapSharedSources) { VRTSource *poSource = nullptr; diff --git a/frmts/vrt/vrtpansharpened.cpp b/frmts/vrt/vrtpansharpened.cpp index 1dc2f8568ebc..e055848eb2fd 100644 --- a/frmts/vrt/vrtpansharpened.cpp +++ b/frmts/vrt/vrtpansharpened.cpp @@ -271,14 +271,14 @@ char **VRTPansharpenedDataset::GetFileList() /* XMLInit() */ /************************************************************************/ -CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, +CPLErr VRTPansharpenedDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn) { return XMLInit(psTree, pszVRTPathIn, nullptr, 0, nullptr); } -CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, +CPLErr VRTPansharpenedDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn, GDALRasterBandH hPanchroBandIn, int nInputSpectralBandsIn, @@ -299,7 +299,7 @@ CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, /* Parse PansharpeningOptions */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psOptions = CPLGetXMLNode(psTree, "PansharpeningOptions"); + const CPLXMLNode *psOptions = CPLGetXMLNode(psTree, "PansharpeningOptions"); if (psOptions == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "Missing PansharpeningOptions"); @@ -325,7 +325,8 @@ CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, if (hPanchroBandIn == nullptr) { - CPLXMLNode *psPanchroBand = CPLGetXMLNode(psOptions, "PanchroBand"); + const CPLXMLNode *psPanchroBand = + CPLGetXMLNode(psOptions, "PanchroBand"); if (psPanchroBand == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "PanchroBand missing"); @@ -433,8 +434,8 @@ CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, } std::vector adfWeights; - CPLXMLNode *psAlgOptions = CPLGetXMLNode(psOptions, "AlgorithmOptions"); - if (psAlgOptions != nullptr) + if (const CPLXMLNode *psAlgOptions = + CPLGetXMLNode(psOptions, "AlgorithmOptions")) { const char *pszWeights = CPLGetXMLValue(psAlgOptions, "Weights", nullptr); @@ -499,7 +500,7 @@ CPLErr VRTPansharpenedDataset::XMLInit(CPLXMLNode *psTree, /* First pass on spectral datasets to check their georeferencing. */ /* -------------------------------------------------------------------- */ int iSpectralBand = 0; - for (CPLXMLNode *psIter = psOptions->psChild; psIter; + for (const CPLXMLNode *psIter = psOptions->psChild; psIter; psIter = psIter->psNext) { GDALDataset *poDataset; @@ -1747,10 +1748,14 @@ CPLErr VRTPansharpenedRasterBand::IRasterIO( /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTPansharpenedRasterBand::SerializeToXML(const char *pszVRTPathIn) +CPLXMLNode * +VRTPansharpenedRasterBand::SerializeToXML(const char *pszVRTPathIn, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { - CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPathIn); + CPLXMLNode *psTree = VRTRasterBand::SerializeToXML( + pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage); /* -------------------------------------------------------------------- */ /* Set subclass. */ diff --git a/frmts/vrt/vrtrasterband.cpp b/frmts/vrt/vrtrasterband.cpp index ffdc2b75ec10..c4f3df26d2bb 100644 --- a/frmts/vrt/vrtrasterband.cpp +++ b/frmts/vrt/vrtrasterband.cpp @@ -343,7 +343,7 @@ VRTParseColorTable(const CPLXMLNode *psColorTable) /************************************************************************/ CPLErr -VRTRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, +VRTRasterBand::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -481,20 +481,18 @@ VRTRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, /* -------------------------------------------------------------------- */ /* Histograms */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psHist = CPLGetXMLNode(psTree, "Histograms"); + const CPLXMLNode *psHist = CPLGetXMLNode(psTree, "Histograms"); if (psHist != nullptr) { - CPLXMLNode *psNext = psHist->psNext; - psHist->psNext = nullptr; - - m_psSavedHistograms = CPLCloneXMLTree(psHist); - psHist->psNext = psNext; + CPLXMLNode sHistTemp = *psHist; + sHistTemp.psNext = nullptr; + m_psSavedHistograms = CPLCloneXMLTree(&sHistTemp); } /* ==================================================================== */ /* Overviews */ /* ==================================================================== */ - CPLXMLNode *psNode = psTree->psChild; + const CPLXMLNode *psNode = psTree->psChild; for (; psNode != nullptr; psNode = psNode->psNext) { @@ -507,7 +505,8 @@ VRTRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, /* Prepare filename. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psFileNameNode = CPLGetXMLNode(psNode, "SourceFilename"); + const CPLXMLNode *psFileNameNode = + CPLGetXMLNode(psNode, "SourceFilename"); const char *pszFilename = psFileNameNode ? CPLGetXMLValue(psFileNameNode, nullptr, nullptr) : nullptr; @@ -557,7 +556,7 @@ VRTRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, /* ==================================================================== */ /* Mask band (specific to that raster band) */ /* ==================================================================== */ - CPLXMLNode *psMaskBandNode = CPLGetXMLNode(psTree, "MaskBand"); + const CPLXMLNode *psMaskBandNode = CPLGetXMLNode(psTree, "MaskBand"); if (psMaskBandNode) psNode = psMaskBandNode->psChild; else @@ -646,7 +645,9 @@ CPLString VRTSerializeNoData(double dfVal, GDALDataType eDataType, /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath) +CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { CPLXMLNode *psTree = @@ -830,9 +831,12 @@ CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath) /* Mask band (specific to that raster band) */ /* ==================================================================== */ + nAccRAMUsage += CPLXMLNodeGetRAMUsageEstimate(psTree); + if (m_poMaskBand != nullptr) { - CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(pszVRTPath); + CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML( + pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage); if (psBandTree != nullptr) { diff --git a/frmts/vrt/vrtrawrasterband.cpp b/frmts/vrt/vrtrawrasterband.cpp index f0ef87399d3d..e5c25753a518 100644 --- a/frmts/vrt/vrtrawrasterband.cpp +++ b/frmts/vrt/vrtrawrasterband.cpp @@ -344,7 +344,7 @@ CPLVirtualMem *VRTRawRasterBand::GetVirtualMemAuto(GDALRWFlag eRWFlag, /************************************************************************/ CPLErr -VRTRawRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, +VRTRawRasterBand::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -429,7 +429,9 @@ VRTRawRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath, /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath) +CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { @@ -444,7 +446,8 @@ CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath) return nullptr; } - CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPath); + CPLXMLNode *psTree = VRTRasterBand::SerializeToXML( + pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage); /* -------------------------------------------------------------------- */ /* Set subclass. */ diff --git a/frmts/vrt/vrtsourcedrasterband.cpp b/frmts/vrt/vrtsourcedrasterband.cpp index b9a12c9256e3..89066707d186 100644 --- a/frmts/vrt/vrtsourcedrasterband.cpp +++ b/frmts/vrt/vrtsourcedrasterband.cpp @@ -1813,7 +1813,7 @@ CPLErr CPL_STDCALL VRTAddSource(VRTSourcedRasterBandH hVRTBand, /************************************************************************/ CPLErr VRTSourcedRasterBand::XMLInit( - CPLXMLNode *psTree, const char *pszVRTPath, + const CPLXMLNode *psTree, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -1830,7 +1830,7 @@ CPLErr VRTSourcedRasterBand::XMLInit( VRTDriver *const poDriver = static_cast(GDALGetDriverByName("VRT")); - for (CPLXMLNode *psChild = psTree->psChild; + for (const CPLXMLNode *psChild = psTree->psChild; psChild != nullptr && poDriver != nullptr; psChild = psChild->psNext) { if (psChild->eType != CXT_Element) @@ -1861,10 +1861,13 @@ CPLErr VRTSourcedRasterBand::XMLInit( /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTSourcedRasterBand::SerializeToXML(const char *pszVRTPath) +CPLXMLNode *VRTSourcedRasterBand::SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { - CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPath); + CPLXMLNode *psTree = VRTRasterBand::SerializeToXML( + pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage); CPLXMLNode *psLastChild = psTree->psChild; while (psLastChild != nullptr && psLastChild->psNext != nullptr) psLastChild = psLastChild->psNext; @@ -1872,19 +1875,45 @@ CPLXMLNode *VRTSourcedRasterBand::SerializeToXML(const char *pszVRTPath) /* -------------------------------------------------------------------- */ /* Process Sources. */ /* -------------------------------------------------------------------- */ + + GIntBig nUsableRAM = -1; + for (int iSource = 0; iSource < nSources; iSource++) { CPLXMLNode *const psXMLSrc = papoSources[iSource]->SerializeToXML(pszVRTPath); - if (psXMLSrc != nullptr) - { - if (psLastChild == nullptr) - psTree->psChild = psXMLSrc; - else - psLastChild->psNext = psXMLSrc; - psLastChild = psXMLSrc; + if (psXMLSrc == nullptr) + break; + + // Creating the CPLXMLNode tree representation of a VRT can easily + // take several times RAM usage than its string serialization, or its + // internal representation in the driver. + // We multiply the estimate by a factor of 2, experimentally found to + // be more realistic than the conservative raw estimate. + nAccRAMUsage += 2 * CPLXMLNodeGetRAMUsageEstimate(psXMLSrc); + if (!bHasWarnedAboutRAMUsage && nAccRAMUsage > 512 * 1024 * 1024) + { + if (nUsableRAM < 0) + nUsableRAM = CPLGetUsablePhysicalRAM(); + if (nUsableRAM > 0 && + nAccRAMUsage > static_cast(nUsableRAM) / 10 * 8) + { + bHasWarnedAboutRAMUsage = true; + CPLError(CE_Warning, CPLE_AppDefined, + "Serialization of this VRT file has already consumed " + "at least %.02f GB of RAM over a total of %.02f. This " + "process may abort", + double(nAccRAMUsage) / (1024 * 1024 * 1024), + double(nUsableRAM) / (1024 * 1024 * 1024)); + } } + + if (psLastChild == nullptr) + psTree->psChild = psXMLSrc; + else + psLastChild->psNext = psXMLSrc; + psLastChild = psXMLSrc; } return psTree; diff --git a/frmts/vrt/vrtsources.cpp b/frmts/vrt/vrtsources.cpp index 8f899cfb2ca9..856927a1300f 100644 --- a/frmts/vrt/vrtsources.cpp +++ b/frmts/vrt/vrtsources.cpp @@ -512,7 +512,7 @@ CPLXMLNode *VRTSimpleSource::SerializeToXML(const char *pszVRTPath) /************************************************************************/ CPLErr -VRTSimpleSource::XMLInit(CPLXMLNode *psSrc, const char *pszVRTPath, +VRTSimpleSource::XMLInit(const CPLXMLNode *psSrc, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -523,7 +523,8 @@ VRTSimpleSource::XMLInit(CPLXMLNode *psSrc, const char *pszVRTPath, /* -------------------------------------------------------------------- */ /* Prepare filename. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psSourceFileNameNode = CPLGetXMLNode(psSrc, "SourceFilename"); + const CPLXMLNode *psSourceFileNameNode = + CPLGetXMLNode(psSrc, "SourceFilename"); const char *pszFilename = psSourceFileNameNode ? CPLGetXMLValue(psSourceFileNameNode, nullptr, "") : ""; @@ -2000,6 +2001,455 @@ CPLErr VRTAveragedSource::GetHistogram( return CE_Failure; } +/************************************************************************/ +/* ==================================================================== */ +/* VRTNoDataFromMaskSource */ +/* ==================================================================== */ +/************************************************************************/ + +/************************************************************************/ +/* VRTNoDataFromMaskSource() */ +/************************************************************************/ + +VRTNoDataFromMaskSource::VRTNoDataFromMaskSource() +{ +} + +/************************************************************************/ +/* XMLInit() */ +/************************************************************************/ + +CPLErr VRTNoDataFromMaskSource::XMLInit( + const CPLXMLNode *psSrc, const char *pszVRTPath, + std::map &oMapSharedSources) + +{ + /* -------------------------------------------------------------------- */ + /* Do base initialization. */ + /* -------------------------------------------------------------------- */ + { + const CPLErr eErr = + VRTSimpleSource::XMLInit(psSrc, pszVRTPath, oMapSharedSources); + if (eErr != CE_None) + return eErr; + } + + if (const char *pszNODATA = CPLGetXMLValue(psSrc, "NODATA", nullptr)) + { + m_bNoDataSet = true; + m_dfNoDataValue = CPLAtofM(pszNODATA); + } + + m_dfMaskValueThreshold = + CPLAtofM(CPLGetXMLValue(psSrc, "MaskValueThreshold", "0")); + + if (const char *pszRemappedValue = + CPLGetXMLValue(psSrc, "RemappedValue", nullptr)) + { + m_bHasRemappedValue = true; + m_dfRemappedValue = CPLAtofM(pszRemappedValue); + } + + return CE_None; +} + +/************************************************************************/ +/* SerializeToXML() */ +/************************************************************************/ + +CPLXMLNode *VRTNoDataFromMaskSource::SerializeToXML(const char *pszVRTPath) + +{ + CPLXMLNode *const psSrc = VRTSimpleSource::SerializeToXML(pszVRTPath); + + if (psSrc == nullptr) + return nullptr; + + CPLFree(psSrc->pszValue); + psSrc->pszValue = CPLStrdup("NoDataFromMaskSource"); + + if (m_bNoDataSet) + { + CPLSetXMLValue(psSrc, "MaskValueThreshold", + CPLSPrintf("%.18g", m_dfMaskValueThreshold)); + + GDALDataType eBandDT = GDT_Unknown; + double dfNoDataValue = m_dfNoDataValue; + const auto kMaxFloat = std::numeric_limits::max(); + if (std::fabs(std::fabs(m_dfNoDataValue) - kMaxFloat) < + 1e-10 * kMaxFloat) + { + auto l_band = GetRasterBand(); + if (l_band) + { + eBandDT = l_band->GetRasterDataType(); + if (eBandDT == GDT_Float32) + { + dfNoDataValue = + GDALAdjustNoDataCloseToFloatMax(m_dfNoDataValue); + } + } + } + CPLSetXMLValue(psSrc, "NODATA", + VRTSerializeNoData(dfNoDataValue, eBandDT, 18).c_str()); + } + + if (m_bHasRemappedValue) + { + CPLSetXMLValue(psSrc, "RemappedValue", + CPLSPrintf("%.18g", m_dfRemappedValue)); + } + + return psSrc; +} + +/************************************************************************/ +/* SetParameters() */ +/************************************************************************/ + +void VRTNoDataFromMaskSource::SetParameters(double dfNoDataValue, + double dfMaskValueThreshold) +{ + m_bNoDataSet = true; + m_dfNoDataValue = dfNoDataValue; + m_dfMaskValueThreshold = dfMaskValueThreshold; + if (!m_bHasRemappedValue) + m_dfRemappedValue = m_dfNoDataValue; +} + +/************************************************************************/ +/* SetParameters() */ +/************************************************************************/ + +void VRTNoDataFromMaskSource::SetParameters(double dfNoDataValue, + double dfMaskValueThreshold, + double dfRemappedValue) +{ + SetParameters(dfNoDataValue, dfMaskValueThreshold); + m_bHasRemappedValue = true; + m_dfRemappedValue = dfRemappedValue; +} + +/************************************************************************/ +/* RasterIO() */ +/************************************************************************/ + +CPLErr VRTNoDataFromMaskSource::RasterIO( + GDALDataType eVRTBandDataType, int nXOff, int nYOff, int nXSize, int nYSize, + void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, + GSpacing nPixelSpace, GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArgIn, WorkingState &oWorkingState) + +{ + if (!m_bNoDataSet) + { + return VRTSimpleSource::RasterIO(eVRTBandDataType, nXOff, nYOff, nXSize, + nYSize, pData, nBufXSize, nBufYSize, + eBufType, nPixelSpace, nLineSpace, + psExtraArgIn, oWorkingState); + } + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + GDALRasterIOExtraArg *psExtraArg = &sExtraArg; + + double dfXOff = nXOff; + double dfYOff = nYOff; + double dfXSize = nXSize; + double dfYSize = nYSize; + if (psExtraArgIn != nullptr && psExtraArgIn->bFloatingPointWindowValidity) + { + dfXOff = psExtraArgIn->dfXOff; + dfYOff = psExtraArgIn->dfYOff; + dfXSize = psExtraArgIn->dfXSize; + dfYSize = psExtraArgIn->dfYSize; + } + + // The window we will actually request from the source raster band. + double dfReqXOff = 0.0; + double dfReqYOff = 0.0; + double dfReqXSize = 0.0; + double dfReqYSize = 0.0; + int nReqXOff = 0; + int nReqYOff = 0; + int nReqXSize = 0; + int nReqYSize = 0; + + // The window we will actual set _within_ the pData buffer. + int nOutXOff = 0; + int nOutYOff = 0; + int nOutXSize = 0; + int nOutYSize = 0; + + bool bError = false; + if (!GetSrcDstWindow(dfXOff, dfYOff, dfXSize, dfYSize, nBufXSize, nBufYSize, + &dfReqXOff, &dfReqYOff, &dfReqXSize, &dfReqYSize, + &nReqXOff, &nReqYOff, &nReqXSize, &nReqYSize, + &nOutXOff, &nOutYOff, &nOutXSize, &nOutYSize, bError)) + { + return bError ? CE_Failure : CE_None; + } + + auto l_band = GetRasterBand(); + if (!l_band) + return CE_Failure; + + /* -------------------------------------------------------------------- */ + /* Allocate temporary buffer(s). */ + /* -------------------------------------------------------------------- */ + const auto eSrcBandDT = l_band->GetRasterDataType(); + const int nSrcBandDTSize = GDALGetDataTypeSizeBytes(eSrcBandDT); + const auto eSrcMaskBandDT = l_band->GetMaskBand()->GetRasterDataType(); + const int nSrcMaskBandDTSize = GDALGetDataTypeSizeBytes(eSrcMaskBandDT); + double dfRemappedValue = m_dfRemappedValue; + if (!m_bHasRemappedValue) + { + if (eSrcBandDT == GDT_Byte && + m_dfNoDataValue >= std::numeric_limits::min() && + m_dfNoDataValue <= std::numeric_limits::max() && + static_cast(m_dfNoDataValue) == m_dfNoDataValue) + { + if (m_dfNoDataValue == std::numeric_limits::max()) + dfRemappedValue = m_dfNoDataValue - 1; + else + dfRemappedValue = m_dfNoDataValue + 1; + } + else if (eSrcBandDT == GDT_UInt16 && + m_dfNoDataValue >= std::numeric_limits::min() && + m_dfNoDataValue <= std::numeric_limits::max() && + static_cast(m_dfNoDataValue) == m_dfNoDataValue) + { + if (m_dfNoDataValue == std::numeric_limits::max()) + dfRemappedValue = m_dfNoDataValue - 1; + else + dfRemappedValue = m_dfNoDataValue + 1; + } + else if (eSrcBandDT == GDT_Int16 && + m_dfNoDataValue >= std::numeric_limits::min() && + m_dfNoDataValue <= std::numeric_limits::max() && + static_cast(m_dfNoDataValue) == m_dfNoDataValue) + { + if (m_dfNoDataValue == std::numeric_limits::max()) + dfRemappedValue = m_dfNoDataValue - 1; + else + dfRemappedValue = m_dfNoDataValue + 1; + } + else + { + constexpr double EPS = 1e-3; + if (m_dfNoDataValue == 0) + dfRemappedValue = EPS; + else + dfRemappedValue = m_dfNoDataValue * (1 + EPS); + } + } + const bool bByteOptim = + (eSrcBandDT == GDT_Byte && eBufType == GDT_Byte && + eSrcMaskBandDT == GDT_Byte && m_dfMaskValueThreshold >= 0 && + m_dfMaskValueThreshold <= 255 && + static_cast(m_dfMaskValueThreshold) == m_dfMaskValueThreshold && + m_dfNoDataValue >= 0 && m_dfNoDataValue <= 255 && + static_cast(m_dfNoDataValue) == m_dfNoDataValue && + dfRemappedValue >= 0 && dfRemappedValue <= 255 && + static_cast(dfRemappedValue) == dfRemappedValue); + GByte *pabyWrkBuffer; + try + { + if (bByteOptim && nOutXOff == 0 && nOutYOff == 0 && + nOutXSize == nBufXSize && nOutYSize == nBufYSize && + eSrcBandDT == eBufType && nPixelSpace == nSrcBandDTSize && + nLineSpace == nPixelSpace * nBufXSize) + { + pabyWrkBuffer = static_cast(pData); + } + else + { + oWorkingState.m_abyWrkBuffer.resize(static_cast(nOutXSize) * + nOutYSize * nSrcBandDTSize); + pabyWrkBuffer = + reinterpret_cast(oWorkingState.m_abyWrkBuffer.data()); + } + oWorkingState.m_abyWrkBufferMask.resize(static_cast(nOutXSize) * + nOutYSize * nSrcMaskBandDTSize); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory when allocating buffers"); + return CE_Failure; + } + + /* -------------------------------------------------------------------- */ + /* Load data. */ + /* -------------------------------------------------------------------- */ + if (!m_osResampling.empty()) + { + psExtraArg->eResampleAlg = GDALRasterIOGetResampleAlg(m_osResampling); + } + else if (psExtraArgIn != nullptr) + { + psExtraArg->eResampleAlg = psExtraArgIn->eResampleAlg; + } + + psExtraArg->bFloatingPointWindowValidity = TRUE; + psExtraArg->dfXOff = dfReqXOff; + psExtraArg->dfYOff = dfReqYOff; + psExtraArg->dfXSize = dfReqXSize; + psExtraArg->dfYSize = dfReqYSize; + + if (l_band->RasterIO(GF_Read, nReqXOff, nReqYOff, nReqXSize, nReqYSize, + pabyWrkBuffer, nOutXSize, nOutYSize, eSrcBandDT, 0, 0, + psExtraArg) != CE_None) + { + return CE_Failure; + } + + if (l_band->GetMaskBand()->RasterIO( + GF_Read, nReqXOff, nReqYOff, nReqXSize, nReqYSize, + oWorkingState.m_abyWrkBufferMask.data(), nOutXSize, nOutYSize, + eSrcMaskBandDT, 0, 0, psExtraArg) != CE_None) + { + return CE_Failure; + } + + /* -------------------------------------------------------------------- */ + /* Do the processing. */ + /* -------------------------------------------------------------------- */ + + GByte *const pabyOut = static_cast(pData) + + nPixelSpace * nOutXOff + + static_cast(nLineSpace) * nOutYOff; + if (bByteOptim) + { + // Special case when everything fits on Byte + const GByte nMaskValueThreshold = + static_cast(m_dfMaskValueThreshold); + const GByte nNoDataValue = static_cast(m_dfNoDataValue); + const GByte nRemappedValue = static_cast(dfRemappedValue); + size_t nSrcIdx = 0; + for (int iY = 0; iY < nOutYSize; iY++) + { + GSpacing nDstOffset = iY * nLineSpace; + for (int iX = 0; iX < nOutXSize; iX++) + { + const GByte nMaskVal = + oWorkingState.m_abyWrkBufferMask[nSrcIdx]; + if (nMaskVal <= nMaskValueThreshold) + { + pabyOut[static_cast(nDstOffset)] = nNoDataValue; + } + else + { + if (pabyWrkBuffer[nSrcIdx] == nNoDataValue) + { + pabyOut[static_cast(nDstOffset)] = + nRemappedValue; + } + else + { + pabyOut[static_cast(nDstOffset)] = + pabyWrkBuffer[nSrcIdx]; + } + } + nDstOffset += nPixelSpace; + nSrcIdx++; + } + } + } + else + { + size_t nSrcIdx = 0; + double dfMaskVal = 0; + const int nBufDTSize = GDALGetDataTypeSizeBytes(eBufType); + std::vector abyDstNoData(nBufDTSize); + GDALCopyWords(&m_dfNoDataValue, GDT_Float64, 0, abyDstNoData.data(), + eBufType, 0, 1); + std::vector abyRemappedValue(nBufDTSize); + GDALCopyWords(&dfRemappedValue, GDT_Float64, 0, abyRemappedValue.data(), + eBufType, 0, 1); + for (int iY = 0; iY < nOutYSize; iY++) + { + GSpacing nDstOffset = iY * nLineSpace; + for (int iX = 0; iX < nOutXSize; iX++) + { + if (eSrcMaskBandDT == GDT_Byte) + { + dfMaskVal = oWorkingState.m_abyWrkBufferMask[nSrcIdx]; + } + else + { + GDALCopyWords(oWorkingState.m_abyWrkBufferMask.data() + + nSrcIdx * nSrcMaskBandDTSize, + eSrcMaskBandDT, 0, &dfMaskVal, GDT_Float64, 0, + 1); + } + void *const pDst = + pabyOut + static_cast(nDstOffset); + if (!(dfMaskVal > m_dfMaskValueThreshold)) + { + memcpy(pDst, abyDstNoData.data(), nBufDTSize); + } + else + { + const void *const pSrc = + pabyWrkBuffer + nSrcIdx * nSrcBandDTSize; + if (eSrcBandDT == eBufType) + { + // coverity[overrun-buffer-arg] + memcpy(pDst, pSrc, nBufDTSize); + } + else + { + GDALCopyWords(pSrc, eSrcBandDT, 0, pDst, eBufType, 0, + 1); + } + if (memcmp(pDst, abyDstNoData.data(), nBufDTSize) == 0) + memcpy(pDst, abyRemappedValue.data(), nBufDTSize); + } + nDstOffset += nPixelSpace; + nSrcIdx++; + } + } + } + + return CE_None; +} + +/************************************************************************/ +/* GetMinimum() */ +/************************************************************************/ + +double VRTNoDataFromMaskSource::GetMinimum(int /* nXSize */, int /* nYSize */, + int *pbSuccess) +{ + *pbSuccess = FALSE; + return 0.0; +} + +/************************************************************************/ +/* GetMaximum() */ +/************************************************************************/ + +double VRTNoDataFromMaskSource::GetMaximum(int /* nXSize */, int /* nYSize */, + int *pbSuccess) +{ + *pbSuccess = FALSE; + return 0.0; +} + +/************************************************************************/ +/* GetHistogram() */ +/************************************************************************/ + +CPLErr VRTNoDataFromMaskSource::GetHistogram( + int /* nXSize */, int /* nYSize */, double /* dfMin */, double /* dfMax */, + int /* nBuckets */, GUIntBig * /* panHistogram */, + int /* bIncludeOutOfRange */, int /* bApproxOK */, + GDALProgressFunc /* pfnProgress */, void * /* pProgressData */) +{ + return CE_Failure; +} + /************************************************************************/ /* ==================================================================== */ /* VRTComplexSource */ @@ -2181,7 +2631,7 @@ CPLXMLNode *VRTComplexSource::SerializeToXML(const char *pszVRTPath) /************************************************************************/ CPLErr -VRTComplexSource::XMLInit(CPLXMLNode *psSrc, const char *pszVRTPath, +VRTComplexSource::XMLInit(const CPLXMLNode *psSrc, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -2198,12 +2648,15 @@ VRTComplexSource::XMLInit(CPLXMLNode *psSrc, const char *pszVRTPath, /* -------------------------------------------------------------------- */ /* Complex parameters. */ /* -------------------------------------------------------------------- */ - if (CPLGetXMLValue(psSrc, "ScaleOffset", nullptr) != nullptr || - CPLGetXMLValue(psSrc, "ScaleRatio", nullptr) != nullptr) + const char *pszScaleOffset = CPLGetXMLValue(psSrc, "ScaleOffset", nullptr); + const char *pszScaleRatio = CPLGetXMLValue(psSrc, "ScaleRatio", nullptr); + if (pszScaleOffset || pszScaleRatio) { m_nProcessingFlags |= PROCESSING_FLAG_SCALING_LINEAR; - m_dfScaleOff = CPLAtof(CPLGetXMLValue(psSrc, "ScaleOffset", "0")); - m_dfScaleRatio = CPLAtof(CPLGetXMLValue(psSrc, "ScaleRatio", "1")); + if (pszScaleOffset) + m_dfScaleOff = CPLAtof(pszScaleOffset); + if (pszScaleRatio) + m_dfScaleRatio = CPLAtof(pszScaleRatio); } else if (CPLGetXMLValue(psSrc, "Exponent", nullptr) != nullptr && CPLGetXMLValue(psSrc, "DstMin", nullptr) != nullptr && @@ -2225,10 +2678,10 @@ VRTComplexSource::XMLInit(CPLXMLNode *psSrc, const char *pszVRTPath, m_dfDstMax = CPLAtof(CPLGetXMLValue(psSrc, "DstMax", "0.0")); } - if (CPLGetXMLValue(psSrc, "NODATA", nullptr) != nullptr) + if (const char *pszNODATA = CPLGetXMLValue(psSrc, "NODATA", nullptr)) { m_nProcessingFlags |= PROCESSING_FLAG_NODATA; - m_osNoDataValueOri = CPLGetXMLValue(psSrc, "NODATA", "0"); + m_osNoDataValueOri = pszNODATA; m_dfNoDataValue = CPLAtofM(m_osNoDataValueOri.c_str()); } @@ -2563,7 +3016,9 @@ CPLErr VRTComplexSource::RasterIOProcessNoData( // Cannot overflow since pData should at least have that number of // elements const size_t nPixelCount = static_cast(nOutXSize) * nOutYSize; - if (nPixelCount > std::numeric_limits::max() / sizeof(SourceDT)) + if (nPixelCount > + static_cast(std::numeric_limits::max()) / + sizeof(SourceDT)) { CPLError(CE_Failure, CPLE_OutOfMemory, "Too large temporary buffer"); @@ -2779,8 +3234,9 @@ CPLErr VRTComplexSource::RasterIOInternal( { // Cannot overflow since pData should at least have that number of // elements - if (nPixelCount > std::numeric_limits::max() / - static_cast(nWordSize)) + if (nPixelCount > + static_cast(std::numeric_limits::max()) / + static_cast(nWordSize)) { CPLError(CE_Failure, CPLE_OutOfMemory, "Too large temporary buffer"); @@ -3199,7 +3655,7 @@ CPLErr VRTFuncSource::GetHistogram( /************************************************************************/ VRTSource * -VRTParseCoreSources(CPLXMLNode *psChild, const char *pszVRTPath, +VRTParseCoreSources(const CPLXMLNode *psChild, const char *pszVRTPath, std::map &oMapSharedSources) { @@ -3220,6 +3676,10 @@ VRTParseCoreSources(CPLXMLNode *psChild, const char *pszVRTPath, { poSource = new VRTComplexSource(); } + else if (EQUAL(psChild->pszValue, "NoDataFromMaskSource")) + { + poSource = new VRTNoDataFromMaskSource(); + } else { CPLError(CE_Failure, CPLE_AppDefined, diff --git a/frmts/vrt/vrtwarped.cpp b/frmts/vrt/vrtwarped.cpp index cc5a93e5bd8d..642241b7392b 100644 --- a/frmts/vrt/vrtwarped.cpp +++ b/frmts/vrt/vrtwarped.cpp @@ -1283,7 +1283,8 @@ CPLErr CPL_STDCALL GDALInitializeWarpedVRT(GDALDatasetH hDS, /* XMLInit() */ /************************************************************************/ -CPLErr VRTWarpedDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) +CPLErr VRTWarpedDataset::XMLInit(const CPLXMLNode *psTree, + const char *pszVRTPathIn) { @@ -1330,7 +1331,8 @@ CPLErr VRTWarpedDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ /* Find the GDALWarpOptions XML tree. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *const psOptionsTree = CPLGetXMLNode(psTree, "GDALWarpOptions"); + const CPLXMLNode *const psOptionsTree = + CPLGetXMLNode(psTree, "GDALWarpOptions"); if (psOptionsTree == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, @@ -1355,14 +1357,16 @@ CPLErr VRTWarpedDataset::XMLInit(CPLXMLNode *psTree, const char *pszVRTPathIn) else pszAbsolutePath = CPLStrdup(pszRelativePath); - CPLSetXMLValue(psOptionsTree, "SourceDataset", pszAbsolutePath); + CPLXMLNode *psOptionsTreeCloned = CPLCloneXMLTree(psOptionsTree); + CPLSetXMLValue(psOptionsTreeCloned, "SourceDataset", pszAbsolutePath); CPLFree(pszAbsolutePath); /* -------------------------------------------------------------------- */ /* And instantiate the warp options, and corresponding warp */ /* operation. */ /* -------------------------------------------------------------------- */ - GDALWarpOptions *psWO = GDALDeserializeWarpOptions(psOptionsTree); + GDALWarpOptions *psWO = GDALDeserializeWarpOptions(psOptionsTreeCloned); + CPLDestroyXMLNode(psOptionsTreeCloned); if (psWO == nullptr) return CE_Failure; @@ -1929,10 +1933,13 @@ CPLErr VRTWarpedRasterBand::IWriteBlock(int nBlockXOff, int nBlockYOff, /* SerializeToXML() */ /************************************************************************/ -CPLXMLNode *VRTWarpedRasterBand::SerializeToXML(const char *pszVRTPathIn) +CPLXMLNode *VRTWarpedRasterBand::SerializeToXML(const char *pszVRTPathIn, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) { - CPLXMLNode *const psTree = VRTRasterBand::SerializeToXML(pszVRTPathIn); + CPLXMLNode *const psTree = VRTRasterBand::SerializeToXML( + pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage); /* -------------------------------------------------------------------- */ /* Set subclass. */ diff --git a/frmts/wms/gdalwmsdataset.cpp b/frmts/wms/gdalwmsdataset.cpp index f2667eb4b794..057efedec3a8 100644 --- a/frmts/wms/gdalwmsdataset.cpp +++ b/frmts/wms/gdalwmsdataset.cpp @@ -471,6 +471,13 @@ CPLErr GDALWMSDataset::Initialize(CPLXMLNode *config, char **l_papszOpenOptions) } m_data_window.m_tlevel = atoi(tlevel); + // Limit to 30 to avoid 1 << m_tlevel overflow + if (m_data_window.m_tlevel < 0 || m_data_window.m_tlevel > 30) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for TileLevel"); + return CE_Failure; + } if (ret == CE_None) { @@ -483,10 +490,43 @@ CPLErr GDALWMSDataset::Initialize(CPLXMLNode *config, char **l_papszOpenOptions) (str_tile_count_x[0] != '\0') && (str_tile_count_y[0] != '\0')) { - int tile_count_x = atoi(str_tile_count_x); - int tile_count_y = atoi(str_tile_count_y); + const int tile_count_x = atoi(str_tile_count_x); + if (tile_count_x <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for TileCountX"); + return CE_Failure; + } + if (tile_count_x > INT_MAX / m_block_size_x || + tile_count_x * m_block_size_x > + INT_MAX / (1 << m_data_window.m_tlevel)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Integer overflow in tile_count_x * " + "m_block_size_x * (1 << " + "m_data_window.m_tlevel)"); + return CE_Failure; + } m_data_window.m_sx = tile_count_x * m_block_size_x * (1 << m_data_window.m_tlevel); + + const int tile_count_y = atoi(str_tile_count_y); + if (tile_count_y <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for TileCountY"); + return CE_Failure; + } + if (tile_count_y > INT_MAX / m_block_size_y || + tile_count_y * m_block_size_y > + INT_MAX / (1 << m_data_window.m_tlevel)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Integer overflow in tile_count_y * " + "m_block_size_y * (1 << " + "m_data_window.m_tlevel)"); + return CE_Failure; + } m_data_window.m_sy = tile_count_y * m_block_size_y * (1 << m_data_window.m_tlevel); } diff --git a/frmts/wms/wmsmetadataset.cpp b/frmts/wms/wmsmetadataset.cpp index f2dd96dba321..58f358f53871 100644 --- a/frmts/wms/wmsmetadataset.cpp +++ b/frmts/wms/wmsmetadataset.cpp @@ -231,13 +231,11 @@ char **GDALWMSMetaDataset::GetMetadata(const char *pszDomain) /* AddSubDataset() */ /************************************************************************/ -void GDALWMSMetaDataset::AddSubDataset(const char *pszLayerName, - const char *pszTitle, - CPL_UNUSED const char *pszAbstract, - const char *pszSRS, const char *pszMinX, - const char *pszMinY, const char *pszMaxX, - const char *pszMaxY, CPLString osFormat, - CPLString osTransparent) +void GDALWMSMetaDataset::AddSubDataset( + const char *pszLayerName, const char *pszTitle, + CPL_UNUSED const char *pszAbstract, const char *pszSRS, const char *pszMinX, + const char *pszMinY, const char *pszMaxX, const char *pszMaxY, + const std::string &osFormat, const std::string &osTransparent) { CPLString osSubdatasetName = "WMS:"; osSubdatasetName += osGetURL; @@ -258,10 +256,11 @@ void GDALWMSMetaDataset::AddSubDataset(const char *pszLayerName, osSubdatasetName, "BBOX", CPLSPrintf("%s,%s,%s,%s", pszMinX, pszMinY, pszMaxX, pszMaxY)); if (!osFormat.empty()) - osSubdatasetName = CPLURLAddKVP(osSubdatasetName, "FORMAT", osFormat); - if (!osTransparent.empty()) osSubdatasetName = - CPLURLAddKVP(osSubdatasetName, "TRANSPARENT", osTransparent); + CPLURLAddKVP(osSubdatasetName, "FORMAT", osFormat.c_str()); + if (!osTransparent.empty()) + osSubdatasetName = CPLURLAddKVP(osSubdatasetName, "TRANSPARENT", + osTransparent.c_str()); if (pszTitle) { diff --git a/frmts/wms/wmsmetadataset.h b/frmts/wms/wmsmetadataset.h index d82283f4d888..de306f3c9e30 100644 --- a/frmts/wms/wmsmetadataset.h +++ b/frmts/wms/wmsmetadataset.h @@ -72,7 +72,8 @@ class GDALWMSMetaDataset final : public GDALPamDataset const char *pszAbstract, const char *pszSRS, const char *pszMinX, const char *pszMinY, const char *pszMaxX, const char *pszMaxY, - CPLString osFormat, CPLString osTransparent); + const std::string &osFormat, + const std::string &osTransparent); void ExploreLayer(CPLXMLNode *psXML, const CPLString &osFormat, diff --git a/frmts/zarr/zarr.h b/frmts/zarr/zarr.h index 139526fb8c69..ae0dc8a9274f 100644 --- a/frmts/zarr/zarr.h +++ b/frmts/zarr/zarr.h @@ -251,6 +251,7 @@ class ZarrSharedResource std::shared_ptr m_poPAM{}; CPLStringList m_aosOpenOptions{}; std::weak_ptr m_poWeakRootGroup{}; + std::set m_oSetArrayInLoading{}; explicit ZarrSharedResource(const std::string &osRootDirectoryName, bool bUpdatable); @@ -313,6 +314,35 @@ class ZarrSharedResource { m_poWeakRootGroup = poRootGroup; } + + bool AddArrayInLoading(const std::string &osZarrayFilename); + void RemoveArrayInLoading(const std::string &osZarrayFilename); + + struct SetFilenameAdder + { + std::shared_ptr m_poSharedResource; + const std::string m_osFilename; + const bool m_bOK; + + SetFilenameAdder( + const std::shared_ptr &poSharedResource, + const std::string &osFilename) + : m_poSharedResource(poSharedResource), m_osFilename(osFilename), + m_bOK(m_poSharedResource->AddArrayInLoading(m_osFilename)) + { + } + + ~SetFilenameAdder() + { + if (m_bOK) + m_poSharedResource->RemoveArrayInLoading(m_osFilename); + } + + bool ok() const + { + return m_bOK; + } + }; }; /************************************************************************/ @@ -518,8 +548,8 @@ class ZarrV2Group final : public ZarrGroupBase std::shared_ptr LoadArray(const std::string &osArrayName, const std::string &osZarrayFilename, const CPLJSONObject &oRoot, - bool bLoadedFromZMetadata, const CPLJSONObject &oAttributes, - std::set &oSetFilenamesInLoading) const; + bool bLoadedFromZMetadata, + const CPLJSONObject &oAttributes) const; std::shared_ptr CreateMDArray( const std::string &osName, @@ -569,10 +599,9 @@ class ZarrV3Group final : public ZarrGroupBase CreateGroup(const std::string &osName, CSLConstList papszOptions = nullptr) override; - std::shared_ptr - LoadArray(const std::string &osArrayName, - const std::string &osZarrayFilename, const CPLJSONObject &oRoot, - std::set &oSetFilenamesInLoading) const; + std::shared_ptr LoadArray(const std::string &osArrayName, + const std::string &osZarrayFilename, + const CPLJSONObject &oRoot) const; std::shared_ptr CreateMDArray( const std::string &osName, diff --git a/frmts/zarr/zarr_array.cpp b/frmts/zarr/zarr_array.cpp index 5a2a2e058afa..f62a43a31311 100644 --- a/frmts/zarr/zarr_array.cpp +++ b/frmts/zarr/zarr_array.cpp @@ -2210,12 +2210,16 @@ bool ZarrArray::CacheTilePresence() "present...", osDirectoryName.c_str()); uint64_t nCounter = 0; + const char chSrcFilenameDirSeparator = + VSIGetDirectorySeparator(osDirectoryName.c_str())[0]; while (const VSIDIREntry *psEntry = VSIGetNextDirEntry(psDir)) { if (!VSI_ISDIR(psEntry->nMode)) { - const CPLStringList aosTokens = - GetTileIndicesFromFilename(psEntry->pszName); + const CPLStringList aosTokens = GetTileIndicesFromFilename( + CPLString(psEntry->pszName) + .replaceAll(chSrcFilenameDirSeparator, '/') + .c_str()); if (aosTokens.size() == static_cast(m_aoDims.size())) { // Get tile indices from filename diff --git a/frmts/zarr/zarr_sharedresource.cpp b/frmts/zarr/zarr_sharedresource.cpp index cd34686da166..b0daddf2b608 100644 --- a/frmts/zarr/zarr_sharedresource.cpp +++ b/frmts/zarr/zarr_sharedresource.cpp @@ -116,9 +116,8 @@ std::shared_ptr ZarrSharedResource::OpenRootGroup() } const std::string osArrayName( CPLGetBasename(m_osRootDirectoryName.c_str())); - std::set oSetFilenamesInLoading; if (!poRG->LoadArray(osArrayName, osZarrayFilename, oRoot, false, - CPLJSONObject(), oSetFilenamesInLoading)) + CPLJSONObject())) return nullptr; return poRG; @@ -184,9 +183,7 @@ std::shared_ptr ZarrSharedResource::OpenRootGroup() const std::string osArrayName( CPLGetBasename(m_osRootDirectoryName.c_str())); poRG_V3->SetExplored(); - std::set oSetFilenamesInLoading; - if (!poRG_V3->LoadArray(osArrayName, osZarrJsonFilename, oRoot, - oSetFilenamesInLoading)) + if (!poRG_V3->LoadArray(osArrayName, osZarrJsonFilename, oRoot)) return nullptr; return poRG_V3; @@ -334,3 +331,37 @@ void ZarrSharedResource::UpdateDimensionSize( } poRG.reset(); } + +/************************************************************************/ +/* ZarrSharedResource::AddArrayInLoading() */ +/************************************************************************/ + +bool ZarrSharedResource::AddArrayInLoading(const std::string &osZarrayFilename) +{ + // Prevent too deep or recursive array loading + if (m_oSetArrayInLoading.find(osZarrayFilename) != + m_oSetArrayInLoading.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Attempt at recursively loading %s", osZarrayFilename.c_str()); + return false; + } + if (m_oSetArrayInLoading.size() == 32) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Too deep call stack in LoadArray()"); + return false; + } + m_oSetArrayInLoading.insert(osZarrayFilename); + return true; +} + +/************************************************************************/ +/* ZarrSharedResource::RemoveArrayInLoading() */ +/************************************************************************/ + +void ZarrSharedResource::RemoveArrayInLoading( + const std::string &osZarrayFilename) +{ + m_oSetArrayInLoading.erase(osZarrayFilename); +} diff --git a/frmts/zarr/zarr_v2_array.cpp b/frmts/zarr/zarr_v2_array.cpp index 7617474a4b48..ada168366aaa 100644 --- a/frmts/zarr/zarr_v2_array.cpp +++ b/frmts/zarr/zarr_v2_array.cpp @@ -1281,45 +1281,14 @@ std::shared_ptr ZarrV2Group::LoadArray(const std::string &osArrayName, const std::string &osZarrayFilename, const CPLJSONObject &oRoot, bool bLoadedFromZMetadata, - const CPLJSONObject &oAttributesIn, - std::set &oSetFilenamesInLoading) const + const CPLJSONObject &oAttributesIn) const { - // Prevent too deep or recursive array loading - if (oSetFilenamesInLoading.find(osZarrayFilename) != - oSetFilenamesInLoading.end()) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Attempt at recursively loading %s", osZarrayFilename.c_str()); - return nullptr; - } - if (oSetFilenamesInLoading.size() == 32) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Too deep call stack in LoadArray()"); - return nullptr; - } - - struct SetFilenameAdder - { - std::set &m_oSetFilenames; - std::string m_osFilename; - - SetFilenameAdder(std::set &oSetFilenamesIn, - const std::string &osFilename) - : m_oSetFilenames(oSetFilenamesIn), m_osFilename(osFilename) - { - m_oSetFilenames.insert(osFilename); - } - - ~SetFilenameAdder() - { - m_oSetFilenames.erase(m_osFilename); - } - }; - - // Add osZarrayFilename to oSetFilenamesInLoading during the scope + // Add osZarrayFilename to m_poSharedResource during the scope // of this function call. - SetFilenameAdder filenameAdder(oSetFilenamesInLoading, osZarrayFilename); + ZarrSharedResource::SetFilenameAdder filenameAdder(m_poSharedResource, + osZarrayFilename); + if (!filenameAdder.ok()) + return nullptr; const auto osFormat = oRoot["zarr_format"].ToString(); if (osFormat != "2") @@ -1412,9 +1381,9 @@ ZarrV2Group::LoadArray(const std::string &osArrayName, const auto arrayDimensionsObj = oAttributes["_ARRAY_DIMENSIONS"]; const auto FindDimension = - [this, &aoDims, bLoadedFromZMetadata, &osArrayName, &oAttributes, - &oSetFilenamesInLoading](const std::string &osDimName, - std::shared_ptr &poDim, int i) + [this, &aoDims, bLoadedFromZMetadata, &osArrayName, + &oAttributes](const std::string &osDimName, + std::shared_ptr &poDim, int i) { auto oIter = m_oMapDimensions.find(osDimName); if (oIter != m_oMapDimensions.end()) @@ -1476,8 +1445,7 @@ ZarrV2Group::LoadArray(const std::string &osArrayName, if (oDoc.Load(osArrayFilenameDim)) { LoadArray(osDimName, osArrayFilenameDim, oDoc.GetRoot(), - false, CPLJSONObject(), - oSetFilenamesInLoading); + false, CPLJSONObject()); } } else @@ -1572,70 +1540,50 @@ ZarrV2Group::LoadArray(const std::string &osArrayName, if (arrayDims[i].GetType() == CPLJSONObject::Type::String) { const auto osDimFullpath = arrayDims[i].ToString(); - auto poDim = poRG->OpenDimensionFromFullname(osDimFullpath); - if (poDim == nullptr) + const std::string osArrayFullname = + (GetFullName() != "/" ? GetFullName() : std::string()) + + '/' + osArrayName; + if (aoDims.size() == 1 && + (osDimFullpath == osArrayFullname || + osDimFullpath == "/" + osArrayFullname)) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find NCZarr dimension %s", - osDimFullpath.c_str()); + // If this is an indexing variable, then fetch the + // dimension type and direction, and patch the dimension + std::string osType; + std::string osDirection; + ZarrArray::GetDimensionTypeDirection( + oAttributes, osType, osDirection); + + auto poDimLocal = std::make_shared( + m_poSharedResource, + std::dynamic_pointer_cast( + m_pSelf.lock()), + GetFullName(), osArrayName, osType, osDirection, + aoDims[i]->GetSize()); + aoDims[i] = poDimLocal; + + m_oMapDimensions[osArrayName] = std::move(poDimLocal); } - else if (poDim->GetSize() != aoDims[i]->GetSize()) + else if (auto poDim = + poRG->OpenDimensionFromFullname(osDimFullpath)) { - CPLError(CE_Failure, CPLE_AppDefined, - "Inconsistency in size between NCZarr " - "dimension %s and regular dimension", - osDimFullpath.c_str()); + if (poDim->GetSize() != aoDims[i]->GetSize()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Inconsistency in size between NCZarr " + "dimension %s and regular dimension", + osDimFullpath.c_str()); + } + else + { + aoDims[i] = std::move(poDim); + } } else { - aoDims[i] = poDim; - - // If this is an indexing variable, then fetch the - // dimension type and direction, and patch the dimension - const std::string osArrayFullname = - (GetFullName() != "/" ? GetFullName() - : std::string()) + - '/' + osArrayName; - if (aoDims.size() == 1 && - osArrayFullname == poDim->GetFullName()) - { - std::string osType; - std::string osDirection; - ZarrArray::GetDimensionTypeDirection( - oAttributes, osType, osDirection); - - std::string osDimParent = osDimFullpath; - const auto nPos = osDimParent.rfind('/'); - if (nPos != std::string::npos) - { - if (nPos == 0) - osDimParent = '/'; - else - osDimParent.resize(nPos); - auto poDimParentGroup = - dynamic_cast( - poRG->OpenGroupFromFullname(osDimParent) - .get()); - if (poDimParentGroup) - { - auto poDimLocal = - std::make_shared( - m_poSharedResource, - std::dynamic_pointer_cast< - ZarrGroupBase>( - poDimParentGroup->m_pSelf - .lock()), - poDimParentGroup->GetFullName(), - poDim->GetName(), osType, - osDirection, poDim->GetSize()); - aoDims[i] = poDimLocal; - - poDimParentGroup - ->m_oMapDimensions[poDim->GetName()] = - std::move(poDimLocal); - } - } - } + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find NCZarr dimension %s", + osDimFullpath.c_str()); } } } @@ -1973,8 +1921,7 @@ ZarrV2Group::LoadArray(const std::string &osArrayName, if (oDoc.Load(osArrayFilenameDim)) { LoadArray(gridMappingName, osArrayFilenameDim, - oDoc.GetRoot(), false, CPLJSONObject(), - oSetFilenamesInLoading); + oDoc.GetRoot(), false, CPLJSONObject()); } } } diff --git a/frmts/zarr/zarr_v2_group.cpp b/frmts/zarr/zarr_v2_group.cpp index e3e8d7fe0c94..4f1d208f4c25 100644 --- a/frmts/zarr/zarr_v2_group.cpp +++ b/frmts/zarr/zarr_v2_group.cpp @@ -151,9 +151,8 @@ std::shared_ptr ZarrV2Group::OpenZarrArray(const std::string &osName, if (!oDoc.Load(osZarrayFilename)) return nullptr; const auto oRoot = oDoc.GetRoot(); - std::set oSetFilenamesInLoading; return LoadArray(osName, osZarrayFilename, oRoot, false, - CPLJSONObject(), oSetFilenamesInLoading); + CPLJSONObject()); } } @@ -332,9 +331,8 @@ void ZarrV2Group::InitFromZMetadata(const CPLJSONObject &obj) CPLFormFilename(poBelongingGroup->m_osDirectoryName.c_str(), osArrayName.c_str(), nullptr), ".zarray", nullptr); - std::set oSetFilenamesInLoading; poBelongingGroup->LoadArray(osArrayName, osZarrayFilename, oArray, true, - oAttributes, oSetFilenamesInLoading); + oAttributes); }; struct ArrayDesc diff --git a/frmts/zarr/zarr_v3_array.cpp b/frmts/zarr/zarr_v3_array.cpp index fd63f629ec27..0058a538c1d3 100644 --- a/frmts/zarr/zarr_v3_array.cpp +++ b/frmts/zarr/zarr_v3_array.cpp @@ -1071,45 +1071,14 @@ static T ParseNoDataComponent(const CPLJSONObject &oObj, bool &bOK) std::shared_ptr ZarrV3Group::LoadArray(const std::string &osArrayName, const std::string &osZarrayFilename, - const CPLJSONObject &oRoot, - std::set &oSetFilenamesInLoading) const + const CPLJSONObject &oRoot) const { - // Prevent too deep or recursive array loading - if (oSetFilenamesInLoading.find(osZarrayFilename) != - oSetFilenamesInLoading.end()) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Attempt at recursively loading %s", osZarrayFilename.c_str()); - return nullptr; - } - if (oSetFilenamesInLoading.size() == 32) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Too deep call stack in LoadArray()"); - return nullptr; - } - - struct SetFilenameAdder - { - std::set &m_oSetFilenames; - std::string m_osFilename; - - SetFilenameAdder(std::set &oSetFilenamesIn, - const std::string &osFilename) - : m_oSetFilenames(oSetFilenamesIn), m_osFilename(osFilename) - { - m_oSetFilenames.insert(osFilename); - } - - ~SetFilenameAdder() - { - m_oSetFilenames.erase(m_osFilename); - } - }; - - // Add osZarrayFilename to oSetFilenamesInLoading during the scope + // Add osZarrayFilename to m_poSharedResource during the scope // of this function call. - SetFilenameAdder filenameAdder(oSetFilenamesInLoading, osZarrayFilename); + ZarrSharedResource::SetFilenameAdder filenameAdder(m_poSharedResource, + osZarrayFilename); + if (!filenameAdder.ok()) + return nullptr; // Warn about unknown members (the spec suggests to error out, but let be // a bit more lenient) @@ -1252,10 +1221,9 @@ ZarrV3Group::LoadArray(const std::string &osArrayName, // Deal with dimension_names const auto dimensionNames = oRoot["dimension_names"]; - const auto FindDimension = - [this, &aoDims, &osArrayName, &oAttributes, - &oSetFilenamesInLoading](const std::string &osDimName, - std::shared_ptr &poDim, int i) + const auto FindDimension = [this, &aoDims, &osArrayName, &oAttributes]( + const std::string &osDimName, + std::shared_ptr &poDim, int i) { auto oIter = m_oMapDimensions.find(osDimName); if (oIter != m_oMapDimensions.end()) @@ -1295,8 +1263,8 @@ ZarrV3Group::LoadArray(const std::string &osArrayName, CPLJSONDocument oDoc; if (oDoc.Load(osArrayFilenameDim)) { - LoadArray(osDimName, osArrayFilenameDim, oDoc.GetRoot(), - oSetFilenamesInLoading); + LoadArray(osDimName, osArrayFilenameDim, + oDoc.GetRoot()); } } else diff --git a/frmts/zarr/zarr_v3_group.cpp b/frmts/zarr/zarr_v3_group.cpp index ae00443685f2..bfa88cf1a8ea 100644 --- a/frmts/zarr/zarr_v3_group.cpp +++ b/frmts/zarr/zarr_v3_group.cpp @@ -75,9 +75,7 @@ std::shared_ptr ZarrV3Group::OpenZarrArray(const std::string &osName, if (!oDoc.Load(osZarrayFilename)) return nullptr; const auto oRoot = oDoc.GetRoot(); - std::set oSetFilenamesInLoading; - return LoadArray(osName, osZarrayFilename, oRoot, - oSetFilenamesInLoading); + return LoadArray(osName, osZarrayFilename, oRoot); } return nullptr; diff --git a/gcore/gdal_misc.cpp b/gcore/gdal_misc.cpp index bb9bb7a7cc50..b219a9437154 100644 --- a/gcore/gdal_misc.cpp +++ b/gcore/gdal_misc.cpp @@ -4125,7 +4125,7 @@ void GDALSerializeGCPListToXML(CPLXMLNode *psParentNode, GDAL_GCP *pasGCPList, /* GDALDeserializeGCPListFromXML() */ /************************************************************************/ -void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, +void GDALDeserializeGCPListFromXML(const CPLXMLNode *psGCPList, GDAL_GCP **ppasGCPList, int *pnGCPCount, OGRSpatialReference **ppoGCP_SRS) { @@ -4168,7 +4168,7 @@ void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, // Count GCPs. int nGCPMax = 0; - for (CPLXMLNode *psXMLGCP = psGCPList->psChild; psXMLGCP != nullptr; + for (const CPLXMLNode *psXMLGCP = psGCPList->psChild; psXMLGCP != nullptr; psXMLGCP = psXMLGCP->psNext) { @@ -4185,7 +4185,7 @@ void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, if (nGCPMax == 0) return; - for (CPLXMLNode *psXMLGCP = psGCPList->psChild; + for (const CPLXMLNode *psXMLGCP = psGCPList->psChild; *ppasGCPList != nullptr && psXMLGCP != nullptr; psXMLGCP = psXMLGCP->psNext) { @@ -4224,14 +4224,15 @@ void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, return true; }; + bool bOK = true; if (!ParseDoubleValue("Pixel", psGCP->dfGCPPixel)) - continue; + bOK = false; if (!ParseDoubleValue("Line", psGCP->dfGCPLine)) - continue; + bOK = false; if (!ParseDoubleValue("X", psGCP->dfGCPX)) - continue; + bOK = false; if (!ParseDoubleValue("Y", psGCP->dfGCPY)) - continue; + bOK = false; const char *pszZ = CPLGetXMLValue(psXMLGCP, "Z", nullptr); if (pszZ == nullptr) { @@ -4245,10 +4246,17 @@ void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, { CPLError(CE_Failure, CPLE_AppDefined, "GCP#Z=%s is an invalid value", pszZ); - continue; + bOK = false; } - (*pnGCPCount)++; + if (!bOK) + { + GDALDeinitGCPs(1, psGCP); + } + else + { + (*pnGCPCount)++; + } } } diff --git a/gcore/gdal_pam.h b/gcore/gdal_pam.h index b40ae21826a7..73ae1e25a939 100644 --- a/gcore/gdal_pam.h +++ b/gcore/gdal_pam.h @@ -129,7 +129,7 @@ class CPL_DLL GDALPamDataset : public GDALDataset GDALDatasetPamInfo *psPam = nullptr; virtual CPLXMLNode *SerializeToXML(const char *); - virtual CPLErr XMLInit(CPLXMLNode *, const char *); + virtual CPLErr XMLInit(const CPLXMLNode *, const char *); virtual CPLErr TryLoadXML(char **papszSiblingFiles = nullptr); virtual CPLErr TrySaveXML(); @@ -278,7 +278,7 @@ class CPL_DLL GDALPamRasterBand : public GDALRasterBand protected: //! @cond Doxygen_Suppress virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath); - virtual CPLErr XMLInit(CPLXMLNode *, const char *); + virtual CPLErr XMLInit(const CPLXMLNode *, const char *); void PamInitialize(); void PamClear(); diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index 60f0be90536b..0368de11c838 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -4056,7 +4056,7 @@ double GDALAdjustNoDataCloseToFloatMax(double dfVal); void GDALSerializeGCPListToXML(CPLXMLNode *psParentNode, GDAL_GCP *pasGCPList, int nGCPCount, const OGRSpatialReference *poGCP_SRS); -void GDALDeserializeGCPListFromXML(CPLXMLNode *psGCPList, +void GDALDeserializeGCPListFromXML(const CPLXMLNode *psGCPList, GDAL_GCP **ppasGCPList, int *pnGCPCount, OGRSpatialReference **ppoGCP_SRS); diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index 63e7d44d46ec..38503ec8d697 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -1094,7 +1094,7 @@ int CPL_STDCALL GDALGetRasterCount(GDALDatasetH hDS) * When a projection definition is not available an empty (but not NULL) * string is returned. * - * \note Startig with GDAL 3.0, this is a compatibility layer around + * \note Starting with GDAL 3.0, this is a compatibility layer around * GetSpatialRef() * * @return a pointer to an internal projection reference string. It should diff --git a/gcore/gdaljp2metadata.cpp b/gcore/gdaljp2metadata.cpp index a92e4dcc94a9..e15e8aee6599 100644 --- a/gcore/gdaljp2metadata.cpp +++ b/gcore/gdaljp2metadata.cpp @@ -1252,6 +1252,32 @@ GDALJP2Box *GDALJP2Metadata::CreateJP2GeoTIFF() return poBox; } +/************************************************************************/ +/* IsSRSCompatible() */ +/************************************************************************/ + +/* Returns true if the SRS can be references through a EPSG code, or encoded + * as a GML SRS + */ +bool GDALJP2Metadata::IsSRSCompatible(const OGRSpatialReference *poSRS) +{ + const char *pszAuthName = poSRS->GetAuthorityName(nullptr); + const char *pszAuthCode = poSRS->GetAuthorityCode(nullptr); + + if (pszAuthName && pszAuthCode && EQUAL(pszAuthName, "epsg")) + { + if (atoi(pszAuthCode)) + return true; + } + + CPLErrorHandlerPusher oErrorHandler(CPLQuietErrorHandler); + CPLErrorStateBackuper oErrorStateBackuper; + char *pszGMLDef = nullptr; + const bool bRet = (poSRS->exportToXML(&pszGMLDef, nullptr) == OGRERR_NONE); + CPLFree(pszGMLDef); + return bRet; +} + /************************************************************************/ /* GetGMLJP2GeoreferencingInfo() */ /************************************************************************/ @@ -1269,43 +1295,28 @@ void GDALJP2Metadata::GetGMLJP2GeoreferencingInfo( bNeedAxisFlip = false; OGRSpatialReference oSRS(m_oSRS); - if (oSRS.IsProjected()) - { - const char *pszAuthName = oSRS.GetAuthorityName("PROJCS"); + const char *pszAuthName = oSRS.GetAuthorityName(nullptr); + const char *pszAuthCode = oSRS.GetAuthorityCode(nullptr); - if (pszAuthName != nullptr && EQUAL(pszAuthName, "epsg")) - { - nEPSGCode = atoi(oSRS.GetAuthorityCode("PROJCS")); - } - } - else if (oSRS.IsGeographic()) + if (pszAuthName && pszAuthCode && EQUAL(pszAuthName, "epsg")) { - const char *pszAuthName = oSRS.GetAuthorityName("GEOGCS"); - - if (pszAuthName != nullptr && EQUAL(pszAuthName, "epsg")) - { - nEPSGCode = atoi(oSRS.GetAuthorityCode("GEOGCS")); - } + nEPSGCode = atoi(pszAuthCode); } - // Save error state as importFromEPSGA() will call CPLReset() - CPLErrorNum errNo = CPLGetLastErrorNo(); - CPLErr eErr = CPLGetLastErrorType(); - CPLString osLastErrorMsg = CPLGetLastErrorMsg(); - - // Determine if we need to flip axis. Reimport from EPSG and make - // sure not to strip axis definitions to determine the axis order. - if (nEPSGCode != 0 && oSRS.importFromEPSGA(nEPSGCode) == OGRERR_NONE) { - if (oSRS.EPSGTreatsAsLatLong() || oSRS.EPSGTreatsAsNorthingEasting()) + CPLErrorStateBackuper oErrorStateBackuper; + // Determine if we need to flip axis. Reimport from EPSG and make + // sure not to strip axis definitions to determine the axis order. + if (nEPSGCode != 0 && oSRS.importFromEPSG(nEPSGCode) == OGRERR_NONE) { - bNeedAxisFlip = true; + if (oSRS.EPSGTreatsAsLatLong() || + oSRS.EPSGTreatsAsNorthingEasting()) + { + bNeedAxisFlip = true; + } } } - // Restore error state - CPLErrorSetState(eErr, errNo, osLastErrorMsg); - /* -------------------------------------------------------------------- */ /* Prepare coverage origin and offset vectors. Take axis */ /* order into account if needed. */ @@ -1369,6 +1380,7 @@ void GDALJP2Metadata::GetGMLJP2GeoreferencingInfo( { char *pszGMLDef = nullptr; + CPLErrorStateBackuper oErrorStateBackuper; if (oSRS.exportToXML(&pszGMLDef, nullptr) == OGRERR_NONE) { char *pszWKT = nullptr; diff --git a/gcore/gdaljp2metadata.h b/gcore/gdaljp2metadata.h index fdc3044f5aff..b3646a7d0fed 100644 --- a/gcore/gdaljp2metadata.h +++ b/gcore/gdaljp2metadata.h @@ -235,6 +235,8 @@ class CPL_DLL GDALJP2Metadata static GDALJP2Box *CreateIPRBox(GDALDataset *poSrcDS); static int IsUUID_MSI(const GByte *abyUUID); static int IsUUID_XMP(const GByte *abyUUID); + + static bool IsSRSCompatible(const OGRSpatialReference *poSRS); }; CPLXMLNode *GDALGetJPEG2000Structure(const char *pszFilename, VSILFILE *fp, diff --git a/gcore/gdalpamdataset.cpp b/gcore/gdalpamdataset.cpp index e672bef224fc..1f0c326762e1 100644 --- a/gcore/gdalpamdataset.cpp +++ b/gcore/gdalpamdataset.cpp @@ -420,14 +420,13 @@ void GDALPamDataset::PamClear() /* XMLInit() */ /************************************************************************/ -CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) +CPLErr GDALPamDataset::XMLInit(const CPLXMLNode *psTree, const char *pszUnused) { /* -------------------------------------------------------------------- */ /* Check for an SRS node. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psSRSNode = CPLGetXMLNode(psTree, "SRS"); - if (psSRSNode) + if (const CPLXMLNode *psSRSNode = CPLGetXMLNode(psTree, "SRS")) { if (psPam->poSRS) psPam->poSRS->Release(); @@ -463,12 +462,12 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) /* -------------------------------------------------------------------- */ /* Check for a GeoTransform node. */ /* -------------------------------------------------------------------- */ - if (strlen(CPLGetXMLValue(psTree, "GeoTransform", "")) > 0) + const char *pszGT = CPLGetXMLValue(psTree, "GeoTransform", ""); + if (strlen(pszGT) > 0) { - const char *pszGT = CPLGetXMLValue(psTree, "GeoTransform", ""); - - char **papszTokens = CSLTokenizeStringComplex(pszGT, ",", FALSE, FALSE); - if (CSLCount(papszTokens) != 6) + const CPLStringList aosTokens( + CSLTokenizeStringComplex(pszGT, ",", FALSE, FALSE)); + if (aosTokens.size() != 6) { CPLError(CE_Warning, CPLE_AppDefined, "GeoTransform node does not have expected six values."); @@ -476,19 +475,15 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) else { for (int iTA = 0; iTA < 6; iTA++) - psPam->adfGeoTransform[iTA] = CPLAtof(papszTokens[iTA]); + psPam->adfGeoTransform[iTA] = CPLAtof(aosTokens[iTA]); psPam->bHaveGeoTransform = TRUE; } - - CSLDestroy(papszTokens); } /* -------------------------------------------------------------------- */ /* Check for GCPs. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psGCPList = CPLGetXMLNode(psTree, "GCPList"); - - if (psGCPList != nullptr) + if (const CPLXMLNode *psGCPList = CPLGetXMLNode(psTree, "GCPList")) { if (psPam->poGCP_SRS) psPam->poGCP_SRS->Release(); @@ -527,8 +522,9 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) // over the root PAMDataset SRS node. // ArcGIS 9.3: GeodataXform as a root element - CPLXMLNode *psGeodataXform = CPLGetXMLNode(psTree, "=GeodataXform"); - CPLXMLNode *psValueAsXML = nullptr; + const CPLXMLNode *psGeodataXform = + CPLGetXMLNode(psTree, "=GeodataXform"); + CPLXMLTreeCloser oTreeValueAsXML(nullptr); if (psGeodataXform != nullptr) { char *apszMD[2]; @@ -543,10 +539,10 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) char **papszXML = oMDMD.GetMetadata("xml:ESRI"); if (CSLCount(papszXML) == 1) { - psValueAsXML = CPLParseXMLString(papszXML[0]); - if (psValueAsXML) + oTreeValueAsXML.reset(CPLParseXMLString(papszXML[0])); + if (oTreeValueAsXML) psGeodataXform = - CPLGetXMLNode(psValueAsXML, "=GeodataXform"); + CPLGetXMLNode(oTreeValueAsXML.get(), "=GeodataXform"); } } @@ -686,14 +682,12 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) } } } - if (psValueAsXML) - CPLDestroyXMLNode(psValueAsXML); } /* -------------------------------------------------------------------- */ /* Process bands. */ /* -------------------------------------------------------------------- */ - for (CPLXMLNode *psBandTree = psTree->psChild; psBandTree != nullptr; + for (const CPLXMLNode *psBandTree = psTree->psChild; psBandTree; psBandTree = psBandTree->psNext) { if (psBandTree->eType != CXT_Element || @@ -719,16 +713,16 @@ CPLErr GDALPamDataset::XMLInit(CPLXMLNode *psTree, const char *pszUnused) /* -------------------------------------------------------------------- */ /* Preserve Array information. */ /* -------------------------------------------------------------------- */ - for (CPLXMLNode *psIter = psTree->psChild; psIter; psIter = psIter->psNext) + for (const CPLXMLNode *psIter = psTree->psChild; psIter; + psIter = psIter->psNext) { if (psIter->eType == CXT_Element && strcmp(psIter->pszValue, "Array") == 0) { - CPLXMLNode *psNextBackup = psIter->psNext; - psIter->psNext = nullptr; + CPLXMLNode sArrayTmp = *psIter; + sArrayTmp.psNext = nullptr; psPam->m_apoOtherNodes.emplace_back( - CPLXMLTreeCloser(CPLCloneXMLTree(psIter))); - psIter->psNext = psNextBackup; + CPLXMLTreeCloser(CPLCloneXMLTree(&sArrayTmp))); } } diff --git a/gcore/gdalpamrasterband.cpp b/gcore/gdalpamrasterband.cpp index 880452399310..a75f1cc3df70 100644 --- a/gcore/gdalpamrasterband.cpp +++ b/gcore/gdalpamrasterband.cpp @@ -372,7 +372,7 @@ void GDALPamRasterBand::PamClear() /* XMLInit() */ /************************************************************************/ -CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, +CPLErr GDALPamRasterBand::XMLInit(const CPLXMLNode *psTree, const char * /* pszUnused */) { PamInitialize(); @@ -387,8 +387,8 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ GDALMajorObject::SetDescription(CPLGetXMLValue(psTree, "Description", "")); - const char *pszNoDataValue = CPLGetXMLValue(psTree, "NoDataValue", nullptr); - if (pszNoDataValue != nullptr) + if (const char *pszNoDataValue = + CPLGetXMLValue(psTree, "NoDataValue", nullptr)) { const char *pszLEHex = CPLGetXMLValue(psTree, "NoDataValue.le_hex_equiv", nullptr); @@ -436,12 +436,10 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, GDALPamRasterBand::SetScale(pszScale ? CPLAtof(pszScale) : 1.0); } - const char *pszUnitType = CPLGetXMLValue(psTree, "UnitType", nullptr); - if (pszUnitType) + if (const char *pszUnitType = CPLGetXMLValue(psTree, "UnitType", nullptr)) GDALPamRasterBand::SetUnitType(pszUnitType); - const char *pszInterp = CPLGetXMLValue(psTree, "ColorInterp", nullptr); - if (pszInterp) + if (const char *pszInterp = CPLGetXMLValue(psTree, "ColorInterp", nullptr)) { GDALPamRasterBand::SetColorInterpretation( GDALGetColorInterpretationByName(pszInterp)); @@ -450,12 +448,11 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ /* Category names. */ /* -------------------------------------------------------------------- */ - const auto psCategoryNames = CPLGetXMLNode(psTree, "CategoryNames"); - if (psCategoryNames) + if (const auto psCategoryNames = CPLGetXMLNode(psTree, "CategoryNames")) { CPLStringList oCategoryNames; - for (CPLXMLNode *psEntry = psCategoryNames->psChild; psEntry != nullptr; + for (const CPLXMLNode *psEntry = psCategoryNames->psChild; psEntry; psEntry = psEntry->psNext) { /* Don't skip tag with empty content */ @@ -475,13 +472,12 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ /* Collect a color table. */ /* -------------------------------------------------------------------- */ - const auto psColorTable = CPLGetXMLNode(psTree, "ColorTable"); - if (psColorTable) + if (const auto psColorTable = CPLGetXMLNode(psTree, "ColorTable")) { GDALColorTable oTable; int iEntry = 0; - for (CPLXMLNode *psEntry = psColorTable->psChild; psEntry != nullptr; + for (const CPLXMLNode *psEntry = psColorTable->psChild; psEntry; psEntry = psEntry->psNext) { if (!(psEntry->eType == CXT_Element && @@ -505,8 +501,7 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ /* Do we have a complete set of stats? */ /* -------------------------------------------------------------------- */ - const char *pszMinimum = CPLGetXMLValue(psTree, "Minimum", nullptr); - if (pszMinimum) + if (const char *pszMinimum = CPLGetXMLValue(psTree, "Minimum", nullptr)) { const char *pszMaximum = CPLGetXMLValue(psTree, "Maximum", nullptr); if (pszMaximum) @@ -517,8 +512,7 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, } } - const char *pszMean = CPLGetXMLValue(psTree, "Mean", nullptr); - if (pszMean) + if (const char *pszMean = CPLGetXMLValue(psTree, "Mean", nullptr)) { const char *pszStandardDeviation = CPLGetXMLValue(psTree, "StandardDeviation", nullptr); @@ -533,26 +527,23 @@ CPLErr GDALPamRasterBand::XMLInit(CPLXMLNode *psTree, /* -------------------------------------------------------------------- */ /* Histograms */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psHist = CPLGetXMLNode(psTree, "Histograms"); - if (psHist != nullptr) + if (const CPLXMLNode *psHist = CPLGetXMLNode(psTree, "Histograms")) { - CPLXMLNode *psNext = psHist->psNext; - psHist->psNext = nullptr; - + CPLXMLNode sHistTemp = *psHist; + sHistTemp.psNext = nullptr; if (psPam->psSavedHistograms != nullptr) { CPLDestroyXMLNode(psPam->psSavedHistograms); psPam->psSavedHistograms = nullptr; } - psPam->psSavedHistograms = CPLCloneXMLTree(psHist); - psHist->psNext = psNext; + psPam->psSavedHistograms = CPLCloneXMLTree(&sHistTemp); } /* -------------------------------------------------------------------- */ /* Raster Attribute Table */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psRAT = CPLGetXMLNode(psTree, "GDALRasterAttributeTable"); - if (psRAT != nullptr) + if (const CPLXMLNode *psRAT = + CPLGetXMLNode(psTree, "GDALRasterAttributeTable")) { delete psPam->poDefaultRAT; auto poNewRAT = new GDALDefaultRasterAttributeTable(); diff --git a/gcore/gdalrasterband.cpp b/gcore/gdalrasterband.cpp index 67bc08496dd0..685b24c5659b 100644 --- a/gcore/gdalrasterband.cpp +++ b/gcore/gdalrasterband.cpp @@ -796,7 +796,7 @@ CPLErr CPL_STDCALL GDALWriteBlock(GDALRasterBandH hBand, int nXOff, int nYOff, * block and so forth. * * @param nYBlockOff the vertical block offset, with zero indicating - * the left most block, 1 the next block and so forth. + * the top most block, 1 the next block and so forth. * * @param pnXValid pointer to an integer in which the number of valid pixels in * the x direction will be stored @@ -804,7 +804,7 @@ CPLErr CPL_STDCALL GDALWriteBlock(GDALRasterBandH hBand, int nXOff, int nYOff, * @param pnYValid pointer to an integer in which the number of valid pixels in * the y direction will be stored * - * @return CE_None if the input parameter are valid, CE_Failure otherwise + * @return CE_None if the input parameters are valid, CE_Failure otherwise * * @since GDAL 2.2 */ diff --git a/gcore/gdalrasterblock.cpp b/gcore/gdalrasterblock.cpp index 48c19b0aec5a..93eea22399ee 100644 --- a/gcore/gdalrasterblock.cpp +++ b/gcore/gdalrasterblock.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include "cpl_atomic_ops.h" #include "cpl_conv.h" @@ -43,7 +44,6 @@ #include "cpl_string.h" #include "cpl_vsi.h" -static bool bCacheMaxInitialized = false; // Will later be overridden by the default 5% if GDAL_CACHEMAX not defined. static GIntBig nCacheMax = 40 * 1024 * 1024; static GIntBig nCacheUsed = 0; @@ -153,10 +153,8 @@ void CPL_STDCALL GDALSetCacheMax64(GIntBig nNewSizeInBytes) } #endif - { - INITIALIZE_LOCK; - } - bCacheMaxInitialized = true; + // To force one-time initialization of nCacheMax if not already done + GDALGetCacheMax64(); nCacheMax = nNewSizeInBytes; /* -------------------------------------------------------------------- */ @@ -237,73 +235,77 @@ int CPL_STDCALL GDALGetCacheMax() GIntBig CPL_STDCALL GDALGetCacheMax64() { - if (!bCacheMaxInitialized) - { + static std::once_flag flagSetupGDALGetCacheMax64; + std::call_once( + flagSetupGDALGetCacheMax64, + []() { - INITIALIZE_LOCK; - } - bSleepsForBockCacheDebug = - CPLTestBool(CPLGetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "NO")); + { + INITIALIZE_LOCK; + } + bSleepsForBockCacheDebug = + CPLTestBool(CPLGetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "NO")); - const char *pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX", "5%"); + const char *pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX", "5%"); - GIntBig nNewCacheMax; - if (strchr(pszCacheMax, '%') != nullptr) - { - GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM(); - if (nUsablePhysicalRAM > 0) + GIntBig nNewCacheMax; + if (strchr(pszCacheMax, '%') != nullptr) { - // For some reason, coverity pretends that this will overflow. - // "Multiply operation overflows on operands - // static_cast( nUsablePhysicalRAM ) and - // CPLAtof(pszCacheMax). Example values for operands: CPLAtof( - // pszCacheMax ) = 2251799813685248, - // static_cast(nUsablePhysicalRAM) = - // -9223372036854775808." coverity[overflow,tainted_data] - double dfCacheMax = static_cast(nUsablePhysicalRAM) * - CPLAtof(pszCacheMax) / 100.0; - if (dfCacheMax >= 0 && dfCacheMax < 1e15) - nNewCacheMax = static_cast(dfCacheMax); + GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM(); + if (nUsablePhysicalRAM > 0) + { + // For some reason, coverity pretends that this will overflow. + // "Multiply operation overflows on operands + // static_cast( nUsablePhysicalRAM ) and + // CPLAtof(pszCacheMax). Example values for operands: CPLAtof( + // pszCacheMax ) = 2251799813685248, + // static_cast(nUsablePhysicalRAM) = + // -9223372036854775808." coverity[overflow,tainted_data] + double dfCacheMax = + static_cast(nUsablePhysicalRAM) * + CPLAtof(pszCacheMax) / 100.0; + if (dfCacheMax >= 0 && dfCacheMax < 1e15) + nNewCacheMax = static_cast(dfCacheMax); + else + nNewCacheMax = nCacheMax; + } else + { + CPLDebug("GDAL", "Cannot determine usable physical RAM."); nNewCacheMax = nCacheMax; + } } else { - CPLDebug("GDAL", "Cannot determine usable physical RAM."); - nNewCacheMax = nCacheMax; - } - } - else - { - nNewCacheMax = CPLAtoGIntBig(pszCacheMax); - if (nNewCacheMax < 100000) - { - if (nNewCacheMax < 0) + nNewCacheMax = CPLAtoGIntBig(pszCacheMax); + if (nNewCacheMax < 100000) { - CPLError(CE_Failure, CPLE_NotSupported, - "Invalid value for GDAL_CACHEMAX. " - "Using default value."); - GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM(); - if (nUsablePhysicalRAM) - nNewCacheMax = nUsablePhysicalRAM / 20; + if (nNewCacheMax < 0) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Invalid value for GDAL_CACHEMAX. " + "Using default value."); + GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM(); + if (nUsablePhysicalRAM) + nNewCacheMax = nUsablePhysicalRAM / 20; + else + { + CPLDebug("GDAL", + "Cannot determine usable physical RAM."); + nNewCacheMax = nCacheMax; + } + } else { - CPLDebug("GDAL", - "Cannot determine usable physical RAM."); - nNewCacheMax = nCacheMax; + nNewCacheMax *= 1024 * 1024; } } - else - { - nNewCacheMax *= 1024 * 1024; - } } - } - nCacheMax = nNewCacheMax; - CPLDebug("GDAL", "GDAL_CACHEMAX = " CPL_FRMT_GIB " MB", - nCacheMax / (1024 * 1024)); - bCacheMaxInitialized = true; - } + nCacheMax = nNewCacheMax; + CPLDebug("GDAL", "GDAL_CACHEMAX = " CPL_FRMT_GIB " MB", + nCacheMax / (1024 * 1024)); + }); + // coverity[overflow_sink] return nCacheMax; } diff --git a/gcore/overview.cpp b/gcore/overview.cpp index 29f23f1f14d3..6eb7dd57b11a 100644 --- a/gcore/overview.cpp +++ b/gcore/overview.cpp @@ -5636,7 +5636,9 @@ CPLErr GDALRegenerateOverviewsMultiBand( /** Undocumented * @param hSrcBand undocumented. - * @param nSampleStep undocumented. + * @param nSampleStep Step between scanlines used to compute statistics. + * When nSampleStep is equal to 1, all scanlines will + * be processed. * @param pdfMean undocumented. * @param pdfStdDev undocumented. * @param pfnProgress undocumented. diff --git a/ogr/ogr_srs_cf1.cpp b/ogr/ogr_srs_cf1.cpp index a67bef2d52e2..8add4d7a6d3b 100644 --- a/ogr/ogr_srs_cf1.cpp +++ b/ogr/ogr_srs_cf1.cpp @@ -1440,8 +1440,7 @@ OGRSpatialReference::exportToCF1(char **ppszGridMappingName, { std::string key{}; std::string valueStr{}; - size_t doubleCount = 0; - double doubles[2] = {0, 0}; + std::vector doubles{}; }; std::vector oParams; @@ -1452,16 +1451,15 @@ OGRSpatialReference::exportToCF1(char **ppszGridMappingName, Value v; v.key = key; v.valueStr = value; - oParams.push_back(v); + oParams.emplace_back(std::move(v)); }; const auto addParamDouble = [&oParams](const char *key, double value) { Value v; v.key = key; - v.doubleCount = 1; - v.doubles[0] = value; - oParams.push_back(v); + v.doubles.push_back(value); + oParams.emplace_back(std::move(v)); }; const auto addParam2Double = @@ -1469,10 +1467,9 @@ OGRSpatialReference::exportToCF1(char **ppszGridMappingName, { Value v; v.key = key; - v.doubleCount = 2; - v.doubles[0] = value1; - v.doubles[1] = value2; - oParams.push_back(v); + v.doubles.push_back(value1); + v.doubles.push_back(value2); + oParams.emplace_back(std::move(v)); }; std::string osCFProjection; @@ -1694,11 +1691,11 @@ OGRSpatialReference::exportToCF1(char **ppszGridMappingName, else { std::string osVal; - for (size_t i = 0; i < param.doubleCount; ++i) + for (const double dfVal : param.doubles) { - if (i > 0) + if (!osVal.empty()) osVal += ','; - osVal += CPLSPrintf("%.18g", param.doubles[i]); + osVal += CPLSPrintf("%.18g", dfVal); } aosKeyValues.AddNameValue(param.key.c_str(), osVal.c_str()); } diff --git a/ogr/ogr_srs_xml.cpp b/ogr/ogr_srs_xml.cpp index d2732338c11d..aecc723d8e24 100644 --- a/ogr/ogr_srs_xml.cpp +++ b/ogr/ogr_srs_xml.cpp @@ -633,8 +633,10 @@ static CPLXMLNode *exportProjCSToXML(const OGRSpatialReference *poSRS) } else { - CPLError(CE_Warning, CPLE_NotSupported, + CPLError(CE_Failure, CPLE_NotSupported, "Unhandled projection method %s", pszProjection); + CPLDestroyXMLNode(psCRS_XML); + return nullptr; } /* -------------------------------------------------------------------- */ @@ -693,6 +695,9 @@ OGRErr OGRSpatialReference::exportToXML(char **ppszRawXML, else return OGRERR_UNSUPPORTED_SRS; + if (!psXMLTree) + return OGRERR_FAILURE; + *ppszRawXML = CPLSerializeXMLTree(psXMLTree); CPLDestroyXMLNode(psXMLTree); diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp index 68b828f95a5b..72598b04e613 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp @@ -252,7 +252,8 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon() break; } } - fields.emplace_back(arrow::field(poFieldDefn->GetNameRef(), dt, + fields.emplace_back(arrow::field(poFieldDefn->GetNameRef(), + std::move(dt), poFieldDefn->IsNullable())); if (poFieldDefn->GetAlternativeNameRef()[0]) bNeedGDALSchema = true; @@ -325,8 +326,9 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon() break; } - std::shared_ptr field(arrow::field( - poGeomFieldDefn->GetNameRef(), dt, poGeomFieldDefn->IsNullable())); + std::shared_ptr field( + arrow::field(poGeomFieldDefn->GetNameRef(), std::move(dt), + poGeomFieldDefn->IsNullable())); if (m_bWriteFieldArrowExtensionName) { auto kvMetadata = field->metadata() @@ -338,7 +340,7 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon() field = field->WithMetadata(kvMetadata); } - fields.emplace_back(field); + fields.emplace_back(std::move(field)); } m_aoEnvelopes.resize(m_poFeatureDefn->GetGeomFieldCount()); diff --git a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp index d83f2219f318..606da4719cec 100644 --- a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp +++ b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp @@ -248,11 +248,15 @@ void OGRCSVLayer::BuildFeatureDefn(const char *pszNfdcGeomField, } // Tokenize without quotes to get the actual values. + VSIRewindL(fpCSV); CSLDestroy(papszTokens); - int l_nFlags = CSLT_HONOURSTRINGS; - if (!bMergeDelimiter) - l_nFlags |= CSLT_ALLOWEMPTYTOKENS; - papszTokens = CSLTokenizeString2(pszLine, szDelimiter, l_nFlags); + papszTokens = + CSVReadParseLine3L(fpCSV, m_nMaxLineSize, szDelimiter, + true, // bHonourStrings + false, // bKeepLeadingAndClosingQuotes + bMergeDelimiter, + true // bSkipBOM + ); nFieldCount = CSLCount(papszTokens); } } diff --git a/ogr/ogrsf_frmts/flatgeobuf/CMakeLists.txt b/ogr/ogrsf_frmts/flatgeobuf/CMakeLists.txt index ec4ec7a2d7f9..7fbbf2050227 100644 --- a/ogr/ogrsf_frmts/flatgeobuf/CMakeLists.txt +++ b/ogr/ogrsf_frmts/flatgeobuf/CMakeLists.txt @@ -11,3 +11,5 @@ add_gdal_driver( gdal_standard_includes(ogr_FlatGeobuf) target_include_directories(ogr_FlatGeobuf PRIVATE $ $) +# Quick and dirty way of modifying the default flatbuffers namespace to gdal_flatbuffers +target_compile_definitions(ogr_FlatGeobuf PRIVATE -Dflatbuffers=gdal_flatbuffers) diff --git a/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobufdataset.cpp b/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobufdataset.cpp index 39261755d63f..75a9afa40e2d 100644 --- a/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobufdataset.cpp +++ b/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobufdataset.cpp @@ -32,6 +32,12 @@ #include "header_generated.h" +// For users not using CMake... +#ifndef flatbuffers +#error \ + "Make sure to build with -Dflatbuffers=gdal_flatbuffers (for example) to avoid potential conflict of flatbuffers" +#endif + using namespace flatbuffers; using namespace FlatGeobuf; diff --git a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h index 2456e58a7fd3..d93290259a84 100644 --- a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h +++ b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h @@ -280,7 +280,7 @@ class GDALGeoPackageDataset final : public OGRSQLiteBaseDataSource, void FixupWrongRTreeTrigger(); void FixupWrongMedataReferenceColumnNameUpdate(); void ClearCachedRelationships(); - void LoadRelationships() const; + void LoadRelationships() const override; void LoadRelationshipsUsingRelatedTablesExtension() const; static std::string GenerateNameForRelationship(const char *pszBaseTableName, @@ -345,12 +345,6 @@ class GDALGeoPackageDataset final : public OGRSQLiteBaseDataSource, bool AddFieldDomain(std::unique_ptr &&domain, std::string &failureReason) override; - std::vector - GetRelationshipNames(CSLConstList papszOptions = nullptr) const override; - - const GDALRelationship * - GetRelationship(const std::string &name) const override; - bool AddRelationship(std::unique_ptr &&relationship, std::string &failureReason) override; diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp index 0763596bc6be..484ea32f100f 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp @@ -1567,6 +1567,7 @@ int GDALGeoPackageDataset::Open(GDALOpenInfo *poOpenInfo, CheckUnknownExtensions(); int bRet = FALSE; + bool bHasGPKGExtRelations = false; if (poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) { m_bHasGPKGGeometryColumns = @@ -1575,6 +1576,7 @@ int GDALGeoPackageDataset::Open(GDALOpenInfo *poOpenInfo, "name = 'gpkg_geometry_columns' AND " "type IN ('table', 'view')", nullptr) == 1; + bHasGPKGExtRelations = HasGpkgextRelationsTable(); } if (m_bHasGPKGGeometryColumns) { @@ -1614,6 +1616,18 @@ int GDALGeoPackageDataset::Open(GDALOpenInfo *poOpenInfo, bHasASpatialOrAttributes = (oResultTable && oResultTable->RowCount() == 1); } + if (bHasGPKGExtRelations) + { + osSQL += "UNION ALL " + "SELECT mapping_table_name, mapping_table_name, 0 as " + "is_spatial, NULL, NULL, 0, 0, 0 AS " + "xmin, 0 AS ymin, 0 AS xmax, 0 AS ymax, 0 AS " + "is_in_gpkg_contents, 'table' AS object_type " + "FROM gpkgext_relations WHERE " + "lower(mapping_table_name) NOT IN (SELECT " + "lower(table_name) FROM " + "gpkg_contents)"; + } if (EQUAL(pszListAllTables, "YES") || (!bHasASpatialOrAttributes && EQUAL(pszListAllTables, "AUTO"))) { @@ -1632,6 +1646,12 @@ int GDALGeoPackageDataset::Open(GDALOpenInfo *poOpenInfo, "'st_geometry_columns', 'geometry_columns') " "AND lower(name) NOT IN (SELECT lower(table_name) FROM " "gpkg_contents)"; + if (bHasGPKGExtRelations) + { + osSQL += " AND lower(name) NOT IN (SELECT " + "lower(mapping_table_name) FROM " + "gpkgext_relations)"; + } } const int nTableLimit = GetOGRTableLimit(); if (nTableLimit > 0) @@ -2018,14 +2038,24 @@ void GDALGeoPackageDataset::ClearCachedRelationships() void GDALGeoPackageDataset::LoadRelationships() const { + m_osMapRelationships.clear(); + + std::vector oExcludedTables; if (HasGpkgextRelationsTable()) { LoadRelationshipsUsingRelatedTablesExtension(); + + for (const auto &oRelationship : m_osMapRelationships) + { + oExcludedTables.emplace_back( + oRelationship.second->GetMappingTableName()); + } } - else - { - LoadRelationshipsFromForeignKeys(); - } + + // Also load relationships defined using foreign keys (i.e. one-to-many + // relationships). Here we must exclude any relationships defined from the + // related tables extension, we don't want them included twice. + LoadRelationshipsFromForeignKeys(oExcludedTables); m_bHasPopulatedRelationships = true; } @@ -9793,45 +9823,6 @@ bool GDALGeoPackageDataset::AddFieldDomain( return true; } -/************************************************************************/ -/* GetRelationshipNames() */ -/************************************************************************/ - -std::vector GDALGeoPackageDataset::GetRelationshipNames( - CPL_UNUSED CSLConstList papszOptions) const - -{ - if (!m_bHasPopulatedRelationships) - LoadRelationships(); - - std::vector oasNames; - oasNames.reserve(m_osMapRelationships.size()); - for (auto it = m_osMapRelationships.begin(); - it != m_osMapRelationships.end(); ++it) - { - oasNames.emplace_back(it->first); - } - return oasNames; -} - -/************************************************************************/ -/* GetRelationship() */ -/************************************************************************/ - -const GDALRelationship * -GDALGeoPackageDataset::GetRelationship(const std::string &name) const - -{ - if (!m_bHasPopulatedRelationships) - LoadRelationships(); - - auto it = m_osMapRelationships.find(name); - if (it == m_osMapRelationships.end()) - return nullptr; - - return it->second.get(); -} - /************************************************************************/ /* AddRelationship() */ /************************************************************************/ @@ -9986,6 +9977,35 @@ bool GDALGeoPackageDataset::AddRelationship( return false; } + /* + * Strictly speaking we should NOT be inserting the mapping table into gpkg_contents. + * The related tables extension explicitly states that the mapping table should only be + * in the gpkgext_relations table and not in gpkg_contents. (See also discussion at + * https://github.com/opengeospatial/geopackage/issues/679). + * + * However, if we don't insert the mapping table into gpkg_contents then it is no longer + * visible to some clients (eg ESRI software only allows opening tables that are present + * in gpkg_contents). So we'll do this anyway, for maximum compatiblity and flexibility. + * + * More related discussion is at https://github.com/OSGeo/gdal/pull/9258 + */ + pszSQL = sqlite3_mprintf( + "INSERT INTO gpkg_contents " + "(table_name,data_type,identifier,description,last_change,srs_id) " + "VALUES " + "('%q','attributes','%q','Mapping table for relationship between " + "%q and %q',%s,0)", + osMappingTableName.c_str(), /*table_name*/ + osMappingTableName.c_str(), /*identifier*/ + osLeftTableName.c_str(), /*description left table name*/ + osRightTableName.c_str(), /*description right table name*/ + GDALGeoPackageDataset::GetCurrentDateEscapedSQL().c_str()); + + // Note -- we explicitly ignore failures here, because hey, we aren't really + // supposed to be adding this table to gpkg_contents anyway! + (void)SQLCommand(hDB, pszSQL); + sqlite3_free(pszSQL); + pszSQL = sqlite3_mprintf( "CREATE INDEX \"idx_%w_base_id\" ON \"%w\" (base_id);", osMappingTableName.c_str(), osMappingTableName.c_str()); diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp index c26f19be2df1..9813599d8ca3 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp @@ -2433,7 +2433,7 @@ OGRErr OGRGeoPackageTableLayer::CreateOrUpsertFeature(OGRFeature *poFeature, #if SQLITE_VERSION_NUMBER >= 3035000L sqlite3_column_int64(m_poInsertStatement, 0) #else - 0 + OGRNullFID #endif : sqlite3_last_insert_rowid(m_poDS->GetDB()); @@ -2446,7 +2446,7 @@ OGRErr OGRGeoPackageTableLayer::CreateOrUpsertFeature(OGRFeature *poFeature, m_poInsertStatement = nullptr; } - if (nFID != 0) + if (nFID != OGRNullFID) { poFeature->SetFID(nFID); if (m_iFIDAsRegularColumnIndex >= 0) diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmldatasource.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmldatasource.cpp index ba20dc1a98b0..88c124205002 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmldatasource.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmldatasource.cpp @@ -729,7 +729,6 @@ SchemaPtr OGRLIBKMLDataSource::FindSchema(const char *pszSchemaUrl) char *pszID = nullptr; char *pszFile = nullptr; char *pszSchemaName = nullptr; - char *pszPound = nullptr; DocumentPtr poKmlDocument = nullptr; SchemaPtr poKmlSchemaResult = nullptr; @@ -746,13 +745,11 @@ SchemaPtr OGRLIBKMLDataSource::FindSchema(const char *pszSchemaUrl) m_poKmlDocKml->IsA(kmldom::Type_Document)) poKmlDocument = AsDocument(m_poKmlDocKml); } - else if ((pszPound = strchr(const_cast(pszSchemaUrl), '#')) != - nullptr) + else if (const char *pszPound = strchr(pszSchemaUrl, '#')) { pszFile = CPLStrdup(pszSchemaUrl); pszID = CPLStrdup(pszPound + 1); - pszPound = strchr(pszFile, '#'); - *pszPound = '\0'; + pszFile[pszPound - pszSchemaUrl] = '\0'; } else { diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp index 40211bdebf4f..d9fc28577613 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp @@ -254,7 +254,7 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( { m_poKmlSchema = nullptr; } - kml2FeatureDef(schema, m_poOgrFeatureDefn); + kml2FeatureDef(std::move(schema), m_poOgrFeatureDefn); } } } diff --git a/ogr/ogrsf_frmts/mssqlspatial/ogr_mssqlspatial.h b/ogr/ogrsf_frmts/mssqlspatial/ogr_mssqlspatial.h index 19fb24c82ce4..85d290be1b99 100644 --- a/ogr/ogrsf_frmts/mssqlspatial/ogr_mssqlspatial.h +++ b/ogr/ogrsf_frmts/mssqlspatial/ogr_mssqlspatial.h @@ -41,6 +41,8 @@ #include "include_msodbcsql.h" #endif +#include + class OGRMSSQLSpatialDataSource; /* layer status */ @@ -598,9 +600,9 @@ class OGRMSSQLSpatialDataSource final : public OGRDataSource // We maintain a list of known SRID to reduce the number of trips to // the database to get SRSes. - int nKnownSRID; - int *panSRID; - OGRSpatialReference **papoSRS; + std::map> + m_oSRSCache{}; OGRMSSQLSpatialTableLayer *poLayerInCopyMode; @@ -666,7 +668,10 @@ class OGRMSSQLSpatialDataSource final : public OGRDataSource static char *LaunderName(const char *pszSrcName); OGRErr InitializeMetadataTables(); - void AddSRIDToCache(int nId, OGRSpatialReference *poSRS); + OGRSpatialReference *AddSRIDToCache( + int nId, + std::unique_ptr + &&poSRS); OGRSpatialReference *FetchSRS(int nId); int FetchSRSId(const OGRSpatialReference *poSRS); diff --git a/ogr/ogrsf_frmts/mssqlspatial/ogrmssqlspatialdatasource.cpp b/ogr/ogrsf_frmts/mssqlspatial/ogrmssqlspatialdatasource.cpp index e09df6fc3f32..c06cb7b2cf67 100644 --- a/ogr/ogrsf_frmts/mssqlspatial/ogrmssqlspatialdatasource.cpp +++ b/ogr/ogrsf_frmts/mssqlspatial/ogrmssqlspatialdatasource.cpp @@ -40,10 +40,6 @@ OGRMSSQLSpatialDataSource::OGRMSSQLSpatialDataSource() : bDSUpdate(false) papoLayers = nullptr; nLayers = 0; - nKnownSRID = 0; - panSRID = nullptr; - papoSRS = nullptr; - poLayerInCopyMode = nullptr; nGeometryFormat = MSSQLGEOMETRY_NATIVE; @@ -90,13 +86,6 @@ OGRMSSQLSpatialDataSource::~OGRMSSQLSpatialDataSource() CPLFree(pszName); CPLFree(pszCatalog); - for (int i = 0; i < nKnownSRID; i++) - { - if (papoSRS[i] != nullptr) - papoSRS[i]->Release(); - } - CPLFree(panSRID); - CPLFree(papoSRS); CPLFree(pszConnection); } /************************************************************************/ @@ -1445,18 +1434,14 @@ OGRSpatialReference *OGRMSSQLSpatialDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - int i; - - for (i = 0; i < nKnownSRID; i++) + auto oIter = m_oSRSCache.find(nId); + if (oIter != m_oSRSCache.end()) { - if (panSRID[i] == nId) - return papoSRS[i]; + return oIter->second.get(); } EndCopy(); - OGRSpatialReference *poSRS = nullptr; - /* -------------------------------------------------------------------- */ /* Try looking up in spatial_ref_sys table */ /* -------------------------------------------------------------------- */ @@ -1470,15 +1455,12 @@ OGRSpatialReference *OGRMSSQLSpatialDataSource::FetchSRS(int nId) { if (oStmt.GetColData(0)) { - poSRS = new OGRSpatialReference(); + auto poSRS = std::unique_ptr( + new OGRSpatialReference()); poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); const char *pszWKT = oStmt.GetColData(0); - if (poSRS->importFromWkt(pszWKT) != OGRERR_NONE) - { - delete poSRS; - poSRS = nullptr; - } - else + if (poSRS->importFromWkt(pszWKT) == OGRERR_NONE) { const char *pszAuthorityName = poSRS->GetAuthorityName(nullptr); @@ -1491,6 +1473,8 @@ OGRSpatialReference *OGRMSSQLSpatialDataSource::FetchSRS(int nId) poSRS->Clear(); poSRS->importFromEPSG(nCode); } + + return AddSRIDToCache(nId, std::move(poSRS)); } } } @@ -1499,26 +1483,16 @@ OGRSpatialReference *OGRMSSQLSpatialDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* Try looking up the EPSG list */ /* -------------------------------------------------------------------- */ - if (!poSRS) + auto poSRS = + std::unique_ptr( + new OGRSpatialReference()); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (poSRS->importFromEPSG(nId) == OGRERR_NONE) { - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (poSRS->importFromEPSG(nId) != OGRERR_NONE) - { - delete poSRS; - poSRS = nullptr; - } + return AddSRIDToCache(nId, std::move(poSRS)); } - /* -------------------------------------------------------------------- */ - /* Add to the cache. */ - /* -------------------------------------------------------------------- */ - if (poSRS) - { - AddSRIDToCache(nId, poSRS); - } - - return poSRS; + return nullptr; } /************************************************************************/ @@ -1528,18 +1502,15 @@ OGRSpatialReference *OGRMSSQLSpatialDataSource::FetchSRS(int nId) /* sure it is freshly created, or add a reference yourself if not. */ /************************************************************************/ -void OGRMSSQLSpatialDataSource::AddSRIDToCache(int nId, - OGRSpatialReference *poSRS) +OGRSpatialReference *OGRMSSQLSpatialDataSource::AddSRIDToCache( + int nId, + std::unique_ptr &&poSRS) { /* -------------------------------------------------------------------- */ /* Add to the cache. */ /* -------------------------------------------------------------------- */ - panSRID = (int *)CPLRealloc(panSRID, sizeof(int) * (nKnownSRID + 1)); - papoSRS = (OGRSpatialReference **)CPLRealloc(papoSRS, sizeof(void *) * - (nKnownSRID + 1)); - panSRID[nKnownSRID] = nId; - papoSRS[nKnownSRID] = poSRS; - nKnownSRID++; + auto oIter = m_oSRSCache.emplace(nId, std::move(poSRS)).first; + return oIter->second.get(); } /************************************************************************/ @@ -1561,15 +1532,15 @@ int OGRMSSQLSpatialDataSource::FetchSRSId(const OGRSpatialReference *poSRS) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - for (int i = 0; i < nKnownSRID; i++) + for (const auto &pair : m_oSRSCache) { - if (papoSRS[i] == poSRS) - return panSRID[i]; + if (pair.second.get() == poSRS) + return pair.first; } - for (int i = 0; i < nKnownSRID; i++) + for (const auto &pair : m_oSRSCache) { - if (papoSRS[i] != nullptr && papoSRS[i]->IsSame(poSRS)) - return panSRID[i]; + if (pair.second != nullptr && pair.second->IsSame(poSRS)) + return pair.first; } OGRSpatialReference oSRS(*poSRS); @@ -1622,9 +1593,11 @@ int OGRMSSQLSpatialDataSource::FetchSRSId(const OGRSpatialReference *poSRS) nSRSId = atoi(oStmt.GetColData(0)); if (nSRSId != 0) { - auto poCachedSRS = new OGRSpatialReference(oSRS); + std::unique_ptr + poCachedSRS(new OGRSpatialReference(oSRS)); poCachedSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - AddSRIDToCache(nSRSId, poCachedSRS); + AddSRIDToCache(nSRSId, std::move(poCachedSRS)); } return nSRSId; } diff --git a/ogr/ogrsf_frmts/mysql/ogr_mysql.h b/ogr/ogrsf_frmts/mysql/ogr_mysql.h index 6b96654bf9e7..71fd55181490 100644 --- a/ogr/ogrsf_frmts/mysql/ogr_mysql.h +++ b/ogr/ogrsf_frmts/mysql/ogr_mysql.h @@ -70,6 +70,8 @@ #include "ogrsf_frmts.h" +#include + class OGRMySQLDataSource; /************************************************************************/ @@ -259,9 +261,9 @@ class OGRMySQLDataSource final : public OGRDataSource // We maintain a list of known SRID to reduce the number of trips to // the database to get SRSes. - int nKnownSRID; - int *panSRID; - OGRSpatialReference **papoSRS; + std::map> + m_oSRSCache{}; OGRMySQLLayer *poLongResultLayer; @@ -280,7 +282,7 @@ class OGRMySQLDataSource final : public OGRDataSource int FetchSRSId(const OGRSpatialReference *poSRS); - OGRSpatialReference *FetchSRS(int nSRSId); + const OGRSpatialReference *FetchSRS(int nSRSId); OGRErr InitializeMetadataTables(); OGRErr UpdateMetadataTables(const char *pszLayerName, diff --git a/ogr/ogrsf_frmts/mysql/ogrmysqldatasource.cpp b/ogr/ogrsf_frmts/mysql/ogrmysqldatasource.cpp index fa7e1aa1cb2f..05fd14918c4f 100644 --- a/ogr/ogrsf_frmts/mysql/ogrmysqldatasource.cpp +++ b/ogr/ogrsf_frmts/mysql/ogrmysqldatasource.cpp @@ -53,8 +53,7 @@ inline void FreeResultAndNullify(MYSQL_RES *&hResult) OGRMySQLDataSource::OGRMySQLDataSource() : papoLayers(nullptr), nLayers(0), pszName(nullptr), bDSUpdate(FALSE), - hConn(nullptr), nKnownSRID(0), panSRID(nullptr), papoSRS(nullptr), - poLongResultLayer(nullptr) + hConn(nullptr), poLongResultLayer(nullptr) { } @@ -76,14 +75,6 @@ OGRMySQLDataSource::~OGRMySQLDataSource() if (hConn != nullptr) mysql_close(hConn); - - for (int i = 0; i < nKnownSRID; i++) - { - if (papoSRS[i] != nullptr) - papoSRS[i]->Release(); - } - CPLFree(panSRID); - CPLFree(papoSRS); } /************************************************************************/ @@ -557,7 +548,7 @@ OGRErr OGRMySQLDataSource::UpdateMetadataTables(const char *pszLayerName, /* OGRSpatialReference, as handles may be cached. */ /************************************************************************/ -OGRSpatialReference *OGRMySQLDataSource::FetchSRS(int nId) +const OGRSpatialReference *OGRMySQLDataSource::FetchSRS(int nId) { if (nId < 0) return nullptr; @@ -565,14 +556,12 @@ OGRSpatialReference *OGRMySQLDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - for (int i = 0; i < nKnownSRID; i++) + auto oIter = m_oSRSCache.find(nId); + if (oIter != m_oSRSCache.end()) { - if (panSRID[i] == nId) - return papoSRS[i]; + return oIter->second.get(); } - OGRSpatialReference *poSRS = nullptr; - // make sure to attempt to free any old results MYSQL_RES *hResult = mysql_store_result(GetConn()); FreeResultAndNullify(hResult); @@ -608,12 +597,12 @@ OGRSpatialReference *OGRMySQLDataSource::FetchSRS(int nId) FreeResultAndNullify(hResult); - poSRS = new OGRSpatialReference(); + std::unique_ptr poSRS( + new OGRSpatialReference()); poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); if (pszWKT == nullptr || poSRS->importFromWkt(pszWKT) != OGRERR_NONE) { - delete poSRS; - poSRS = nullptr; + poSRS.reset(); } CPLFree(pszWKT); @@ -635,14 +624,8 @@ OGRSpatialReference *OGRMySQLDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* Add to the cache. */ /* -------------------------------------------------------------------- */ - panSRID = (int *)CPLRealloc(panSRID, sizeof(int) * (nKnownSRID + 1)); - papoSRS = (OGRSpatialReference **)CPLRealloc(papoSRS, sizeof(void *) * - (nKnownSRID + 1)); - panSRID[nKnownSRID] = nId; - papoSRS[nKnownSRID] = poSRS; - nKnownSRID++; - - return poSRS; + oIter = m_oSRSCache.emplace(nId, std::move(poSRS)).first; + return oIter->second.get(); } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/oci/ogr_oci.h b/ogr/ogrsf_frmts/oci/ogr_oci.h index 9375aa47d977..329c9cef0c68 100644 --- a/ogr/ogrsf_frmts/oci/ogr_oci.h +++ b/ogr/ogrsf_frmts/oci/ogr_oci.h @@ -34,6 +34,8 @@ #include "oci.h" #include "cpl_error.h" +#include + /* -------------------------------------------------------------------- */ /* Low level Oracle spatial declarations. */ /* -------------------------------------------------------------------- */ @@ -564,9 +566,9 @@ class OGROCIDataSource final : public OGRDataSource // We maintain a list of known SRID to reduce the number of trips to // the database to get SRSes. - int nKnownSRID; - int *panSRID; - OGRSpatialReference **papoSRS; + std::map> + m_oSRSCache{}; public: OGROCIDataSource(); diff --git a/ogr/ogrsf_frmts/oci/ogrocidatasource.cpp b/ogr/ogrsf_frmts/oci/ogrocidatasource.cpp index 3d2a45e49ed2..4b45d965bb92 100644 --- a/ogr/ogrsf_frmts/oci/ogrocidatasource.cpp +++ b/ogr/ogrsf_frmts/oci/ogrocidatasource.cpp @@ -59,9 +59,6 @@ OGROCIDataSource::OGROCIDataSource() bDSUpdate = FALSE; bNoLogging = FALSE; poSession = nullptr; - papoSRS = nullptr; - panSRID = nullptr; - nKnownSRID = 0; } /************************************************************************/ @@ -81,13 +78,6 @@ OGROCIDataSource::~OGROCIDataSource() CPLFree(papoLayers); - for (i = 0; i < nKnownSRID; i++) - { - papoSRS[i]->Release(); - } - CPLFree(papoSRS); - CPLFree(panSRID); - if (poSession != nullptr) delete poSession; } @@ -801,12 +791,10 @@ OGRSpatialReference *OGROCIDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - int i; - - for (i = 0; i < nKnownSRID; i++) + auto oIter = m_oSRSCache.find(nId); + if (oIter != m_oSRSCache.end()) { - if (panSRID[i] == nId) - return papoSRS[i]; + return oIter->second.get(); } /* -------------------------------------------------------------------- */ @@ -830,11 +818,11 @@ OGRSpatialReference *OGROCIDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* Turn into a spatial reference. */ /* -------------------------------------------------------------------- */ - OGRSpatialReference *poSRS = new OGRSpatialReference(); + std::unique_ptr poSRS( + new OGRSpatialReference()); poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); if (poSRS->importFromWkt(papszResult[0]) != OGRERR_NONE) { - delete poSRS; return nullptr; } @@ -853,10 +841,10 @@ OGRSpatialReference *OGROCIDataSource::FetchSRS(int nId) const char *const apszOptions[] = { "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr}; if (oSRS_EPSG.importFromEPSG(nId) == OGRERR_NONE && - oSRS_EPSG.IsSame(poSRS, apszOptions)) + oSRS_EPSG.IsSame(poSRS.get(), apszOptions)) { *poSRS = oSRS_EPSG; - return poSRS; + return poSRS.release(); } } @@ -865,7 +853,7 @@ OGRSpatialReference *OGROCIDataSource::FetchSRS(int nId) /* authority. */ /* -------------------------------------------------------------------- */ int bGotEPSGMapping = FALSE; - for (i = 0; anEPSGOracleMapping[i] != 0; i += 2) + for (int i = 0; anEPSGOracleMapping[i] != 0; i += 2) { if (anEPSGOracleMapping[i] == nId) { @@ -890,15 +878,8 @@ OGRSpatialReference *OGROCIDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* Add to the cache. */ /* -------------------------------------------------------------------- */ - panSRID = (int *)CPLRealloc(panSRID, sizeof(int) * (nKnownSRID + 1)); - papoSRS = (OGRSpatialReference **)CPLRealloc(papoSRS, sizeof(void *) * - (nKnownSRID + 1)); - panSRID[nKnownSRID] = nId; - papoSRS[nKnownSRID] = poSRS; - - nKnownSRID++; - - return poSRS; + oIter = m_oSRSCache.emplace(nId, std::move(poSRS)).first; + return oIter->second.get(); } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/odbc/ogr_odbc.h b/ogr/ogrsf_frmts/odbc/ogr_odbc.h index 75d37a829c4a..3688dad950c6 100644 --- a/ogr/ogrsf_frmts/odbc/ogr_odbc.h +++ b/ogr/ogrsf_frmts/odbc/ogr_odbc.h @@ -33,6 +33,8 @@ #include "ogrsf_frmts.h" #include "cpl_odbc.h" #include "cpl_error.h" + +#include #include /************************************************************************/ @@ -190,11 +192,15 @@ class OGRODBCDataSource final : public OGRDataSource CPLODBCSession oSession; +#if 0 + // NOTE: nothing uses the SRS cache currently. Hence disabled. + // We maintain a list of known SRID to reduce the number of trips to // the database to get SRSes. - int nKnownSRID; - int *panSRID; - OGRSpatialReference **papoSRS; + std::map> + m_oSRSCache{}; +#endif // set of all lowercase table names. Note that this is only used when // opening MDB datasources, not generic ODBC ones. diff --git a/ogr/ogrsf_frmts/odbc/ogrodbcdatasource.cpp b/ogr/ogrsf_frmts/odbc/ogrodbcdatasource.cpp index 261b9b001b9f..1e56c07fd572 100644 --- a/ogr/ogrsf_frmts/odbc/ogrodbcdatasource.cpp +++ b/ogr/ogrsf_frmts/odbc/ogrodbcdatasource.cpp @@ -37,8 +37,7 @@ /************************************************************************/ OGRODBCDataSource::OGRODBCDataSource() - : papoLayers(nullptr), nLayers(0), pszName(nullptr), nKnownSRID(0), - panSRID(nullptr), papoSRS(nullptr) + : papoLayers(nullptr), nLayers(0), pszName(nullptr) { } @@ -55,14 +54,6 @@ OGRODBCDataSource::~OGRODBCDataSource() delete papoLayers[i]; CPLFree(papoLayers); - - for (int i = 0; i < nKnownSRID; i++) - { - if (papoSRS[i] != nullptr) - papoSRS[i]->Release(); - } - CPLFree(panSRID); - CPLFree(papoSRS); } /************************************************************************/ @@ -415,6 +406,9 @@ int OGRODBCDataSource::Open(GDALOpenInfo *poOpenInfo) CSLDestroy(papszTables); CSLDestroy(papszGeomCol); +#if 0 + // NOTE: nothing uses the SRS cache currently. Hence disabled. + /* -------------------------------------------------------------------- */ /* If no explicit list of tables was given, check for a list in */ /* a geometry_columns table. */ @@ -439,11 +433,6 @@ int OGRODBCDataSource::Open(GDALOpenInfo *poOpenInfo) oSRSList.GetCommand()); if (oSRSList.ExecuteSQL()) { - int nRows = 256; // A reasonable number of SRIDs to start from - panSRID = (int *)CPLMalloc(nRows * sizeof(int)); - papoSRS = (OGRSpatialReference **)CPLMalloc( - nRows * sizeof(OGRSpatialReference *)); - while (oSRSList.Fetch()) { const char *pszSRID = oSRSList.GetColData(pszSRIDCol); @@ -454,29 +443,18 @@ int OGRODBCDataSource::Open(GDALOpenInfo *poOpenInfo) if (pszSRText) { - if (nKnownSRID > nRows) - { - nRows *= 2; - panSRID = - (int *)CPLRealloc(panSRID, nRows * sizeof(int)); - papoSRS = (OGRSpatialReference **)CPLRealloc( - papoSRS, nRows * sizeof(OGRSpatialReference *)); - } - panSRID[nKnownSRID] = atoi(pszSRID); - papoSRS[nKnownSRID] = new OGRSpatialReference(); - papoSRS[nKnownSRID]->SetAxisMappingStrategy( + std::unique_ptr poSRS(new OGRSpatialReference()); + poSRS->SetAxisMappingStrategy( OAMS_TRADITIONAL_GIS_ORDER); - if (papoSRS[nKnownSRID]->importFromWkt(pszSRText) != - OGRERR_NONE) + if (poSRS->importFromWkt(pszSRText) == OGRERR_NONE ) { - delete papoSRS[nKnownSRID]; - continue; + m_oSRSCache[atoi(pszSRID)] = std::move(poSRS); } - nKnownSRID++; } } } } +#endif if (pszSRIDCol) CPLFree(pszSRIDCol); diff --git a/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp b/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp index e74cdb191eaa..62a4ff213bd3 100644 --- a/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp +++ b/ogr/ogrsf_frmts/ods/ogrodsdatasource.cpp @@ -72,6 +72,7 @@ OGRODSLayer::OGRODSLayer(OGRODSDataSource *poDSIn, const char *pszName, bUpdated(CPL_TO_BOOL(bUpdatedIn)), bHasHeaderLine(false), m_poAttrQueryODS(nullptr) { + SetAdvertizeUTF8(true); } /************************************************************************/ @@ -846,7 +847,6 @@ void OGRODSDataSource::endElementTable( reinterpret_cast(poCurLayer) ->SetUpdatable(bUpdatable); - reinterpret_cast(poCurLayer)->SetAdvertizeUTF8(true); reinterpret_cast(poCurLayer)->SetUpdated(false); } diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdb_relationship.h b/ogr/ogrsf_frmts/openfilegdb/filegdb_relationship.h index ca6f1205958e..9381458e71c4 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdb_relationship.h +++ b/ogr/ogrsf_frmts/openfilegdb/filegdb_relationship.h @@ -242,7 +242,7 @@ ParseXMLRelationshipDef(const std::string &domainDef) } else { - poRelationship->SetRelatedTableType("feature"); + poRelationship->SetRelatedTableType("features"); } return poRelationship; } diff --git a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp index d44057dc1da5..92841fc6e83b 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp @@ -4069,18 +4069,18 @@ bool OGROSMDataSource::TransferToDiskIfNecesserary() VSIFCloseL(m_fpNodes); m_fpNodes = nullptr; - CPLString osNewTmpDBName; - osNewTmpDBName = CPLGenerateTempFilename("osm_tmp_nodes"); + const std::string osNewTmpDBName( + CPLGenerateTempFilename("osm_tmp_nodes")); CPLDebug("OSM", "%s too big for RAM. Transferring it onto disk in %s", m_osNodesFilename.c_str(), osNewTmpDBName.c_str()); - if (CPLCopyFile(osNewTmpDBName, m_osNodesFilename) != 0) + if (CPLCopyFile(osNewTmpDBName.c_str(), m_osNodesFilename) != 0) { CPLError(CE_Failure, CPLE_AppDefined, "Cannot copy %s to %s", m_osNodesFilename.c_str(), osNewTmpDBName.c_str()); - VSIUnlink(osNewTmpDBName); + VSIUnlink(osNewTmpDBName.c_str()); m_bStopParsing = true; return false; } diff --git a/ogr/ogrsf_frmts/pg/ogr_pg.h b/ogr/ogrsf_frmts/pg/ogr_pg.h index ba22baef4980..557b1d84f231 100644 --- a/ogr/ogrsf_frmts/pg/ogr_pg.h +++ b/ogr/ogrsf_frmts/pg/ogr_pg.h @@ -196,7 +196,7 @@ class OGRPGLayer CPL_NON_FINAL : public OGRLayer static char *GeometryToBYTEA(const OGRGeometry *, int nPostGISMajor, int nPostGISMinor); static GByte *BYTEAToGByteArray(const char *pszBytea, int *pnLength); - static OGRGeometry *BYTEAToGeometry(const char *, int bIsPostGIS1); + static OGRGeometry *BYTEAToGeometry(const char *); Oid GeometryToOID(OGRGeometry *); OGRGeometry *OIDToGeometry(Oid); @@ -638,6 +638,11 @@ class OGRPGDataSource final : public OGRDataSource bool m_bHasGeometryColumns = false; bool m_bHasSpatialRefSys = false; + bool HavePostGIS() const + { + return bHavePostGIS; + } + int GetUndefinedSRID() const { return nUndefinedSRID; diff --git a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp index cf8dd3780c27..8b6a8db7e528 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp @@ -260,16 +260,6 @@ static void OGRPGTableEntryAddGeomColumn( psTableEntry->nGeomColumnCount++; } -static void -OGRPGTableEntryAddGeomColumn(PGTableEntry *psTableEntry, - const PGGeomColumnDesc *psGeomColumnDesc) -{ - OGRPGTableEntryAddGeomColumn( - psTableEntry, psGeomColumnDesc->pszName, psGeomColumnDesc->pszGeomType, - psGeomColumnDesc->GeometryTypeFlags, psGeomColumnDesc->nSRID, - psGeomColumnDesc->ePostgisType, psGeomColumnDesc->bNullable); -} - static void OGRPGFreeTableEntry(void *_psTableEntry) { PGTableEntry *psTableEntry = static_cast(_psTableEntry); @@ -781,17 +771,6 @@ int OGRPGDataSource::Open(const char *pszNewName, int bUpdate, int bTestOpen, isdigit(static_cast(pszSpace[1]))) { OGRPGDecodeVersionString(&sPostgreSQLVersion, pszSpace + 1); -#if defined(BINARY_CURSOR_ENABLED) - if (sPostgreSQLVersion.nMajor == 7 && sPostgreSQLVersion.nMinor < 4) - { - /* We don't support BINARY CURSOR for PostgreSQL < 7.4. */ - /* The binary protocol for arrays seems to be different from - * later versions */ - CPLDebug("PG", "BINARY cursor will finally NOT be used because " - "version < 7.4"); - bUseBinaryCursor = FALSE; - } -#endif } } OGRPGClearResult(hResult); @@ -870,19 +849,6 @@ int OGRPGDataSource::Open(const char *pszNewName, int bUpdate, int bTestOpen, } #endif -#ifdef notdef - /* This would be the quickest fix... instead, ogrpglayer has been updated to - * support */ - /* bytea hex format */ - if (sPostgreSQLVersion.nMajor >= 9) - { - // Starting with PostgreSQL 9.0, the default output format for values - // of type bytea is hex, whereas we traditionally expect escape. - hResult = OGRPG_PQexec(hPGConn, "SET bytea_output TO escape"); - OGRPGClearResult(hResult); - } -#endif - /* -------------------------------------------------------------------- */ /* Test to see if this database instance has support for the */ /* PostGIS Geometry type. If so, disable sequential scanning */ @@ -936,21 +902,6 @@ int OGRPGDataSource::Open(const char *pszNewName, int bUpdate, int bTestOpen, OGRPGDecodeVersionString(&sPostGISVersion, pszVer); } OGRPGClearResult(hResult); - - if (sPostGISVersion.nMajor == 0 && sPostGISVersion.nMinor < 8) - { - // Turning off sequential scans for PostGIS < 0.8 - hResult = OGRPG_PQexec(hPGConn, "SET ENABLE_SEQSCAN = OFF"); - - CPLDebug("PG", "SET ENABLE_SEQSCAN=OFF"); - } - else - { - // PostGIS >=0.8 is correctly integrated with query planner, - // thus PostgreSQL will use indexes whenever appropriate. - hResult = OGRPG_PQexec(hPGConn, "SET ENABLE_SEQSCAN = ON"); - } - OGRPGClearResult(hResult); } m_bHasGeometryColumns = @@ -1260,7 +1211,7 @@ void OGRPGDataSource::LoadTables() if (bHavePostGIS && !bListAllTables) { osCommand.Printf( - "SELECT c.relname, n.nspname, c.relkind, g.f_geometry_column, " + "SELECT c.relname, n.nspname, g.f_geometry_column, " "g.type, g.coord_dimension, g.srid, %d, a.attnotnull, " "d.description, c.oid as oid, a.attnum as attnum " "FROM pg_class c " @@ -1278,7 +1229,7 @@ void OGRPGDataSource::LoadTables() if (bHaveGeography) osCommand += CPLString().Printf( - "UNION SELECT c.relname, n.nspname, c.relkind, " + "UNION SELECT c.relname, n.nspname, " "g.f_geography_column, " "g.type, g.coord_dimension, g.srid, %d, a.attnotnull, " "d.description, c.oid as oid, a.attnum as attnum " @@ -1298,7 +1249,7 @@ void OGRPGDataSource::LoadTables() osCommand += " ORDER BY oid, attnum"; } else - osCommand.Printf("SELECT c.relname, n.nspname, c.relkind FROM " + osCommand.Printf("SELECT c.relname, n.nspname FROM " "pg_class c, pg_namespace n " "WHERE (c.relkind in (%s) AND c.relname !~ '^pg_' " "AND c.relnamespace=n.oid)", @@ -1324,7 +1275,6 @@ void OGRPGDataSource::LoadTables() { const char *pszTable = PQgetvalue(hResult, iRecord, 0); const char *pszSchemaName = PQgetvalue(hResult, iRecord, 1); - const char *pszRelkind = PQgetvalue(hResult, iRecord, 2); const char *pszGeomColumnName = nullptr; const char *pszGeomType = nullptr; const char *pszDescription = nullptr; @@ -1335,23 +1285,15 @@ void OGRPGDataSource::LoadTables() PostgisType ePostgisType = GEOM_TYPE_UNKNOWN; if (bHavePostGIS && !bListAllTables) { - pszGeomColumnName = PQgetvalue(hResult, iRecord, 3); - pszGeomType = PQgetvalue(hResult, iRecord, 4); + pszGeomColumnName = PQgetvalue(hResult, iRecord, 2); + pszGeomType = PQgetvalue(hResult, iRecord, 3); bHasM = pszGeomType[strlen(pszGeomType) - 1] == 'M'; - nGeomCoordDimension = atoi(PQgetvalue(hResult, iRecord, 5)); - nSRID = atoi(PQgetvalue(hResult, iRecord, 6)); + nGeomCoordDimension = atoi(PQgetvalue(hResult, iRecord, 4)); + nSRID = atoi(PQgetvalue(hResult, iRecord, 5)); ePostgisType = static_cast( - atoi(PQgetvalue(hResult, iRecord, 7))); - bNullable = EQUAL(PQgetvalue(hResult, iRecord, 8), "f"); - pszDescription = PQgetvalue(hResult, iRecord, 9); - - /* We cannot reliably find geometry columns of a view that is */ - /* based on a table that inherits from another one, wit that */ - /* method, so give up, and let - * OGRPGTableLayer::ReadTableDefinition() */ - /* do the job */ - if (pszRelkind[0] == 'v' && sPostGISVersion.nMajor < 2) - pszGeomColumnName = nullptr; + atoi(PQgetvalue(hResult, iRecord, 6))); + bNullable = EQUAL(PQgetvalue(hResult, iRecord, 7), "f"); + pszDescription = PQgetvalue(hResult, iRecord, 8); } if (EQUAL(pszTable, "spatial_ref_sys") || @@ -1407,123 +1349,6 @@ void OGRPGDataSource::LoadTables() /* -------------------------------------------------------------------- */ OGRPGClearResult(hResult); - - /* With PostGIS 2.0, we don't need to query base tables of inherited */ - /* tables */ - if (bHavePostGIS && !bListAllTables && sPostGISVersion.nMajor < 2) - { - /* ------------------------------------------------------------------ - */ - /* Fetch inherited tables */ - /* ------------------------------------------------------------------ - */ - hResult = OGRPG_PQexec( - hPGConn, - "SELECT c1.relname AS derived, c2.relname AS parent, n.nspname " - "FROM pg_class c1, pg_class c2, pg_namespace n, pg_inherits i " - "WHERE i.inhparent = c2.oid AND i.inhrelid = c1.oid AND " - "c1.relnamespace=n.oid " - "AND c1.relkind in ('r', 'v') AND c1.relnamespace=n.oid AND " - "c2.relkind in ('r','v') " - "AND c2.relname !~ '^pg_' AND c2.relnamespace=n.oid"); - - if (!hResult || PQresultStatus(hResult) != PGRES_TUPLES_OK) - { - OGRPGClearResult(hResult); - - CPLError(CE_Failure, CPLE_AppDefined, "%s", - PQerrorMessage(hPGConn)); - goto end; - } - - /* ------------------------------------------------------------------ - */ - /* Parse the returned table list */ - /* ------------------------------------------------------------------ - */ - bool bHasDoneSomething = false; - do - { - /* Iterate over the tuples while we have managed to resolved at - * least one */ - /* table to its table parent with a geometry */ - /* For example if we have C inherits B and B inherits A, where A - * is a base table with a geometry */ - /* The first pass will add B to the set of tables */ - /* The second pass will add C to the set of tables */ - - bHasDoneSomething = false; - - for (int iRecord = 0; iRecord < PQntuples(hResult); iRecord++) - { - const char *pszTable = PQgetvalue(hResult, iRecord, 0); - const char *pszParentTable = - PQgetvalue(hResult, iRecord, 1); - const char *pszSchemaName = PQgetvalue(hResult, iRecord, 2); - - PGTableEntry *psEntry = OGRPGFindTableEntry( - hSetTables, pszTable, pszSchemaName); - /* We must be careful that a derived table can have its own - * geometry column(s) */ - /* and some inherited from another table */ - if (psEntry == nullptr || !psEntry->bDerivedInfoAdded) - { - PGTableEntry *psParentEntry = OGRPGFindTableEntry( - hSetTables, pszParentTable, pszSchemaName); - if (psParentEntry != nullptr) - { - /* The parent table of this table is already in the - * set, so we */ - /* can now add the table in the set if it was not in - * it already */ - - bHasDoneSomething = true; - - if (psEntry == nullptr) - psEntry = - OGRPGAddTableEntry(hSetTables, pszTable, - pszSchemaName, nullptr); - - for (int iGeomColumn = 0; - iGeomColumn < psParentEntry->nGeomColumnCount; - iGeomColumn++) - { - papsTables = - static_cast(CPLRealloc( - papsTables, sizeof(PGTableEntry *) * - (nTableCount + 1))); - papsTables[nTableCount] = - static_cast( - CPLCalloc(1, sizeof(PGTableEntry))); - papsTables[nTableCount]->pszTableName = - CPLStrdup(pszTable); - papsTables[nTableCount]->pszSchemaName = - CPLStrdup(pszSchemaName); - OGRPGTableEntryAddGeomColumn( - papsTables[nTableCount], - &psParentEntry - ->pasGeomColumns[iGeomColumn]); - nTableCount++; - - OGRPGTableEntryAddGeomColumn( - psEntry, - &psParentEntry - ->pasGeomColumns[iGeomColumn]); - } - - psEntry->bDerivedInfoAdded = true; - } - } - } - } while (bHasDoneSomething); - - /* -------------------------------------------------------------------- - */ - /* Cleanup */ - /* -------------------------------------------------------------------- - */ - OGRPGClearResult(hResult); - } } /* -------------------------------------------------------------------- */ @@ -1666,19 +1491,6 @@ OGRErr OGRPGDataSource::DeleteLayer(int iLayer) SoftStartTransaction(); - if (bHavePostGIS && sPostGISVersion.nMajor < 2) - { - // This is unnecessary if the layer is not a geometry table, - // or an inherited geometry table but it should not hurt. - osCommand.Printf( - "DELETE FROM geometry_columns WHERE f_table_name='%s' and " - "f_table_schema='%s'", - osTableName.c_str(), osSchemaName.c_str()); - - PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - OGRPGClearResult(hResult); - } - osCommand.Printf("DROP TABLE %s.%s CASCADE", OGRPGEscapeColumnName(osSchemaName).c_str(), OGRPGEscapeColumnName(osTableName).c_str()); @@ -2085,7 +1897,7 @@ OGRLayer *OGRPGDataSource::ICreateLayer(const char *pszLayerName, osFIDColumnNameEscaped.c_str(), pszSerialType, osFIDColumnNameEscaped.c_str()); } - osCreateTable = osCommand; + osCreateTable = std::move(osCommand); } const char *pszSI = @@ -2114,33 +1926,6 @@ OGRLayer *OGRPGDataSource::ICreateLayer(const char *pszLayerName, bCreateSpatialIndex = false; } - CPLString osEscapedTableNameSingleQuote = - OGRPGEscapeString(hPGConn, pszTableName); - const char *pszEscapedTableNameSingleQuote = - osEscapedTableNameSingleQuote.c_str(); - CPLString osEscapedSchemaNameSingleQuote = - OGRPGEscapeString(hPGConn, pszSchemaName); - const char *pszEscapedSchemaNameSingleQuote = - osEscapedSchemaNameSingleQuote.c_str(); - - if (eType != wkbNone && bHavePostGIS && sPostGISVersion.nMajor <= 1) - { - /* Sometimes there is an old cruft entry in the geometry_columns - * table if things were not properly cleaned up before. We make - * an effort to clean out such cruft. - * Note: PostGIS 2.0 defines geometry_columns as a view (no clean up is - * needed) - */ - CPLString osCommand; - osCommand.Printf("DELETE FROM geometry_columns WHERE f_table_name = %s " - "AND f_table_schema = %s", - pszEscapedTableNameSingleQuote, - pszEscapedSchemaNameSingleQuote); - - PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - OGRPGClearResult(hResult); - } - if (!bDeferredCreation) { SoftStartTransaction(); @@ -2164,49 +1949,6 @@ OGRLayer *OGRPGDataSource::ICreateLayer(const char *pszLayerName, OGRPGClearResult(hResult); - /* -------------------------------------------------------------------- - */ - /* Eventually we should be adding this table to a table of */ - /* "geometric layers", capturing the WKT projection, and */ - /* perhaps some other housekeeping. */ - /* -------------------------------------------------------------------- - */ - if (eType != wkbNone && bHavePostGIS && - !EQUAL(pszGeomType, "geography") && sPostGISVersion.nMajor <= 1) - { - int dim = 2; - if (GeometryTypeFlags & OGRGeometry::OGR_G_3D) - dim++; - if (GeometryTypeFlags & OGRGeometry::OGR_G_MEASURED) - dim++; - osCommand.Printf("SELECT AddGeometryColumn(%s,%s,%s,%d,'%s',%d)", - pszEscapedSchemaNameSingleQuote, - pszEscapedTableNameSingleQuote, - OGRPGEscapeString(hPGConn, pszGFldName).c_str(), - nSRSId, pszGeometryType, dim); - - hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - - if (!hResult || PQresultStatus(hResult) != PGRES_TUPLES_OK) - { - CPLError(CE_Failure, CPLE_AppDefined, - "AddGeometryColumn failed for layer %s, layer " - "creation has failed.", - pszLayerName); - - CPLFree(pszTableName); - CPLFree(pszSchemaName); - - OGRPGClearResult(hResult); - - SoftRollbackTransaction(); - - return nullptr; - } - - OGRPGClearResult(hResult); - } - if (eType != wkbNone && bHavePostGIS && bCreateSpatialIndex) { /* -------------------------------------------------------------------- @@ -2483,6 +2225,7 @@ OGRLayer *OGRPGDataSource::GetLayerByName(const char *pszNameIn) if (!osSchemaName.has_value()) { CPLFree(pszNameWithoutBracket); + CPLFree(pszGeomColumnName); return nullptr; } pszSchemaName = CPLStrdup(osSchemaName->c_str()); diff --git a/ogr/ogrsf_frmts/pg/ogrpglayer.cpp b/ogr/ogrsf_frmts/pg/ogrpglayer.cpp index bdbc936a5fd9..53bfaac881c9 100644 --- a/ogr/ogrsf_frmts/pg/ogrpglayer.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpglayer.cpp @@ -649,17 +649,14 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, STARTS_WITH(pszVal, "\\x00") || STARTS_WITH(pszVal, "\\x01"))) { - poGeom = BYTEAToGeometry( - pszVal, (poDS->sPostGISVersion.nMajor < 2)); + poGeom = BYTEAToGeometry(pszVal); } else { const GByte *pabyVal = reinterpret_cast(pszVal); OGRGeometryFactory::createFromWkb( - pabyVal, nullptr, &poGeom, nLength, - (poDS->sPostGISVersion.nMajor < 2) ? wkbVariantPostGIS1 - : wkbVariantOldOgc); + pabyVal, nullptr, &poGeom, nLength, wkbVariantOldOgc); } if (poGeom != nullptr) @@ -686,8 +683,7 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, // Potentially dangerous to modify the result of PQgetvalue... nLength = CPLBase64DecodeInPlace(const_cast(pabyData)); OGRGeometry *poGeom = OGRGeometryFromEWKB( - const_cast(pabyData), nLength, nullptr, - poDS->sPostGISVersion.nMajor < 2); + const_cast(pabyData), nLength, nullptr, false); if (poGeom != nullptr) { @@ -721,15 +717,13 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, { GByte *pabyEWKB = BYTEAToGByteArray(pabyData, &nLength); poGeom = - OGRGeometryFromEWKB(pabyEWKB, nLength, nullptr, - poDS->sPostGISVersion.nMajor < 2); + OGRGeometryFromEWKB(pabyEWKB, nLength, nullptr, false); CPLFree(pabyEWKB); } else if (nLength >= 2 && (STARTS_WITH_CI(pabyData, "00") || STARTS_WITH_CI(pabyData, "01"))) { - poGeom = OGRGeometryFromHexEWKB( - pabyData, nullptr, poDS->sPostGISVersion.nMajor < 2); + poGeom = OGRGeometryFromHexEWKB(pabyData, nullptr, false); } else { @@ -738,7 +732,7 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, poGeom = OGRGeometryFromEWKB( const_cast( reinterpret_cast(pabyData)), - nLength, nullptr, poDS->sPostGISVersion.nMajor < 2); + nLength, nullptr, false); } if (poGeom != nullptr) @@ -773,8 +767,7 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, if (STARTS_WITH_CI(pszPostSRID, "00") || STARTS_WITH_CI(pszPostSRID, "01")) { - poGeometry = OGRGeometryFromHexEWKB( - pszWKT, nullptr, poDS->sPostGISVersion.nMajor < 2); + poGeometry = OGRGeometryFromHexEWKB(pszWKT, nullptr, false); } else OGRGeometryFactory::createFromWkt(pszPostSRID, nullptr, @@ -814,14 +807,12 @@ OGRFeature *OGRPGLayer::RecordToFeature(PGresult *hResult, const GByte *pabyData = reinterpret_cast(pszData); poGeometry = - OGRGeometryFromEWKB(pabyData, nLength, NULL, - poDS->sPostGISVersion.nMajor < 2); + OGRGeometryFromEWKB(pabyData, nLength, NULL, false); } if (poGeometry == nullptr) #endif { - poGeometry = BYTEAToGeometry( - pszData, (poDS->sPostGISVersion.nMajor < 2)); + poGeometry = BYTEAToGeometry(pszData); } } @@ -1732,7 +1723,7 @@ GByte *OGRPGLayer::BYTEAToGByteArray(const char *pszBytea, int *pnLength) /* BYTEAToGeometry() */ /************************************************************************/ -OGRGeometry *OGRPGLayer::BYTEAToGeometry(const char *pszBytea, int bIsPostGIS1) +OGRGeometry *OGRPGLayer::BYTEAToGeometry(const char *pszBytea) { if (pszBytea == nullptr) @@ -1743,8 +1734,7 @@ OGRGeometry *OGRPGLayer::BYTEAToGeometry(const char *pszBytea, int bIsPostGIS1) OGRGeometry *poGeometry = nullptr; OGRGeometryFactory::createFromWkb(pabyWKB, nullptr, &poGeometry, nLen, - (bIsPostGIS1) ? wkbVariantPostGIS1 - : wkbVariantOldOgc); + wkbVariantOldOgc); CPLFree(pabyWKB); return poGeometry; @@ -1813,9 +1803,7 @@ OGRGeometry *OGRPGLayer::OIDToGeometry(Oid oid) OGRGeometry *poGeometry = nullptr; OGRGeometryFactory::createFromWkb(pabyWKB, nullptr, &poGeometry, nBytes, - poDS->sPostGISVersion.nMajor < 2 - ? wkbVariantPostGIS1 - : wkbVariantOldOgc); + wkbVariantOldOgc); CPLFree(pabyWKB); @@ -1840,10 +1828,8 @@ Oid OGRPGLayer::GeometryToOID(OGRGeometry *poGeometry) GByte *pabyWKB = static_cast(VSI_MALLOC_VERBOSE(nWkbSize)); if (pabyWKB == nullptr) return 0; - if (poGeometry->exportToWkb(wkbNDR, pabyWKB, - (poDS->sPostGISVersion.nMajor < 2) - ? wkbVariantPostGIS1 - : wkbVariantOldOgc) != OGRERR_NONE) + if (poGeometry->exportToWkb(wkbNDR, pabyWKB, wkbVariantOldOgc) != + OGRERR_NONE) return 0; Oid oid = lo_creat(hPGConn, INV_READ | INV_WRITE); @@ -1937,14 +1923,11 @@ OGRErr OGRPGLayer::GetExtent(int iGeomField, OGREnvelope *psExtent, int bForce) OGRPGGeomFieldDefn *poGeomFieldDefn = poFeatureDefn->GetGeomFieldDefn(iGeomField); - const char *pszExtentFct = - poDS->sPostGISVersion.nMajor >= 2 ? "ST_Extent" : "Extent"; - if (TestCapability(OLCFastGetExtent)) { /* Do not take the spatial filter into account */ osCommand.Printf( - "SELECT %s(%s) FROM %s AS ogrpgextent", pszExtentFct, + "SELECT ST_Extent(%s) FROM %s AS ogrpgextent", OGRPGEscapeColumnName(poGeomFieldDefn->GetNameRef()).c_str(), GetFromClauseForGetExtent().c_str()); } @@ -1953,8 +1936,8 @@ OGRErr OGRPGLayer::GetExtent(int iGeomField, OGREnvelope *psExtent, int bForce) /* Probably not very efficient, but more efficient than client-side * implementation */ osCommand.Printf( - "SELECT %s(ST_GeomFromWKB(ST_AsBinary(%s))) FROM %s AS ogrpgextent", - pszExtentFct, + "SELECT ST_Extent(ST_GeomFromWKB(ST_AsBinary(%s))) FROM %s AS " + "ogrpgextent", OGRPGEscapeColumnName(poGeomFieldDefn->GetNameRef()).c_str(), GetFromClauseForGetExtent().c_str()); } @@ -2001,14 +1984,11 @@ OGRErr OGRPGLayer::GetExtent3D(int iGeomField, OGREnvelope3D *psExtent3D, OGRPGGeomFieldDefn *poGeomFieldDefn = poFeatureDefn->GetGeomFieldDefn(iGeomField); - const char *pszExtentFct = - poDS->sPostGISVersion.nMajor >= 2 ? "ST_3DExtent" : "ST_Extent3D"; - if (TestCapability(OLCFastGetExtent3D)) { /* Do not take the spatial filter into account */ osCommand.Printf( - "SELECT %s(%s) FROM %s AS ogrpgextent", pszExtentFct, + "SELECT ST_Extent(%s) FROM %s AS ogrpgextent", OGRPGEscapeColumnName(poGeomFieldDefn->GetNameRef()).c_str(), GetFromClauseForGetExtent().c_str()); } @@ -2017,8 +1997,8 @@ OGRErr OGRPGLayer::GetExtent3D(int iGeomField, OGREnvelope3D *psExtent3D, /* Probably not very efficient, but more efficient than client-side * implementation */ osCommand.Printf( - "SELECT %s(ST_GeomFromWKB(ST_AsBinary(%s))) FROM %s AS ogrpgextent", - pszExtentFct, + "SELECT ST_Extent(ST_GeomFromWKB(ST_AsBinary(%s))) FROM %s AS " + "ogrpgextent", OGRPGEscapeColumnName(poGeomFieldDefn->GetNameRef()).c_str(), GetFromClauseForGetExtent().c_str()); } @@ -2072,14 +2052,14 @@ OGRErr OGRPGLayer::RunGetExtentRequest(OGREnvelope &sExtent, strncpy(szVals, ptr, ptrEndParenthesis - ptr); szVals[ptrEndParenthesis - ptr] = '\0'; - char **papszTokens = CSLTokenizeString2(szVals, " ,", CSLT_HONOURSTRINGS); - int nTokenCnt = poDS->sPostGISVersion.nMajor >= 1 ? 4 : 6; + const CPLStringList aosTokens( + CSLTokenizeString2(szVals, " ,", CSLT_HONOURSTRINGS)); + constexpr int nTokenCnt = 4; - if (CSLCount(papszTokens) != nTokenCnt) + if (aosTokens.size() != nTokenCnt) { CPLError(CE_Failure, CPLE_IllegalArg, "Bad extent representation: '%s'", pszBox); - CSLDestroy(papszTokens); OGRPGClearResult(hResult); return OGRERR_FAILURE; @@ -2091,12 +2071,11 @@ OGRErr OGRPGLayer::RunGetExtentRequest(OGREnvelope &sExtent, // => X2 index calculated as nTokenCnt/2 // Y2 index calculated as nTokenCnt/2+1 - sExtent.MinX = CPLAtof(papszTokens[0]); - sExtent.MinY = CPLAtof(papszTokens[1]); - sExtent.MaxX = CPLAtof(papszTokens[nTokenCnt / 2]); - sExtent.MaxY = CPLAtof(papszTokens[nTokenCnt / 2 + 1]); + sExtent.MinX = CPLAtof(aosTokens[0]); + sExtent.MinY = CPLAtof(aosTokens[1]); + sExtent.MaxX = CPLAtof(aosTokens[nTokenCnt / 2]); + sExtent.MaxY = CPLAtof(aosTokens[nTokenCnt / 2 + 1]); - CSLDestroy(papszTokens); OGRPGClearResult(hResult); return OGRERR_NONE; diff --git a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp index 1795238d8245..925c01ddeda8 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp @@ -670,36 +670,18 @@ int OGRPGTableLayer::ReadTableDefinition() /* Identify the integer primary key. */ /* -------------------------------------------------------------------- */ - const char *pszTypnameEqualsAnyClause = - poDS->sPostgreSQLVersion.nMajor == 7 && - poDS->sPostgreSQLVersion.nMinor <= 3 - ? "ANY(SELECT '{int2, int4, int8, serial, bigserial}')" - : "ANY(ARRAY['int2','int4','int8','serial','bigserial'])"; - - const char *pszAttnumEqualAnyIndkey = - poDS->sPostgreSQLVersion.nMajor > 8 || - (poDS->sPostgreSQLVersion.nMajor == 8 && - poDS->sPostgreSQLVersion.nMinor >= 2) - ? "a.attnum = ANY(i.indkey)" - : "(i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR " - "i.indkey[2]=a.attnum " - "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR " - "i.indkey[5]=a.attnum " - "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR " - "i.indkey[8]=a.attnum " - "OR i.indkey[9]=a.attnum)"; - - /* See #1889 for why we don't use 'AND a.attnum = ANY(i.indkey)' */ osCommand.Printf( - "SELECT a.attname, a.attnum, t.typname, t.typname = %s AS isfid " + "SELECT a.attname, a.attnum, t.typname, " + "t.typname = ANY(ARRAY['int2','int4','int8','serial','bigserial']) AS " + "isfid " "FROM pg_attribute a " "JOIN pg_type t ON t.oid = a.atttypid " "JOIN pg_index i ON i.indrelid = a.attrelid " "WHERE a.attnum > 0 AND a.attrelid = %u " "AND i.indisprimary = 't' " "AND t.typname !~ '^geom' " - "AND %s ORDER BY a.attnum", - pszTypnameEqualsAnyClause, nTableOID, pszAttnumEqualAnyIndkey); + "AND a.attnum = ANY(i.indkey) ORDER BY a.attnum", + nTableOID); PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); @@ -1124,9 +1106,8 @@ void OGRPGTableLayer::BuildWhere() CPLsnprintf(szBox3D_2, sizeof(szBox3D_2), "%.18g %.18g", sEnvelope.MaxX, sEnvelope.MaxY); osWHERE.Printf( - "WHERE %s && %s('BOX3D(%s, %s)'::box3d,%d) ", + "WHERE %s && ST_SetSRID('BOX3D(%s, %s)'::box3d,%d) ", OGRPGEscapeColumnName(poGeomFieldDefn->GetNameRef()).c_str(), - (poDS->sPostGISVersion.nMajor >= 2) ? "ST_SetSRID" : "SetSRID", szBox3D_1, szBox3D_2, poGeomFieldDefn->nSRSId); } @@ -1269,45 +1250,22 @@ CPLString OGRPGTableLayer::BuildFields() if (poGeomFieldDefn->ePostgisType == GEOM_TYPE_GEOMETRY) { - if (poDS->sPostGISVersion.nMajor < 0 || poDS->bUseBinaryCursor) + if (!poDS->HavePostGIS() || poDS->bUseBinaryCursor) { osFieldList += osEscapedGeom; } else if (CPLTestBool(CPLGetConfigOption("PG_USE_BASE64", "NO"))) { - if (poDS->sPostGISVersion.nMajor >= 2) - osFieldList += "encode(ST_AsEWKB("; - else - osFieldList += "encode(AsEWKB("; + osFieldList += "encode(ST_AsEWKB("; osFieldList += osEscapedGeom; osFieldList += "), 'base64') AS "; osFieldList += OGRPGEscapeColumnName( CPLSPrintf("EWKBBase64_%s", poGeomFieldDefn->GetNameRef())); } - else if (poDS->sPostGISVersion.nMajor > 1 || - (poDS->sPostGISVersion.nMajor == 1 && - poDS->sPostGISVersion.nMinor >= 1)) - /* perhaps works also for older version, but I didn't check - */ - { - /* This will return EWKB in an hex encoded form */ - osFieldList += osEscapedGeom; - } - else if (poDS->sPostGISVersion.nMajor >= 1) - { - osFieldList += "AsEWKT("; - osFieldList += osEscapedGeom; - osFieldList += ") AS "; - osFieldList += OGRPGEscapeColumnName( - CPLSPrintf("AsEWKT_%s", poGeomFieldDefn->GetNameRef())); - } else { - osFieldList += "AsText("; + /* This will return EWKB in an hex encoded form */ osFieldList += osEscapedGeom; - osFieldList += ") AS "; - osFieldList += OGRPGEscapeColumnName( - CPLSPrintf("AsText_%s", poGeomFieldDefn->GetNameRef())); } } else if (poGeomFieldDefn->ePostgisType == GEOM_TYPE_GEOGRAPHY) @@ -2123,14 +2081,10 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) osCommand.Printf("INSERT INTO %s DEFAULT VALUES", pszSqlTableName); int bReturnRequested = FALSE; - /* RETURNING is only available since Postgres 8.2 */ /* We only get the FID, but we also could add the unset fields to get */ /* the default values */ if (bRetrieveFID && pszFIDColumn != nullptr && - poFeature->GetFID() == OGRNullFID && - (poDS->sPostgreSQLVersion.nMajor >= 9 || - (poDS->sPostgreSQLVersion.nMajor == 8 && - poDS->sPostgreSQLVersion.nMinor >= 2))) + poFeature->GetFID() == OGRNullFID) { bReturnRequested = TRUE; osCommand += " RETURNING "; @@ -3457,13 +3411,8 @@ void OGRPGTableLayer::ResolveSRID(const OGRPGGeomFieldDefn *poGFldDefn) if (nSRSId <= 0 && poGFldDefn->ePostgisType == GEOM_TYPE_GEOMETRY && poDS->sPostGISVersion.nMajor >= 0) { - const char *psGetSRIDFct = - poDS->sPostGISVersion.nMajor >= 2 ? "ST_SRID" : "getsrid"; - CPLString osGetSRID; - osGetSRID += "SELECT "; - osGetSRID += psGetSRIDFct; - osGetSRID += "("; + osGetSRID += "SELECT ST_SRID("; osGetSRID += OGRPGEscapeColumnName(poGFldDefn->GetNameRef()); osGetSRID += ") FROM "; osGetSRID += pszSqlTableName; @@ -3865,7 +3814,7 @@ OGRErr OGRPGTableLayer::RunDeferredCreationIfNecessary() { OGRPGGeomFieldDefn *poGeomField = poFeatureDefn->GetGeomFieldDefn(i); - if (poDS->sPostGISVersion.nMajor >= 2 || + if (poDS->HavePostGIS() || poGeomField->ePostgisType == GEOM_TYPE_GEOGRAPHY) { const char *pszGeometryType = @@ -3919,21 +3868,6 @@ OGRErr OGRPGTableLayer::RunDeferredCreationIfNecessary() } m_aosDeferredCommentOnColumns.clear(); - // For PostGIS 1.X, use AddGeometryColumn() to create geometry columns - if (poDS->sPostGISVersion.nMajor < 2) - { - for (int i = 0; i < poFeatureDefn->GetGeomFieldCount(); i++) - { - OGRPGGeomFieldDefn *poGeomField = - poFeatureDefn->GetGeomFieldDefn(i); - if (poGeomField->ePostgisType == GEOM_TYPE_GEOMETRY && - RunAddGeometryColumn(poGeomField) != OGRERR_NONE) - { - return OGRERR_FAILURE; - } - } - } - if (bCreateSpatialIndexFlag) { for (int i = 0; i < poFeatureDefn->GetGeomFieldCount(); i++) diff --git a/ogr/ogrsf_frmts/pmtiles/ogr_pmtiles.h b/ogr/ogrsf_frmts/pmtiles/ogr_pmtiles.h index 2e1394204d88..3b689bcc229c 100644 --- a/ogr/ogrsf_frmts/pmtiles/ogr_pmtiles.h +++ b/ogr/ogrsf_frmts/pmtiles/ogr_pmtiles.h @@ -202,7 +202,9 @@ class OGRPMTilesTileIterator int m_nCurY = -1; // for sanity checks. Must be increasing when walking through entries - uint64_t m_nLastTileId = 0; + static constexpr uint64_t INVALID_LAST_TILE_ID = + std::numeric_limits::max(); + uint64_t m_nLastTileId = INVALID_LAST_TILE_ID; // Computed values from zoom leven and min/max x/y uint64_t m_nMinTileId = std::numeric_limits::max(); diff --git a/ogr/ogrsf_frmts/pmtiles/ogrpmtilestileiterator.cpp b/ogr/ogrsf_frmts/pmtiles/ogrpmtilestileiterator.cpp index 674f685e6f62..25e40f4dedce 100644 --- a/ogr/ogrsf_frmts/pmtiles/ogrpmtilestileiterator.cpp +++ b/ogr/ogrsf_frmts/pmtiles/ogrpmtilestileiterator.cpp @@ -250,7 +250,7 @@ pmtiles::entry_zxy OGRPMTilesTileIterator::GetNextTile(uint32_t *pnRunLength) static_cast(m_nZoomLevel), m_nCurX, m_nCurY); m_nMaxTileId = m_nMinTileId; - m_nLastTileId = 0; + m_nLastTileId = INVALID_LAST_TILE_ID; while (m_aoStack.size() > 1) m_aoStack.pop(); const int nMinEntryIdx = find_tile_idx_lesser_or_equal( @@ -323,7 +323,8 @@ pmtiles::entry_zxy OGRPMTilesTileIterator::GetNextTile(uint32_t *pnRunLength) break; } - if (sContext.sEntries[0].tile_id <= m_nLastTileId) + if (m_nLastTileId != INVALID_LAST_TILE_ID && + sContext.sEntries[0].tile_id <= m_nLastTileId) { CPLError(CE_Failure, CPLE_AppDefined, "Non increasing tile_id"); diff --git a/ogr/ogrsf_frmts/shape/COPYING b/ogr/ogrsf_frmts/shape/COPYING deleted file mode 100644 index 0b643ac83c8b..000000000000 --- a/ogr/ogrsf_frmts/shape/COPYING +++ /dev/null @@ -1,483 +0,0 @@ - - GNU LIBRARY GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1991 Free Software Foundation, Inc. - 675 Mass Ave, Cambridge, MA 02139, USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the library GPL. It is - numbered 2 because it goes with version 2 of the ordinary GPL.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Library General Public License, applies to some -specially designated Free Software Foundation software, and to any -other libraries whose authors decide to use it. You can use it for -your libraries, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if -you distribute copies of the library, or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link a program with the library, you must provide -complete object files to the recipients so that they can relink them -with the library, after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - Our method of protecting your rights has two steps: (1) copyright -the library, and (2) offer you this license which gives you legal -permission to copy, distribute and/or modify the library. - - Also, for each distributor's protection, we want to make certain -that everyone understands that there is no warranty for this free -library. If the library is modified by someone else and passed on, we -want its recipients to know that what they have is not the original -version, so that any problems introduced by others will not reflect on -the original authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that companies distributing free -software will individually obtain patent licenses, thus in effect -transforming the program into proprietary software. To prevent this, -we have made it clear that any patent must be licensed for everyone's -free use or not licensed at all. - - Most GNU software, including some libraries, is covered by the ordinary -GNU General Public License, which was designed for utility programs. This -license, the GNU Library General Public License, applies to certain -designated libraries. This license is quite different from the ordinary -one; be sure to read it in full, and don't assume that anything in it is -the same as in the ordinary license. - - The reason we have a separate public license for some libraries is that -they blur the distinction we usually make between modifying or adding to a -program and simply using it. Linking a program with a library, without -changing the library, is in some sense simply using the library, and is -analogous to running a utility program or application program. However, in -a textual and legal sense, the linked executable is a combined work, a -derivative of the original library, and the ordinary General Public License -treats it as such. - - Because of this blurred distinction, using the ordinary General -Public License for libraries did not effectively promote software -sharing, because most developers did not use the libraries. We -concluded that weaker conditions might promote sharing better. - - However, unrestricted linking of non-free programs would deprive the -users of those programs of all benefit from the free status of the -libraries themselves. This Library General Public License is intended to -permit developers of non-free programs to use free libraries, while -preserving your freedom as a user of such programs to change the free -libraries that are incorporated in them. (We have not seen how to achieve -this as regards changes in header files, but we have achieved it as regards -changes in the actual functions of the Library.) The hope is that this -will lead to faster development of free libraries. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, while the latter only -works together with the library. - - Note that it is possible for a library to be covered by the ordinary -General Public License rather than by this special one. - - GNU LIBRARY GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library which -contains a notice placed by the copyright holder or other authorized -party saying it may be distributed under the terms of this Library -General Public License (also called "this License"). Each licensee is -addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also compile or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - c) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - d) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the source code distributed need not include anything that is normally -distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Library General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - Appendix: How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Library General Public - License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Library General Public License for more details. - - You should have received a copy of the GNU Library General Public - License along with this library; if not, write to the Free - Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! - diff --git a/ogr/ogrsf_frmts/shape/ogrshapedriver.cpp b/ogr/ogrsf_frmts/shape/ogrshapedriver.cpp index d7833892fde3..69d671b29750 100644 --- a/ogr/ogrsf_frmts/shape/ogrshapedriver.cpp +++ b/ogr/ogrsf_frmts/shape/ogrshapedriver.cpp @@ -50,19 +50,28 @@ static int OGRShapeDriverIdentify(GDALOpenInfo *poOpenInfo) if (!poOpenInfo->bStatOK) return FALSE; if (poOpenInfo->bIsDirectory) - return -1; // Unsure. + { + if (STARTS_WITH(poOpenInfo->pszFilename, "/vsizip/") && + (strstr(poOpenInfo->pszFilename, ".shp.zip") || + strstr(poOpenInfo->pszFilename, ".SHP.ZIP"))) + { + return TRUE; + } + + return GDAL_IDENTIFY_UNKNOWN; // Unsure. + } if (poOpenInfo->fpL == nullptr) { return FALSE; } - CPLString osExt(CPLGetExtension(poOpenInfo->pszFilename)); - if (EQUAL(osExt, "SHP") || EQUAL(osExt, "SHX")) + const std::string osExt(CPLGetExtension(poOpenInfo->pszFilename)); + if (EQUAL(osExt.c_str(), "SHP") || EQUAL(osExt.c_str(), "SHX")) { return poOpenInfo->nHeaderBytes >= 4 && (memcmp(poOpenInfo->pabyHeader, "\x00\x00\x27\x0A", 4) == 0 || memcmp(poOpenInfo->pabyHeader, "\x00\x00\x27\x0D", 4) == 0); } - if (EQUAL(osExt, "DBF")) + if (EQUAL(osExt.c_str(), "DBF")) { if (poOpenInfo->nHeaderBytes < 32) return FALSE; @@ -82,8 +91,8 @@ static int OGRShapeDriverIdentify(GDALOpenInfo *poOpenInfo) return FALSE; return TRUE; } - if (EQUAL(osExt, "shz") || - (EQUAL(osExt, "zip") && + if (EQUAL(osExt.c_str(), "shz") || + (EQUAL(osExt.c_str(), "zip") && (CPLString(poOpenInfo->pszFilename).endsWith(".shp.zip") || CPLString(poOpenInfo->pszFilename).endsWith(".SHP.ZIP")))) { @@ -95,7 +104,7 @@ static int OGRShapeDriverIdentify(GDALOpenInfo *poOpenInfo) if (!STARTS_WITH(poOpenInfo->pszFilename, "/vsitar/") && EQUAL(CPLGetFilename(poOpenInfo->pszFilename), ".cur_input")) { - return -1; + return GDAL_IDENTIFY_UNKNOWN; } #endif return FALSE; diff --git a/ogr/ogrsf_frmts/sqlite/CMakeLists.txt b/ogr/ogrsf_frmts/sqlite/CMakeLists.txt index d296fe5e15fb..14f9f6af311e 100644 --- a/ogr/ogrsf_frmts/sqlite/CMakeLists.txt +++ b/ogr/ogrsf_frmts/sqlite/CMakeLists.txt @@ -95,8 +95,7 @@ if (GDAL_USE_SPATIALITE) endif () # Test sqlite3 extension -find_path(SQLITE3EXT_INCLUDE_DIR NAMES sqlite3ext.h) -if (SQLITE3EXT_INCLUDE_DIR) +if (HAVE_SQLITE3EXT_H AND BUILD_SHARED_LIBS) add_library(my_test_sqlite3_ext MODULE my_test_sqlite3_ext.c) gdal_standard_includes(my_test_sqlite3_ext) get_target_property(PLUGIN_OUTPUT_DIR ${GDAL_LIB_TARGET_NAME} PLUGIN_OUTPUT_DIR) diff --git a/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h b/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h index f933afb954e4..da32aa6a63f8 100644 --- a/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h +++ b/ogr/ogrsf_frmts/sqlite/ogr_sqlite.h @@ -641,11 +641,14 @@ class OGRSQLiteDataSource final : public OGRSQLiteBaseDataSource // We maintain a list of known SRID to reduce the number of trips to // the database to get SRSes. - int m_nKnownSRID = 0; - int *m_panSRID = nullptr; - OGRSpatialReference **m_papoSRS = nullptr; + std::map> + m_oSRSCache{}; - void AddSRIDToCache(int nId, OGRSpatialReference *poSRS); + OGRSpatialReference *AddSRIDToCache( + int nId, + std::unique_ptr + &&poSRS); bool m_bHaveGeometryColumns = false; bool m_bIsSpatiaLiteDB = false; @@ -789,12 +792,6 @@ class OGRSQLiteDataSource final : public OGRSQLiteBaseDataSource return m_bHaveGeometryColumns; } - std::vector - GetRelationshipNames(CSLConstList papszOptions = nullptr) const override; - - const GDALRelationship * - GetRelationship(const std::string &name) const override; - bool AddRelationship(std::unique_ptr &&relationship, std::string &failureReason) override; bool ValidateRelationship(const GDALRelationship *poRelationship, diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h b/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h index 0be1906e8ccf..8031fd796fbd 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitebase.h @@ -216,7 +216,13 @@ class OGRSQLiteBaseDataSource CPL_NON_FINAL : public GDALPamDataset OGRErr PragmaCheck(const char *pszPragma, const char *pszExpected, int nRowsExpected); - void LoadRelationshipsFromForeignKeys() const; + virtual void LoadRelationships() const; + void LoadRelationshipsFromForeignKeys( + const std::vector &excludedTables) const; + std::vector + GetRelationshipNames(CSLConstList papszOptions = nullptr) const override; + const GDALRelationship * + GetRelationship(const std::string &name) const override; bool IsSpatialiteLoaded(); static int MakeSpatialiteVersionNumber(int x, int y, int z) diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp index 50a56f5ad9cf..bddea3f46272 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp @@ -275,45 +275,6 @@ int OGRSQLiteBaseDataSource::GetSpatialiteVersionNumber() return v; } -/************************************************************************/ -/* GetRelationshipNames() */ -/************************************************************************/ - -std::vector OGRSQLiteDataSource::GetRelationshipNames( - CPL_UNUSED CSLConstList papszOptions) const - -{ - if (!m_bHasPopulatedRelationships) - LoadRelationshipsFromForeignKeys(); - - std::vector oasNames; - oasNames.reserve(m_osMapRelationships.size()); - for (auto it = m_osMapRelationships.begin(); - it != m_osMapRelationships.end(); ++it) - { - oasNames.emplace_back(it->first); - } - return oasNames; -} - -/************************************************************************/ -/* GetRelationship() */ -/************************************************************************/ - -const GDALRelationship * -OGRSQLiteDataSource::GetRelationship(const std::string &name) const - -{ - if (!m_bHasPopulatedRelationships) - LoadRelationshipsFromForeignKeys(); - - auto it = m_osMapRelationships.find(name); - if (it == m_osMapRelationships.end()) - return nullptr; - - return it->second.get(); -} - /************************************************************************/ /* AddRelationship() */ /************************************************************************/ @@ -704,18 +665,28 @@ OGRErr OGRSQLiteBaseDataSource::PragmaCheck(const char *pszPragma, } /************************************************************************/ -/* LoadRelationshipsFromForeignKeys() */ +/* LoadRelationships() */ /************************************************************************/ -void OGRSQLiteBaseDataSource::LoadRelationshipsFromForeignKeys() const +void OGRSQLiteBaseDataSource::LoadRelationships() const { m_osMapRelationships.clear(); + LoadRelationshipsFromForeignKeys({}); + m_bHasPopulatedRelationships = true; +} + +/************************************************************************/ +/* LoadRelationshipsFromForeignKeys() */ +/************************************************************************/ + +void OGRSQLiteBaseDataSource::LoadRelationshipsFromForeignKeys( + const std::vector &excludedTables) const +{ if (hDB) { - auto oResult = SQLQuery( - hDB, + std::string osSQL = "SELECT m.name, p.id, p.seq, p.\"table\" AS base_table_name, " "p.\"from\", p.\"to\", " "p.on_delete FROM sqlite_master m " @@ -728,8 +699,27 @@ void OGRSQLiteBaseDataSource::LoadRelationshipsFromForeignKeys() const // Same with Spatialite system tables "AND base_table_name NOT IN ('geometry_columns', " "'spatial_ref_sys', 'views_geometry_columns', " - "'virts_geometry_columns') " - "ORDER BY m.name"); + "'virts_geometry_columns') "; + if (!excludedTables.empty()) + { + std::string oExcludedTablesList; + for (const auto &osExcludedTable : excludedTables) + { + oExcludedTablesList += !oExcludedTablesList.empty() ? "," : ""; + char *pszEscapedName = + sqlite3_mprintf("'%q'", osExcludedTable.c_str()); + oExcludedTablesList += pszEscapedName; + sqlite3_free(pszEscapedName); + } + + osSQL += "AND base_table_name NOT IN (" + oExcludedTablesList + + ")" + " AND m.name NOT IN (" + + oExcludedTablesList + ") "; + } + osSQL += "ORDER BY m.name"; + + auto oResult = SQLQuery(hDB, osSQL.c_str()); if (!oResult) { @@ -806,9 +796,50 @@ void OGRSQLiteBaseDataSource::LoadRelationshipsFromForeignKeys() const std::move(poRelationship); } } + } +} + +/************************************************************************/ +/* GetRelationshipNames() */ +/************************************************************************/ + +std::vector OGRSQLiteBaseDataSource::GetRelationshipNames( + CPL_UNUSED CSLConstList papszOptions) const - m_bHasPopulatedRelationships = true; +{ + if (!m_bHasPopulatedRelationships) + { + LoadRelationships(); + } + + std::vector oasNames; + oasNames.reserve(m_osMapRelationships.size()); + for (auto it = m_osMapRelationships.begin(); + it != m_osMapRelationships.end(); ++it) + { + oasNames.emplace_back(it->first); } + return oasNames; +} + +/************************************************************************/ +/* GetRelationship() */ +/************************************************************************/ + +const GDALRelationship * +OGRSQLiteBaseDataSource::GetRelationship(const std::string &name) const + +{ + if (!m_bHasPopulatedRelationships) + { + LoadRelationships(); + } + + auto it = m_osMapRelationships.find(name); + if (it == m_osMapRelationships.end()) + return nullptr; + + return it->second.get(); } /***********************************************************************/ @@ -910,13 +941,7 @@ CPLErr OGRSQLiteDataSource::Close() CPLFree(m_papoLayers); - for (int i = 0; i < m_nKnownSRID; i++) - { - if (m_papoSRS[i] != nullptr) - m_papoSRS[i]->Release(); - } - CPLFree(m_panSRID); - CPLFree(m_papoSRS); + m_oSRSCache.clear(); if (!CloseDB()) eErr = CE_Failure; @@ -4102,17 +4127,15 @@ const char *OGRSQLiteDataSource::GetSRTEXTColName() /* sure it is freshly created, or add a reference yourself if not. */ /************************************************************************/ -void OGRSQLiteDataSource::AddSRIDToCache(int nId, OGRSpatialReference *poSRS) +OGRSpatialReference *OGRSQLiteDataSource::AddSRIDToCache( + int nId, + std::unique_ptr &&poSRS) { /* -------------------------------------------------------------------- */ /* Add to the cache. */ /* -------------------------------------------------------------------- */ - m_panSRID = (int *)CPLRealloc(m_panSRID, sizeof(int) * (m_nKnownSRID + 1)); - m_papoSRS = (OGRSpatialReference **)CPLRealloc( - m_papoSRS, sizeof(void *) * (m_nKnownSRID + 1)); - m_panSRID[m_nKnownSRID] = nId; - m_papoSRS[m_nKnownSRID] = poSRS; - m_nKnownSRID++; + auto oIter = m_oSRSCache.emplace(nId, std::move(poSRS)).first; + return oIter->second.get(); } /************************************************************************/ @@ -4132,15 +4155,15 @@ int OGRSQLiteDataSource::FetchSRSId(const OGRSpatialReference *poSRS) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - for (int i = 0; i < m_nKnownSRID; i++) + for (const auto &pair : m_oSRSCache) { - if (m_papoSRS[i] == poSRS) - return m_panSRID[i]; + if (pair.second.get() == poSRS) + return pair.first; } - for (int i = 0; i < m_nKnownSRID; i++) + for (const auto &pair : m_oSRSCache) { - if (m_papoSRS[i] != nullptr && m_papoSRS[i]->IsSame(poSRS)) - return m_panSRID[i]; + if (pair.second != nullptr && pair.second->IsSame(poSRS)) + return pair.first; } /* -------------------------------------------------------------------- */ @@ -4251,10 +4274,12 @@ int OGRSQLiteDataSource::FetchSRSId(const OGRSpatialReference *poSRS) if (nSRSId != m_nUndefinedSRID) { - auto poCachedSRS = new OGRSpatialReference(oSRS); + std::unique_ptr + poCachedSRS(new OGRSpatialReference(oSRS)); poCachedSRS->SetAxisMappingStrategy( OAMS_TRADITIONAL_GIS_ORDER); - AddSRIDToCache(nSRSId, poCachedSRS); + AddSRIDToCache(nSRSId, std::move(poCachedSRS)); } return nSRSId; @@ -4352,7 +4377,12 @@ int OGRSQLiteDataSource::FetchSRSId(const OGRSpatialReference *poSRS) sqlite3_finalize(hSelectStmt); if (nSRSId != m_nUndefinedSRID) - AddSRIDToCache(nSRSId, new OGRSpatialReference(oSRS)); + { + auto poSRSClone = std::unique_ptr( + new OGRSpatialReference(oSRS)); + AddSRIDToCache(nSRSId, std::move(poSRSClone)); + } return nSRSId; } @@ -4584,9 +4614,10 @@ int OGRSQLiteDataSource::FetchSRSId(const OGRSpatialReference *poSRS) if (nSRSId != m_nUndefinedSRID) { - auto poCachedSRS = new OGRSpatialReference(std::move(oSRS)); + std::unique_ptr + poCachedSRS(new OGRSpatialReference(std::move(oSRS))); poCachedSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - AddSRIDToCache(nSRSId, poCachedSRS); + AddSRIDToCache(nSRSId, std::move(poCachedSRS)); } return nSRSId; @@ -4609,10 +4640,10 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* First, we look through our SRID cache, is it there? */ /* -------------------------------------------------------------------- */ - for (int i = 0; i < m_nKnownSRID; i++) + auto oIter = m_oSRSCache.find(nId); + if (oIter != m_oSRSCache.end()) { - if (m_panSRID[i] == nId) - return m_papoSRS[i]; + return oIter->second.get(); } /* -------------------------------------------------------------------- */ @@ -4622,7 +4653,7 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) char **papszResult = nullptr; int nRowCount = 0; int nColCount = 0; - OGRSpatialReference *poSRS = nullptr; + std::unique_ptr poSRS; CPLString osCommand; osCommand.Printf("SELECT srtext FROM spatial_ref_sys WHERE srid = %d " @@ -4649,12 +4680,11 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) /* Translate into a spatial reference. */ /* -------------------------------------------------------------------- */ - poSRS = new OGRSpatialReference(); + poSRS.reset(new OGRSpatialReference()); poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); if (poSRS->importFromWkt(osWKT.c_str()) != OGRERR_NONE) { - delete poSRS; - poSRS = nullptr; + poSRS.reset(); } } @@ -4706,7 +4736,7 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) const char *pszWKT = (pszSRTEXTColName != nullptr) ? papszRow[3] : nullptr; - poSRS = new OGRSpatialReference(); + poSRS.reset(new OGRSpatialReference()); poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); /* Try first from EPSG code */ @@ -4729,8 +4759,7 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) } else { - delete poSRS; - poSRS = nullptr; + poSRS.reset(); } sqlite3_free_table(papszResult); @@ -4756,9 +4785,7 @@ OGRSpatialReference *OGRSQLiteDataSource::FetchSRS(int nId) /* -------------------------------------------------------------------- */ /* Add to the cache. */ /* -------------------------------------------------------------------- */ - AddSRIDToCache(nId, poSRS); - - return poSRS; + return AddSRIDToCache(nId, std::move(poSRS)); } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp b/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp index c485582f1a2f..9f730184bf1f 100644 --- a/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp +++ b/ogr/ogrsf_frmts/xlsx/ogrxlsxdatasource.cpp @@ -52,6 +52,7 @@ OGRXLSXLayer::OGRXLSXLayer(OGRXLSXDataSource *poDSIn, const char *pszFilename, poDS(poDSIn), osFilename(pszFilename), bUpdated(CPL_TO_BOOL(bUpdatedIn)), bHasHeaderLine(false) { + SetAdvertizeUTF8(true); } /************************************************************************/ @@ -822,7 +823,6 @@ void OGRXLSXDataSource::endElementTable(CPL_UNUSED const char *pszNameIn) if (poCurLayer) { ((OGRMemLayer *)poCurLayer)->SetUpdatable(CPL_TO_BOOL(bUpdatable)); - ((OGRMemLayer *)poCurLayer)->SetAdvertizeUTF8(true); ((OGRXLSXLayer *)poCurLayer)->SetUpdated(false); } diff --git a/port/cpl_conv.cpp b/port/cpl_conv.cpp index 99f17ed953c2..3491cd6386a5 100644 --- a/port/cpl_conv.cpp +++ b/port/cpl_conv.cpp @@ -83,6 +83,10 @@ #endif #include +#if __cplusplus >= 202002L +#include // For std::endian +#endif + #include "cpl_config.h" #include "cpl_multiproc.h" #include "cpl_string.h" @@ -186,8 +190,6 @@ void *CPLMalloc(size_t nSize) if (nSize == 0) return nullptr; - CPLVerifyConfiguration(); - if ((nSize >> (8 * sizeof(nSize) - 1)) != 0) { // coverity[dead_error_begin] @@ -1564,33 +1566,36 @@ int CPLPrintTime(char *pszBuffer, int nMaxLen, const char *pszFormat, void CPLVerifyConfiguration() { - static bool verified = false; - if (verified) - { - return; - } - verified = true; - /* -------------------------------------------------------------------- */ /* Verify data types. */ /* -------------------------------------------------------------------- */ - CPL_STATIC_ASSERT(sizeof(GInt32) == 4); - CPL_STATIC_ASSERT(sizeof(GInt16) == 2); - CPL_STATIC_ASSERT(sizeof(GByte) == 1); + static_assert(sizeof(short) == 2); // We unfortunately rely on this + static_assert(sizeof(int) == 4); // We unfortunately rely on this + static_assert(sizeof(float) == 4); // We unfortunately rely on this + static_assert(sizeof(double) == 8); // We unfortunately rely on this + static_assert(sizeof(GInt64) == 8); + static_assert(sizeof(GInt32) == 4); + static_assert(sizeof(GInt16) == 2); + static_assert(sizeof(GByte) == 1); /* -------------------------------------------------------------------- */ /* Verify byte order */ /* -------------------------------------------------------------------- */ - GInt32 nTest = 1; - #ifdef CPL_LSB - if (reinterpret_cast(&nTest)[0] != 1) +#if __cplusplus >= 202002L + static_assert(std::endian::native == std::endian::little); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) + static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__); #endif -#ifdef CPL_MSB - if (reinterpret_cast(&nTest)[3] != 1) +#elif defined(CPL_MSB) +#if __cplusplus >= 202002L + static_assert(std::endian::native == std::endian::big); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + static_assert(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__); +#endif +#else +#error "CPL_LSB or CPL_MSB must be defined" #endif - CPLError(CE_Fatal, CPLE_AppDefined, - "CPLVerifyConfiguration(): byte order set wrong."); } #ifdef DEBUG_CONFIG_OPTIONS diff --git a/port/cpl_error.cpp b/port/cpl_error.cpp index 20536f0cfd6a..a08d55f2a181 100644 --- a/port/cpl_error.cpp +++ b/port/cpl_error.cpp @@ -879,8 +879,8 @@ CPLErr CPL_STDCALL CPLGetLastErrorType() * been cleared by CPLErrorReset(). The returned pointer is to an internal * string that should not be altered or freed. * - * @return the last error message, or NULL if there is no posted error - * message. + * @return the last error message, or an empty string ("") if there is no + * posted error message. */ const char *CPL_STDCALL CPLGetLastErrorMsg() diff --git a/port/cpl_minixml.cpp b/port/cpl_minixml.cpp index bb623779619e..d4690a4e0fbf 100644 --- a/port/cpl_minixml.cpp +++ b/port/cpl_minixml.cpp @@ -2262,6 +2262,41 @@ void CPLCleanXMLElementName(char *pszTarget) } } +/************************************************************************/ +/* CPLXMLNodeGetRAMUsageEstimate() */ +/************************************************************************/ + +static size_t CPLXMLNodeGetRAMUsageEstimate(const CPLXMLNode *psNode, + bool bVisitSiblings) +{ + size_t nRet = sizeof(CPLXMLNode); + // malloc() aligns on 16-byte boundaries on 64 bit. + nRet += std::max(2 * sizeof(void *), strlen(psNode->pszValue) + 1); + if (bVisitSiblings) + { + for (const CPLXMLNode *psIter = psNode->psNext; psIter; + psIter = psIter->psNext) + { + nRet += CPLXMLNodeGetRAMUsageEstimate(psIter, false); + } + } + if (psNode->psChild) + { + nRet += CPLXMLNodeGetRAMUsageEstimate(psNode->psChild, true); + } + return nRet; +} + +/** Return a conservative estimate of the RAM usage of this node, its children + * and siblings. The returned values is in bytes. + * + * @since 3.9 + */ +size_t CPLXMLNodeGetRAMUsageEstimate(const CPLXMLNode *psNode) +{ + return CPLXMLNodeGetRAMUsageEstimate(psNode, true); +} + /************************************************************************/ /* CPLXMLTreeCloser::getDocumentElement() */ /************************************************************************/ diff --git a/port/cpl_minixml.h b/port/cpl_minixml.h index d0b57194a13e..d0cdcffdcd26 100644 --- a/port/cpl_minixml.h +++ b/port/cpl_minixml.h @@ -177,6 +177,8 @@ CPLXMLNode CPL_DLL *CPLParseXMLFile(const char *pszFilename); int CPL_DLL CPLSerializeXMLTreeToFile(const CPLXMLNode *psTree, const char *pszFilename); +size_t CPL_DLL CPLXMLNodeGetRAMUsageEstimate(const CPLXMLNode *psNode); + CPL_C_END #if defined(__cplusplus) && !defined(CPL_SUPRESS_CPLUSPLUS) diff --git a/port/cpl_multiproc.cpp b/port/cpl_multiproc.cpp index 2658ac1aa5de..65384486c29b 100644 --- a/port/cpl_multiproc.cpp +++ b/port/cpl_multiproc.cpp @@ -358,16 +358,16 @@ int CPLCreateOrAcquireMutexEx(CPLMutex **phMutex, double dfWaitInSeconds, /************************************************************************/ #ifdef MUTEX_NONE -static int CPLCreateOrAcquireMutexInternal(CPLLock **phLock, - double dfWaitInSeconds, - CPLLockType eType) +static bool CPLCreateOrAcquireMutexInternal(CPLLock **phLock, + double dfWaitInSeconds, + CPLLockType eType) { return false; } #else -static int CPLCreateOrAcquireMutexInternal(CPLLock **phLock, - double dfWaitInSeconds, - CPLLockType eType) +static bool CPLCreateOrAcquireMutexInternal(CPLLock **phLock, + double dfWaitInSeconds, + CPLLockType eType) { bool bSuccess = false; @@ -1462,35 +1462,31 @@ int CPLCreateOrAcquireMutexEx(CPLMutex **phMutex, double dfWaitInSeconds, int nOptions) { - bool bSuccess = false; - pthread_mutex_lock(&global_mutex); if (*phMutex == nullptr) { *phMutex = CPLCreateMutexInternal(true, nOptions); - bSuccess = *phMutex != nullptr; + const bool bSuccess = *phMutex != nullptr; pthread_mutex_unlock(&global_mutex); + if (!bSuccess) + return false; } else { pthread_mutex_unlock(&global_mutex); - - bSuccess = CPL_TO_BOOL(CPLAcquireMutex(*phMutex, dfWaitInSeconds)); } - return bSuccess; + return CPL_TO_BOOL(CPLAcquireMutex(*phMutex, dfWaitInSeconds)); } /************************************************************************/ /* CPLCreateOrAcquireMutexInternal() */ /************************************************************************/ -static int CPLCreateOrAcquireMutexInternal(CPLLock **phLock, - double dfWaitInSeconds, - CPLLockType eType) +static bool CPLCreateOrAcquireMutexInternal(CPLLock **phLock, + double dfWaitInSeconds, + CPLLockType eType) { - bool bSuccess = false; - pthread_mutex_lock(&global_mutex); if (*phLock == nullptr) { @@ -1507,18 +1503,17 @@ static int CPLCreateOrAcquireMutexInternal(CPLLock **phLock, *phLock = nullptr; } } - bSuccess = *phLock != nullptr; + const bool bSuccess = *phLock != nullptr; pthread_mutex_unlock(&global_mutex); + if (!bSuccess) + return false; } else { pthread_mutex_unlock(&global_mutex); - - bSuccess = - CPL_TO_BOOL(CPLAcquireMutex((*phLock)->u.hMutex, dfWaitInSeconds)); } - return bSuccess; + return CPL_TO_BOOL(CPLAcquireMutex((*phLock)->u.hMutex, dfWaitInSeconds)); } /************************************************************************/ @@ -1624,20 +1619,23 @@ static CPLMutex *CPLCreateMutexInternal(bool bAlreadyInGlobalLock, int nOptions) psItem->nOptions = nOptions; CPLInitMutex(psItem); - // Mutexes are implicitly acquired when created. - CPLAcquireMutex(reinterpret_cast(psItem), 0.0); - return reinterpret_cast(psItem); } CPLMutex *CPLCreateMutex() { - return CPLCreateMutexInternal(false, CPL_MUTEX_RECURSIVE); + CPLMutex *mutex = CPLCreateMutexInternal(false, CPL_MUTEX_RECURSIVE); + if (mutex) + CPLAcquireMutex(mutex, 0); + return mutex; } CPLMutex *CPLCreateMutexEx(int nOptions) { - return CPLCreateMutexInternal(false, nOptions); + CPLMutex *mutex = CPLCreateMutexInternal(false, nOptions); + if (mutex) + CPLAcquireMutex(mutex, 0); + return mutex; } /************************************************************************/ diff --git a/port/cpl_path.cpp b/port/cpl_path.cpp index 9c298967abd8..ec5efeaca505 100644 --- a/port/cpl_path.cpp +++ b/port/cpl_path.cpp @@ -53,12 +53,6 @@ constexpr int CPL_PATH_BUF_SIZE = 2048; constexpr int CPL_PATH_BUF_COUNT = 10; -#if defined(_WIN32) -constexpr char SEP_STRING[] = "\\"; -#else -constexpr char SEP_STRING[] = "/"; -#endif - static const char *CPLStaticBufferTooSmall(char *pszStaticResult) { CPLError(CE_Failure, CPLE_AppDefined, "Destination buffer too small"); @@ -486,36 +480,6 @@ const char *CPLResetExtension(const char *pszPath, const char *pszExt) return pszStaticResult; } -/************************************************************************/ -/* RequiresUnixPathSeparator() */ -/************************************************************************/ - -#if defined(_WIN32) -static bool RequiresUnixPathSeparator(const char *pszPath) -{ - return strcmp(pszPath, "/vsimem") == 0 || STARTS_WITH(pszPath, "http://") || - STARTS_WITH(pszPath, "https://") || - STARTS_WITH(pszPath, "/vsimem/") || - STARTS_WITH(pszPath, "/vsicurl/") || - STARTS_WITH(pszPath, "/vsicurl_streaming/") || - STARTS_WITH(pszPath, "/vsis3/") || - STARTS_WITH(pszPath, "/vsis3_streaming/") || - STARTS_WITH(pszPath, "/vsigs/") || - STARTS_WITH(pszPath, "/vsigs_streaming/") || - STARTS_WITH(pszPath, "/vsiaz/") || - STARTS_WITH(pszPath, "/vsiaz_streaming/") || - STARTS_WITH(pszPath, "/vsiadls/") || - STARTS_WITH(pszPath, "/vsioss/") || - STARTS_WITH(pszPath, "/vsioss_streaming/") || - STARTS_WITH(pszPath, "/vsiswift/") || - STARTS_WITH(pszPath, "/vsiswift_streaming/") || - STARTS_WITH(pszPath, "/vsihdfs/") || - STARTS_WITH(pszPath, "/vsiwebhdfs/") || - STARTS_WITH(pszPath, "/vsizip/") || - STARTS_WITH(pszPath, "/vsi7z/") || STARTS_WITH(pszPath, "/vsirar/"); -} -#endif - /************************************************************************/ /* CPLFormFilename() */ /************************************************************************/ @@ -596,22 +560,13 @@ const char *CPLFormFilename(const char *pszPath, const char *pszBasename, else { nLenPath = nLenPathOri; - pszAddedPathSep = SEP_STRING; + pszAddedPathSep = VSIGetDirectorySeparator(pszPath); } } else if (nLenPath > 0 && pszPath[nLenPath - 1] != '/' && pszPath[nLenPath - 1] != '\\') { -#if defined(_WIN32) - // FIXME? Would be better to ask the filesystems what it - // prefers as directory separator? - if (RequiresUnixPathSeparator(pszPath)) - pszAddedPathSep = "/"; - else -#endif - { - pszAddedPathSep = SEP_STRING; - } + pszAddedPathSep = VSIGetDirectorySeparator(pszPath); } if (pszExtension == nullptr) @@ -785,17 +740,7 @@ const char *CPLProjectRelativeFilename(const char *pszProjectDir, if (pszProjectDir[strlen(pszProjectDir) - 1] != '/' && pszProjectDir[strlen(pszProjectDir) - 1] != '\\') { - // FIXME: Better to ask the filesystems what it - // prefers as directory separator? - const char *pszAddedPathSep = nullptr; -#if defined(_WIN32) - if (RequiresUnixPathSeparator(pszStaticResult)) - pszAddedPathSep = "/"; - else -#endif - { - pszAddedPathSep = SEP_STRING; - } + const char *pszAddedPathSep = VSIGetDirectorySeparator(pszProjectDir); if (CPLStrlcat(pszStaticResult, pszAddedPathSep, CPL_PATH_BUF_SIZE) >= static_cast(CPL_PATH_BUF_SIZE)) return CPLStaticBufferTooSmall(pszStaticResult); diff --git a/port/cpl_vsi.h b/port/cpl_vsi.h index e14b3cca37e0..cb0c293caf0c 100644 --- a/port/cpl_vsi.h +++ b/port/cpl_vsi.h @@ -380,6 +380,8 @@ char CPL_DLL **VSIReadDirRecursive(const char *pszPath); char CPL_DLL **VSIReadDirEx(const char *pszPath, int nMaxFiles); char CPL_DLL **VSISiblingFiles(const char *pszPath); +const char CPL_DLL *VSIGetDirectorySeparator(const char *pszPath); + /** Opaque type for a directory iterator */ typedef struct VSIDIR VSIDIR; diff --git a/port/cpl_vsi_virtual.h b/port/cpl_vsi_virtual.h index cb25db618305..847b270cb602 100644 --- a/port/cpl_vsi_virtual.h +++ b/port/cpl_vsi_virtual.h @@ -305,6 +305,17 @@ class CPL_DLL VSIFilesystemHandler "Duplicate() not supported on this file system"); return nullptr; } + + /** Return the directory separator. + * + * Default is forward slash. The only exception currently is the Windows + * file system which returns anti-slash, unless the specified path is of the + * form "{drive_letter}:/{rest_of_the_path}". + */ + virtual const char *GetDirectorySeparator(CPL_UNUSED const char *pszPath) + { + return "/"; + } }; #endif /* #ifndef DOXYGEN_SKIP */ diff --git a/port/cpl_vsil.cpp b/port/cpl_vsil.cpp index 1034583cdd28..ef01860e6656 100644 --- a/port/cpl_vsil.cpp +++ b/port/cpl_vsil.cpp @@ -155,6 +155,27 @@ char **VSISiblingFiles(const char *pszFilename) return poFSHandler->SiblingFiles(pszFilename); } +/************************************************************************/ +/* VSIGetDirectorySeparator() */ +/************************************************************************/ + +/** Return the directory separator for the specified path. + * + * Default is forward slash. The only exception currently is the Windows + * file system which returns anti-slash, unless the specified path is of the + * form "{drive_letter}:/{rest_of_the_path}". + * + * @since 3.9 + */ +const char *VSIGetDirectorySeparator(const char *pszPath) +{ + if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://")) + return "/"; + + VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath); + return poFSHandler->GetDirectorySeparator(pszPath); +} + /************************************************************************/ /* VSIReadRecursive() */ /************************************************************************/ @@ -171,6 +192,11 @@ char **VSISiblingFiles(const char *pszFilename) * Note that no error is issued via CPLError() if the directory path is * invalid, though NULL is returned. * + * Note: since GDAL 3.9, for recursive mode, the directory separator will no + * longer be always forward slash, but will be the one returned by + * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows + * file systems. + * * @param pszPathIn the relative, or absolute path of a directory to read. * UTF-8 encoded. * @@ -183,11 +209,7 @@ char **VSISiblingFiles(const char *pszFilename) char **VSIReadDirRecursive(const char *pszPathIn) { -#if defined(_WIN32) - const char SEP = pszPathIn[0] == '\\' ? '\\' : '/'; -#else - constexpr char SEP = '/'; -#endif + const char SEP = VSIGetDirectorySeparator(pszPathIn)[0]; const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr}; VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions); @@ -295,6 +317,11 @@ VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth, * The returned entry remains valid until the next call to VSINextDirEntry() * or VSICloseDir() with the same handle. * + * Note: since GDAL 3.9, for recursive mode, the directory separator will no + * longer be always forward slash, but will be the one returned by + * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows + * file systems. + * * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL. * * @return a entry, or NULL if there is no more entry in the directory. This @@ -1375,11 +1402,7 @@ bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget, GDALProgressFunc pProgressFunc, void *pProgressData, char ***ppapszOutputs) { -#if defined(_WIN32) - const char SOURCE_SEP = pszSource[0] == '\\' ? '\\' : '/'; -#else - constexpr char SOURCE_SEP = '/'; -#endif + const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0]; if (ppapszOutputs) { @@ -1664,11 +1687,7 @@ VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth, const VSIDIREntry *VSIDIRGeneric::NextDirEntry() { -#if defined(_WIN32) - const char SEP = osRootPath[0] == '\\' ? '\\' : '/'; -#else - constexpr char SEP = '/'; -#endif + const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0]; begin: if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0) @@ -1816,11 +1835,7 @@ int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname) osDirnameWithoutEndSlash.resize(osDirnameWithoutEndSlash.size() - 1); } -#if defined(_WIN32) - const char SEP = pszDirname[0] == '\\' ? '\\' : '/'; -#else - constexpr char SEP = '/'; -#endif + const char SEP = VSIGetDirectorySeparator(pszDirname)[0]; CPLStringList aosOptions; auto poDir = diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index 9063da67925e..122b46bccb34 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -310,6 +310,19 @@ static std::string VSICurlGetURLFromFilename( if (!STARTS_WITH(pszFilename, "/vsicurl/") && !STARTS_WITH(pszFilename, "/vsicurl?")) return pszFilename; + + if (pbPlanetaryComputerURLSigning) + { + // It may be more convenient sometimes to store Planetary Computer URL + // signing as a per-path specific option rather than capturing it in + // the filename with the &pc_url_signing=yes option. + if (CPLTestBool(VSIGetPathSpecificOption( + pszFilename, "VSICURL_PC_URL_SIGNING", "FALSE"))) + { + *pbPlanetaryComputerURLSigning = true; + } + } + pszFilename += strlen("/vsicurl/"); if (!STARTS_WITH(pszFilename, "http://") && !STARTS_WITH(pszFilename, "https://") && @@ -1974,6 +1987,22 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", static_cast(response_code), szCurlErrBuf); } + else if (response_code == 416) /* Range Not Satisfiable */ + { + if (sWriteFuncData.pBuffer) + { + CPLError( + CE_Failure, CPLE_AppDefined, + "%d: Range downloading not supported by this server: %s", + static_cast(response_code), sWriteFuncData.pBuffer); + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, + "%d: Range downloading not supported by this server", + static_cast(response_code)); + } + } if (!oFileProp.bHasComputedFileSize && startOffset == 0) { oFileProp.bHasComputedFileSize = true; diff --git a/port/cpl_vsil_s3.cpp b/port/cpl_vsil_s3.cpp index ae7381360b57..2880714b27d0 100644 --- a/port/cpl_vsil_s3.cpp +++ b/port/cpl_vsil_s3.cpp @@ -3963,7 +3963,8 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, struct ChunkToCopy { - std::string osFilename{}; + std::string osSrcFilename{}; + std::string osDstFilename{}; GIntBig nMTime = 0; std::string osETag{}; vsi_l_offset nTotalSize = 0; @@ -4078,6 +4079,14 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, std::string osTargetDir; // set in the VSI_ISDIR(sSource.st_mode) case std::string osTarget; // set in the !(VSI_ISDIR(sSource.st_mode)) case + const auto NormalizeDirSeparatorForDstFilename = + [&osSource, &osTargetDir](const std::string &s) -> std::string + { + return CPLString(s).replaceAll( + VSIGetDirectorySeparator(osSource.c_str()), + VSIGetDirectorySeparator(osTargetDir.c_str())); + }; + if (VSI_ISDIR(sSource.st_mode)) { osTargetDir = pszTarget; @@ -4109,15 +4118,16 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, const auto entry = VSIGetNextDirEntry(poTargetDir.get()); if (!entry) break; + const auto osDstName = + NormalizeDirSeparatorForDstFilename(entry->pszName); if (VSI_ISDIR(entry->nMode)) { - oSetTargetSubdirs.insert(entry->pszName); + oSetTargetSubdirs.insert(osDstName); } else { oMapExistingTargetFiles.insert( - std::pair(entry->pszName, - *entry)); + std::pair(osDstName, *entry)); } } poTargetDir.reset(); @@ -4142,11 +4152,13 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, break; if (VSI_ISDIR(entry->nMode)) { - if (oSetTargetSubdirs.find(entry->pszName) == + const auto osDstName = + NormalizeDirSeparatorForDstFilename(entry->pszName); + if (oSetTargetSubdirs.find(osDstName) == oSetTargetSubdirs.end()) { const std::string osTargetSubdir(CPLFormFilename( - osTargetDir.c_str(), entry->pszName, nullptr)); + osTargetDir.c_str(), osDstName.c_str(), nullptr)); aoSetDirsToCreate.insert(osTargetSubdir); } } @@ -4165,7 +4177,9 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, return false; } ChunkToCopy chunk; - chunk.osFilename = entry->pszName; + chunk.osSrcFilename = entry->pszName; + chunk.osDstFilename = + NormalizeDirSeparatorForDstFilename(entry->pszName); chunk.nMTime = entry->nMTime; chunk.nTotalSize = entry->nSize; chunk.osETag = @@ -4212,12 +4226,12 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, continue; const std::string osSubSource( CPLFormFilename(osSourceWithoutSlash.c_str(), - chunk.osFilename.c_str(), nullptr)); + chunk.osSrcFilename.c_str(), nullptr)); const std::string osSubTarget(CPLFormFilename( - osTargetDir.c_str(), chunk.osFilename.c_str(), nullptr)); + osTargetDir.c_str(), chunk.osDstFilename.c_str(), nullptr)); bool bSkip = false; const auto oIterExistingTarget = - oMapExistingTargetFiles.find(chunk.osFilename); + oMapExistingTargetFiles.find(chunk.osDstFilename); if (oIterExistingTarget != oMapExistingTargetFiles.end() && oIterExistingTarget->second.nSize == chunk.nTotalSize) { @@ -4334,9 +4348,9 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, CPLAssert(chunk.nStartOffset == 0); const std::string osSubSource( CPLFormFilename(osSourceWithoutSlash.c_str(), - chunk.osFilename.c_str(), nullptr)); + chunk.osSrcFilename.c_str(), nullptr)); const std::string osSubTarget(CPLFormFilename( - osTargetDir.c_str(), chunk.osFilename.c_str(), nullptr)); + osTargetDir.c_str(), chunk.osDstFilename.c_str(), nullptr)); // coverity[divide_by_zero] void *pScaledProgress = GDALCreateScaledProgress( double(nAccSize) / nTotalSize, @@ -4630,12 +4644,12 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, queue->osTargetDir.empty() ? queue->osSource.c_str() : CPLFormFilename(queue->osSourceDir.c_str(), - chunk.osFilename.c_str(), nullptr)); + chunk.osSrcFilename.c_str(), nullptr)); const std::string osSubTarget( queue->osTargetDir.empty() ? queue->osTarget.c_str() : CPLFormFilename(queue->osTargetDir.c_str(), - chunk.osFilename.c_str(), nullptr)); + chunk.osDstFilename.c_str(), nullptr)); ProgressData progressData; progressData.nFileSize = chunk.nSize; diff --git a/port/cpl_vsil_win32.cpp b/port/cpl_vsil_win32.cpp index 634c5f36e28a..12dd4e015b1a 100644 --- a/port/cpl_vsil_win32.cpp +++ b/port/cpl_vsil_win32.cpp @@ -83,6 +83,14 @@ class VSIWin32FilesystemHandler final : public VSIFilesystemHandler virtual bool IsLocal(const char *pszPath) override; std::string GetCanonicalFilename(const std::string &osFilename) const override; + + const char *GetDirectorySeparator(const char *pszPath) override + { + // Return forward slash for paths of the form + // "{drive_letter}:/{rest_of_the_path}", and backslash otherwise. + return (pszPath[0] && pszPath[1] == ':' && pszPath[2] == '/') ? "/" + : "\\"; + } }; /************************************************************************/ @@ -830,14 +838,23 @@ int VSIWin32FilesystemHandler::Stat(const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags) { - (void)nFlags; - #if defined(_MSC_VER) || __MSVCRT_VERSION__ >= 0x0601 if (CPLTestBool(CPLGetConfigOption("GDAL_FILENAME_IS_UTF8", "YES"))) { wchar_t *pwszFilename = CPLRecodeToWChar(pszFilename, CPL_ENC_UTF8, CPL_ENC_UCS2); + if (nFlags == VSI_STAT_EXISTS_FLAG) + { + memset(pStatBuf, 0, sizeof(VSIStatBufL)); + const int nResult = + (GetFileAttributesW(pwszFilename) == INVALID_FILE_ATTRIBUTES) + ? -1 + : 0; + CPLFree(pwszFilename); + return nResult; + } + int nResult = _wstat64(pwszFilename, pStatBuf); // If _wstat64() fails and the original name is not an extended one, @@ -879,6 +896,7 @@ int VSIWin32FilesystemHandler::Stat(const char *pszFilename, else #endif { + (void)nFlags; return (VSI_STAT64(pszFilename, pStatBuf)); } } diff --git a/scripts/cppcheck.sh b/scripts/cppcheck.sh index 7f66b755f7e5..7927eb6b70f8 100755 --- a/scripts/cppcheck.sh +++ b/scripts/cppcheck.sh @@ -98,6 +98,7 @@ for dirname in alg port gcore ogr frmts gnm apps fuzzers; do -D__x86_64__ \ -DFLT_EVAL_METHOD \ -DKDU_HAS_ROI_RECT \ + -Dflatbuffers=gdal_flatbuffers \ --include="${CPL_CONFIG_H}" \ --include=port/cpl_port.h \ -I "${CPL_CONFIG_H_DIR}" \ diff --git a/swig/include/python/docs/gdal_band_docs.i b/swig/include/python/docs/gdal_band_docs.i new file mode 100644 index 000000000000..6685bff03e3e --- /dev/null +++ b/swig/include/python/docs/gdal_band_docs.i @@ -0,0 +1,706 @@ +%feature("docstring") GDALRasterBandShadow " + +Python proxy of a :cpp:class:`GDALRasterBand`. + +"; + +%extend GDALRasterBandShadow { + +%feature("docstring") Checksum " + +Computes a checksum from a region of a RasterBand. +See :cpp:func:`GDALChecksumImage`. + +Parameters +---------- +xoff : int, default=0 + The pixel offset to left side of the region of the band to + be read. This would be zero to start from the left side. +yoff : int, default=0 + The line offset to top side of the region of the band to + be read. This would be zero to start from the top side. +xsize : int, optional + The number of pixels to read in the x direction. By default, + equal to the number of columns in the raster. +ysize : int, optional + The number of rows to read in the y direction. By default, + equal to the number of bands in the raster. + +Returns +------- +int + checksum value, or -1 in case of error + +"; + +%feature("docstring") ComputeBandStats " + +Computes the mean and standard deviation of values in this Band. +See :cpp:func:`GDALComputeBandStats`. + +Parameters +---------- +samplestep : int, default=1 + Step between scanlines used to compute statistics. + +Returns +------- +tuple + tuple of length 2 with value of mean and standard deviation + +See Also +-------- +:py:meth:`ComputeRasterMinMax` +:py:meth:`ComputeStatistics` +:py:meth:`GetMaximum` +:py:meth:`GetMinimum` +:py:meth:`GetStatistics` +:py:meth:`SetStatistics` +"; + +%feature("docstring") CreateMaskBand " + +Add a mask band to the current band. +See :cpp:func:`GDALRasterBand::CreateMaskBand`. + +Parameters +---------- +nFlags : int + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +"; + +%feature("docstring") DeleteNoDataValue " + +Remove the nodata value for this band. + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +"; + +%feature("docstring") Fill " + +Fill this band with a constant value. +See :cpp:func:`GDALRasterBand::Fill`. + +Parameters +---------- +real_fill : float + real component of the fill value +imag_fill : float, default = 0.0 + imaginary component of the fill value + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +"; + +%feature("docstring") FlushCache " + +Flush raster data cache. +See :cpp:func:`GDALRasterBand::FlushCache`. +"; + +%feature("docstring") GetActualBlockSize " + +Fetch the actual block size for a given block offset. +See :cpp:func:`GDALRasterBand::GetActualBlockSize`. + +Parameters +---------- +nXBlockOff : int + the horizontal block offset for which to calculate the + number of valid pixels, with zero indicating the left most block, 1 the next + block and so forth. +nYBlockOff : int + the vertical block offset, with zero indicating + the top most block, 1 the next block and so forth. + +Returns +------- +tuple + tuple with the x and y dimensions of the block +"; + +%feature("docstring") GetBand " + +Return the index of this band. +See :cpp:func:`GDALRasterBand::GetBand`. + +Returns +------- +int + the (1-based) index of this band +"; + +%feature("docstring") GetBlockSize " + +Fetch the natural block size of this band. +See :cpp:func:`GDALRasterBand::GetBlockSize`. + +Returns +------- +list + list with the x and y dimensions of a block +"; + +%feature("docstring") GetCategoryNames " + +Fetch the list of category names for this raster. +See :cpp:func:`GDALRasterBand::GetCategoryNames`. + +Returns +------- +list + A list of category names, or ``None`` +"; + +%feature("docstring") GetColorInterpretation " + +Get the :cpp:enum:`GDALColorInterp` value for this band. +See :cpp:func:`GDALRasterBand::GetColorInterpretation`. + +Returns +------- +int +"; + +%feature("docstring") GetColorTable " + +Get the color table associated with this band. +See :cpp:func:`GDALRasterBand::GetColorTable`. + +Returns +------- +ColorTable or ``None`` +"; + +%feature("docstring") GetDataCoverageStatus " + +Determine whether a sub-window of the Band contains only data, only empty blocks, or a mix of both. +See :cpp:func:`GDALRasterBand::GetDataCoverageStatus`. + +Parameters +---------- +nXOff : int +nYOff : int +nXSize : int +nYSize : int +nMaskFlagStop : int, default=0 + +Returns +------- +list + First value represents a bitwise-or value of the following constants + - :py:const:`gdalconst.GDAL_DATA_COVERAGE_STATUS_DATA` + - :py:const:`gdalconst.GDAL_DATA_COVERAGE_STATUS_EMPTY` + - :py:const:`gdalconst.GDAL_DATA_COVERAGE_STATUS_UNIMPLEMENTED` + Second value represents the approximate percentage in [0, 100] of pixels in the window that have valid values + +Examples +-------- +>>> import numpy as np +>>> # Create a raster with four blocks +>>> ds = gdal.GetDriverByName('GTiff').Create('test.tif', 64, 64, options = {'SPARSE_OK':True, 'TILED':True, 'BLOCKXSIZE':32, 'BLOCKYSIZE':32}) +>>> band = ds.GetRasterBand(1) +>>> # Write some data to upper-left block +>>> band.WriteArray(np.array([[1, 2], [3, 4]])) +0 +>>> # Check status of upper-left block +>>> flags, pct = band.GetDataCoverageStatus(0, 0, 32, 32) +>>> flags == gdal.GDAL_DATA_COVERAGE_STATUS_DATA +True +>>> pct +100.0 +>>> # Check status of upper-right block +>>> flags, pct = band.GetDataCoverageStatus(32, 0, 32, 32) +>>> flags == gdal.GDAL_DATA_COVERAGE_STATUS_EMPTY +True +>>> pct +0.0 +>>> # Check status of window touching all four blocks +>>> flags, pct = band.GetDataCoverageStatus(16, 16, 32, 32) +>>> flags == gdal.GDAL_DATA_COVERAGE_STATUS_DATA | gdal.GDAL_DATA_COVERAGE_STATUS_EMPTY +True +>>> pct +25.0 +"; + +%feature("docstring") GetDataset " + +Fetch the :py:class:`Dataset` associated with this Band. +See :cpp:func:`GDALRasterBand::GetDataset`. +"; + +%feature("docstring") GetDefaultHistogram " + +Fetch the default histogram for this band. +See :cpp:func:`GDALRasterBand::GetDefaultHistogram`. + +Returns +------- +list + List with the following four elements: + - lower bound of histogram + - upper bound of histogram + - number of buckets in histogram + - tuple with counts for each bucket +"; + +%feature("docstring") GetHistogram " + +Compute raster histogram. +See :cpp:func:`GDALRasterBand::GetHistogram`. + +Parameters +---------- +min : float, default=-0.05 + the lower bound of the histogram +max : float, default=255.5 + the upper bound of the histogram +buckets : int, default=256 + the number of buckets int he histogram +include_out_of_range : bool, default=False + if ``True``, add out-of-range values into the first and last buckets +approx_ok : bool, default=True + if ``True``, compute an approximate histogram by using subsampling or overviews +callback : function, optional + A progress callback function +callback_data: optional + Optional data to be passed to callback function + +Returns +------- +list + list with length equal to ``buckets``. If ``approx_ok`` is ``False``, each + the value of each list item will equal the number of pixels in that bucket. + +Examples +-------- +>>> import numpy as np +>>> ds = gdal.GetDriverByName('MEM').Create('', 10, 10, eType=gdal.GDT_Float32) +>>> ds.WriteArray(np.random.normal(size=100).reshape(10, 10)) +0 +>>> ds.GetRasterBand(1).GetHistogram(min=-3.5, max=3.5, buckets=13, approx_ok=False) +[0, 0, 3, 9, 13, 12, 25, 22, 9, 6, 0, 1, 0] # random +"; + +%feature("docstring") GetMaskBand " + +Return the mask band associated with this band. +See :cpp:func:`GDALRasterBand::GetMaskBand`. + +Returns +------- +Band + +"; + +%feature("docstring") GetMaskFlags " + +Return the status flags of the mask band. +See :cpp:func:`GDALRasterBand::GetMaskFlags`. + +Returns +------- +int + +Examples +-------- +>>> import numpy as np +>>> ds = gdal.GetDriverByName('MEM').Create('', 10, 10) +>>> band = ds.GetRasterBand(1) +>>> band.GetMaskFlags() == gdal.GMF_ALL_VALID +True +>>> band.SetNoDataValue(22) +0 +>>> band.WriteArray(np.array([[22]])) +0 +>>> band.GetMaskBand().ReadAsArray(win_xsize=2,win_ysize=2) +array([[ 0, 255], + [255, 255]], dtype=uint8) +>>> band.GetMaskFlags() == gdal.GMF_NODATA +True + +"; + +%feature("docstring") GetMaximum " + +Fetch a previously stored maximum value for this band. +See :cpp:func:`GDALRasterBand::GetMaximum`. + +Returns +------- +float + The stored maximum value, or ``None`` if no value + has been stored. + +"; + +%feature("docstring") GetMinimum " + +Fetch a previously stored maximum value for this band. +See :cpp:func:`GDALRasterBand::GetMinimum`. + +Returns +------- +float + The stored minimum value, or ``None`` if no value + has been stored. + +"; + +%feature("docstring") GetNoDataValueAsInt64 " + +Fetch the nodata value for this band. +See :cpp:func:`GDALRasterBand::GetNoDataValueAsInt64`. + +Returns +------- +int + The nodata value, or ``None`` if it has not been set or + the data type of this band is not :py:const:`gdal.GDT_Int64`. + +"; + +%feature("docstring") GetNoDataValueAsUInt64 " + +Fetch the nodata value for this band. +See :cpp:func:`GDALRasterBand::GetNoDataValueAsUInt64`. + +Returns +------- +int + The nodata value, or ``None`` if it has not been set or + the data type of this band is not :py:const:`gdal.GDT_UInt64`. + +"; + +%feature("docstring") GetOffset " + +Fetch the raster value offset. +See :cpp:func:`GDALRasterBand::GetOffset`. + +Returns +------- +double + The offset value, or ``0.0``. + +"; + +%feature("docstring") GetOverview " + +Fetch a raster overview. +See :cpp:func:`GDALRasterBand::GetOverview`. + +Parameters +---------- +i : int + Overview index between 0 and ``GetOverviewCount() - 1``. + +Returns +------- +Band + +"; + +%feature("docstring") GetOverviewCount " + +Return the number of overview layers available. +See :cpp:func:`GDALRasterBand::GetOverviewCount`. + +Returns +------- +int + +"; + +%feature("docstring") GetRasterCategoryNames " + +Fetch the list of category names for this band. +See :cpp:func:`GDALRasterBand::GetCategoryNames`. + +Returns +------- +list + The list of names, or ``None`` if no names exist. + +"; + +%feature("docstring") GetRasterColorInterpretation " + +Return the color interpretation code for this band. +See :cpp:func:`GDALRasterBand::GetColorInterpretation`. + +Returns +------- +int + The color interpretation code (default :py:const:`gdal.GCI_Undefined`) + +"; + +%feature("docstring") GetRasterColorTable " + +Fetch the color table associated with this band. +See :cpp:func:`GDALRasterBand::GetColorTable`. + +Returns +------- +ColorTable + The :py:class:`ColorTable`, or ``None`` if it has not been defined. +"; + +%feature("docstring") GetScale " + +Fetch the band scale value. +See :cpp:func:`GDALRasterBand::GetScale`. + +Returns +------- +double + The scale value, or ``1.0``. +"; + +%feature("docstring") GetStatistics " + +Return the minimum, maximum, mean, and standard deviation of all pixel values +in this band. +See :cpp:func:`GDALRasterBand::GetStatistics` + +Parameters +---------- +approx_ok : bool + If ``True``, allow overviews or a subset of image tiles to be used in + computing the statistics. +force : bool + If ``False``, only return a result if it can be obtained without scanning + the image, i.e. from pre-existing metadata. + +Returns +------- +list + a list with the min, max, mean, and standard deviation of values + in the Band. + +See Also +-------- +:py:meth:`ComputeBandStats` +:py:meth:`ComputeRasterMinMax` +:py:meth:`GetMaximum` +:py:meth:`GetMinimum` +:py:meth:`GetStatistics` +"; + +%feature("docstring") GetUnitType " + +Return a name for the units of this raster's values. +See :cpp:func:`GDALRasterBand::GetUnitType`. + +Returns +------- +str + +Examples +-------- +>>> ds = gdal.GetDriverByName('MEM').Create('', 10, 10) +>>> ds.GetRasterBand(1).SetUnitType('ft') +0 +>>> ds.GetRasterBand(1).GetUnitType() +'ft' +"; + +%feature("docstring") HasArbitraryOverviews " + +Check for arbitrary overviews. +See :cpp:func:`GDALRasterBand::HasArbitraryOverviews`. + +Returns +------- +bool +"; + +%feature("docstring") IsMaskBand " + +Returns whether the band is a mask band. +See :cpp:func:`GDALRasterBand::IsMaskBand`. + +Returns +------- +bool +"; + +%feature("docstring") SetCategoryNames " + +Set the category names for this band. +See :cpp:func:`GDALRasterBand::SetCategoryNames`. + +Parameters +---------- +papszCategoryNames : list + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. +"; + +%feature("docstring") SetColorInterpretation " + +Set color interpretation of the band +See :cpp:func:`GDALRasterBand::SetColorInterpretation`. + +Parameters +---------- +val : int + A color interpretation code such as :py:const:`gdal.GCI_RedBand` + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. +"; + +%feature("docstring") SetColorTable " + +Set the raster color table. +See :cpp:func:`GDALRasterBand::SetColorTable`. + +Parameters +---------- +arg : ColorTable + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. +"; + +%feature("docstring") SetDefaultHistogram " + +Set default histogram. +See :cpp:func:`GDALRasterBand::SetDefaultHistogram`. + +Parameters +---------- +min : float + minimum value +max : float + maximum value +buckets_in : list + list of pixel counts for each bucket + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +See Also +-------- +:py:meth:`SetHistogram` +"; + +%feature("docstring") SetOffset " + +Set scaling offset. +See :cpp:func:`GDALRasterBand::SetOffset`. + +Parameters +---------- +val : float + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +See Also +-------- +:py:meth:`SetScale` +"; + + +%feature("docstring") SetRasterColorTable " +Deprecated. Alternate name for :py:meth:`SetColorTable`. +"; + +%feature("docstring") SetRasterColorInterpretation " +Deprecated. Alternate name for :py:meth:`SetColorInterpretation`. +"; + +%feature("docstring") SetRasterCategoryNames " +Deprecated. Alternate name for :py:meth:`SetCategoryNames`. +"; + +%feature("docstring") SetScale " +Set scaling ratio. +See :cpp:func:`GDALRasterBand::SetScale`. + +Parameters +---------- +val : float + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +See Also +-------- +:py:meth:`SetOffset` +"; + +%feature("docstring") SetStatistics " + +Set statistics on band. +See :cpp:func:`GDALRasterBand::SetStatistics`. + +Parameters +---------- +min : float +max : float +mean : float +stdev : float + +Returns +------- +int: + :py:const:`CE_None` on apparent success or :py:const:`CE_Failure` on + failure. This method cannot detect whether metadata will be properly saved and + so may return :py:const:`gdal.`CE_None` even if the statistics will never be + saved. + +See Also +-------- +:py:meth:`ComputeBandStats` +:py:meth:`ComputeRasterMinMax` +:py:meth:`ComputeStatistics` +:py:meth:`GetMaximum` +:py:meth:`GetMinimum` +:py:meth:`GetStatistics` +"; + +%feature("docstring") SetUnitType " + +Set unit type. +See :cpp:func:`GDALRasterBand::SetUnitType`. + +Parameters +---------- +val : str + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. +"; + +} diff --git a/swig/include/python/docs/gdal_dataset_docs.i b/swig/include/python/docs/gdal_dataset_docs.i index 5849b542f626..0a4ec8c07d8f 100644 --- a/swig/include/python/docs/gdal_dataset_docs.i +++ b/swig/include/python/docs/gdal_dataset_docs.i @@ -1,5 +1,5 @@ %feature("docstring") GDALDatasetShadow " -Python proxy of a raster :cpp:class:`GDALDataset`. +Python proxy of a :cpp:class:`GDALDataset`. Since GDAL 3.8, a Dataset can be used as a context manager. When exiting the context, the Dataset will be closed and @@ -8,14 +8,718 @@ data will be written to disk. %extend GDALDatasetShadow { +%feature("docstring") AbortSQL " + +Abort any SQL statement running in the data store. + +Not implemented by all drivers. See :cpp:func:`GDALDataset::AbortSQL`. + +Returns +------- +:py:const:`ogr.OGRERR_NONE` on success or :py:const:`ogr.OGRERR_UNSUPPORTED_OPERATION` if AbortSQL is not supported for this dataset. +"; + + +%feature("docstring") AddBand " + +Adds a band to a :py:class:`Dataset`. + +Not supported by all drivers. + +Parameters +----------- +datatype: int + the data type of the pixels in the new band +options: dict/list + an optional dict or list of format-specific ``NAME=VALUE`` option strings. + +Returns +------- +int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + +Examples +-------- +>>> ds=gdal.GetDriverByName('MEM').Create('', 10, 10) +>>> ds.RasterCount +1 +>>> ds.AddBand(gdal.GDT_Float32) +0 +>>> ds.RasterCount +2 +"; + +%feature("docstring") AddFieldDomain " + +Add a :py:class:`ogr.FieldDomain` to the dataset. + +Only a few drivers support this operation. See :cpp:func:`GDALDataset::AddFieldDomain`. + +Parameters +---------- +fieldDomain : ogr.FieldDomain + The field domain to add + +Returns +-------- +bool: + ``True`` if the field domain was added, ``False`` in case of error. + + +"; + +%feature("docstring") AddRelationship " + +Add a :py:class:`Relationship` to the dataset. + +See :cpp:func:`GDALDataset::AddRelationship`. + +Parameters +---------- +relationship : Relationship + The relationship to add + +Returns +------- +bool: + ``True`` if the field domain was added, ``False`` in case of error. + +"; + +%feature("docstring") AdviseRead " + +Advise driver of upcoming read requests. + +See :cpp:func:`GDALDataset::AdviseRead`. + +"; + +%feature("docstring") BuildOverviews " + +Build raster overview(s) for all bands. + +See :cpp:func:`GDALDataset::BuildOverviews` + +Parameters +---------- +resampling : str, optional + The resampling method to use. See :cpp:func:`GDALDataset::BuildOveriews`. +overviewlist : list + A list of overview levels (decimation factors) to build, or an + empty list to clear existing overviews. +callback : function, optional + A progress callback function +callback_data: optional + Optional data to be passed to callback function +options : dict/list, optional + A dict or list of key=value options + +Returns +------- +:py:const:`CE_Failure` if an error occurs, otherwise :py:const:`CE_None`. + +Examples +-------- +>>> import numpy as np +>>> ds = gdal.GetDriverByName('GTiff').Create('test.tif', 12, 12) +>>> ds.GetRasterBand(1).WriteArray(np.arange(12*12).reshape((12, 12))) +0 +>>> ds.BuildOverviews('AVERAGE', [2, 4]) +0 +>>> ds.GetRasterBand(1).GetOverviewCount() +2 +>>> ds.BuildOverviews(overviewlist=[]) +0 +>>> ds.GetRasterBand(1).GetOverviewCount() +0 +"; + +%feature("docstring") ClearStatistics " + +Clear statistics + +See :cpp:func:`GDALDatset::ClearStatistics`. + +"; + %feature("docstring") Close " Closes opened dataset and releases allocated resources. This method can be used to force the dataset to close when one more references to the dataset are still -reachable. If Close is never called, the dataset will +reachable. If :py:meth:`Close` is never called, the dataset will be closed automatically during garbage collection. -" -} +In most cases, it is preferable to open or create a dataset +using a context manager instead of calling :py:meth:`Close` +directly. + +"; + +%feature("docstring") CommitTransaction " +Commits a transaction, for `Datasets` that support transactions. + +See :cpp:func:`GDALDataset::CommitTransaction`. +"; + +%feature("docstring") CopyLayer " + +Duplicate an existing :py:class:`ogr.Layer`. + +See :cpp:func:`GDALDAtaset::CopyLayer`. + +Parameters +---------- +src_layer : ogr.Layer + source layer +new_name : str + name of the layer to create +options : dict/list + a dict or list of name=value driver-specific creation options + +Returns +------- +ogr.Layer, or ``None`` if an error occurs +"; + +%feature("docstring") CreateLayer " + +Create a new layer in a vector Dataset. + +Parameters +---------- +name : string + the name for the new layer. This should ideally not + match any existing layer on the datasource. +srs : osr.SpatialReference, default=None + the coordinate system to use for the new layer, or ``None`` if + no coordinate system is available. +geom_type : int, default = :py:const:`ogr.wkbUnknown` + geometry type for the layer. Use :py:const:`ogr.wkbUnknown` if there + are no constraints on the types geometry to be written. +options : dict/list, optional + Driver-specific dict or list of name=value options + +Returns +------- +ogr.Layer or ``None`` on failure. + + +Examples +-------- +>>> ds = gdal.GetDriverByName('GPKG').Create('test.gpkg', 0, 0) +>>> ds.GetLayerCount() +0 +>>> lyr = ds.CreateLayer('poly', geom_type=ogr.wkbPolygon) +>>> ds.GetLayerCount() +1 + +"; + +%feature("docstring") CreateMaskBand " + +Adds a mask band to the dataset. + +See :cpp:func:`GDALDataset::CreateMaskBand`. + +Parameters +---------- +flags : int + +Returns +------- +int + :py:const:`CE_Failure` if an error occurs, otherwise :py:const:`CE_None`. + +"; + +%feature("docstring") DeleteFieldDomain " + +Removes a field domain from the Dataset. + +Parameters +---------- +name : str + Name of the field domain to delete + +Returns +------- +bool + ``True`` if the field domain was removed, otherwise ``False``. + +"; + +%feature("docstring") DeleteRelationship " + +Removes a relationship from the Dataset. + +Parameters +---------- +name : str + Name of the relationship to remove. + +Returns +------- +bool + ``True`` if the relationship was removed, otherwise ``False``. + + +"; + +%feature("docstring") FlushCache " + +Flush all write-cached data to disk. + +See :cpp:func:`GDALDataset::FlushCache`. + +Returns +------- +int + `gdal.CE_None` in case of success +"; + + +%feature("docstring") GetDriver " + +Fetch the driver used to open or create this :py:class:`Dataset`. + +"; + +%feature("docstring") GetFieldDomain " + +Get a field domain from its name. + +Parameters +---------- +name: str + The name of the field domain + +Returns +------- +ogr.FieldDomain, or ``None`` if it is not found. +"; + +%feature("docstring") GetFieldDomainNames " + +Get a list of the names of all field domains stored in the dataset. + +Parameters +---------- +options: dict/list, optional + Driver-specific options determining how attributes should + be retrieved. + +Returns +------- +list, or ``None`` if no field domains are stored in the dataset. +"; + +%feature("docstring") GetFileList " + +Returns a list of files believed to be part of this dataset. +See :cpp:func:`GDALGetFileList`. + +"; + +%feature("docstring") GetGCPCount " + +Get number of GCPs. See :cpp:func:`GDALGetGCPCount`. + +Returns +-------- +int + +"; + +%feature("docstring") GetGCPProjection " + +Return a WKT representation of the GCP spatial reference. + +Returns +-------- +string + +"; + +%feature("docstring") GetGCPSpatialRef " + +Get output spatial reference system for GCPs. + +See :cpp:func:`GDALGetGCPSpatialRef` + +"; + +%feature("docstring") GetGCPs " + +Get the GCPs. See :cpp:func:`GDALGetGCPs`. + +Returns +-------- +tuple + a tuple of :py:class:`GCP` objects. + +"; + +%feature("docstring") GetLayerByIndex " + +Fetch a layer by index. + +Parameters +---------- +index : int + A layer number between 0 and ``GetLayerCount() - 1`` + +Returns +------- +ogr.Layer + +"; + + +%feature("docstring") GetLayerByNAme " + +Fetch a layer by name. + +Parameters +---------- +layer_name : str + +Returns +------- +ogr.Layer + +"; + +%feature("docstring") GetLayerCount " +Get the number of layers in this dataset. + +Returns +------- +int + +"; + + +%feature("docstring") GetNextFeature " + +Fetch the next available feature from this dataset. + +This method is intended for the few drivers where +:py:meth:`OGRLayer.GetNextFeature` is not efficient, but in general +:py:meth:`OGRLayer.GetNextFeature` is a more natural API. + +See :cpp:func:`GDALDataset::GetNextFeature`. + +Returns +------- +ogr.Feature + +"; + +%feature("docstring") GetProjection " + +Return a WKT representation of the dataset spatial reference. +Equivalent to :py:meth:`GetProjectionRef`. + +Returns +------- +str + +"; + +%feature("docstring") GetProjectionRef " + +Return a WKT representation of the dataset spatial reference. + +Returns +------- +str + +"; + +%feature("docstring") GetGeoTransform " + +Fetch the affine transformation coefficients. + +See :cpp:func:`GDALGetGeoTransform`. + +Parameters +----------- +can_return_null : bool, default=False + if ``True``, return ``None`` instead of the default transformation + if the transformation for this :py:class:`Dataset` has not been defined. + +Returns +------- +tuple: + a 6-member tuple representing the transformation coefficients + + +"; + +%feature("docstring") GetRasterBand " + +Fetch a :py:class:`Band` band from a :py:class:`Dataset`. See :cpp:func:`GDALGetRasterBand`. + +Parameters +----------- +nBand : int + the index of the band to fetch, from 1 to :py:attr:`RasterCount` + +Returns +-------- +Band: + the :py:class:`Band`, or ``None`` on error. + +"; + +%feature("docstring") GetRelationship " + +Get a relationship from its name. + +Returns +------- +Relationship, or ``None`` if not found. +"; + +%feature("docstring") GetRelationshipNames " + +Get a list of the names of all relationships stored in the dataset. + +Parameters +---------- +options : dict/list, optional + driver-specific options determining how the relationships shoudl be retrieved + +"; + +%feature("docstring") GetRootGroup " + +Return the root :py:class:`Group` of this dataset. +Only value for multidimensional datasets. + +Returns +------- +Group + +"; + +%feature("docstring") GetSpatialRef " + +Fetch the spatial reference for this dataset. + +Returns +-------- +osr.SpatialReference + +"; + +%feature("docstring") GetStyleTable " + +Returns dataset style table. + +Returns +------- +ogr.StyleTable + +"; + +%feature("docstring") IsLayerPrivate " + +Parameters +---------- +index : int + Index o layer to check + +Returns +------- +bool + ``True`` if the layer is a private or system table, ``False`` otherwise + + +"; + +%feature("docstring") RasterCount " + +The number of bands in this dataset. + +"; + +%feature("docstring") RasterXSize " + +Raster width in pixels. See :cpp:func:`GDALGetRasterXSize`. + +"; + +%feature("docstring") RasterYSize " + +Raster height in pixels. See :cpp:func:`GDALGetRasterYSize`. + +"; + +%feature("docstring") ResetReading " + +Reset feature reading to start on the first feature. + +This affects :py:meth:`GetNextFeature`. + +Depending on drivers, this may also have the side effect of calling +:py:meth:`OGRLayer.ResetReading` on the layers of this dataset. + +"; + +%feature("docstring") RollbackTransaction " + +Roll back a Dataset to its state before the start of the current transaction. + +For datasets that support transactions. + +Returns +------- +int + If no transaction is active, or the rollback fails, will return + :py:const:`OGRERR_FAILURE`. Datasources which do not support transactions will + always return :py:const:`OGRERR_UNSUPPORTED_OPERATION`. + +"; + +%feature("docstring") SetGCPs " +"; + +%feature("docstring") SetGeoTransform " + +Set the affine transformation coefficients. + +See :py:meth:`GetGeoTransform` for details on the meaning of the coefficients. + +Parameters +---------- +argin : tuple + +Returns +------- +:py:const:`CE_Failure` if an error occurs, otherwise :py:const:`CE_None`. + +"; + +%feature("docstring") SetProjection " + +Set the spatial reference system for this dataset. + +See :cpp:func:`GDALDataset::SetProjection`. + +Parameters +---------- +prj: + The projection string in OGC WKT or PROJ.4 format + +Returns +------- +:py:const:`CE_Failure` if an error occurs, otherwise :py:const:`CE_None`. + +"; + +%feature("docstring") SetSpatialRef " + +Set the spatial reference system for this dataset. + +Parameters +---------- +srs : SpatialReference + +Returns +------- +:py:const:`CE_Failure` if an error occurs, otherwise :py:const:`CE_None`. + +"; + +%feature("docstring") SetStyleTable " + +Set dataset style table + +Parameters +---------- +table : ogr.StyleTable +"; + +%feature("docstring") StartTransaction " + +Creates a transaction. See :cpp:func:`GDALDataset::StartTransaction`. + +Returns +------- +int + If starting the transaction fails, will return + :py:const:`ogr.OGRERR_FAILURE`. Datasources which do not support transactions will + always return :py:const:`OGRERR_UNSUPPORTED_OPERATION`. + +"; + +%feature("docstring") TestCapability " + +Test if a capability is available. + +Parameters +---------- +cap : str + Name of the capability (e.g., :py:const:`ogr.ODsCTransactions`) + +Returns +------- +bool + ``True`` if the capability is available, ``False`` if invalid or unavailable + +Examples +-------- +>>> ds = gdal.GetDriverByName('ESRI Shapefile').Create('test.shp', 0, 0, 0, gdal.GDT_Unknown) +>>> ds.TestCapability(ogr.ODsCTransactions) +False +>>> ds.TestCapability(ogr.ODsCMeasuredGeometries) +True +>>> ds.TestCapability(gdal.GDsCAddRelationship) +False + +"; + +%feature("docstring") UpdateFieldDomain " + +Update an existing field domain by replacing its definition. + +The existing field domain with matching name will be replaced. + +Requires the :py:const:`ogr.ODsCUpdateFieldDomain` datasset capability. + +Parameters +---------- +fieldDomain : ogr.FieldDomain + Updated field domain. + +Returns +------- +bool + ``True`` in case of success + +"; + +%feature("docstring") UpdateRelationship " + +Update an existing relationship by replacing its definition. + +The existing relationship with matching name will be replaced. + +Requires the :py:const:`gdal.GDsCUpdateFieldDomain` dataset capability. + +Parameters +---------- +relationship : Relationship + Updated relationship + +Returns +------- +bool + ``True`` in case of success + +"; + +} diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 756b9e760427..947d77e04c01 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -7,6 +7,7 @@ %feature("autodoc"); +%include "gdal_band_docs.i" %include "gdal_dataset_docs.i" %init %{ @@ -523,6 +524,10 @@ void wrapper_VSIGetMemFileBuffer(const char *utf8_path, GByte **out, vsi_l_offse buf_string, buf_xsize=None, buf_ysize=None, buf_type=None, buf_pixel_space=None, buf_line_space=None ): + """ + Write the contents of a buffer to a dataset. + + """ if buf_xsize is None: buf_xsize = xsize @@ -549,9 +554,82 @@ void wrapper_VSIGetMemFileBuffer(const char *utf8_path, GByte **out, vsi_l_offse resample_alg=gdalconst.GRIORA_NearestNeighbour, callback=None, callback_data=None): - """ Reading a chunk of a GDAL band into a numpy array. The optional (buf_xsize,buf_ysize,buf_type) - parameters should generally not be specified if buf_obj is specified. The array is returned""" - + """ + Read a window of this raster band into a NumPy array. + + Parameters + ---------- + xoff : float, default=0 + The pixel offset to left side of the region of the band to + be read. This would be zero to start from the left side. + yoff : float, default=0 + The line offset to top side of the region of the band to + be read. This would be zero to start from the top side. + win_xsize : float, optional + The number of pixels to read in the x direction. By default, + equal to the number of columns in the raster. + win_ysize : float, optional + The number of rows to read in the y direction. By default, + equal to the number of bands in the raster. + buf_xsize : int, optional + The number of columns in the returned array. If not equal + to ``win_xsize``, the returned values will be determined + by ``resample_alg``. + buf_ysize : int, optional + The number of rows in the returned array. If not equal + to ``win_ysize``, the returned values will be determined + by ``resample_alg``. + buf_type : int, optional + The data type of the returned array + buf_obj : np.ndarray, optional + Optional buffer into which values will be read. If ``buf_obj`` + is specified, then ``buf_xsize``/``buf_ysize``/``buf_type`` + should generally not be specified. + resample_alg : int, default = :py:const:`gdal.GRIORA_NearestNeighbour`. + Specifies the resampling algorithm to use when the size of + the read window and the buffer are not equal. + callback : function, optional + A progress callback function + callback_data: optional + Optional data to be passed to callback function + + Returns + ------- + np.ndarray + + Examples + -------- + >>> import numpy as np + >>> ds = gdal.GetDriverByName("GTiff").Create("test.tif", 4, 4, eType=gdal.GDT_Float32) + >>> ds.WriteArray(np.arange(16).reshape(4, 4)) + 0 + >>> band = ds.GetRasterBand(1) + >>> # Reading an entire band + >>> band.ReadAsArray() + array([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]], dtype=float32) + >>> # Reading a window of a band + >>> band.ReadAsArray(xoff=2, yoff=2, win_xsize=2, win_ysize=2) + array([[10., 11.], + [14., 15.]], dtype=float32) + >>> # Reading a band into a new buffer at higher resolution + >>> band.ReadAsArray(xoff=0.5, yoff=0.5, win_xsize=2.5, win_ysize=2.5, buf_xsize=5, buf_ysize=5) + array([[ 0., 1., 1., 2., 2.], + [ 4., 5., 5., 6., 6.], + [ 4., 5., 5., 6., 6.], + [ 8., 9., 9., 10., 10.], + [ 8., 9., 9., 10., 10.]], dtype=float32) + >>> # Reading a band into an existing buffer at lower resolution + >>> band.ReadAsArray(buf_xsize=2, buf_ysize=2, buf_type=gdal.GDT_Float64, resample_alg=gdal.GRIORA_Average) + array([[ 2.5, 4.5], + [10.5, 12.5]]) + >>> buf = np.zeros((2,2)) + >>> band.ReadAsArray(buf_obj=buf) + array([[ 5., 7.], + [13., 15.]]) + """ from osgeo import gdal_array return gdal_array.BandReadAsArray(self, xoff, yoff, @@ -565,6 +643,31 @@ void wrapper_VSIGetMemFileBuffer(const char *utf8_path, GByte **out, vsi_l_offse resample_alg=gdalconst.GRIORA_NearestNeighbour, callback=None, callback_data=None): + """ + Write the contents of a NumPy array to a Band. + + Parameters + ---------- + array : np.ndarray + Two-dimensional array containing values to write + xoff : int, default=0 + The pixel offset to left side of the region of the band to + be written. This would be zero to start from the left side. + yoff : int, default=0 + The line offset to top side of the region of the band to + be written. This would be zero to start from the top side. + resample_alg : int, default = :py:const:`gdal.GRIORA_NearestNeighbour` + Resampling algorithm. Placeholder argument, not currently supported. + callback : function, optional + A progress callback function + callback_data: optional + Optional data to be passed to callback function + + Returns + ------- + int: + Error code, or ``gdal.CE_None`` if no error occurred. + """ from osgeo import gdal_array return gdal_array.BandWriteArray(self, array, xoff, yoff, @@ -649,7 +752,36 @@ void wrapper_VSIGetMemFileBuffer(const char *utf8_path, GByte **out, vsi_l_offse %feature("shadow") ComputeStatistics %{ def ComputeStatistics(self, *args, **kwargs) -> "CPLErr": - """ComputeStatistics(Band self, bool approx_ok, callback=None, callback_data=None) -> CPLErr""" + """ComputeStatistics(Band self, bool approx_ok, callback=None, callback_data=None) -> CPLErr + + Compute image statistics. + See :cpp:func:`GDALRasterBand::ComputeStatistics`. + + Parameters + ---------- + approx_ok : bool + If ``True``, compute statistics based on overviews or a + subset of tiles. + callback : function, optional + A progress callback function + callback_data: optional + Optional data to be passed to callback function + + Returns + ------- + list + a list with the min, max, mean, and standard deviation of values + in the Band. + + See Also + -------- + :py:meth:`ComputeBandStats` + :py:meth:`ComputeRasterMinMax` + :py:meth:`GetMaximum` + :py:meth:`GetMinimum` + :py:meth:`GetStatistics` + :py:meth:`SetStatistics` + """ if len(args) == 1: kwargs["approx_ok"] = args[0] @@ -669,7 +801,17 @@ def ComputeStatistics(self, *args, **kwargs) -> "CPLErr": %feature("shadow") GetNoDataValue %{ def GetNoDataValue(self): - """GetNoDataValue(Band self) -> value """ + """GetNoDataValue(Band self) -> value + + Fetch the nodata value for this band. + Unlike :cpp:func:`GDALRasterBand::GetNoDataValue`, this + method handles 64-bit integer data types. + + Returns + ------- + float/int + The nodata value, or ``None`` if it has not been set. + """ if self.DataType == gdalconst.GDT_Int64: return _gdal.Band_GetNoDataValueAsInt64(self) @@ -683,7 +825,23 @@ def GetNoDataValue(self): %feature("shadow") SetNoDataValue %{ def SetNoDataValue(self, value) -> "CPLErr": - """SetNoDataValue(Band self, value) -> CPLErr""" + """SetNoDataValue(Band self, value) -> CPLErr + + Set the nodata value for this band. + Unlike :cpp:func:`GDALRasterBand::SetNoDataValue`, this + method handles 64-bit integer types. + + Parameters + ---------- + value : float/int + The nodata value to set + + Returns + ------- + int: + :py:const:`CE_None` on success or :py:const:`CE_Failure` on failure. + + """ if self.DataType == gdalconst.GDT_Int64: return _gdal.Band_SetNoDataValueAsInt64(self, value) @@ -696,7 +854,33 @@ def SetNoDataValue(self, value) -> "CPLErr": %feature("shadow") ComputeRasterMinMax %{ def ComputeRasterMinMax(self, *args, **kwargs): - """ComputeRasterMinMax(Band self, bool approx_ok=False, bool can_return_none=False) -> (min, max) or None""" + """ComputeRasterMinMax(Band self, bool approx_ok=False, bool can_return_none=False) -> (min, max) or None + + Computes the minimum and maximum values for this Band. + See :cpp:func:`GDALComputeRasterMinMax`. + + Parameters + ---------- + approx_ok : bool, default=False + If ``False``, read all pixels in the band. If ``True``, check + :py:meth:`GetMinimum`/:py:meth:`GetMaximum` or read a subsample. + can_return_none : bool, default=False + If ``True``, return ``None`` on error. Otherwise, return a tuple + with NaN values. + + Returns + ------- + tuple + + See Also + -------- + :py:meth:`ComputeBandStats` + :py:meth:`ComputeStatistics` + :py:meth:`GetMaximum` + :py:meth:`GetMinimum` + :py:meth:`GetStatistics` + :py:meth:`SetStatistics` + """ if len(args) == 1: kwargs["approx_ok"] = args[0] @@ -845,8 +1029,92 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, callback_data=None, interleave='band', band_list=None): - """ Reading a chunk of a GDAL band into a numpy array. The optional (buf_xsize,buf_ysize,buf_type) - parameters should generally not be specified if buf_obj is specified. The array is returned""" + """ + Read a window from raster bands into a NumPy array. + + Parameters + ---------- + xoff : float, default=0 + The pixel offset to left side of the region of the band to + be read. This would be zero to start from the left side. + yoff : float, default=0 + The line offset to top side of the region of the band to + be read. This would be zero to start from the top side. + xsize : float, optional + The number of pixels to read in the x direction. By default, + equal to the number of columns in the raster. + ysize : float, optional + The number of rows to read in the y direction. By default, + equal to the number of bands in the raster. + buf_xsize : int, optional + The number of columns in the returned array. If not equal + to ``win_xsize``, the returned values will be determined + by ``resample_alg``. + buf_ysize : int, optional + The number of rows in the returned array. If not equal + to ``win_ysize``, the returned values will be determined + by ``resample_alg``. + buf_type : int, optional + The data type of the returned array + buf_obj : np.ndarray, optional + Optional buffer into which values will be read. If ``buf_obj`` + is specified, then ``buf_xsize``/``buf_ysize``/``buf_type`` + should generally not be specified. + resample_alg : int, default = :py:const:`gdal.GRIORA_NearestNeighbour`. + Specifies the resampling algorithm to use when the size of + the read window and the buffer are not equal. + callback : function, optional + A progress callback function + callback_data: optional + Optional data to be passed to callback function + band_list : list, optional + Indexes of bands from which data should be read. By default, + data will be read from all bands. + + Returns + ------- + np.ndarray + + Examples + -------- + >>> ds = gdal.GetDriverByName("GTiff").Create("test.tif", 4, 4, bands=2) + >>> ds.WriteArray(np.arange(32).reshape(2, 4, 4)) + 0 + >>> ds.ReadAsArray() + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]], + [[16, 17, 18, 19], + [20, 21, 22, 23], + [24, 25, 26, 27], + [28, 29, 30, 31]]], dtype=uint8) + >>> ds.ReadAsArray(xoff=2, yoff=2, xsize=2, ysize=2) + array([[[10, 11], + [14, 15]], + [[26, 27], + [30, 31]]], dtype=uint8) + >>> ds.ReadAsArray(buf_xsize=2, buf_ysize=2, buf_type=gdal.GDT_Float64, resample_alg=gdal.GRIORA_Average) + array([[[ 3., 5.], + [11., 13.]], + [[19., 21.], + [27., 29.]]]) + >>> buf = np.zeros((2,2,2)) + >>> ds.ReadAsArray(buf_obj=buf) + array([[[ 5., 7.], + [13., 15.]], + [[21., 23.], + [29., 31.]]]) + >>> ds.ReadAsArray(band_list=[2,1]) + array([[[16, 17, 18, 19], + [20, 21, 22, 23], + [24, 25, 26, 27], + [28, 29, 30, 31]], + [[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]], dtype=uint8) + """ from osgeo import gdal_array return gdal_array.DatasetReadAsArray(self, xoff, yoff, xsize, ysize, buf_obj, @@ -863,6 +1131,74 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, resample_alg=gdalconst.GRIORA_NearestNeighbour, callback=None, callback_data=None): + """ + Write the contents of a NumPy array to a Dataset. + + Parameters + ---------- + array : np.ndarray + Two- or three-dimensional array containing values to write + xoff : int, default=0 + The pixel offset to left side of the region of the band to + be written. This would be zero to start from the left side. + yoff : int, default=0 + The line offset to top side of the region of the band to + be written. This would be zero to start from the top side. + band_list : list, optional + Indexes of bands to which data should be written. By default, + it is assumed that the Dataset contains the same number of + bands as levels in ``array``. + interleave : str, default="band" + Interleaving, "band" or "pixel". For band-interleaved writing, + ``array`` should have shape ``(nband, ny, nx)``. For pixel- + interleaved-writing, ``array`` should have shape + ``(ny, nx, nbands)``. + resample_alg : int, default = :py:const:`gdal.GRIORA_NearestNeighbour` + Resampling algorithm. Placeholder argument, not currently supported. + callback : function, optional + A progress callback function + callback_data: optional + Optional data to be passed to callback function + + Returns + ------- + int: + Error code, or ``gdal.CE_None`` if no error occurred. + + Examples + -------- + + >>> import numpy as np + >>> + >>> nx = 4 + >>> ny = 3 + >>> nbands = 2 + >>> with gdal.GetDriverByName("GTiff").Create("band3_px.tif", nx, ny, bands=nbands) as ds: + ... data = np.arange(nx*ny*nbands).reshape(ny,nx,nbands) + ... ds.WriteArray(data, interleave="pixel") + ... ds.ReadAsArray() + ... + 0 + array([[[ 0, 2, 4, 6], + [ 8, 10, 12, 14], + [16, 18, 20, 22]], + [[ 1, 3, 5, 7], + [ 9, 11, 13, 15], + [17, 19, 21, 23]]], dtype=uint8) + >>> with gdal.GetDriverByName("GTiff").Create("band3_band.tif", nx, ny, bands=nbands) as ds: + ... data = np.arange(nx*ny*nbands).reshape(nbands, ny, nx) + ... ds.WriteArray(data, interleave="band") + ... ds.ReadAsArray() + ... + 0 + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]], dtype=uint8) + """ + from osgeo import gdal_array return gdal_array.DatasetWriteArray(self, array, xoff, yoff, @@ -993,6 +1329,15 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, return gdal_array.VirtualMemGetArray( virtualmem ) def GetSubDatasets(self): + """ + Return a list of Subdatasets. + + + Returns + ------- + list + + """ sd_list = [] sd = self.GetMetadata('SUBDATASETS') @@ -1034,7 +1379,18 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, return _gdal.Dataset_BeginAsyncReader(self, xoff, yoff, xsize, ysize, buf_obj, buf_xsize, buf_ysize, buf_type, band_list, 0, 0, 0, options) def GetLayer(self, iLayer=0): - """Return the layer given an index or a name""" + """ + Get the indicated layer from the Dataset + + Parameters + ---------- + value : int/str + Name or 0-based index of the layer to delete. + + Returns + ------- + ogr.Layer, or ``None`` on error + """ _WarnIfUserHasNotSpecifiedIfUsingOgrExceptions() @@ -1046,7 +1402,21 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, raise TypeError("Input %s is not of String or Int type" % type(iLayer)) def DeleteLayer(self, value): - """Deletes the layer given an index or layer name""" + """ + Delete the indicated layer from the Dataset. + + Parameters + ---------- + value : int/str + Name or 0-based index of the layer to delete. + + Returns + ------- + int + :py:const:`ogr.OGRERR_NONE` on success or + :py:const:`ogr.OGRERR_UNSUPPORTED_OPERATION` if DeleteLayer is not supported + for this dataset. + """ if isinstance(value, str): for i in range(self.GetLayerCount()): name = self.GetLayer(i).GetName() @@ -1059,6 +1429,19 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, raise TypeError("Input %s is not of String or Int type" % type(value)) def SetGCPs(self, gcps, wkt_or_spatial_ref): + """ + Assign GCPs. + + See :cpp:func:`GDALSetGCPs`. + + Parameters + ---------- + gcps : list + a list of :py:class:`GCP` objects + wkt_or_spatial_ref : str/osr.SpatialReference + spatial reference of the GCPs + """ + if isinstance(wkt_or_spatial_ref, str): return self._SetGCPs(gcps, wkt_or_spatial_ref) else: @@ -1108,10 +1491,10 @@ def ExecuteSQL(self, statement, spatialFilter=None, dialect="", keep_ref_on_ds=F - None (or an exception if exceptions are enabled) for statements that are in error - or None for statements that have no results set, - - or a ogr.Layer handle representing a results set from the query. + - or a :py:class:`ogr.Layer` handle representing a results set from the query. - Note that this ogr.Layer is in addition to the layers in the data store - and must be released with ReleaseResultSet() before the data source is closed + Note that this :py:class:`ogr.Layer` is in addition to the layers in the data store + and must be released with :py:meth:`ReleaseResultSet` before the data source is closed (destroyed). Starting with GDAL 3.7, this method can also be used as a context manager, @@ -1175,14 +1558,14 @@ def ExecuteSQL(self, statement, spatialFilter=None, dialect="", keep_ref_on_ds=F def ReleaseResultSet(self, sql_lyr): """ReleaseResultSet(self, sql_lyr: ogr.Layer) - Release ogr.Layer returned by ExecuteSQL() (when not called as an execution manager) + Release :py:class:`ogr.Layer` returned by :py:meth:`ExecuteSQL` (when not called as a context manager) The sql_lyr object is invalidated after this call. Parameters ---------- sql_lyr: - ogr.Layer got with ExecuteSQL() + :py:class:`ogr.Layer` got with :py:meth:`ExecuteSQL` """ if sql_lyr and not hasattr(sql_lyr, "_to_release"): @@ -1628,7 +2011,8 @@ def InfoOptions(options=None, format='text', deserialize=True, if '-json' in new_options: format = 'json' else: - new_options = options + import copy + new_options = copy.copy(options) if format == 'json': new_options += ['-json'] elif format != "text": @@ -1747,7 +2131,8 @@ def VectorInfoOptions(options=None, if '-json' in new_options: format = 'json' else: - new_options = options + import copy + new_options = copy.copy(options) if format == 'json': new_options += ['-json'] elif format != "text": @@ -1824,7 +2209,8 @@ def MultiDimInfoOptions(options=None, detailed=False, array=None, arrayoptions=N if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if detailed: new_options += ['-detailed'] if array: @@ -1984,7 +2370,8 @@ def TranslateOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if outputType != gdalconst.GDT_Unknown: @@ -2250,7 +2637,8 @@ def WarpOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if srcBands: for b in srcBands: new_options += ['-srcband', str(b)] @@ -2596,7 +2984,8 @@ def VectorTranslateOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-f', format] if srcSRS is not None: @@ -2865,7 +3254,8 @@ def DEMProcessingOptions(options=None, colorFilename=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if creationOptions is not None: @@ -2997,7 +3387,8 @@ def NearblackOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if creationOptions is not None: @@ -3142,7 +3533,8 @@ def GridOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if outputType != gdalconst.GDT_Unknown: @@ -3309,7 +3701,8 @@ def RasterizeOptions(options=None, format=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if outputType != gdalconst.GDT_Unknown: @@ -3498,7 +3891,8 @@ def FootprintOptions(options=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if bands is not None: @@ -3665,6 +4059,7 @@ def BuildVRTOptions(options=None, srcNodata=None, VRTNodata=None, hideNodata=None, + nodataMaxMaskThreshold=None, strict=False, callback=None, callback_data=None): """Create a BuildVRTOptions() object that can be passed to gdal.BuildVRT() @@ -3702,6 +4097,8 @@ def BuildVRTOptions(options=None, nodata values at the VRT band level. hideNodata: whether to make the VRT band not report the NoData value. + nodataMaxMaskThreshold: + value of the mask band of a source below which the source band values should be replaced by VRTNodata (or 0 if not specified) strict: set to True if warnings should be failures callback: @@ -3720,7 +4117,8 @@ def BuildVRTOptions(options=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if resolution is not None: new_options += ['-resolution', str(resolution)] if outputBounds is not None: @@ -3749,6 +4147,8 @@ def BuildVRTOptions(options=None, new_options += ['-srcnodata', str(srcNodata)] if VRTNodata is not None: new_options += ['-vrtnodata', str(VRTNodata)] + if nodataMaxMaskThreshold is not None: + new_options += ['-nodata_max_mask_threshold', str(nodataMaxMaskThreshold)] if hideNodata: new_options += ['-hidenodata'] if strict: @@ -3814,6 +4214,7 @@ def TileIndexOptions(options=None, maxPixelSize=None, format=None, layerName=None, + layerCreationOptions=None, locationFieldName="location", outputSRS=None, writeAbsolutePath=None, @@ -3848,6 +4249,8 @@ def TileIndexOptions(options=None, output format ("ESRI Shapefile", "GPKG", etc...) layerName: output layer name + layerCreationOptions: + list or dict of layer creation options locationFieldName: Specifies the name of the field in the resulting vector dataset where the path of the input dataset will be stored. The default field name is "location". Can be set to None to disable creation of such field. outputSRS: @@ -3890,7 +4293,8 @@ def TileIndexOptions(options=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if overwrite: new_options += ['-overwrite'] if recursive: @@ -3909,6 +4313,15 @@ def TileIndexOptions(options=None, new_options += ['-f', format] if layerName is not None: new_options += ['-lyr_name', layerName] + + if layerCreationOptions is not None: + if isinstance(layerCreationOptions, dict): + for k, v in layerCreationOptions.items(): + new_options += ['-lco', f'{k}={v}'] + else: + for opt in layerCreationOptions: + new_options += ['-lco', opt] + if locationFieldName is not None: new_options += ['-tileindex', locationFieldName] if outputSRS is not None: @@ -4046,7 +4459,8 @@ def MultiDimTranslateOptions(options=None, format=None, creationOptions=None, if isinstance(options, str): new_options = ParseCommandLine(options) else: - new_options = options + import copy + new_options = copy.copy(options) if format is not None: new_options += ['-of', format] if creationOptions is not None: @@ -4178,9 +4592,9 @@ def config_options(options, thread_local=True): ---------- options: dict Dictionary of configuration options passed as key, value - thread_local: bool + thread_local: bool, default=True Whether the configuration options should be only set on the current - thread. The default is True. + thread. Returns ------- @@ -4189,8 +4603,8 @@ def config_options(options, thread_local=True): Example ------- - with gdal.config_options({"GDAL_NUM_THREADS": "ALL_CPUS"}): - gdal.Warp("out.tif", "in.tif", dstSRS="EPSG:4326") + >>> with gdal.config_options({"GDAL_NUM_THREADS": "ALL_CPUS"}): + ... gdal.Warp("out.tif", "in.tif", dstSRS="EPSG:4326") """ get_config_option = GetThreadLocalConfigOption if thread_local else GetGlobalConfigOption set_config_option = SetThreadLocalConfigOption if thread_local else SetConfigOption @@ -4215,9 +4629,9 @@ def config_option(key, value, thread_local=True): Name of the configuration option value: str Value of the configuration option - thread_local: bool + thread_local: bool, default=True Whether the configuration option should be only set on the current - thread. The default is True. + thread. Returns ------- @@ -4226,8 +4640,8 @@ def config_option(key, value, thread_local=True): Example ------- - with gdal.config_option("GDAL_NUM_THREADS", "ALL_CPUS"): - gdal.Warp("out.tif", "in.tif", dstSRS="EPSG:4326") + >>> with gdal.config_option("GDAL_NUM_THREADS", "ALL_CPUS"): + ... gdal.Warp("out.tif", "in.tif", dstSRS="EPSG:4326") """ return config_options({key: value}, thread_local=thread_local) @@ -4243,8 +4657,8 @@ def quiet_errors(): Example ------- - with gdal.ExceptionMgr(useExceptions=False), gdal.quiet_errors(): - gdal.Error(gdal.CE_Failure, gdal.CPLE_AppDefined, "you will never see me") + >>> with gdal.ExceptionMgr(useExceptions=False), gdal.quiet_errors(): + ... gdal.Error(gdal.CE_Failure, gdal.CPLE_AppDefined, "you will never see me") """ PushErrorHandler("CPLQuietErrorHandler") try: diff --git a/swig/python/CMakeLists.txt b/swig/python/CMakeLists.txt index 88adc00807f9..8db7378db034 100644 --- a/swig/python/CMakeLists.txt +++ b/swig/python/CMakeLists.txt @@ -31,6 +31,7 @@ set(GDAL_PYTHON_CSOURCES ${PROJECT_SOURCE_DIR}/swig/include/python/python_exceptions.i ${PROJECT_SOURCE_DIR}/swig/include/python/python_strings.i ${PROJECT_SOURCE_DIR}/swig/include/python/typemaps_python.i + ${PROJECT_SOURCE_DIR}/swig/include/python/docs/gdal_band_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/gdal_dataset_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_datasource_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_driver_docs.i diff --git a/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py b/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py index 07a4c556a012..4f9198d0e654 100644 --- a/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py +++ b/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py @@ -1574,7 +1574,10 @@ def create_overview_tile( gdal.Unlink(aux_xml) if options.verbose: - logger.debug(f"\tbuild from zoom {base_tz}, tiles: %s" % ",".join(base_tiles)) + logger.debug( + f"\tbuild from zoom {base_tz}, tiles: %s" + % ",".join(["(%d, %d)" % (t[0], t[1]) for t in base_tiles]) + ) # Create a KML file for this tile. if tile_job_info.kml: