diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 106f2c6a576b..000000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,40 +0,0 @@ - - -## Expected behavior and actual behavior. - -For example (please modify !!!): I expected to be able to open this raster file (with a link to -the raster file, or it as an attachment) and it returns an error message -instead. - -## Steps to reproduce the problem. - -For example (please modify !!!): "gdalinfo myfile" - -## Operating system - -For example (please modify !!!): Ubuntu 16.04 64 bit - -## GDAL version and provenance - -For example (please modify !!!): the 2.2.3 version from ubuntugis-unstable PPA diff --git a/.github/ISSUE_TEMPLATE/10_bug_report.yml b/.github/ISSUE_TEMPLATE/10_bug_report.yml new file mode 100644 index 000000000000..e87303029ac9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/10_bug_report.yml @@ -0,0 +1,46 @@ +name: Bug report +description: Create a bug report. +labels: + - 'Bug' +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report correctly. Do NOT use GitHub to post any questions or support requests! They will be closed immediately and ignored. + + The GDAL project is made of contributions from various individuals and organizations, each with their own focus. The issue you are facing is not necessarily in the priority list of those contributors and consequently there is no guarantee that it will be addressed in a timely manner. If this bug report or feature request is high-priority for you, and you cannot address it yourself, we suggest engaging a GDAL developer or support organisation and financially sponsoring a fix. + + - type: textarea + id: what + attributes: + label: What is the bug? + description: | + If you think there is an issue with coordinate order in GDAL 3.0, then it is likely an intended behavior. See https://github.com/OSGeo/gdal/issues/1974 for more explanations. + validations: + required: true + + - type: textarea + id: steps + attributes: + label: Steps to reproduce the issue + description: | + Steps, code extract and/or sample datasets to reproduce the behavior. + validations: + required: true + + - type: textarea + id: about-info + attributes: + label: Versions and provenance + description: | + Please indicate the operating system (e.g "Windows 11", or "Linux Ubuntu 22.04") and the GDAL version, for example by pasting the output of ``gdalinfo --version``. Please also indicate how you got the GDAL binary. For example: self-built, package coming from the Linux distribution, from OSGeo4W, from Conda-Forge, from vcpkg, etc. + Note that the GDAL project supports only the current version https://gdal.org/download.html#current-release so it should be used for testing. Or at least the newest easily available version for your platform. + validations: + required: true + + - type: textarea + id: additional-context + attributes: + label: Additional context + description: | + Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/20_feature_request.yml b/.github/ISSUE_TEMPLATE/20_feature_request.yml new file mode 100644 index 000000000000..c9b7d0c11d10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/20_feature_request.yml @@ -0,0 +1,25 @@ +name: Feature request +description: Suggest a feature idea. +labels: + - 'enhancement' +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request correctly. + + - type: textarea + id: what + attributes: + label: Feature description + description: A clear and concise description of what you want to happen. + validations: + required: true + + - type: textarea + id: Additional + attributes: + label: Additional context + description: | + Add any other context about the feature request here. Open source is community driven, please consider a way to support this work either by hiring developers, supporting the GDAL project, find someone to submit a pull request. + If the change required is important, you should consider writing a [GDAL RFC](https://gdal.org/development/rfc/index.html) or hiring someone to, and announce your work on the maiing list. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..879d3256f9b9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,10 @@ +blank_issues_enabled: false + +contact_links: + + - name: Question + url: https://gis.stackexchange.com/ + about: > + Questions should go to the gdal-dev mailing list at https://lists.osgeo.org/mailman/listinfo/gdal-dev + or other support forums such as https://gis.stackexchange.com/. + GitHub issues are for bug reports and suggestions for new features. diff --git a/.github/workflows/android_cmake.yml b/.github/workflows/android_cmake.yml index 6d1f232161ab..efc7f8bfd2c2 100644 --- a/.github/workflows/android_cmake.yml +++ b/.github/workflows/android_cmake.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 id: cache with: path: | diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 436e145f36f6..c6c73bb69187 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -29,7 +29,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index b48f1d5c9281..d0aa3d14695f 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Setup cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 id: cache with: path: ${{ github.workspace }}/.ccache @@ -43,10 +43,7 @@ jobs: ${{ runner.os }}-${{ env.cache-name }} - name: Install dependency run: | - # This currently fails as of https://lists.osgeo.org/pipermail/ubuntu/2023-October/002046.html - # sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6B827C12C2D425E227EDCA75089EBE08314DF160 - sudo add-apt-repository -y http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu/ + sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable sudo apt-get update sudo apt-get install -y -q bison libaec-dev libjpeg-dev libgif-dev liblzma-dev libzstd-dev libgeos-dev git \ libcurl4-gnutls-dev libproj-dev libxml2-dev libxerces-c-dev libnetcdf-dev netcdf-bin \ @@ -144,12 +141,12 @@ jobs: - name: Install pdfium run: | - wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ + wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && sudo chown -R root:root install \ && sudo mv install/lib/* /usr/lib/ \ && sudo mv install/include/* /usr/include/ \ - && sudo rm -rf install-ubuntu2004-rev5952.tar.gz install \ + && sudo rm -rf install-ubuntu2004-rev6309.tar.gz install \ && sudo apt-get update -y \ && sudo apt-get install -y --fix-missing --no-install-recommends liblcms2-dev - name: Configure ccache @@ -328,7 +325,7 @@ jobs: mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2 mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil - name: Setup cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 id: cache with: path: ${{ github.workspace }}\.ccache @@ -412,7 +409,7 @@ jobs: shell: pwsh run: | echo "JAVA_HOME=$env:JAVA_HOME_11_X64" >> %GITHUB_ENV% - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3 with: activate-environment: gdalenv miniforge-variant: Mambaforge @@ -420,8 +417,6 @@ jobs: use-mamba: true auto-update-conda: true use-only-tar-bz2: false - - run: | - cmake --version - name: Install dependency shell: bash -l {0} run: | @@ -432,12 +427,16 @@ jobs: cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \ sqlite tiledb zstd cryptopp cgal doxygen librttopo libkml openssl xz \ openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \ - arrow-cpp pyarrow libaec + arrow-cpp pyarrow libaec cmake + - name: Check CMake version + shell: bash -l {0} + run: | + cmake --version - name: Install pdfium shell: bash -l {0} run: | - curl -LOs https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-win10-vs2019-x64-rev5952.zip - unzip install-win10-vs2019-x64-rev5952.zip + curl -LOs https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-win10-vs2019-x64-rev6309.zip + unzip install-win10-vs2019-x64-rev6309.zip mv install install-pdfium - name: Remove conflicting libraries @@ -455,7 +454,7 @@ jobs: # Build PDF driver as plugin due to the PDFium build including libopenjp2 symbols which would conflict with external libopenjp2 run: | mkdir -p $GITHUB_WORKSPACE/build - cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA_PREFIX}" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON + cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config Release -j 2 @@ -507,7 +506,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3 with: activate-environment: gdalenv miniforge-variant: Mambaforge @@ -515,17 +514,19 @@ jobs: use-mamba: true auto-update-conda: true use-only-tar-bz2: false - - run: | - cmake --version - name: Install dependency shell: bash -l {0} run: | - conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml + conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml cmake + - name: Check CMake version + shell: bash -l {0} + run: | + cmake --version - name: Configure shell: bash -l {0} run: | mkdir -p $GITHUB_WORKSPACE/build - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 @@ -533,12 +534,12 @@ jobs: shell: bash -l {0} run: | rm -f build/CMakeCache.txt - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Configure with even less dependencies, and disabling all optional drivers shell: bash -l {0} run: | rm -f build/CMakeCache.txt - cmake -A ${architecture} -G "${generator}" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DWERROR_DEV_FLAG="-Werror=dev" + cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DWERROR_DEV_FLAG="-Werror=dev" - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 @@ -573,7 +574,7 @@ jobs: - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Setup cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 id: cache with: path: ${{ github.workspace }}/.ccache @@ -652,7 +653,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3 with: activate-environment: gdalenv python-version: 3.9 diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 91dddf6bd13b..c0a53bca8c6b 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -83,7 +83,7 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 doxygen: runs-on: ubuntu-latest diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 53999c4c3829..ec48de50551c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,8 +1,12 @@ name: "CodeQL" on: - push: - pull_request: + push: + paths-ignore: + - 'doc/**' + pull_request: + paths-ignore: + - 'doc/**' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} @@ -41,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 + uses: github/codeql-action/init@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -57,6 +61,7 @@ jobs: - name: Install dependencies run: | + sudo apt-get update sudo apt-get install -y ccache cmake g++ swig python3-numpy libproj-dev libqhull-dev sudo apt-get install -y \ libblosc-dev \ @@ -107,7 +112,7 @@ jobs: - name: Restore build cache if: matrix.language == 'c-cpp' id: restore-cache - uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/restore@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} @@ -138,12 +143,12 @@ jobs: - name: Save build cache if: matrix.language == 'c-cpp' - uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/save@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 + uses: github/codeql-action/analyze@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index e0d044655ccf..bec14fb0f946 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: true matrix: - platform: ['ubuntu-latest','windows-latest','macos-latest'] + platform: ['ubuntu-latest','windows-latest','macos-latest','macos-14'] env: GHA_CI_PLATFORM: ${{ matrix.platform }} @@ -42,12 +42,12 @@ jobs: if: matrix.platform == 'windows-latest' - name: Cache Conda Environment - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-${{ steps.get-date.outputs.today }}-conda-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3 with: #miniforge-variant: Mambaforge miniforge-version: latest @@ -72,7 +72,7 @@ jobs: source ../ci/travis/conda/compile.sh working-directory: ./gdal-feedstock - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: ${{ matrix.platform }}-conda-package path: ./gdal-feedstock/packages/ diff --git a/.github/workflows/delete_untagged_containers.yml b/.github/workflows/delete_untagged_containers.yml index b6321b784857..4510b0c21ca7 100644 --- a/.github/workflows/delete_untagged_containers.yml +++ b/.github/workflows/delete_untagged_containers.yml @@ -14,6 +14,7 @@ jobs: delete-untagged-containers: name: Delete all containers from gdal-deps without tags runs-on: ubuntu-latest + if: github.repository == 'OSGeo/gdal' steps: - name: Delete all containers from gdal-deps without tags uses: Chizkiyahu/delete-untagged-ghcr-action@bbbab219998078a91c9b283dac9389b825894603 # v3.2.0 diff --git a/.github/workflows/doc_build.yml b/.github/workflows/doc_build.yml index 05dbf587d7d7..0b93863643c9 100644 --- a/.github/workflows/doc_build.yml +++ b/.github/workflows/doc_build.yml @@ -81,15 +81,15 @@ jobs: # run: | # make spelling # working-directory: ./doc - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: PDF path: doc/build/latex/gdal.pdf - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: HTML path: doc/build/html/* - #- uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + #- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 # with: # name: Misspelled # path: doc/build/spelling/output.txt diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 22d6dbbcf307..64a90f5c5db4 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -210,7 +210,7 @@ jobs: # different architecture. - name: Restore build cache id: restore-cache - uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/restore@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} @@ -258,7 +258,7 @@ jobs: ccache -s - name: Save build cache - uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/save@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} @@ -322,14 +322,14 @@ jobs: docker push ${CONTAINER_NAME_FULL} - name: Upload coverage artifacts - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: ${{ matrix.id == 'coverage' }} with: name: coverage_index.html path: build-${{ matrix.id }}/coverage_html/index.html - name: Upload coverage artifacts - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: ${{ matrix.id == 'coverage' }} with: name: HTML diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1dc6e6d7e747..4ab2d0be775b 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3 with: channels: conda-forge auto-update-conda: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d245855bdc21..8bd577c9892f 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -63,7 +63,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: SARIF file path: results.sarif @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 + uses: github/codeql-action/upload-sarif@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 with: sarif_file: results.sarif diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index 4e1ea8e7926b..b2f08f6a7d78 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -107,7 +107,7 @@ jobs: # different architecture. - name: Restore build cache id: restore-cache - uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/restore@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} @@ -147,7 +147,7 @@ jobs: ccache -s - name: Save build cache - uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/save@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 with: path: ${{ github.workspace }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/ubuntu_20.04/Dockerfile.ci b/.github/workflows/ubuntu_20.04/Dockerfile.ci index 54798996984c..58b223bcb2b3 100644 --- a/.github/workflows/ubuntu_20.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_20.04/Dockerfile.ci @@ -182,12 +182,12 @@ RUN mkdir geos \ && rm -rf geos # Install pdfium -RUN wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ +RUN wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && chown -R root:root install \ && mv install/lib/* /usr/lib/ \ && mv install/include/* /usr/include/ \ - && rm -rf install-ubuntu2004-rev5952.tar.gz install + && rm -rf install-ubuntu2004-rev6309.tar.gz install # HANA: client side # Install hdbsql tool diff --git a/.github/workflows/ubuntu_22.04/Dockerfile.ci b/.github/workflows/ubuntu_22.04/Dockerfile.ci index d257da558c10..a983e3aceb49 100644 --- a/.github/workflows/ubuntu_22.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_22.04/Dockerfile.ci @@ -112,6 +112,13 @@ RUN curl -L -O https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_A && rm -rf FileGDB_API-64gcc51 \ && ldconfig +# Oracle : client side (proprietary software) +RUN curl -L -O https://download.oracle.com/otn_software/linux/instantclient/199000/instantclient-basic-linux.x64-19.9.0.0.0dbru.zip \ + && curl -L -O https://download.oracle.com/otn_software/linux/instantclient/199000/instantclient-sdk-linux.x64-19.9.0.0.0dbru.zip \ + && unzip instantclient-basic-linux.x64-19.9.0.0.0dbru.zip -d /opt \ + && unzip instantclient-sdk-linux.x64-19.9.0.0.0dbru.zip -d opt \ + && apt-get install -y libaio1 + COPY requirements.txt /tmp/ RUN python3 -m pip install -U -r /tmp/requirements.txt diff --git a/.github/workflows/ubuntu_22.04/build.sh b/.github/workflows/ubuntu_22.04/build.sh index b797888baf72..79057fbf42ba 100755 --- a/.github/workflows/ubuntu_22.04/build.sh +++ b/.github/workflows/ubuntu_22.04/build.sh @@ -2,11 +2,15 @@ set -e +LD_LIBRARY_PATH="/opt/instantclient_19_9:/opt/instantclient_19_9/lib:${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH + cmake ${GDAL_SOURCE_DIR:=..} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_FLAGS="-Werror" \ -DCMAKE_CXX_FLAGS="-Werror" \ -DUSE_CCACHE=ON \ + -DOracle_ROOT=/opt/instantclient_19_9 \ -DGDAL_USE_GEOTIFF_INTERNAL:BOOL=ON \ -DGDAL_USE_TIFF_INTERNAL:BOOL=ON diff --git a/.github/workflows/ubuntu_22.04/test.sh b/.github/workflows/ubuntu_22.04/test.sh index 5addcb8694eb..9af1dd53f7b7 100755 --- a/.github/workflows/ubuntu_22.04/test.sh +++ b/.github/workflows/ubuntu_22.04/test.sh @@ -4,6 +4,9 @@ set -e . ../scripts/setdevenv.sh +LD_LIBRARY_PATH="/opt/instantclient_19_9:/opt/instantclient_19_9/lib:${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH + export PYTEST="python3 -m pytest -vv -p no:sugar --color=no" # Run C++ tests diff --git a/alg/gdal_alg.h b/alg/gdal_alg.h index ee15033a7810..e76b76f1adb7 100644 --- a/alg/gdal_alg.h +++ b/alg/gdal_alg.h @@ -257,6 +257,8 @@ GDALSuggestedWarpOutput(GDALDatasetH hSrcDS, GDALTransformerFunc pfnTransformer, /** Flag for GDALSuggestedWarpOutput2() to ask to round-up output size */ #define GDAL_SWO_ROUND_UP_SIZE 0x1 +/** Flag for GDALSuggestedWarpOutput2() to ask to force square pixels */ +#define GDAL_SWO_FORCE_SQUARE_PIXEL 0x2 CPLErr CPL_DLL CPL_STDCALL GDALSuggestedWarpOutput2( GDALDatasetH hSrcDS, GDALTransformerFunc pfnTransformer, diff --git a/alg/gdalpansharpen.cpp b/alg/gdalpansharpen.cpp index beccf909ec8a..952f0f5b7d06 100644 --- a/alg/gdalpansharpen.cpp +++ b/alg/gdalpansharpen.cpp @@ -307,7 +307,7 @@ GDALPansharpenOperation::Initialize(const GDALPansharpenOptions *psOptionsIn) CPLError(CE_Failure, CPLE_AppDefined, "Dimensions of input spectral band %d different from " "first spectral band", - i); + i + 1); return CE_Failure; } @@ -317,8 +317,8 @@ GDALPansharpenOperation::Initialize(const GDALPansharpenOptions *psOptionsIn) { CPLError(CE_Failure, CPLE_AppDefined, "Cannot retrieve dataset associated with " - "ahInputSpectralBands[%i]", - i); + "input spectral band %d", + i + 1); return CE_Failure; } // Make sure that the band is really a first level child of the owning dataset @@ -334,16 +334,16 @@ GDALPansharpenOperation::Initialize(const GDALPansharpenOptions *psOptionsIn) { CPLError( CE_Failure, CPLE_AppDefined, - "ahInputSpectralBands[%d] band has no associated geotransform", - i); + "input spectral band %d band has no associated geotransform", + i + 1); return CE_Failure; } if (adfMSGT != adfRefMSGT) { CPLError(CE_Failure, CPLE_AppDefined, - "ahInputSpectralBands[%d] band has a different " - "geotransform than ahInputSpectralBands[0]", - i); + "input spectral band %d has a different " + "geotransform than the first spectral band", + i + 1); return CE_Failure; } diff --git a/alg/gdaltransformer.cpp b/alg/gdaltransformer.cpp index 496c20ea8995..f0950fd5d389 100644 --- a/alg/gdaltransformer.cpp +++ b/alg/gdaltransformer.cpp @@ -75,6 +75,18 @@ static void *GDALCreateApproxTransformer2(GDALTransformerFunc pfnRawTransformer, double dfMaxErrorForward, double dfMaxErrorReverse); +static bool IsTransformer(void *hTransformerArg, const char *pszClassName) +{ + if (!hTransformerArg) + return false; + // All transformers should have a GDALTransformerInfo member as their first members + GDALTransformerInfo *psInfo = + static_cast(hTransformerArg); + return memcmp(psInfo->abySignature, GDAL_GTI2_SIGNATURE, + strlen(GDAL_GTI2_SIGNATURE)) == 0 && + strcmp(psInfo->pszClassName, pszClassName) == 0; +} + /************************************************************************/ /* ==================================================================== */ /* GDALGenImgProjTransformer */ @@ -106,6 +118,9 @@ typedef struct // GDALRefreshGenImgProjTransformer() must do something or not. bool bCheckWithInvertPROJ; + // Set to TRUE when the transformation pipline is a custom one. + bool bHasCustomTransformationPipeline; + } GDALGenImgProjTransformInfo; /************************************************************************/ @@ -396,8 +411,9 @@ static int GDALSuggestedWarpOutput2_MustAdjustForBottomBorder( * @param pnLines int in which the suggest pixel height of output is returned. * @param padfExtent Four entry array to return extents as (xmin, ymin, xmax, * ymax). - * @param nOptions Options. Zero or GDAL_SWO_ROUND_UP_SIZE to ask *pnPixels - * and *pnLines to be rounded up instead of being rounded to the closes integer. + * @param nOptions Options flags. Zero or GDAL_SWO_ROUND_UP_SIZE to ask *pnPixels + * and *pnLines to be rounded up instead of being rounded to the closes integer, or + * GDAL_SWO_FORCE_SQUARE_PIXEL to indicate that the generated pixel size is a square. * * @return CE_None if successful or CE_Failure otherwise. */ @@ -411,15 +427,19 @@ CPLErr CPL_STDCALL GDALSuggestedWarpOutput2(GDALDatasetH hSrcDS, { VALIDATE_POINTER1(hSrcDS, "GDALSuggestedWarpOutput2", CE_Failure); + const bool bIsGDALGenImgProjTransform{ + pTransformArg && + IsTransformer(pTransformArg, "GDALGenImgProjTransformer")}; + /* -------------------------------------------------------------------- */ /* Setup sample points all around the edge of the input raster. */ /* -------------------------------------------------------------------- */ - if (pfnTransformer == GDALGenImgProjTransform) + if (bIsGDALGenImgProjTransform) { // In case CHECK_WITH_INVERT_PROJ has been modified. GDALRefreshGenImgProjTransformer(pTransformArg); } - else if (pfnTransformer == GDALApproxTransform) + else if (IsTransformer(pTransformArg, "GDALApproxTransformer")) { // In case CHECK_WITH_INVERT_PROJ has been modified. GDALRefreshApproxTransformer(pTransformArg); @@ -428,6 +448,83 @@ CPLErr CPL_STDCALL GDALSuggestedWarpOutput2(GDALDatasetH hSrcDS, const int nInXSize = GDALGetRasterXSize(hSrcDS); const int nInYSize = GDALGetRasterYSize(hSrcDS); + /* ------------------------------------------------------------- */ + /* Special case for warping on the same (or null) CRS. */ + /* ------------------------------------------------------------- */ + if ((!nOptions || (nOptions & GDAL_SWO_FORCE_SQUARE_PIXEL) == 0) && + pTransformArg && bIsGDALGenImgProjTransform) + { + double adfGeoTransform[6]; + + if (GDALGetGeoTransform(hSrcDS, adfGeoTransform) == CE_None && + adfGeoTransform[2] == 0.0 && adfGeoTransform[4] == 0.0) + { + GDALGenImgProjTransformInfo *psInfo{ + static_cast(pTransformArg)}; + + if (psInfo && !psInfo->pSrcTransformer && + !psInfo->bHasCustomTransformationPipeline && + !psInfo->pDstTransformer && + psInfo->adfDstGeoTransform[0] == 0 && + psInfo->adfDstGeoTransform[1] == 1 && + psInfo->adfDstGeoTransform[2] == 0 && + psInfo->adfDstGeoTransform[3] == 0 && + psInfo->adfDstGeoTransform[4] == 0 && + psInfo->adfDstGeoTransform[5] == 1) + { + const OGRSpatialReference *poSourceCRS = nullptr; + const OGRSpatialReference *poTargetCRS = nullptr; + + if (psInfo->pReprojectArg) + { + const GDALReprojectionTransformInfo *psRTI = + static_cast( + psInfo->pReprojectArg); + poSourceCRS = psRTI->poForwardTransform->GetSourceCS(); + poTargetCRS = psRTI->poForwardTransform->GetTargetCS(); + } + + if ((!poSourceCRS && !poTargetCRS) || + (poSourceCRS && poTargetCRS && + poSourceCRS->IsSame(poTargetCRS))) + { + + const bool bNorthUp{adfGeoTransform[5] < 0.0}; + + memcpy(padfGeoTransformOut, adfGeoTransform, + sizeof(double) * 6); + + if (!bNorthUp) + { + padfGeoTransformOut[3] = + padfGeoTransformOut[3] + + nInYSize * padfGeoTransformOut[5]; + padfGeoTransformOut[5] = -padfGeoTransformOut[5]; + } + + *pnPixels = nInXSize; + *pnLines = nInYSize; + + // Calculate extent from hSrcDS + if (padfExtent) + { + padfExtent[0] = adfGeoTransform[0]; + padfExtent[1] = + adfGeoTransform[3] + nInYSize * adfGeoTransform[5]; + padfExtent[2] = + adfGeoTransform[0] + nInXSize * adfGeoTransform[1]; + padfExtent[3] = adfGeoTransform[3]; + if (!bNorthUp) + { + std::swap(padfExtent[1], padfExtent[3]); + } + } + return CE_None; + } + } + } + } + const int N_PIXELSTEP = 50; int nSteps = static_cast( static_cast(std::min(nInYSize, nInXSize)) / N_PIXELSTEP + 0.5); @@ -1256,6 +1353,7 @@ static GDALGenImgProjTransformInfo *GDALCreateGenImgProjTransformerInternal() psInfo->sTI.pfnCreateSimilar = GDALCreateSimilarGenImgProjTransformer; psInfo->bCheckWithInvertPROJ = GetCurrentCheckWithInvertPROJ(); + psInfo->bHasCustomTransformationPipeline = false; return psInfo; } @@ -2471,6 +2569,11 @@ void *GDALCreateGenImgProjTransformer2(GDALDatasetH hSrcDS, GDALDatasetH hDstDS, : nullptr, aosOptions.List()); + if (pszCO) + { + psInfo->bHasCustomTransformationPipeline = true; + } + if (psInfo->pReprojectArg == nullptr) { GDALDestroyGenImgProjTransformer(psInfo); diff --git a/apps/data/gdalinfo_output.schema.json b/apps/data/gdalinfo_output.schema.json index 04055fcdabe9..166a490d9e26 100644 --- a/apps/data/gdalinfo_output.schema.json +++ b/apps/data/gdalinfo_output.schema.json @@ -176,6 +176,7 @@ } }, "size": { + "$comment": "note that the order of items in side is width,height", "$ref": "#/definitions/arrayOfTwoIntegers" }, "coordinateSystem": { @@ -306,6 +307,7 @@ }, "proj:shape": { + "$comment": "note that the order of items in proj:shape is height,width starting with GDAL 3.8.5 (previous versions ordered it wrongly as width,height)", "title": "Shape", "type": "array", "minItems": 2, diff --git a/apps/gdalbuildvrt_bin.cpp b/apps/gdalbuildvrt_bin.cpp index 84feff19088d..0437ca0f3335 100644 --- a/apps/gdalbuildvrt_bin.cpp +++ b/apps/gdalbuildvrt_bin.cpp @@ -56,6 +56,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg) " [-srcnodata \"[ ]...\"] [-vrtnodata " "\"[ ]...\"\n" " [-ignore_srcmaskband]\n" + " [-nodata_max_mask_threshold ]\n" " [-a_srs ]\n" " [-r " "{nearest|bilinear|cubic|cubicspline|lanczos|average|mode}]\n" diff --git a/apps/gdalbuildvrt_lib.cpp b/apps/gdalbuildvrt_lib.cpp index 93d79017ad3f..b07f416cf5a9 100644 --- a/apps/gdalbuildvrt_lib.cpp +++ b/apps/gdalbuildvrt_lib.cpp @@ -42,8 +42,9 @@ #include #include -#include #include +#include +#include #include "commonutils.h" #include "cpl_conv.h" @@ -249,6 +250,8 @@ class VRTBuilder char *pszResampling = nullptr; char **papszOpenOptions = nullptr; bool bUseSrcMaskBand = true; + bool bNoDataFromMask = false; + double dfMaskValueThreshold = 0; /* Internal variables */ char *pszProjectionRef = nullptr; @@ -284,6 +287,7 @@ class VRTBuilder int bAllowProjectionDifference, int bAddAlpha, int bHideNoData, int nSubdataset, const char *pszSrcNoData, const char *pszVRTNoData, bool bUseSrcMaskBand, + bool bNoDataFromMask, double dfMaskValueThreshold, const char *pszOutputSRS, const char *pszResampling, const char *const *papszOpenOptionsIn); @@ -305,7 +309,8 @@ VRTBuilder::VRTBuilder( double maxYIn, int bSeparateIn, int bAllowProjectionDifferenceIn, int bAddAlphaIn, int bHideNoDataIn, int nSubdatasetIn, const char *pszSrcNoDataIn, const char *pszVRTNoDataIn, - bool bUseSrcMaskBandIn, const char *pszOutputSRSIn, + bool bUseSrcMaskBandIn, bool bNoDataFromMaskIn, + double dfMaskValueThresholdIn, const char *pszOutputSRSIn, const char *pszResamplingIn, const char *const *papszOpenOptionsIn) : bStrict(bStrictIn) { @@ -365,6 +370,8 @@ VRTBuilder::VRTBuilder( pszOutputSRS = (pszOutputSRSIn) ? CPLStrdup(pszOutputSRSIn) : nullptr; pszResampling = (pszResamplingIn) ? CPLStrdup(pszResamplingIn) : nullptr; bUseSrcMaskBand = bUseSrcMaskBandIn; + bNoDataFromMask = bNoDataFromMaskIn; + dfMaskValueThreshold = dfMaskValueThresholdIn; } /************************************************************************/ @@ -563,11 +570,14 @@ std::string VRTBuilder::AnalyseRaster(GDALDatasetH hDS, double ds_minY = ds_maxY + GDALGetRasterYSize(hDS) * padfGeoTransform[GEOTRSFRM_NS_RES]; - const int _nBands = GDALGetRasterCount(hDS); + int _nBands = GDALGetRasterCount(hDS); if (_nBands == 0) { return "Dataset has no bands"; } + if (bNoDataFromMask && + poDS->GetRasterBand(_nBands)->GetColorInterpretation() == GCI_AlphaBand) + _nBands--; GDALRasterBand *poFirstBand = poDS->GetRasterBand(1); poFirstBand->GetBlockSize(&psDatasetProperties->nBlockXSize, @@ -1298,8 +1308,20 @@ void VRTBuilder::CreateVRTNonSeparate(VRTDatasetH hVRTDS) static_cast(hVRTBand); VRTSimpleSource *poSimpleSource; - if (bAllowSrcNoData && - psDatasetProperties->abHasNoData[nSelBand - 1]) + if (bNoDataFromMask) + { + auto poNoDataFromMaskSource = new VRTNoDataFromMaskSource(); + poSimpleSource = poNoDataFromMaskSource; + poNoDataFromMaskSource->SetParameters( + (nVRTNoDataCount > 0) + ? ((j < nVRTNoDataCount) + ? padfVRTNoData[j] + : padfVRTNoData[nVRTNoDataCount - 1]) + : 0, + dfMaskValueThreshold); + } + else if (bAllowSrcNoData && + psDatasetProperties->abHasNoData[nSelBand - 1]) { auto poComplexSource = new VRTComplexSource(); poSimpleSource = poComplexSource; @@ -1747,6 +1769,8 @@ struct GDALBuildVRTOptions char *pszResampling; char **papszOpenOptions; bool bUseSrcMaskBand; + bool bNoDataFromMask; + double dfMaskValueThreshold; /*! allow or suppress progress monitor and other non-error output */ int bQuiet; @@ -1925,7 +1949,8 @@ GDALDatasetH GDALBuildVRT(const char *pszDest, int nSrcCount, psOptions->bSeparate, psOptions->bAllowProjectionDifference, psOptions->bAddAlpha, psOptions->bHideNoData, psOptions->nSubdataset, psOptions->pszSrcNoData, psOptions->pszVRTNoData, - psOptions->bUseSrcMaskBand, psOptions->pszOutputSRS, + psOptions->bUseSrcMaskBand, psOptions->bNoDataFromMask, + psOptions->dfMaskValueThreshold, psOptions->pszOutputSRS, psOptions->pszResampling, psOptions->papszOpenOptions); GDALDatasetH hDstDS = static_cast( @@ -1996,6 +2021,8 @@ GDALBuildVRTOptionsNew(char **papszArgv, psOptions->pfnProgress = GDALDummyProgress; psOptions->pProgressData = nullptr; psOptions->bUseSrcMaskBand = true; + psOptions->bNoDataFromMask = false; + psOptions->dfMaskValueThreshold = 0; psOptions->bStrict = false; /* -------------------------------------------------------------------- */ @@ -2182,6 +2209,12 @@ GDALBuildVRTOptionsNew(char **papszArgv, { psOptions->bUseSrcMaskBand = false; } + else if (EQUAL(papszArgv[iArg], "-nodata_max_mask_threshold") && + iArg + 1 < argc) + { + psOptions->bNoDataFromMask = true; + psOptions->dfMaskValueThreshold = CPLAtofM(papszArgv[++iArg]); + } else if (papszArgv[iArg][0] == '-') { CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", diff --git a/apps/gdalinfo_lib.cpp b/apps/gdalinfo_lib.cpp index d72cbbe036ab..13524b5b507e 100644 --- a/apps/gdalinfo_lib.cpp +++ b/apps/gdalinfo_lib.cpp @@ -290,30 +290,31 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) GDALGetDriverShortName(hDriver), GDALGetDriverLongName(hDriver)); } - // The list of files of a raster FileGDB is not super useful and potentially - // super long, so omit it, unless the -json mode is enabled - char **papszFileList = - (!bJson && EQUAL(GDALGetDriverShortName(hDriver), "OpenFileGDB")) - ? nullptr - : GDALGetFileList(hDataset); - - if (papszFileList == nullptr || *papszFileList == nullptr) + if (psOptions->bShowFileList) { - if (bJson) + // The list of files of a raster FileGDB is not super useful and potentially + // super long, so omit it, unless the -json mode is enabled + char **papszFileList = + (!bJson && EQUAL(GDALGetDriverShortName(hDriver), "OpenFileGDB")) + ? nullptr + : GDALGetFileList(hDataset); + + if (!papszFileList || *papszFileList == nullptr) { - json_object *poFiles = json_object_new_array(); - json_object_object_add(poJsonObject, "files", poFiles); + if (bJson) + { + json_object *poFiles = json_object_new_array(); + json_object_object_add(poJsonObject, "files", poFiles); + } + else + { + Concat(osStr, psOptions->bStdoutOutput, + "Files: none associated\n"); + } } else { - Concat(osStr, psOptions->bStdoutOutput, "Files: none associated\n"); - } - } - else - { - if (bJson) - { - if (psOptions->bShowFileList) + if (bJson) { json_object *poFiles = json_object_new_array(); @@ -327,36 +328,47 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) json_object_object_add(poJsonObject, "files", poFiles); } - } - else - { - Concat(osStr, psOptions->bStdoutOutput, "Files: %s\n", - papszFileList[0]); - if (psOptions->bShowFileList) + else { + Concat(osStr, psOptions->bStdoutOutput, "Files: %s\n", + papszFileList[0]); for (int i = 1; papszFileList[i] != nullptr; i++) Concat(osStr, psOptions->bStdoutOutput, " %s\n", papszFileList[i]); } } + CSLDestroy(papszFileList); } - CSLDestroy(papszFileList); if (bJson) { - json_object *poSize = json_object_new_array(); - json_object *poSizeX = - json_object_new_int(GDALGetRasterXSize(hDataset)); - json_object *poSizeY = - json_object_new_int(GDALGetRasterYSize(hDataset)); - - json_object_array_add(poSize, poSizeX); - json_object_array_add(poSize, poSizeY); - - json_object *poStacSize = nullptr; - json_object_deep_copy(poSize, &poStacSize, nullptr); - json_object_object_add(poJsonObject, "size", poSize); - json_object_object_add(poStac, "proj:shape", poStacSize); + { + json_object *poSize = json_object_new_array(); + json_object *poSizeX = + json_object_new_int(GDALGetRasterXSize(hDataset)); + json_object *poSizeY = + json_object_new_int(GDALGetRasterYSize(hDataset)); + + // size is X, Y ordered + json_object_array_add(poSize, poSizeX); + json_object_array_add(poSize, poSizeY); + + json_object_object_add(poJsonObject, "size", poSize); + } + + { + json_object *poStacSize = json_object_new_array(); + json_object *poSizeX = + json_object_new_int(GDALGetRasterXSize(hDataset)); + json_object *poSizeY = + json_object_new_int(GDALGetRasterYSize(hDataset)); + + // ... but ... proj:shape is Y, X ordered. + json_object_array_add(poStacSize, poSizeY); + json_object_array_add(poStacSize, poSizeX); + + json_object_object_add(poStac, "proj:shape", poStacSize); + } } else { diff --git a/apps/gdaltindex_bin.cpp b/apps/gdaltindex_bin.cpp index dd1f56814fb3..25a5929bf5d2 100644 --- a/apps/gdaltindex_bin.cpp +++ b/apps/gdaltindex_bin.cpp @@ -51,7 +51,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg) " [-skip_different_projection] [-t_srs ]\n" " [-src_srs_name field_name] [-src_srs_format " "{AUTO|WKT|EPSG|PROJ}]\n" - " [-lyr_name ]\n" + " [-lyr_name ] [-lco =]...\n" " [-gti_filename ]\n" " [-tr ] [-te " "]\n" diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 59a6bbe124eb..79770cd2030b 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -75,6 +75,7 @@ struct GDALTileIndexOptions std::string osFormat{}; std::string osIndexLayerName{}; std::string osLocationField = "location"; + CPLStringList aosLCO{}; std::string osTargetSRS{}; bool bWriteAbsolutePath = false; bool bSkipDifferentProjection = false; @@ -504,9 +505,9 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, oSRS = *poSrcSRS; } - poLayer = poTileIndexDS->CreateLayer(osLayerName.c_str(), - oSRS.IsEmpty() ? nullptr : &oSRS, - wkbPolygon, nullptr); + poLayer = poTileIndexDS->CreateLayer( + osLayerName.c_str(), oSRS.IsEmpty() ? nullptr : &oSRS, wkbPolygon, + psOptions->aosLCO.List()); if (!poLayer) return nullptr; @@ -1193,6 +1194,11 @@ GDALTileIndexOptionsNew(char **papszArgv, CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); psOptions->osLocationField = papszArgv[++iArg]; } + else if (strcmp(papszArgv[iArg], "-lco") == 0) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + psOptions->aosLCO.AddString(papszArgv[++iArg]); + } else if (strcmp(papszArgv[iArg], "-t_srs") == 0) { CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); diff --git a/apps/gdaltransform.cpp b/apps/gdaltransform.cpp index fe2a5cb2509b..e161638b9432 100644 --- a/apps/gdaltransform.cpp +++ b/apps/gdaltransform.cpp @@ -67,7 +67,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg = nullptr) " [-ct ] [-order ] [-tps] [-rpc] [-geoloc] \n" " [-gcp [elevation]]... " "[-output_xy]\n" - " [srcfile [dstfile]]\n" + " [ []]\n" "\n"); if (pszErrorMsg != nullptr) diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 8d924e07bd8d..695c2bcd4b59 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -2705,8 +2705,8 @@ static GDALDatasetH GDALWarpDirect(const char *pszDest, GDALDatasetH hDstDS, // Check if transformation is inversible { - double dfX = GDALGetRasterXSize(hDstDS) / 2; - double dfY = GDALGetRasterYSize(hDstDS) / 2; + double dfX = GDALGetRasterXSize(hDstDS) / 2.0; + double dfY = GDALGetRasterYSize(hDstDS) / 2.0; double dfZ = 0; int bSuccess = false; const auto nErrorCounterBefore = CPLGetErrorCounter(); @@ -2798,12 +2798,10 @@ static GDALDatasetH GDALWarpDirect(const char *pszDest, GDALDatasetH hDstDS, * would be for this */ /* input dataset */ double adfSuggestedGeoTransform[6]; - double adfExtent[4]; int nPixels, nLines; - if (GDALSuggestedWarpOutput2(hSrcDS, pfnTransformer, - hTransformArg, - adfSuggestedGeoTransform, &nPixels, - &nLines, adfExtent, 0) == CE_None) + if (GDALSuggestedWarpOutput( + hSrcDS, pfnTransformer, hTransformArg, + adfSuggestedGeoTransform, &nPixels, &nLines) == CE_None) { dfTargetRatio = 1.0 / adfSuggestedGeoTransform[1]; } @@ -3977,9 +3975,13 @@ static GDALDatasetH GDALWarpCreateOutput( // For sum, round-up dimension, to be sure that the output extent // includes all source pixels, to have the sum preserving property. - const int nOptions = (psOptions->eResampleAlg == GRA_Sum) - ? GDAL_SWO_ROUND_UP_SIZE - : 0; + int nOptions = (psOptions->eResampleAlg == GRA_Sum) + ? GDAL_SWO_ROUND_UP_SIZE + : 0; + if (psOptions->bSquarePixels) + { + nOptions |= GDAL_SWO_FORCE_SQUARE_PIXEL; + } if (GDALSuggestedWarpOutput2(hSrcDS, psInfo->pfnTransform, hTransformArg, adfThisGeoTransform, &nThisPixels, &nThisLines, adfExtent, @@ -4818,16 +4820,56 @@ static void RemoveZeroWidthSlivers(OGRGeometry *poGeom) const OGRwkbGeometryType eType = wkbFlatten(poGeom->getGeometryType()); if (eType == wkbMultiPolygon) { - for (auto poSubGeom : *(poGeom->toMultiPolygon())) + auto poMP = poGeom->toMultiPolygon(); + int nNumGeometries = poMP->getNumGeometries(); + for (int i = 0; i < nNumGeometries; /* incremented in loop */) { - RemoveZeroWidthSlivers(poSubGeom); + auto poPoly = poMP->getGeometryRef(i); + RemoveZeroWidthSlivers(poPoly); + if (poPoly->IsEmpty()) + { + CPLDebug("WARP", + "RemoveZeroWidthSlivers: removing empty polygon"); + poMP->removeGeometry(i, /* bDelete = */ true); + --nNumGeometries; + } + else + { + ++i; + } } } else if (eType == wkbPolygon) { - for (auto poSubGeom : *(poGeom->toPolygon())) + auto poPoly = poGeom->toPolygon(); + if (auto poExteriorRing = poPoly->getExteriorRing()) { - RemoveZeroWidthSlivers(poSubGeom); + RemoveZeroWidthSlivers(poExteriorRing); + if (poExteriorRing->getNumPoints() < 4) + { + poPoly->empty(); + return; + } + } + int nNumInteriorRings = poPoly->getNumInteriorRings(); + for (int i = 0; i < nNumInteriorRings; /* incremented in loop */) + { + auto poRing = poPoly->getInteriorRing(i); + RemoveZeroWidthSlivers(poRing); + if (poRing->getNumPoints() < 4) + { + CPLDebug( + "WARP", + "RemoveZeroWidthSlivers: removing empty interior ring"); + constexpr int OFFSET_EXTERIOR_RING = 1; + poPoly->removeRing(i + OFFSET_EXTERIOR_RING, + /* bDelete = */ true); + --nNumInteriorRings; + } + else + { + ++i; + } } } else if (eType == wkbLineString) diff --git a/apps/ogr2ogr_bin.cpp b/apps/ogr2ogr_bin.cpp index 9558e220967f..18fc96b38e1b 100644 --- a/apps/ogr2ogr_bin.cpp +++ b/apps/ogr2ogr_bin.cpp @@ -386,10 +386,9 @@ MAIN_START(nArgc, papszArgv) { GDALDriverManager *poDM = GetGDALDriverManager(); - fprintf(stderr, - "FAILURE:\n" - "Unable to open datasource `%s' with the following drivers.\n", - psOptionsForBinary->osDataSource.c_str()); + CPLError(CE_Failure, CPLE_AppDefined, + "Unable to open datasource `%s' with the following drivers.", + psOptionsForBinary->osDataSource.c_str()); for (int iDriver = 0; iDriver < poDM->GetDriverCount(); iDriver++) { diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 4a74a870a731..a3c404c53133 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -4324,6 +4324,21 @@ SetupTargetLayer::Setup(OGRLayer *poSrcLayer, const char *pszNewLayerName, bPreserveFID = false; } } + // Detect scenario of converting from GPX to a format like GPKG + // Cf https://github.com/OSGeo/gdal/issues/9225 + else if (!bPreserveFID && !m_bUnsetFid && !bAppend && + m_poSrcDS->GetDriver() && + EQUAL(m_poSrcDS->GetDriver()->GetDescription(), "GPX") && + pszDestCreationOptions && + (strstr(pszDestCreationOptions, "='FID'") != nullptr || + strstr(pszDestCreationOptions, "=\"FID\"") != nullptr) && + CSLFetchNameValue(m_papszLCO, "FID") == nullptr) + { + CPLDebug("GDALVectorTranslate", + "Forcing -preserve_fid because source is GPX and layers " + "have FID cross references"); + bPreserveFID = true; + } // Detect scenario of converting GML2 with fid attribute to GPKG else if (EQUAL(m_poDstDS->GetDriver()->GetDescription(), "GPKG") && CSLFetchNameValue(m_papszLCO, "FID") == nullptr) diff --git a/autotest/alg/warp.py b/autotest/alg/warp.py index b128930a9e9c..2729fd4f3264 100755 --- a/autotest/alg/warp.py +++ b/autotest/alg/warp.py @@ -1669,3 +1669,74 @@ def test_warp_average_oversampling(): assert math.isnan(got_data[(y + 4) * 14 + (14 - 1 - x)]) for x in range(6): assert got_data[(y + 4) * 14 + (x + 4)] == 3.0 + + +############################################################################### +# Test bug fix for https://github.com/qgis/QGIS/issues/56288 + + +@pytest.mark.require_driver("XYZ") +def test_non_square(): + + # bottom up + content = """y x z +30.0 10.0 1 +30.0 10.2 2 +30.0 10.4 3 +30.1 10.0 4 +30.1 10.2 5 +30.1 10.4 6 +30.2 10.0 7 +30.2 10.2 8 +30.2 10.4 9 +""" + + with gdaltest.tempfile("/vsimem/grid.xyz", content): + ds = gdal.Open("/vsimem/grid.xyz") + assert ds.RasterXSize == 3 and ds.RasterYSize == 3 + assert ds.GetGeoTransform() == pytest.approx((9.9, 0.2, 0.0, 29.95, 0.0, 0.1)) + ulx, xres, xskew, uly, yskew, yres = ds.GetGeoTransform() + lrx = ulx + (ds.RasterXSize * xres) + lry = uly + (ds.RasterYSize * yres) + assert lrx == pytest.approx(10.5) + assert lry == pytest.approx(30.25) + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert struct.unpack("b" * (3 * 3), ds.ReadRaster()) == ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + ) + + warped = gdal.AutoCreateWarpedVRT(ds) + assert warped.GetRasterBand(1).DataType == gdal.GDT_Byte + assert ( + warped.RasterXSize == ds.RasterXSize + and warped.RasterYSize == ds.RasterYSize + ) + assert warped.GetGeoTransform() == pytest.approx( + (9.9, 0.2, 0.0, 30.25, 0.0, -0.1) + ) + assert struct.unpack("b" * (3 * 3), warped.ReadRaster()) == ( + 7, + 8, + 9, + 4, + 5, + 6, + 1, + 2, + 3, + ) + + # Test extent calculation + res = gdal.SuggestedWarpOutput(ds, []) + assert res.ymin == pytest.approx(29.95) + assert res.ymax == pytest.approx(30.25) + assert res.xmin == pytest.approx(9.9) + assert res.xmax == pytest.approx(10.5) diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index c833e87483ac..5c655d4b0cb4 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -57,6 +57,7 @@ #include "cpl_threadsafe_queue.hpp" #include +#include #include #include #include @@ -3127,6 +3128,7 @@ TEST_F(test_cpl, cpl_minixml) CPLXMLNode *psElt = CPLCreateXMLElementAndValue(psRoot, "Elt", "value"); CPLAddXMLAttributeAndValue(psElt, "attr1", "val1"); CPLAddXMLAttributeAndValue(psElt, "attr2", "val2"); + EXPECT_GE(CPLXMLNodeGetRAMUsageEstimate(psRoot), 0); char *str = CPLSerializeXMLTree(psRoot); CPLDestroyXMLNode(psRoot); ASSERT_STREQ( @@ -4889,4 +4891,137 @@ TEST_F(test_cpl, VSIGetCanonicalFilename) VSIUnlink(osLC.c_str()); } +TEST_F(test_cpl, CPLStrtod) +{ + { + const char *pszVal = "5"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 5.0); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + + { + const char *pszVal = "5 foo"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 5.0); + EXPECT_EQ(pszEnd, pszVal + 1); + } + + { + const char *pszVal = "foo"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), 0.0); + EXPECT_EQ(pszEnd, pszVal); + } + + { + const char *pszVal = "-inf"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + -std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "-Inf"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + -std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "-INF"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + -std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "-Infinity"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + -std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "-1.#INF"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + -std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + + { + const char *pszVal = "inf"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "Inf"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "INF"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "Infinity"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "1.#INF"; + char *pszEnd = nullptr; + EXPECT_EQ(CPLStrtod(pszVal, &pszEnd), + std::numeric_limits::infinity()); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + + { + const char *pszVal = "-1.#QNAN"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "-1.#IND"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "1.#QNAN"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "1.#SNAN"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "NaN"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } + { + const char *pszVal = "nan"; + char *pszEnd = nullptr; + EXPECT_TRUE(std::isnan(CPLStrtod(pszVal, &pszEnd))); + EXPECT_EQ(pszEnd, pszVal + strlen(pszVal)); + } +} + } // namespace diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp index a79a83ec21fc..5828da330ef5 100644 --- a/autotest/cpp/test_gdal.cpp +++ b/autotest/cpp/test_gdal.cpp @@ -2969,7 +2969,7 @@ TEST_F(test_gdal, GDALDatasetReportError) CPLPushErrorHandler(CPLQuietErrorHandler); poSrcDS->ReportError("%foo", CE_Warning, CPLE_AppDefined, "bar"); CPLPopErrorHandler(); - EXPECT_STREQ(CPLGetLastErrorMsg(), "bar"); + EXPECT_STREQ(CPLGetLastErrorMsg(), "%foo: bar"); CPLPushErrorHandler(CPLQuietErrorHandler); poSrcDS->ReportError( diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index f6abca4c25c0..0bc59ed1c072 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -2586,3 +2586,43 @@ def test_vrt_read_compute_statistics_approximate(tmp_vsimem): assert "STATISTICS_MEAN" in md assert "STATISTICS_STDDEV" in md assert md["STATISTICS_VALID_PERCENT"] == "100" + + +############################################################################### +# Test that when generating virtual overviews we select them as close as +# possible to source ones + + +def test_vrt_read_virtual_overviews_match_src_overviews(tmp_vsimem): + + gtiff_filename = str(tmp_vsimem / "tmp.tif") + ds = gdal.GetDriverByName("GTiff").Create( + gtiff_filename, 4095, 2047, options=["SPARSE_OK=YES"] + ) + ds.BuildOverviews("NONE", [2, 4, 8]) + band = ds.GetRasterBand(1) + # The below assert is a pre-condition to test the behavior of the + # "vrt://{gtiff_filename}?outsize=200%,200%". If we decided to round + # differently overview dataset sizes in the GTiff driver, we'd need to + # change this source dataset + assert band.GetOverview(0).XSize == 2048 + assert band.GetOverview(0).YSize == 1024 + assert band.GetOverview(1).XSize == 1024 + assert band.GetOverview(1).YSize == 512 + ds = None + + vrt_ds = gdal.Open(f"vrt://{gtiff_filename}?outsize=200%,200%") + assert vrt_ds.RasterXSize == 4095 * 2 + assert vrt_ds.RasterYSize == 2047 * 2 + vrt_band = vrt_ds.GetRasterBand(1) + assert vrt_band.XSize == 4095 * 2 + assert vrt_band.YSize == 2047 * 2 + assert vrt_band.GetOverviewCount() == 3 + # We reuse the dimension of the source dataset + assert vrt_band.GetOverview(0).XSize == 4095 + assert vrt_band.GetOverview(0).YSize == 2047 + # We reuse the dimension of the overviews of the source dataset + assert vrt_band.GetOverview(1).XSize == 2048 + assert vrt_band.GetOverview(1).YSize == 1024 + assert vrt_band.GetOverview(2).XSize == 1024 + assert vrt_band.GetOverview(2).YSize == 512 diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index a750d6ab50fe..69493680ff2f 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -768,15 +768,30 @@ def test_vsicurl_planetary_computer_url_signing(server): ) with webserver.install_http_handler(handler): - with gdaltest.config_option( - "VSICURL_PC_SAS_SIGN_HREF_URL", - "http://localhost:%d/pc_sas_sign_href?href=" % server.port, - ): - statres = gdal.VSIStatL( - "/vsicurl?pc_url_signing=yes&url=http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" - % server.port + try: + gdal.SetPathSpecificOption( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing" + % server.port, + "VSICURL_PC_URL_SIGNING", + "YES", + ) + + with gdaltest.config_option( + "VSICURL_PC_SAS_SIGN_HREF_URL", + "http://localhost:%d/pc_sas_sign_href?href=" % server.port, + ): + statres = gdal.VSIStatL( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" + % server.port + ) + assert statres.size == 3 + finally: + gdal.SetPathSpecificOption( + "/vsicurl/http://localhost:%d/test_vsicurl_planetary_computer_url_signing" + % server.port, + "VSICURL_PC_URL_SIGNING", + None, ) - assert statres.size == 3 # Check that signing request is done since it has expired gdal.VSICurlClearCache() @@ -788,14 +803,14 @@ def test_vsicurl_planetary_computer_url_signing(server): % server.port, 200, {}, - '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin?my_token"}' + '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin?my_token2"}' % server.port, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, - {"Content-Length": "3"}, + {"Content-Length": "4"}, ) with webserver.install_http_handler(handler): @@ -807,7 +822,7 @@ def test_vsicurl_planetary_computer_url_signing(server): "/vsicurl?pc_url_signing=yes&url=http://localhost:%d/test_vsicurl_planetary_computer_url_signing.bin" % server.port ) - assert statres.size == 3 + assert statres.size == 4 # Check that signing request is not needed gdal.VSICurlClearCache() @@ -815,7 +830,7 @@ def test_vsicurl_planetary_computer_url_signing(server): handler = webserver.SequentialHandler() handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, {"Content-Length": "3"}, ) @@ -838,12 +853,12 @@ def test_vsicurl_planetary_computer_url_signing(server): % server.port, 200, {}, - '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing2.bin?my_token2"}' + '{"msft:expiry":"9999-01-01T00:00:00","href":"http://localhost:%d/test_vsicurl_planetary_computer_url_signing2.bin?my_token3"}' % server.port, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing2.bin?my_token2", + "/test_vsicurl_planetary_computer_url_signing2.bin?my_token3", 200, {"Content-Length": "4"}, ) @@ -866,14 +881,14 @@ def test_vsicurl_planetary_computer_url_signing(server): handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing.bin?my_token", + "/test_vsicurl_planetary_computer_url_signing.bin?my_token2", 200, {"Content-Length": "3"}, ) handler.add( "HEAD", - "/test_vsicurl_planetary_computer_url_signing2.bin?my_token2", + "/test_vsicurl_planetary_computer_url_signing2.bin?my_token3", 200, {"Content-Length": "4"}, ) diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index 6e1353c11eeb..0ba955c2f316 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -937,8 +937,9 @@ def test_vsifile_opendir(basepath): for i in range(4): entry = gdal.GetNextDirEntry(d) assert entry - if entry.name == "test": - entries_found.append(entry.name) + name = entry.name.replace("\\", "/") + if name == "test": + entries_found.append(name) assert (entry.mode & 32768) != 0 assert entry.modeKnown assert entry.size == 3 @@ -946,14 +947,14 @@ def test_vsifile_opendir(basepath): assert entry.mtime != 0 assert entry.mtimeKnown assert not entry.extra - elif entry.name == "subdir": - entries_found.append(entry.name) + elif name == "subdir": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2": - entries_found.append(entry.name) + elif name == "subdir/subdir2": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2/test2": - entries_found.append(entry.name) + elif name == "subdir/subdir2/test2": + entries_found.append(name) assert (entry.mode & 32768) != 0 else: assert False, entry.name @@ -970,19 +971,20 @@ def test_vsifile_opendir(basepath): for i in range(4): entry = gdal.GetNextDirEntry(d) assert entry - if entry.name == "test": - entries_found.append(entry.name) + name = entry.name.replace("\\", "/") + if name == "test": + entries_found.append(name) assert (entry.mode & 32768) != 0 if os.name == "posix" and basepath == "tmp/": assert entry.size == 0 - elif entry.name == "subdir": - entries_found.append(entry.name) + elif name == "subdir": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2": - entries_found.append(entry.name) + elif name == "subdir/subdir2": + entries_found.append(name) assert (entry.mode & 16384) != 0 - elif entry.name == "subdir/subdir2/test2": - entries_found.append(entry.name) + elif name == "subdir/subdir2/test2": + entries_found.append(name) assert (entry.mode & 32768) != 0 if os.name == "posix" and basepath == "tmp/": assert entry.size == 0 @@ -1009,7 +1011,10 @@ def test_vsifile_opendir(basepath): # Depth 1 files = set( - [l_entry.name for l_entry in gdal.listdir(basepath + "/vsifile_opendir", 1)] + [ + l_entry.name.replace("\\", "/") + for l_entry in gdal.listdir(basepath + "/vsifile_opendir", 1) + ] ) assert files == set(["test", "subdir", "subdir/subdir2"]) @@ -1030,18 +1035,19 @@ def test_vsifile_opendir(basepath): entry = gdal.GetNextDirEntry(d) assert entry.name == "subdir" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2" + sep = "\\" if "\\" in entry.name else "/" + assert entry.name.replace("\\", "/") == "subdir/subdir2" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2/test2" + assert entry.name.replace("\\", "/") == "subdir/subdir2/test2" entry = gdal.GetNextDirEntry(d) assert not entry gdal.CloseDir(d) - d = gdal.OpenDir(basepath + "/vsifile_opendir", -1, ["PREFIX=subdir/sub"]) + d = gdal.OpenDir(basepath + "/vsifile_opendir", -1, ["PREFIX=subdir" + sep + "sub"]) entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2" + assert entry.name.replace("\\", "/") == "subdir/subdir2" entry = gdal.GetNextDirEntry(d) - assert entry.name == "subdir/subdir2/test2" + assert entry.name.replace("\\", "/") == "subdir/subdir2/test2" entry = gdal.GetNextDirEntry(d) assert not entry gdal.CloseDir(d) diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index 106869ab86cc..cacec03b5b45 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -3734,17 +3734,58 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp prefix_path = "\\\\?\\" + tmp_path_str - f = gdal.VSIFOpenL(prefix_path + "\\testsync.txt", "wb") - assert f - gdal.VSIFCloseL(f) - - # S3 to local: S3 file is newer + # S3 to local gdal.VSICurlClearCache() handler = webserver.SequentialHandler() - handler.add( "GET", - "/out/testsync.txt", + "/bucket/", + 200, + {}, + """ + + + + false + + subdir/ + 2037-01-01T00:00:01.000Z + 0 + + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/", + 200, + {}, + """ + + + + false + + subdir/ + 2037-01-01T00:00:01.000Z + 0 + + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/subdir/testsync.txt", 206, { "Content-Length": "3", @@ -3755,15 +3796,35 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp ) handler.add( "GET", - "/out/testsync.txt", + "/bucket/?delimiter=%2F&prefix=subdir%2F", + 200, + {}, + """ + + subdir/ + + false + + subdir/testsync.txt + 2037-01-01T00:00:01.000Z + 3 + + + """, + ) + handler.add( + "GET", + "/bucket/subdir/testsync.txt", 200, {"Content-Length": "3", "Last-Modified": "Mon, 01 Jan 2037 00:00:01 GMT"}, "foo", ) with webserver.install_http_handler(handler): - assert gdal.Sync("/vsis3/out/testsync.txt", prefix_path, options=options) + assert gdal.Sync("/vsis3/bucket/", prefix_path, options=options) - # Local to S3: S3 file is newer + assert gdal.VSIStatL(prefix_path + "\\subdir\\testsync.txt") is not None + + # Local to S3 gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add("GET", "/out/", 404) @@ -3779,14 +3840,12 @@ def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp false - testsync.txt - 2037-01-01T00:00:01.000Z - 3 """, ) - handler.add("PUT", "/out/testsync.txt", 200) + handler.add("PUT", "/out/subdir/", 200) + handler.add("PUT", "/out/subdir/testsync.txt", 200) with webserver.install_http_handler(handler): assert gdal.Sync(prefix_path + "\\", "/vsis3/out/", options=options) diff --git a/autotest/gdrivers/data/ers/references_ecw.ecw b/autotest/gdrivers/data/ers/references_ecw.ecw new file mode 100644 index 000000000000..ade6498f1606 Binary files /dev/null and b/autotest/gdrivers/data/ers/references_ecw.ecw differ diff --git a/autotest/gdrivers/data/ers/references_ecw.ers b/autotest/gdrivers/data/ers/references_ecw.ers new file mode 100644 index 000000000000..7c16a1b4b20d --- /dev/null +++ b/autotest/gdrivers/data/ers/references_ecw.ers @@ -0,0 +1,14 @@ +DatasetHeader Begin + Version = "6.2" + Name = "references_ecw.ers" + DataFile = "references_ecw.ecw" + DataSetType = Translated + DataType = Raster + ByteOrder = LSBFirst + RasterInfo Begin + CellType = Unsigned8BitInteger + NrOfLines = 512 + NrOfCellsPerLine = 519 + NrOfBands = 1 + RasterInfo End +DatasetHeader End diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data new file mode 100644 index 000000000000..000d0eb8d63d --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble.http_data @@ -0,0 +1,195 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:24 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Tue, 11 Feb 2025 05:49:25 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 7564 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 171838 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "Information about the Blue Marble Next Generation (2004) data (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "Information about the Blue Marble Next Generation (2004) data (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/mapml", + "title" : "Information about the Blue Marble Next Generation (2004) data (as MapML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=mapml" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "Information about the Blue Marble Next Generation (2004) data (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/dggs-list", + "title" : "Discrete Global Grid Systems for Blue Marble Next Generation (2004)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/dggs" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "application/json", + "title" : "Schema (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "text/plain", + "title" : "Schema (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/schema", + "type" : "text/html", + "title" : "Schema (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/schema?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "application/json", + "title" : "Queryables (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "text/plain", + "title" : "Queryables (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/queryables", + "type" : "text/html", + "title" : "Queryables (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/queryables?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage", + "type" : "image/png", + "title" : "Blue Marble Next Generation (2004) (as PNG; Note: requesting large extent may result in generalized data)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage?f=png" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage", + "type" : "image/tiff; application=geotiff", + "title" : "Blue Marble Next Generation (2004) (as GeoTIFF; Note: requesting large extent may result in generalized data)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage?f=tif" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage-domainset", + "type" : "application/json", + "title" : "Blue Marble Next Generation (2004) (domain set of the coverage for this collection)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage/domainset?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/coverage-rangetype", + "type" : "application/json", + "title" : "Blue Marble Next Generation (2004) (range type of the coverage for this collection)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/coverage/rangetype?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/png", + "title" : "Default map (as PNG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.png" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/jpeg", + "title" : "Default map (as JPG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.jpg" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/map", + "type" : "image/tif", + "title" : "Default map (as GeoTIFF)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map.tif" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets available for this dataset (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets available for this dataset (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets available for this dataset (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles?f=html" + }, + { + "rel" : "styles", + "type" : "text/html", + "title" : "Styles for Blue Marble Next Generation (2004) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=html" + }, + { + "rel" : "styles", + "type" : "application/json", + "title" : "Styles for Blue Marble Next Generation (2004) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=json" + }, + { + "rel" : "styles", + "type" : "text/plain", + "title" : "Styles for Blue Marble Next Generation (2004) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles?f=econ" + } + ], + "title" : "Blue Marble Next Generation (2004)", + "extent" : { + "spatial" : { + "bbox" : [ [ -180, -90, 180, 90 ] ], + "grid" : [ + { + "cellsCount" : 131072, + "resolution" : 0.0027465820312 + }, + { + "cellsCount" : 65536, + "resolution" : 0.0027465820312 + } + ] + }, + "temporal" : { + "interval" : [ [ "2004-01", "2004-12" ] ], + "grid" : { + "cellsCount" : 12, + "resolution" : "P1M" + } + } + }, + "crs" : [ + "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "http://www.opengis.net/def/crs/EPSG/0/4326", + "http://www.opengis.net/def/crs/EPSG/0/3857", + "http://www.opengis.net/def/crs/EPSG/0/3395" + ], + "storageCrs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "id" : "blueMarble", + "dataType" : "map", + "attribution" : "NASA Earth Observatory", + "minScaleDenominator" : 1091957.5469310893677, + "minCellSize" : 0.0027465820312 +} diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data new file mode 100644 index 000000000000..0760475e8302 Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.jpg.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data new file mode 100644 index 000000000000..ff0efadec872 Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.png.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data new file mode 100644 index 000000000000..70944fa0c06f Binary files /dev/null and b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_0_0_0.tif.http_data differ diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data new file mode 100644 index 000000000000..d044f8585f0a --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_WorldMercatorWGS84Quad_f_json.http_data @@ -0,0 +1,138 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:25 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Thu, 13 Feb 2025 04:01:39 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 5775 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 5505 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3395", + "dataType" : "map", + "tileMatrixSetLimits" : [ + { "tileMatrix" : "0", "minTileRow" : 0, "maxTileRow" : 0, "minTileCol" : 0, "maxTileCol" : 0 }, + { "tileMatrix" : "1", "minTileRow" : 0, "maxTileRow" : 1, "minTileCol" : 0, "maxTileCol" : 1 }, + { "tileMatrix" : "2", "minTileRow" : 0, "maxTileRow" : 3, "minTileCol" : 0, "maxTileCol" : 3 }, + { "tileMatrix" : "3", "minTileRow" : 0, "maxTileRow" : 7, "minTileCol" : 0, "maxTileCol" : 7 }, + { "tileMatrix" : "4", "minTileRow" : 0, "maxTileRow" : 15, "minTileCol" : 0, "maxTileCol" : 15 }, + { "tileMatrix" : "5", "minTileRow" : 0, "maxTileRow" : 31, "minTileCol" : 0, "maxTileCol" : 31 }, + { "tileMatrix" : "6", "minTileRow" : 0, "maxTileRow" : 63, "minTileCol" : 0, "maxTileCol" : 63 }, + { "tileMatrix" : "7", "minTileRow" : 0, "maxTileRow" : 127, "minTileCol" : 0, "maxTileCol" : 127 }, + { "tileMatrix" : "8", "minTileRow" : 0, "maxTileRow" : 255, "minTileCol" : 0, "maxTileCol" : 255 }, + { "tileMatrix" : "9", "minTileRow" : 0, "maxTileRow" : 511, "minTileCol" : 0, "maxTileCol" : 511 } + ], + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "The JSON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "The ECON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "The HTML representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?=html" + }, + { + "rel" : "alternate", + "type" : "application/json+tile", + "title" : "The TileJSON representation of the WorldMercatorWGS84Quad map tileset for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=tilejson" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldMercatorWGS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldMercatorWGS84Quad" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/geodata", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble" + }, + { + "rel" : "item", + "type" : "application/vnd.gnosis-map-tile", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as GNOSIS Map Tiles)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.gmt", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/png", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as PNG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.png", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/jpeg", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as JPG)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.jpg", + "templated" : true + }, + { + "rel" : "item", + "type" : "image/tiff; application=geotiff", + "title" : "WorldMercatorWGS84Quad map tiles for blueMarble (as GeoTIFF)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad/{tileMatrix}/{tileRow}/{tileCol}.tif", + "templated" : true + } + ], + "layers" : [ + { + "id" : "blueMarble", + "dataType" : "map", + "minScaleDenominator" : 1091957.5469310893677, + "minCellSize" : 305.748113140705, + "maxTileMatrix" : "9", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/geodata", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble" + } + ], + "propertiesSchema" : { + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "type" : "object", + "properties" : { "a" : { + "title" : "Alpha channel", + "type" : "integer", + "x-ogc-property-seq" : 4 + }, "b" : { + "title" : "Blue channel", + "type" : "integer", + "x-ogc-property-seq" : 3 + }, "g" : { + "title" : "Green channel", + "type" : "integer", + "x-ogc-property-seq" : 2 + }, "r" : { + "title" : "Red channel", + "type" : "integer", + "x-ogc-property-seq" : 1 + } } + } + } + ], + "centerPoint" : { + "coordinates" : [ 0, 0 ], + "tileMatrix" : "4", + "scaleDenominator" : 34942641.501794859767, + "cellSize" : 9783.9396205025605, + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + } +} diff --git a/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data new file mode 100644 index 000000000000..1694f8e8e181 --- /dev/null +++ b/autotest/gdrivers/data/ogcapi/request_collections_blueMarble_map_tiles_f_json.http_data @@ -0,0 +1,433 @@ +HTTP/1.1 200 OK +Date: Wed, 14 Feb 2024 05:33:25 GMT +Server: Apache/2.4.52 (Ubuntu) +Expires: Thu, 13 Feb 2025 04:01:39 GMT +Access-Control-Allow-Origin: * +Vary: Accept,Accept-Encoding,Prefer +Content-Length: 20038 +Access-Control-Allow-Headers: x-requested-with, Content-Type, origin, authorization, acc$ +Age: 5505 +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: application/json + +{ + "links" : [ + { + "rel" : "self", + "type" : "application/json", + "title" : "The JSON representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=json" + }, + { + "rel" : "alternate", + "type" : "text/plain", + "title" : "The ECON representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=econ" + }, + { + "rel" : "alternate", + "type" : "text/html", + "title" : "The HTML representation of the available map tilesets for blueMarble", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (default style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (default style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (default style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/default/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (evi style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (evi style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (evi style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (nir style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (nir style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (nir style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/nir/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (scl style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (scl style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (scl style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/scl/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (ndvi style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (ndvi style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (ndvi style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/ndvi/map/tiles?f=html" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "application/json", + "title" : "Map tilesets for blueMarble (evi2 style) (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=json" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/plain", + "title" : "Map tilesets for blueMarble (evi2 style) (as ECON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=econ" + }, + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tilesets-map", + "type" : "text/html", + "title" : "Map tilesets for blueMarble (evi2 style) (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/styles/evi2/map/tiles?f=html" + } + ], + "tilesets" : [ + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/CDB1GlobalGrid", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/4326", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "CDB1GlobalGridTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/CDB1GlobalGrid" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "CDB1GlobalGrid map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "CDB1GlobalGrid map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "CDB1GlobalGrid map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/CDB1GlobalGrid?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GlobalCRS84Pixel", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GlobalCRS84PixelTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GlobalCRS84Pixel" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GlobalCRS84Pixel map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Pixel?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GlobalCRS84Scale", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GlobalCRS84ScaleTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GlobalCRS84Scale" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GlobalCRS84Scale map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GlobalCRS84Scale map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GlobalCRS84Scale map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GlobalCRS84Scale?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GNOSISGlobalGrid", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/4326", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GNOSISGlobalGridTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GNOSISGlobalGrid" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GNOSISGlobalGrid map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GNOSISGlobalGrid?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/GoogleCRS84Quad", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "GoogleCRS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/GoogleCRS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "GoogleCRS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "GoogleCRS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "GoogleCRS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/GoogleCRS84Quad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/ISEA9R", + "crs" : "http://www.opengis.net/def/crs/OGC/0/153456", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "ISEA9RTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/ISEA9R" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "ISEA9R map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "ISEA9R map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "ISEA9R map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/ISEA9R?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3857", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WebMercatorQuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WebMercatorQuad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WebMercatorQuad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WebMercatorQuad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WebMercatorQuad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WebMercatorQuad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad", + "crs" : "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldCRS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldCRS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WorldCRS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WorldCRS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WorldCRS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldCRS84Quad?f=tilejson" + } + ] + }, + { + "title" : "Blue Marble Next Generation (2004)", + "tileMatrixSetURI" : "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldMercatorWGS84Quad", + "crs" : "http://www.opengis.net/def/crs/EPSG/0/3395", + "dataType" : "map", + "links" : [ + { + "rel" : "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type" : "application/json", + "title" : "WorldMercatorWGS84QuadTileMatrixSet definition (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/tileMatrixSets/WorldMercatorWGS84Quad" + }, + { + "rel" : "self", + "type" : "application/json", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (as JSON)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=json" + }, + { + "rel" : "self", + "type" : "text/html", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (as HTML)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=html" + }, + { + "rel" : "self", + "type" : "application/json+tilejson", + "title" : "WorldMercatorWGS84Quad map tileset for blueMarble (in TileJSON format)", + "href" : "http://127.0.0.1:8080/fakeogcapi/collections/blueMarble/map/tiles/WorldMercatorWGS84Quad?f=tilejson" + } + ] + } + ] +} diff --git a/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf b/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf new file mode 100644 index 000000000000..5c942a9c5c31 Binary files /dev/null and b/autotest/gdrivers/data/pdf/layer_with_same_name_on_different_pages.pdf differ diff --git a/autotest/gdrivers/dimap.py b/autotest/gdrivers/dimap.py index 95a7e066e702..9758e256362d 100755 --- a/autotest/gdrivers/dimap.py +++ b/autotest/gdrivers/dimap.py @@ -145,6 +145,14 @@ def test_dimap_2_single_component(): "GSD_ALONG_TRACK": "CENTER_GSD_ALONG_TRACK", "IMAGE_ORIENTATION": "CENTER_IMAGE_ORIENTATION", "AZIMUTH_ANGLE": "CENTER_AZIMUTH_ANGLE", + "RADIOMETRIC_RADIOMETRIC_PROCESSING": "BASIC", + "RADIOMETRIC_INTER_DETECTOR_NORMALIZATION": "false", + "RADIOMETRIC_DETECTORS_INTERPOLATION": "true", + "RADIOMETRIC_STRAYLIGHT_CORRECTION": "false", + "RADIOMETRIC_VCTI_CORRECTION": "false", + "RADIOMETRIC_INTER_ARRAY_RECONSTRUCTION": "true", + "RADIOMETRIC_RADIOMETRIC_STRETCH": "false", + "RADIOMETRIC_OUT_OF_ORDER_THRESHOLD": "0.5", } assert md == expected_md, "metadata wrong." diff --git a/autotest/gdrivers/ecw.py b/autotest/gdrivers/ecw.py index 388e31663c75..008057c1c11e 100755 --- a/autotest/gdrivers/ecw.py +++ b/autotest/gdrivers/ecw.py @@ -2210,6 +2210,66 @@ def test_ecw_read_uint32_jpeg2000(): ) +############################################################################### +# Test unsupported XML SRS + + +def test_jp2ecw_unsupported_srs_for_gmljp2(tmp_vsimem): + + if gdaltest.jp2ecw_drv is None or not has_write_support(): + pytest.skip("ECW write support not available") + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate(filename, "data/byte.tif", outputSRS=wkt, format="JP2ECW") + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() + + ############################################################################### diff --git a/autotest/gdrivers/ers.py b/autotest/gdrivers/ers.py index 82046df69d14..701580704ac9 100755 --- a/autotest/gdrivers/ers.py +++ b/autotest/gdrivers/ers.py @@ -319,6 +319,16 @@ def test_ers_recursive_opening(): ds.GetFileList() +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/9352 + + +@pytest.mark.require_driver("ECW") +def test_ers_open_data_file_ecw(): + + assert gdal.Open("data/ers/references_ecw.ers") + + ############################################################################### # Cleanup diff --git a/autotest/gdrivers/georaster.py b/autotest/gdrivers/georaster.py index 661fffe4beab..d5e312164dcb 100755 --- a/autotest/gdrivers/georaster.py +++ b/autotest/gdrivers/georaster.py @@ -54,7 +54,7 @@ def get_connection_str(): if oci_dsname is None: # TODO: Spelling - informe? return "" - return "geor:" + oci_dsname.split(":")[1] + return "geor:" + oci_dsname[oci_dsname.index(":") + 1 :] ############################################################################### @@ -488,6 +488,37 @@ def test_georaster_4bit(): # +def test_georaster_genstats(): + if gdaltest.oci_ds is None: + pytest.skip() + + ds_src = gdal.Open("data/byte.tif") + + ds = gdaltest.georasterDriver.CreateCopy( + get_connection_str() + ",GDAL_TEST,RASTER", + ds_src, + 1, + [ + "DESCRIPTION=(id number, raster sdo_georaster)", + "INSERT=(1014, sdo_geor.init('GDAL_TEST_RDT',1014))", + "NBITS=4", + "GENSTATS=TRUE", + ], + ) + + ds_name = ds.GetDescription() + + ds = None + + tst = gdaltest.GDALTest("GeoRaster", ds_name, 1, 2578, filename_absolute=1) + + tst.testOpen() + + +############################################################################### +# + + def test_georaster_cleanup(): if gdaltest.oci_ds is None: pytest.skip() diff --git a/autotest/gdrivers/jp2kak.py b/autotest/gdrivers/jp2kak.py index 209032e4e7c6..c8f14d7aa6eb 100755 --- a/autotest/gdrivers/jp2kak.py +++ b/autotest/gdrivers/jp2kak.py @@ -33,7 +33,7 @@ import gdaltest import pytest -from osgeo import gdal +from osgeo import gdal, osr pytestmark = pytest.mark.require_driver("JP2KAK") @@ -953,3 +953,60 @@ def find_xml_node(ar, element_name, only_attributes=False): if found is not None: return found return None + + +############################################################################### +# Test unsupported XML SRS + + +def test_jp2kak_unsupported_srs_for_gmljp2(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate(filename, "data/byte.tif", outputSRS=wkt, format="JP2KAK") + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() diff --git a/autotest/gdrivers/jp2openjpeg.py b/autotest/gdrivers/jp2openjpeg.py index e38952725803..feaddd440910 100755 --- a/autotest/gdrivers/jp2openjpeg.py +++ b/autotest/gdrivers/jp2openjpeg.py @@ -3850,3 +3850,76 @@ def test_jp2openjpeg_vrt_protocol(): webserver.server_stop(webserver_process, webserver_port) gdal.VSICurlClearCache() + + +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/9236 + + +def test_jp2openjpeg_limit_resolution_count_from_image_size(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + assert gdal.Translate(filename, "data/byte.tif", width=1024, height=7) + + # Check number of resolutions + ret = gdal.GetJPEG2000StructureAsString(filename, ["ALL=YES"]) + assert '2' in ret + + +############################################################################### +# Test unsupported XML SRS + + +def test_jp2openjpeg_unsupported_srs_for_gmljp2(tmp_vsimem): + + filename = str(tmp_vsimem / "out.jp2") + # There is no EPSG code and Albers Equal Area is not supported by OGRSpatialReference::exportToXML() + wkt = """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + gdal.ErrorReset() + assert gdal.Translate( + filename, "data/byte.tif", outputSRS=wkt, format="JP2OpenJPEG" + ) + assert gdal.GetLastErrorMsg() == "" + ds = gdal.Open(filename) + ref_srs = osr.SpatialReference() + ref_srs.ImportFromWkt(wkt) + assert ds.GetSpatialRef().IsSame(ref_srs) + # Check that we do *not* have a GMLJP2 box + assert "xml:gml.root-instance" not in ds.GetMetadataDomainList() diff --git a/autotest/gdrivers/ogcapi.py b/autotest/gdrivers/ogcapi.py index d078ca0bb81d..f12a239ec7ed 100644 --- a/autotest/gdrivers/ogcapi.py +++ b/autotest/gdrivers/ogcapi.py @@ -438,3 +438,42 @@ def test_ogc_api_raster_tiles(): assert ds.GetRasterBand(1).ReadBlock( ds.RasterXSize // 2 // 256, ds.RasterYSize // 2 // 256 ) + + +@pytest.mark.parametrize( + "image_format,raster_count,statistics", + ( + ("AUTO", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("PNG", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("PNG_PREFERRED", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ("JPEG", 3, [0.0, 255.0, 83.83631896972656, 30.486497283147653]), + ("JPEG_PREFERRED", 3, [0.0, 255.0, 83.83631896972656, 30.486497283147653]), + ("GEOTIFF", 4, [0.0, 255.0, 83.8084411621094, 30.532715248645687]), + ), +) +@pytest.mark.require_driver("WMS") +def test_ogc_api_raster_tiles_format(image_format, raster_count, statistics): + + ds = gdal.OpenEx( + f"OGCAPI:http://127.0.0.1:{gdaltest.webserver_port}/fakeogcapi/collections/blueMarble", + gdal.OF_RASTER, + open_options=[ + "API=TILES", + "CACHE=NO", + "TILEMATRIXSET=WorldMercatorWGS84Quad", + f"IMAGE_FORMAT={image_format}", + ], + ) + + assert ds is not None + + assert ds.RasterCount == raster_count + assert ds.RasterXSize == 131072 + assert ds.GetRasterBand(1).GetOverviewCount() == 9 + + # For some reason these tests fail on Github on Ubuntu 20.04 gcc and Ubuntu 20.04 coverage, but + # pass on all other builds. + # assert ds.RasterYSize == 1586181 + # assert ds.GetRasterBand(1).GetStatistics(True, True) == statistics + + del ds diff --git a/autotest/gdrivers/pdf.py b/autotest/gdrivers/pdf.py index 02e4c8b5fef3..7db4786de764 100755 --- a/autotest/gdrivers/pdf.py +++ b/autotest/gdrivers/pdf.py @@ -30,6 +30,7 @@ ############################################################################### import os +import shutil import sys import gdaltest @@ -1363,6 +1364,48 @@ def test_pdf_layers(poppler_or_pdfium): assert not (cs4 == cs1 or cs4 == cs2), "did not get expected checksum" +############################################################################### +# Check layers support + + +def test_pdf_layers_with_same_name_on_different_pages(poppler_or_pdfium): + + ds = gdal.Open("data/pdf/layer_with_same_name_on_different_pages.pdf") + layers = ds.GetMetadata_List("LAYERS") + assert layers == [ + "LAYER_00_NAME=Map_Frame (page 1)", + "LAYER_01_NAME=Map_Frame.Map (page 1)", + "LAYER_02_NAME=Map_Frame.Map.States (page 1)", + "LAYER_03_NAME=Map_Frame (page 2)", + "LAYER_04_NAME=Map_Frame.Map (page 2)", + "LAYER_05_NAME=Map_Frame.Map.States (page 2)", + "LAYER_06_NAME=Map_Frame (page 3)", + "LAYER_07_NAME=Map_Frame.Map (page 3)", + "LAYER_08_NAME=Map_Frame.Map.States (page 3)", + "LAYER_09_NAME=Map_Frame (page 4)", + "LAYER_10_NAME=Map_Frame.Map (page 4)", + "LAYER_11_NAME=Map_Frame.Map.States (page 4)", + ] + + for page in (1, 2, 3, 4): + ds = gdal.Open( + f"PDF:{page}:data/pdf/layer_with_same_name_on_different_pages.pdf" + ) + layers = ds.GetMetadata_List("LAYERS") + assert layers == [ + "LAYER_00_NAME=Map_Frame", + "LAYER_01_NAME=Map_Frame.Map", + "LAYER_02_NAME=Map_Frame.Map.States", + ] + cs_all = ds.GetRasterBand(1).Checksum() + + ds = gdal.OpenEx( + f"PDF:{page}:data/pdf/layer_with_same_name_on_different_pages.pdf", + open_options=["LAYERS_OFF=Map_Frame.Map.States"], + ) + assert ds.GetRasterBand(1).Checksum() != cs_all + + ############################################################################### # Test MARGIN, EXTRA_STREAM, EXTRA_LAYER_NAME and EXTRA_IMAGES options @@ -1957,6 +2000,24 @@ def test_pdf_metadata(poppler_or_pdfium): gdal.GetDriverByName("PDF").Delete("tmp/pdf_metadata.pdf") +############################################################################### +# Test PAM support with subdatasets + + +def test_pdf_pam_subdatasets(poppler_or_pdfium, tmp_path): + + tmpfilename = str(tmp_path / "test_pdf_pam_subdatasets.pdf") + shutil.copy("data/pdf/byte_and_rgbsmall_2pages.pdf", tmpfilename) + + ds = gdal.Open("PDF:1:" + tmpfilename) + ds.GetRasterBand(1).ComputeStatistics(False) + ds = None + assert gdal.VSIStatL(tmpfilename + ".aux.xml") + ds = gdal.Open("PDF:1:" + tmpfilename) + assert ds.GetRasterBand(1).GetMetadataItem("STATISTICS_MINIMUM") is not None + ds = None + + ############################################################################### # Test PAM georef support diff --git a/autotest/gdrivers/vrtderived.py b/autotest/gdrivers/vrtderived.py index e02b4c459f37..b5fd42965881 100755 --- a/autotest/gdrivers/vrtderived.py +++ b/autotest/gdrivers/vrtderived.py @@ -1023,6 +1023,51 @@ def identity(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize _validate(xml) +############################################################################### + + +@pytest.mark.parametrize("dtype", range(1, gdal.GDT_TypeCount)) +def test_vrt_derived_dtype(tmp_vsimem, dtype): + pytest.importorskip("numpy") + + input_fname = tmp_vsimem / "input.tif" + + nx = 1 + ny = 1 + + with gdal.GetDriverByName("GTiff").Create( + input_fname, nx, ny, 1, eType=gdal.GDT_Int8 + ) as input_ds: + input_ds.GetRasterBand(1).Fill(1) + gt = input_ds.GetGeoTransform() + + vrt_xml = f""" + + {', '.join([str(x) for x in gt])} + + Python + identity + + + + {input_fname} + 1 + + + + """ + + with gdal.config_option("GDAL_VRT_ENABLE_PYTHON", "YES"): + with gdal.Open(vrt_xml) as vrt_ds: + arr = vrt_ds.ReadAsArray() + if dtype not in {gdal.GDT_CInt16, gdal.GDT_CInt32}: + assert arr[0, 0] == 1 + assert vrt_ds.GetRasterBand(1).DataType == dtype + + ############################################################################### # Cleanup. diff --git a/autotest/gdrivers/vrtpansharpen.py b/autotest/gdrivers/vrtpansharpen.py index 58d158734d22..5215a6a25c10 100755 --- a/autotest/gdrivers/vrtpansharpen.py +++ b/autotest/gdrivers/vrtpansharpen.py @@ -2304,3 +2304,72 @@ def test_vrtpansharpen_out_of_order_input_bands_and_nodata(): cs2 = [vrt_ds.GetRasterBand(i + 1).Checksum() for i in range(vrt_ds.RasterCount)] assert cs2 == cs[::-1] + + +############################################################################### +# Test open options for input bands + + +def test_vrtpansharpen_open_options_input_bands(): + def my_handler(typ, errno, msg): + msgs.append(msg) + + msgs = [] + with gdaltest.error_handler(my_handler): + gdal.Open( + """ + + + tmp/small_world_pan.tif + + foo + + 1 + + + data/small_world.tif + 1 + + + data/small_world.tif + 2 + + + data/small_world.tif + 3 + + + """ + ) + # Not the prettiest way to check that open options are used, but that does the job... + assert "small_world_pan.tif: Invalid value for NUM_THREADS: foo" in msgs + + msgs = [] + with gdaltest.error_handler(my_handler): + gdal.Open( + """ + + + tmp/small_world_pan.tif + 1 + + + data/small_world.tif + + foo + + 1 + + + data/small_world.tif + 2 + + + data/small_world.tif + 3 + + + """ + ) + # Not the prettiest way to check that open options are used, but that does the job... + assert "small_world.tif: Invalid value for NUM_THREADS: foo" in msgs diff --git a/autotest/ogr/data/csv/header_with_line_break.csv b/autotest/ogr/data/csv/header_with_line_break.csv new file mode 100644 index 000000000000..e56f470e181b --- /dev/null +++ b/autotest/ogr/data/csv/header_with_line_break.csv @@ -0,0 +1,6 @@ +Column one,Column two,"Column with a +line break",Column three,"Another +line break",Column four,Column five +1,2,3,4,5,6,7 +4,5,6,7,8,9,10 +7,8,9,10,11,12,13 diff --git a/autotest/ogr/data/kml/gx_track_without_when.kml b/autotest/ogr/data/kml/gx_track_without_when.kml new file mode 100644 index 000000000000..2994cbb83278 --- /dev/null +++ b/autotest/ogr/data/kml/gx_track_without_when.kml @@ -0,0 +1,17 @@ + + + + + + -122.207881 37.371915 156.000000 + -122.205712 37.373288 152.000000 + -122.204678 37.373939 147.000000 + -122.203572 37.374630 142.199997 + -122.203451 37.374706 141.800003 + -122.203329 37.374780 141.199997 + -122.203207 37.374857 140.199997 + + + + diff --git a/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles b/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles new file mode 100644 index 000000000000..44057c2036e3 Binary files /dev/null and b/autotest/ogr/data/pmtiles/subset7_truncated.pmtiles differ diff --git a/autotest/ogr/ogr_csv.py b/autotest/ogr/ogr_csv.py index b8efff32ca28..9008bec734ba 100755 --- a/autotest/ogr/ogr_csv.py +++ b/autotest/ogr/ogr_csv.py @@ -2988,6 +2988,37 @@ def test_ogr_csv_getextent3d(tmp_vsimem): ############################################################################### +def test_ogr_csv_read_header_with_line_break(): + + ds = ogr.Open("data/csv/header_with_line_break.csv") + lyr = ds.GetLayer(0) + lyr_defn = lyr.GetLayerDefn() + assert [ + lyr_defn.GetFieldDefn(i).GetName() for i in range(lyr_defn.GetFieldCount()) + ] == [ + "Column one", + "Column two", + "Column with a\nline break", + "Column three", + "Another\nline break", + "Column four", + "Column five", + ] + f = lyr.GetNextFeature() + assert [f.GetField(i) for i in range(lyr_defn.GetFieldCount())] == [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + ] + + +############################################################################### + + if __name__ == "__main__": gdal.UseExceptions() if len(sys.argv) != 2: diff --git a/autotest/ogr/ogr_fgdb.py b/autotest/ogr/ogr_fgdb.py index da1a1ea8dd3e..f0c483500dcd 100755 --- a/autotest/ogr/ogr_fgdb.py +++ b/autotest/ogr/ogr_fgdb.py @@ -2875,7 +2875,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "my forward path label" assert rel.GetBackwardPathLabel() == "my backward path label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -2887,7 +2887,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -2901,7 +2901,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["destination_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -2913,7 +2913,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -2925,7 +2925,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_many_to_many") assert rel is not None @@ -2939,7 +2939,7 @@ def test_ogr_filegdb_read_relationships(openfilegdb_drv, fgdb_drv): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["dest_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_geom.py b/autotest/ogr/ogr_geom.py index 9cfe8bd5f1b3..b9aeee360974 100755 --- a/autotest/ogr/ogr_geom.py +++ b/autotest/ogr/ogr_geom.py @@ -2874,14 +2874,14 @@ def test_ogr_geom_getcurvegeometry(): # Add repeated point at end of line g2 = g1.GetLinearGeometry() - g2.AddPoint_2D(2 - 1e-9, 0) + g2.AddPoint_2D(2 - 1e-14, 0) g3 = g2.GetCurveGeometry() assert g3.Equals(g1) # Add repeated point at end of line g2 = g1.GetLinearGeometry() g2_new = ogr.Geometry(ogr.wkbLineString) - g2_new.AddPoint_2D(0, 1e-9) + g2_new.AddPoint_2D(0, 1e-14) for i in range(g2.GetPointCount()): g2_new.AddPoint_2D(g2.GetX(i), g2.GetY(i)) g3 = g2_new.GetCurveGeometry() @@ -3019,6 +3019,20 @@ def test_ogr_geom_getcurvegeometry_issue_8332(): ogrtest.check_feature_geometry(g3, expected_g, 1e-10) +############################################################################### + + +def test_ogr_geom_getcurvegeometry_issue9382(): + + wkt = "POLYGON ((299517.68 6695650.45,299519.85 6695648.29,299522.85 6695645.24,299523.253879511 6695644.9271687,299523.639375687 6695644.59194529,299524.00524874 6695644.23540783,299524.350321919 6695643.8587029,299524.673485444 6695643.46304214,299524.973699995 6695643.04969797,299525.25 6695642.62,299525.27 6695642.6,299525.28 6695642.57,299525.581819649 6695642.02405493,299525.844959794 6695641.4584502,299526.068147436 6695640.87592268,299526.250302851 6695640.27928972,299526.390544832 6695639.67143822,299526.488194942 6695639.05530858,299526.542780772 6695638.43388152,299526.554038241 6695637.81016326,299526.521912903 6695637.18717098,299526.446560174 6695636.56791901,299526.328344583 6695635.95540261,299526.167838007 6695635.35258508,299525.965816945 6695634.76238274,299525.723258704 6695634.18765068,299525.441336721 6695633.63116956,299525.121414825 6695633.09563136,299524.765040724 6695632.58362663,299524.373938437 6695632.09763257,299523.95 6695631.64,299522.14 6695633.39,299522.32 6695633.59,299522.48 6695633.77,299522.63 6695633.96,299522.77 6695634.15,299522.9 6695634.35,299523.03 6695634.55,299523.15 6695634.76,299523.26 6695634.98,299523.37 6695635.19,299523.46 6695635.41,299523.55 6695635.64,299523.63 6695635.86,299523.7 6695636.09,299523.76 6695636.32,299523.81 6695636.56,299523.86 6695636.79,299523.9 6695637.03,299523.92 6695637.27,299523.94 6695637.51,299523.95 6695637.71,299523.95 6695637.97,299523.94 6695638.23,299523.93 6695638.47,299523.9 6695638.71,299523.87 6695638.95,299523.82 6695639.18,299523.77 6695639.42,299523.71 6695639.65,299523.64 6695639.88,299523.56 6695640.11,299523.47 6695640.33,299523.38 6695640.55,299523.28 6695640.77,299523.17 6695640.98,299523.05 6695641.19,299522.92 6695641.4,299522.79 6695641.6,299522.65 6695641.79,299522.5 6695641.98,299522.34 6695642.16,299522.21 6695642.31,299522.08 6695642.45,299515.85 6695648.75,299515.41 6695649.19,299514.69 6695649.95,299513.99 6695650.71,299513.29 6695651.49,299512.902658228 6695651.94,299516.254162679 6695651.94,299517.68 6695650.45))" + g = ogr.CreateGeometryFromWkt(wkt) + g = g.GetCurveGeometry() + + expected_wkt = "CURVEPOLYGON (COMPOUNDCURVE ((299517.68 6695650.45,299519.85 6695648.29,299522.85 6695645.24),CIRCULARSTRING (299522.85 6695645.24,299524.00524874 6695644.23540783,299525.25 6695642.62),(299525.25 6695642.62,299525.27 6695642.6,299525.28 6695642.57),CIRCULARSTRING (299525.28 6695642.57,299526.521912903 6695637.18717098,299523.95 6695631.64),(299523.95 6695631.64,299522.14 6695633.39,299522.32 6695633.59,299522.48 6695633.77,299522.63 6695633.96,299522.77 6695634.15,299522.9 6695634.35,299523.03 6695634.55,299523.15 6695634.76,299523.26 6695634.98,299523.37 6695635.19,299523.46 6695635.41,299523.55 6695635.64,299523.63 6695635.86,299523.7 6695636.09,299523.76 6695636.32,299523.81 6695636.56,299523.86 6695636.79,299523.9 6695637.03,299523.92 6695637.27,299523.94 6695637.51,299523.95 6695637.71,299523.95 6695637.97,299523.94 6695638.23,299523.93 6695638.47,299523.9 6695638.71,299523.87 6695638.95,299523.82 6695639.18,299523.77 6695639.42,299523.71 6695639.65,299523.64 6695639.88,299523.56 6695640.11,299523.47 6695640.33,299523.38 6695640.55,299523.28 6695640.77,299523.17 6695640.98,299523.05 6695641.19,299522.92 6695641.4,299522.79 6695641.6,299522.65 6695641.79,299522.5 6695641.98,299522.34 6695642.16,299522.21 6695642.31,299522.08 6695642.45,299515.85 6695648.75,299515.41 6695649.19,299514.69 6695649.95,299513.99 6695650.71,299513.29 6695651.49,299512.902658228 6695651.94,299516.254162679 6695651.94,299517.68 6695650.45)))" + expected_g = ogr.CreateGeometryFromWkt(expected_wkt) + ogrtest.check_feature_geometry(g, expected_g, 1e-10) + + ############################################################################### # Test OGR_GT_ functions diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index 6d73b0106f0c..6d58c77795dd 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -2295,6 +2295,30 @@ def test_ogr_gpkg_22(tmp_vsimem): assert f.GetFID() == 1234567890123 +############################################################################### +# Test creating a feature with FID 0 + + +def test_ogr_gpkg_create_with_fid_0(tmp_vsimem): + + fname = tmp_vsimem / "test_ogr_gpkg_create_with_fid_0.gpkg" + + ds = gdaltest.gpkg_dr.CreateDataSource(fname) + lyr = ds.CreateLayer("test") + + feat = ogr.Feature(lyr.GetLayerDefn()) + feat.SetFID(0) + lyr.CreateFeature(feat) + feat = None + + ds = None + + ds = ogr.Open(fname) + lyr = ds.GetLayerByName("test") + f = lyr.GetNextFeature() + assert f.GetFID() == 0 + + ############################################################################### # Test not nullable fields @@ -7008,6 +7032,13 @@ def test_ogr_gpkg_relations(tmp_vsimem, tmp_path): assert rel.GetRightMappingTableFields() == ["related_id"] assert rel.GetRelatedTableType() == "attributes" + # ensure that the mapping table, which is present in gpkgext_relations but + # NOT gpkg_contents can be opened as a layer + lyr = ds.GetLayer("my_mapping_table") + assert lyr is not None + assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "base_id" + assert lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "related_id" + lyr = ds.GetLayer("a") lyr.Rename("a_renamed") lyr.AlterFieldDefn( @@ -7086,6 +7117,33 @@ def test_ogr_gpkg_relations(tmp_vsimem, tmp_path): assert rel.GetRightMappingTableFields() == ["related_id"] assert rel.GetRelatedTableType() == "features" + # a one-to-many relationship defined using foreign key constraints + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + ds.ExecuteSQL( + "CREATE TABLE test_relation_a(artistid INTEGER PRIMARY KEY, artistname TEXT)" + ) + ds.ExecuteSQL( + "CREATE TABLE test_relation_b(trackid INTEGER, trackname TEXT, trackartist INTEGER, FOREIGN KEY(trackartist) REFERENCES test_relation_a(artistid))" + ) + ds = None + + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + assert ds.GetRelationshipNames() == [ + "custom_type", + "test_relation_a_test_relation_b", + ] + assert ds.GetRelationship("custom_type") is not None + rel = ds.GetRelationship("test_relation_a_test_relation_b") + assert rel is not None + assert rel.GetName() == "test_relation_a_test_relation_b" + assert rel.GetLeftTableName() == "test_relation_a" + assert rel.GetRightTableName() == "test_relation_b" + assert rel.GetCardinality() == gdal.GRC_ONE_TO_MANY + assert rel.GetType() == gdal.GRT_ASSOCIATION + assert rel.GetLeftTableFields() == ["artistid"] + assert rel.GetRightTableFields() == ["trackartist"] + assert rel.GetRelatedTableType() == "features" + ds = None @@ -7260,6 +7318,31 @@ def get_query_row_count(query): == 1 ) + # validate mapping table was created + assert get_query_row_count("SELECT * FROM 'origin_table_dest_table'") == 0 + # validate mapping table is present in gpkg_contents + assert ( + get_query_row_count( + "SELECT * FROM gpkg_contents WHERE table_name='origin_table_dest_table' AND data_type='attributes'" + ) + == 1 + ) + # force delete from gpkg_contents, and then ensure that we CAN successfully + # load layers which are present ONLY in gpkgext_relations but NOT + # gpkg_contents (i.e. datasources which follow the Related Tables specification + # exactly) + ds.ExecuteSQL( + "DELETE FROM gpkg_contents WHERE table_name='origin_table_dest_table'" + ) + + ds = gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) + # ensure that the mapping table, which is present in gpkgext_relations but + # NOT gpkg_contents can be opened as a layer + lyr = ds.GetLayer("origin_table_dest_table") + assert lyr is not None + assert lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "base_id" + assert lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "related_id" + lyr = ds.CreateLayer("origin_table2", geom_type=ogr.wkbNone) fld_defn = ogr.FieldDefn("o_pkey", ogr.OFTInteger) assert lyr.CreateField(fld_defn) == ogr.OGRERR_NONE diff --git a/autotest/ogr/ogr_gpx.py b/autotest/ogr/ogr_gpx.py index f9bff84c1dc7..3af9c0a10a8c 100755 --- a/autotest/ogr/ogr_gpx.py +++ b/autotest/ogr/ogr_gpx.py @@ -572,3 +572,22 @@ def test_ogr_gpx_N_MAX_LINKS(): lyr = ds.GetLayerByName("track_points") f = lyr.GetNextFeature() assert f["link3_href"] is None + + +############################################################################### +# Test preservation of FID when converting to GPKG +# (https://github.com/OSGeo/gdal/issues/9225) + + +@pytest.mark.require_driver("GPKG") +def test_ogr_gpx_convert_to_gpkg(tmp_vsimem): + + outfilename = str(tmp_vsimem / "out.gpkg") + gdal.VectorTranslate(outfilename, "data/gpx/test.gpx") + + ds = ogr.Open(outfilename) + lyr = ds.GetLayer("tracks") + f = lyr.GetNextFeature() + assert f.GetFID() == 0 + f = lyr.GetNextFeature() + assert f.GetFID() == 1 diff --git a/autotest/ogr/ogr_ili.py b/autotest/ogr/ogr_ili.py index 67155ba7ab56..b545d4756dbb 100755 --- a/autotest/ogr/ogr_ili.py +++ b/autotest/ogr/ogr_ili.py @@ -34,6 +34,7 @@ pytestmark = [ pytest.mark.require_driver("Interlis 1"), + pytest.mark.require_driver("Interlis 2"), pytest.mark.random_order(disabled=True), ] @@ -1197,3 +1198,22 @@ def test_ogr_interlis_arc2(): for i in range(feat.GetGeomFieldCount()): geom = feat.GetGeomFieldRef(i) ogrtest.check_feature_geometry(geom, geom_field_values[i]) + + +############################################################################### +# Test failure in creation of ILI2 dataset + + +def test_ogr_interlis2_create_file_error(): + + with pytest.raises( + Exception, match="model file not specified in destination filename" + ): + ogr.GetDriverByName("Interlis 2").CreateDataSource("tmp/out.xtf") + + with pytest.raises( + Exception, match="Failed to create XTF file /i_do/not/exist/out.xtf" + ): + ogr.GetDriverByName("Interlis 2").CreateDataSource( + "/i_do/not/exist/out.xtf,data/ili/ch.bazl.sicherheitszonenplan.oereb_20131118.imd" + ) diff --git a/autotest/ogr/ogr_libkml.py b/autotest/ogr/ogr_libkml.py index 6f09117f1056..e46390cb1783 100755 --- a/autotest/ogr/ogr_libkml.py +++ b/autotest/ogr/ogr_libkml.py @@ -2193,3 +2193,21 @@ def test_ogr_libkml_write_reproject(tmp_vsimem): ds = None gdal.Unlink(outfilename) + + +############################################################################### +# Test reading a gx:Track without elements + + +def test_ogr_libkml_gx_track_without_when(): + if not ogrtest.have_read_libkml: + pytest.skip() + + ds = ogr.Open("data/kml/gx_track_without_when.kml") + lyr = ds.GetLayer(0) + feat = lyr.GetNextFeature() + ogrtest.check_feature_geometry( + feat, + "LINESTRING Z (-122.207881 37.371915 156,-122.205712 37.373288 152,-122.204678 37.373939 147,-122.203572 37.37463 142.199997,-122.203451 37.374706 141.800003,-122.203329 37.37478 141.199997,-122.203207 37.374857 140.199997)", + ) + ds = None diff --git a/autotest/ogr/ogr_ods.py b/autotest/ogr/ogr_ods.py index 6977cd3fa980..16fccb20fbd3 100755 --- a/autotest/ogr/ogr_ods.py +++ b/autotest/ogr/ogr_ods.py @@ -59,6 +59,8 @@ def ogr_ods_check(ds): assert lyr.TestCapability("foo") == 0 + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 + lyr = ds.GetLayer(6) assert lyr.GetName() == "Feuille7", "bad layer name" @@ -363,6 +365,7 @@ def test_ogr_ods_8(): drv = ogr.GetDriverByName("ODS") ds = drv.CreateDataSource("/vsimem/ogr_ods_8.ods") lyr = ds.CreateLayer("foo") + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 lyr.CreateField(ogr.FieldDefn("Field1", ogr.OFTInteger64)) f = ogr.Feature(lyr.GetLayerDefn()) f.SetField(0, 1) diff --git a/autotest/ogr/ogr_openfilegdb.py b/autotest/ogr/ogr_openfilegdb.py index 901e10ae3990..c9c2ad519b6c 100755 --- a/autotest/ogr/ogr_openfilegdb.py +++ b/autotest/ogr/ogr_openfilegdb.py @@ -2386,7 +2386,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "my forward path label" assert rel.GetBackwardPathLabel() == "my backward path label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -2398,7 +2398,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -2412,7 +2412,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["destination_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -2424,7 +2424,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -2436,7 +2436,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["parent_pk"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_many_to_many") assert rel is not None @@ -2450,7 +2450,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["origin_foreign_key"] assert rel.GetRightTableFields() == ["parent_pk"] assert rel.GetRightMappingTableFields() == ["dest_foreign_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_openfilegdb_write.py b/autotest/ogr/ogr_openfilegdb_write.py index 7e105bfe8823..eadd7147c8a0 100755 --- a/autotest/ogr/ogr_openfilegdb_write.py +++ b/autotest/ogr/ogr_openfilegdb_write.py @@ -54,7 +54,10 @@ def setup_driver(): if filegdb_driver is not None: filegdb_driver.Deregister() - yield + with gdaltest.config_option( + "OGR_OPENFILEGDB_ERROR_ON_INCONSISTENT_BUFFER_MAX_SIZE", "YES" + ): + yield if filegdb_driver is not None: print("Reregistering FileGDB driver") @@ -2833,7 +2836,7 @@ def test_ogr_openfilegdb_write_relationships(): assert retrieved_rel.GetRightTableFields() == ["dest_pkey"] assert retrieved_rel.GetForwardPathLabel() == "fwd label" assert retrieved_rel.GetBackwardPathLabel() == "backward label" - assert retrieved_rel.GetRelatedTableType() == "feature" + assert retrieved_rel.GetRelatedTableType() == "features" items_lyr = ds.GetLayerByName("GDB_Items") f = items_lyr.GetFeature(8) @@ -3182,7 +3185,7 @@ def test_ogr_openfilegdb_write_relationships(): assert retrieved_rel.GetRightTableFields() == ["dest_pkey"] assert retrieved_rel.GetForwardPathLabel() == "my new fwd label" assert retrieved_rel.GetBackwardPathLabel() == "my new backward label" - assert retrieved_rel.GetRelatedTableType() == "feature" + assert retrieved_rel.GetRelatedTableType() == "features" # change relationship tables lyr = ds.CreateLayer("new_origin_table", geom_type=ogr.wkbNone) @@ -4493,3 +4496,37 @@ def test_ogr_openfilegdb_write_new_datetime_types(tmp_vsimem): f = sql_lyr.GetNextFeature() assert f["MIN_timestamp_offset"] == "1900/12/31 14:01:01" assert f["MAX_timestamp_offset"] == "2023/12/30 14:01:01" + + +############################################################################### +# Test updating an existing feature with one whose m_nRowBlobLength is +# larger than m_nHeaderBufferMaxSize + + +def test_ogr_openfilegdb_write_update_feature_larger(tmp_vsimem): + + filename = str(tmp_vsimem / "out.gdb") + ds = ogr.GetDriverByName("OpenFileGDB").CreateDataSource(filename) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + lyr = ds.CreateLayer("test", srs, ogr.wkbLineString) + f = ogr.Feature(lyr.GetLayerDefn()) + g = ogr.Geometry(ogr.wkbLineString) + g.SetPoint_2D(10, 0, 0) + f.SetGeometry(g) + lyr.CreateFeature(f) + ds = None + + ds = ogr.Open(filename, update=1) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + g = ogr.Geometry(ogr.wkbLineString) + g.SetPoint_2D(999, 0, 0) + f.SetGeometry(g) + lyr.SetFeature(f) + ds = None + + ds = ogr.Open(filename) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f.GetGeometryRef().GetGeometryRef(0).GetPointCount() == 1000 diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index 42e2a9c3106b..40c7d5c8a3dc 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -307,12 +307,6 @@ def pg_quote_with_E(pg_autotest_ds): return True -@pytest.fixture(scope="module") -def pg_retrieve_fid(pg_version): - - return pg_version >= (8, 2) - - ############################################################################### # Create table from data/poly.shp @@ -2023,7 +2017,7 @@ def test_ogr_pg_39(pg_ds): @only_with_postgis -def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): +def test_ogr_pg_39_bis(pg_ds, pg_has_postgis): schema = current_schema(pg_ds) @@ -2038,10 +2032,6 @@ def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): # Create a view pg_ds.ExecuteSQL("CREATE VIEW testview AS SELECT * FROM inherited") - if pg_postgis_version[0] < 2: - pg_ds.ExecuteSQL( - f"INSERT INTO geometry_columns VALUES ( '', '{schema}', 'testview', 'wkb_geometry', 2, -1, 'POINT') " - ) pg_ds.ExecuteSQL( "INSERT INTO inherited (col1, wkb_geometry) VALUES ( 'a', GeomFromEWKT('POINT (0 1)') )" ) @@ -2069,10 +2059,7 @@ def test_ogr_pg_39_bis(pg_ds, pg_has_postgis, pg_postgis_version): ), ("bad geometry %s" % feat.GetGeometryRef().ExportToWkt()) # Test another geometry column - if pg_postgis_version[0] < 2: - pg_ds.ExecuteSQL( - f"INSERT INTO geometry_columns VALUES ( '', '{schema}', 'testview', 'point25D', 3, -1, 'POINT') " - ) + pg_ds.ExecuteSQL( "UPDATE inherited SET \"point25D\" = GeomFromEWKT('POINT (0 1 2)') " ) @@ -2796,12 +2783,9 @@ def test_ogr_pg_53_bis(tmp_path, pg_ds): @only_with_postgis -def test_ogr_pg_54(pg_ds, pg_postgis_version): +def test_ogr_pg_54(pg_ds): - if pg_postgis_version[0] >= 2: - sql_lyr = pg_ds.ExecuteSQL("SELECT ST_AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") - else: - sql_lyr = pg_ds.ExecuteSQL("SELECT AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") + sql_lyr = pg_ds.ExecuteSQL("SELECT ST_AsEWKB(GeomFromEWKT('POINT (0 1 2)'))") feat = sql_lyr.GetNextFeature() pg_ds.ReleaseResultSet(sql_lyr) @@ -3538,7 +3522,7 @@ def test_ogr_pg_70bis(pg_ds, pg_postgis_schema): @only_with_postgis -def test_ogr_pg_71(pg_ds, pg_postgis_version): +def test_ogr_pg_71(pg_ds): curve_lyr = pg_ds.CreateLayer("test_curve") curve_lyr2 = pg_ds.CreateLayer( @@ -3574,25 +3558,7 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): "GEOMETRYCOLLECTION (CIRCULARSTRING (0 1,2 3,4 5),COMPOUNDCURVE ((0 1,2 3,4 5)),CURVEPOLYGON ((0 0,0 1,1 1,1 0,0 0)),MULTICURVE ((0 0,1 1)),MULTISURFACE (((0 0,0 10,10 10,10 0,0 0))))", ]: - # would cause PostGIS 1.X to crash - if pg_postgis_version[0] < 2 and wkt == "CURVEPOLYGON EMPTY": - continue - # Parsing error of WKT by PostGIS 1.X - if ( - pg_postgis_version[0] < 2 - and "MULTICURVE" in wkt - and "CIRCULARSTRING" in wkt - ): - continue - postgis_in_wkt = wkt - while True: - z_pos = postgis_in_wkt.find("Z ") - # PostGIS 1.X doesn't like Z in WKT - if pg_postgis_version[0] < 2 and z_pos >= 0: - postgis_in_wkt = postgis_in_wkt[0:z_pos] + postgis_in_wkt[z_pos + 2 :] - else: - break # Test parsing PostGIS WKB lyr = pg_ds.ExecuteSQL("SELECT ST_GeomFromText('%s')" % postgis_in_wkt) @@ -3604,18 +3570,11 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): pg_ds.ReleaseResultSet(lyr) expected_wkt = wkt - if pg_postgis_version[0] < 2 and "EMPTY" in wkt: - expected_wkt = "GEOMETRYCOLLECTION EMPTY" assert out_wkt == expected_wkt # Test parsing PostGIS WKT - if pg_postgis_version[0] >= 2: - fct = "ST_AsText" - else: - fct = "AsEWKT" - lyr = pg_ds.ExecuteSQL( - "SELECT %s(ST_GeomFromText('%s'))" % (fct, postgis_in_wkt) + "SELECT ST_AsText(ST_GeomFromText('%s'))" % (postgis_in_wkt) ) f = lyr.GetNextFeature() g = f.GetGeometryRef() @@ -3625,8 +3584,6 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): pg_ds.ReleaseResultSet(lyr) expected_wkt = wkt - if pg_postgis_version[0] < 2 and "EMPTY" in wkt: - expected_wkt = "GEOMETRYCOLLECTION EMPTY" assert out_wkt == expected_wkt g = ogr.CreateGeometryFromWkt(wkt) @@ -3643,13 +3600,9 @@ def test_ogr_pg_71(pg_ds, pg_postgis_version): assert ret == 0, wkt fid = f.GetFID() - # AsEWKT() in PostGIS 1.X does not like CIRCULARSTRING EMPTY - if pg_postgis_version[0] < 2 and "CIRCULARSTRING" in wkt and "EMPTY" in wkt: - continue - lyr = pg_ds.ExecuteSQL( - "SELECT %s(wkb_geometry) FROM %s WHERE ogc_fid = %d" - % (fct, active_lyr.GetName(), fid) + "SELECT ST_AsText(wkb_geometry) FROM %s WHERE ogc_fid = %d" + % (active_lyr.GetName(), fid) ) f = lyr.GetNextFeature() g = f.GetGeometryRef() @@ -4165,14 +4118,14 @@ def ogr_pg_76_get_transaction_state(ds): ) -def test_ogr_pg_76(pg_ds, pg_postgis_version, use_postgis): +def test_ogr_pg_76(pg_ds, use_postgis): assert pg_ds.TestCapability(ogr.ODsCTransactions) == 1 level = int(pg_ds.GetMetadataItem("nSoftTransactionLevel", "_DEBUG_")) assert level == 0 - if use_postgis and pg_postgis_version[0] >= 2: + if use_postgis: pg_ds.StartTransaction() lyr = pg_ds.CreateLayer("will_not_be_created", options=["OVERWRITE=YES"]) lyr.CreateField(ogr.FieldDefn("foo", ogr.OFTString)) @@ -4505,10 +4458,7 @@ def test_ogr_pg_77(pg_ds, tmp_path): @only_with_postgis -def test_ogr_pg_78(pg_ds, pg_postgis_version): - - if pg_postgis_version[0] < 2: - pytest.skip("Requires PostGIS >= 2.0") +def test_ogr_pg_78(pg_ds): pg_ds.ExecuteSQL("CREATE TABLE ogr_pg_78 (ID INTEGER PRIMARY KEY)") pg_ds.ExecuteSQL("ALTER TABLE ogr_pg_78 ADD COLUMN my_geom GEOMETRY") @@ -4802,7 +4752,7 @@ def ogr_pg_83_ids(param): ], ids=ogr_pg_83_ids, ) -def test_ogr_pg_83(pg_ds, pg_postgis_version, geom_type, options, wkt, expected_wkt): +def test_ogr_pg_83(pg_ds, geom_type, options, wkt, expected_wkt): lyr = pg_ds.CreateLayer("ogr_pg_83", geom_type=geom_type, options=options) f = ogr.Feature(lyr.GetLayerDefn()) @@ -4821,12 +4771,8 @@ def test_ogr_pg_83(pg_ds, pg_postgis_version, geom_type, options, wkt, expected_ if "GEOM_TYPE=geography" in options: return - # Cannot do AddGeometryColumn( 'GEOMETRYM', 3 ) with PostGIS 2, and doesn't accept inserting a XYM geometry - if ( - pg_postgis_version[0] >= 2 - and geom_type == ogr.wkbUnknown - and options == ["DIM=XYM"] - ): + # Cannot do AddGeometryColumn( 'GEOMETRYM', 3 ) with PostGIS >= 2, and doesn't accept inserting a XYM geometry + if geom_type == ogr.wkbUnknown and options == ["DIM=XYM"]: return lyr = pg_ds.CreateLayer( diff --git a/autotest/ogr/ogr_pgeo.py b/autotest/ogr/ogr_pgeo.py index e9e147445335..66f4a3f39aed 100755 --- a/autotest/ogr/ogr_pgeo.py +++ b/autotest/ogr/ogr_pgeo.py @@ -910,7 +910,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetRightMappingTableFields() is None assert rel.GetForwardPathLabel() == "forward label" assert rel.GetBackwardPathLabel() == "backward label" - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_one_to_many") assert rel is not None @@ -922,7 +922,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_ASSOCIATION assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("simple_many_to_many") assert rel is not None @@ -936,7 +936,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == ["rel_pk"] assert rel.GetRightTableFields() == ["primary_key"] assert rel.GetRightMappingTableFields() == ["rel_primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_one") assert rel is not None @@ -948,7 +948,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetType() == gdal.GRT_COMPOSITE assert rel.GetLeftTableFields() == ["pk"] assert rel.GetRightTableFields() == ["primary_key"] - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("composite_one_to_many") assert rel is not None @@ -962,7 +962,7 @@ def test_ogr_openfilegdb_read_relationships(): assert rel.GetLeftMappingTableFields() == None assert rel.GetRightTableFields() == ["primary_key"] assert rel.GetRightMappingTableFields() == None - assert rel.GetRelatedTableType() == "feature" + assert rel.GetRelatedTableType() == "features" rel = ds.GetRelationship("points__ATTACHREL") assert rel is not None diff --git a/autotest/ogr/ogr_pmtiles.py b/autotest/ogr/ogr_pmtiles.py index 0757e95a9d59..81361f82def5 100755 --- a/autotest/ogr/ogr_pmtiles.py +++ b/autotest/ogr/ogr_pmtiles.py @@ -617,3 +617,15 @@ def test_ogr_pmtiles_read_corrupted_min_zoom_larger_than_30(): ds = ogr.Open(tmpfilename) assert gdal.GetLastErrorMsg() == "Clamping max_zoom from 255 to 30" assert ds.GetMetadataItem("ZOOM_LEVEL") == "30" + + +############################################################################### + + +def test_ogr_pmtiles_read_with_many_directories(): + + ds = gdal.OpenEx( + "data/pmtiles/subset7_truncated.pmtiles", open_options=["ZOOM_LEVEL=0"] + ) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() != 0 diff --git a/autotest/ogr/ogr_rfc41.py b/autotest/ogr/ogr_rfc41.py index 37f0096dd0c3..9befbaa54667 100755 --- a/autotest/ogr/ogr_rfc41.py +++ b/autotest/ogr/ogr_rfc41.py @@ -400,7 +400,6 @@ def test_ogr_rfc41_6(): geomfield = ogr.GeomFieldDefn("geomfield", ogr.wkbPolygon) geomfield.SetSpatialRef(sr) lyr.CreateGeomField(geomfield) - lyr.GetLayerDefn().GetGeomFieldDefn(0).SetName("geomfield") lyr.CreateField(ogr.FieldDefn("intfield", ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn("wkt", ogr.OFTString)) feat = ogr.Feature(lyr.GetLayerDefn()) diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index de8ec7f0b78b..8037b6e6f08d 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -696,9 +696,15 @@ def test_ogr_sql_28(): ds = ogr.GetDriverByName("Memory").CreateDataSource("my_ds") lyr = ds.CreateLayer("my_layer") - lyr.GetLayerDefn().GetGeomFieldDefn(0).SetName( - "geom" - ) # a bit border line but OK for Memory driver... + + new_geom_field_defn = ogr.GeomFieldDefn("geom", ogr.wkbUnknown) + assert ( + lyr.AlterGeomFieldDefn( + 0, new_geom_field_defn, ogr.ALTER_GEOM_FIELD_DEFN_NAME_FLAG + ) + == ogr.OGRERR_NONE + ) + field_defn = ogr.FieldDefn("strfield", ogr.OFTString) lyr.CreateField(field_defn) field_defn = ogr.FieldDefn("intfield", ogr.OFTInteger) diff --git a/autotest/ogr/ogr_xlsx.py b/autotest/ogr/ogr_xlsx.py index 39971b128d96..164bdd835d29 100755 --- a/autotest/ogr/ogr_xlsx.py +++ b/autotest/ogr/ogr_xlsx.py @@ -59,6 +59,8 @@ def ogr_xlsx_check(ds): assert lyr.TestCapability("foo") == 0 + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 + lyr = ds.GetLayer(6) assert lyr.GetName() == "Feuille7", "bad layer name" @@ -277,6 +279,7 @@ def test_ogr_xlsx_8(): ds = ogr.GetDriverByName("XLSX").CreateDataSource("/vsimem/ogr_xlsx_8.xlsx") lyr = ds.CreateLayer("foo") + assert lyr.TestCapability(ogr.OLCStringsAsUTF8) == 1 for i in range(30): lyr.CreateField(ogr.FieldDefn("Field%d" % (i + 1))) f = ogr.Feature(lyr.GetLayerDefn()) diff --git a/autotest/osr/osr_xml.py b/autotest/osr/osr_xml.py index f804aec67ded..41ef4d1377bf 100755 --- a/autotest/osr/osr_xml.py +++ b/autotest/osr/osr_xml.py @@ -31,6 +31,7 @@ import re import gdaltest +import pytest from osgeo import osr @@ -200,3 +201,58 @@ def test_osr_xml_2(): expected = re.sub(r' gml:id="[^"]*"', "", expected, 0) assert got == expected + + +############################################################################### +# Test the osr.SpatialReference.ExportToXML() function. +# + + +def test_osr_xml_export_failure(): + + srs = osr.SpatialReference() + srs.ImportFromWkt( + """PROJCRS["Africa_Albers_Equal_Area_Conic", + BASEGEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]], + CONVERSION["Albers Equal Area", + METHOD["Albers Equal Area", + ID["EPSG",9822]], + PARAMETER["Latitude of false origin",0, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8821]], + PARAMETER["Longitude of false origin",25, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8822]], + PARAMETER["Latitude of 1st standard parallel",20, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8823]], + PARAMETER["Latitude of 2nd standard parallel",-23, + ANGLEUNIT["degree",0.0174532925199433], + ID["EPSG",8824]], + PARAMETER["Easting at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8826]], + PARAMETER["Northing at false origin",0, + LENGTHUNIT["metre",1], + ID["EPSG",8827]]], + CS[Cartesian,2], + AXIS["easting",east, + ORDER[1], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]], + AXIS["northing",north, + ORDER[2], + LENGTHUNIT["metre",1, + ID["EPSG",9001]]]]""" + ) + + with pytest.raises( + Exception, match="Unhandled projection method Albers_Conic_Equal_Area" + ): + srs.ExportToXML() diff --git a/autotest/pyscripts/gdal2tiles/test_logger.py b/autotest/pyscripts/gdal2tiles/test_logger.py new file mode 100644 index 000000000000..de9cd087b1bd --- /dev/null +++ b/autotest/pyscripts/gdal2tiles/test_logger.py @@ -0,0 +1,72 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: gdal2tiles.py testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import pytest + +from osgeo import gdal +from osgeo_utils import gdal2tiles + + +def test_gdal2tiles_logger(): + + if gdal.GetDriverByName("PNG") is None: + pytest.skip("PNG driver is missing") + + gdal2tiles.main( + argv=[ + "gdal2tiles", + "--verbose", + "-z", + "13-14", + "../../gcore/data/byte.tif", + "/vsimem/gdal2tiles", + ] + ) + + assert set(gdal.ReadDirRecursive("/vsimem/gdal2tiles")) == set( + [ + "13/", + "13/1418/", + "13/1418/4916.png", + "13/1419/", + "13/1419/4916.png", + "14/", + "14/2837/", + "14/2837/9833.png", + "14/2838/", + "14/2838/9833.png", + "googlemaps.html", + "leaflet.html", + "openlayers.html", + "tilemapresource.xml", + ] + ) + gdal.RmdirRecursive("/vsimem/gdal2tiles") diff --git a/autotest/pyscripts/gdal2tiles/test_vsimem.py b/autotest/pyscripts/gdal2tiles/test_vsimem.py index c6c61ce946d3..340e0238f7c4 100644 --- a/autotest/pyscripts/gdal2tiles/test_vsimem.py +++ b/autotest/pyscripts/gdal2tiles/test_vsimem.py @@ -40,7 +40,9 @@ def test_gdal2tiles_vsimem(): if gdal.GetDriverByName("PNG") is None: pytest.skip("PNG driver is missing") - gdal2tiles.main(argv=["-q", "../../gcore/data/byte.tif", "/vsimem/gdal2tiles"]) + gdal2tiles.main( + argv=["gdal2tiles", "-q", "../../gcore/data/byte.tif", "/vsimem/gdal2tiles"] + ) assert set(gdal.ReadDirRecursive("/vsimem/gdal2tiles")) == set( [ diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index c49c816f67fc..94f449d1e282 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -748,3 +748,158 @@ def test_gdalbuildvrt_lib_stable_average(): vrt_gt = vrt_ds.GetGeoTransform() assert vrt_gt == gt + + +############################################################################### + + +def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgba(tmp_vsimem): + + ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 4) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + # Test remapping of second valid pixel at 0 to 1 + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x00") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x02\x02") + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x03\x03") + ds.GetRasterBand(4).WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.GetRasterBand(4).SetColorInterpretation(gdal.GCI_AlphaBand) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128, VRTNodata=0) + assert vrt_ds.RasterCount == 3 + assert vrt_ds.GetRasterBand(1).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x00\x01" + assert vrt_ds.GetRasterBand(2).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(2).ReadRaster() == b"\x00\x02" + assert vrt_ds.GetRasterBand(3).GetNoDataValue() == 0 + assert vrt_ds.GetRasterBand(3).ReadRaster() == b"\x00\x03" + + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (0, 1) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128.5, VRTNodata=0) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\x00\x01" + + # VRTNodata=255, test remapping of 255 to 254 + ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 2) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\xFF") + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand) + + vrt_ds = gdal.BuildVRT("", [ds], nodataMaxMaskThreshold=128, VRTNodata=255) + assert vrt_ds.GetRasterBand(1).ReadRaster() == b"\xFF\xFE" + + +############################################################################### + + +def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgb_mask(tmp_vsimem): + + # UInt16, VRTNodata=0 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 3, gdal.GDT_UInt16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 1, 0)) + ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 2, 2)) + ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 3, 2)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=0) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (0, 1) + + assert struct.unpack( + "B" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Byte) + ) == (0, 1) + + # UInt16, VRTNodata=65535 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_UInt16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("H" * 2, 1, 65535)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=65535 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "H" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_UInt16) + ) == (65535, 65534) + + # Int16, VRTNodata=-32768 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Int16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("h" * 2, 1, -32768)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=-32768 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (-32768, -32767) + + # Int16, VRTNodata=32767 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Int16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("h" * 2, 1, 32767)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT( + vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=32767 + ) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "h" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Int16) + ) == (32767, 32766) + + # Float32, VRTNodata=0 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1, gdal.GDT_Float32) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, struct.pack("f" * 2, 1, 0)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 2, 1, b"\x00\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=0) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "f" * 2, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Float32) + ) == pytest.approx((0.0, 0.001)) + + # Float32, VRTNodata=1 + src_filename = str(tmp_vsimem / "src.tif") + ds = gdal.GetDriverByName("GTiff").Create(src_filename, 3, 1, 1, gdal.GDT_Float32) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, struct.pack("f" * 3, 0, 1, 2)) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 3, 1, b"\x00\xFF\xFF") + ds.Close() + + vrt_filename = str(tmp_vsimem / "test.vrt") + gdal.BuildVRT(vrt_filename, [src_filename], nodataMaxMaskThreshold=128, VRTNodata=1) + vrt_ds = gdal.Open(vrt_filename) + assert struct.unpack( + "f" * 3, vrt_ds.GetRasterBand(1).ReadRaster(buf_type=gdal.GDT_Float32) + ) == pytest.approx((1.0, 1.001, 2.0)) diff --git a/autotest/utilities/test_gdalinfo_lib.py b/autotest/utilities/test_gdalinfo_lib.py index a170b880affc..af751909b8cd 100755 --- a/autotest/utilities/test_gdalinfo_lib.py +++ b/autotest/utilities/test_gdalinfo_lib.py @@ -263,3 +263,16 @@ def test_gdalinfo_lib_json_projjson_no_epsg(): ret = gdal.Info(ds, options="-json") assert ret["stac"]["proj:epsg"] is None assert ret["stac"]["proj:wkt2"] is not None + + +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/9337 + + +def test_gdalinfo_lib_json_proj_shape(): + + width = 2 + height = 1 + ds = gdal.GetDriverByName("MEM").Create("", width, height) + ret = gdal.Info(ds, options="-json") + assert ret["stac"]["proj:shape"] == [height, width] diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index 5edd1027b92f..1ca76128ea5f 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -286,10 +286,12 @@ def test_gdaltindex_lib_gti_non_xml(tmp_path, four_tiles): colorInterpretation="gray", mask=True, metadataOptions={"foo": "bar"}, + layerCreationOptions=["FID=my_fid"], ) ds = ogr.Open(index_filename) lyr = ds.GetLayer(0) + assert lyr.GetFIDColumn() == "my_fid" assert lyr.GetMetadataItem("RESX") == "60" assert lyr.GetMetadataItem("RESY") == "60" assert lyr.GetMetadataItem("MINX") == "0" diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 07ccacfc5c09..e22997b97f66 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -31,6 +31,7 @@ ############################################################################### import collections +import json import shutil import struct @@ -3004,6 +3005,94 @@ def test_gdalwarp_lib_cutline_zero_width_sliver(tmp_vsimem): assert ds is not None +############################################################################### +# Test cutline with zero-width sliver + + +def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_polygon(tmp_vsimem): + + geojson = { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [-101.43346, 36.91886], + [-101.43337, 36.91864], + [-101.43332, 36.91865], + [-101.43342, 36.91888], + [-101.43346, 36.91886], + ] + ], + # The below polygon has a zero-width sliver + [ + [ + [-101.4311, 36.91909], + [-101.43106, 36.91913], + [-101.43111, 36.91908], + [-101.4311, 36.91909], + ] + ], + ], + } + gdal.FileFromMemBuffer(tmp_vsimem / "cutline.geojson", json.dumps(geojson)) + src_ds = gdal.GetDriverByName("MEM").Create("", 100, 100) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + src_ds.SetSpatialRef(srs) + src_ds.SetGeoTransform([-101.5, 0.1, 0, 37, 0, -0.1]) + ds = gdal.Warp( + "", src_ds, format="MEM", cutlineDSName=tmp_vsimem / "cutline.geojson" + ) + assert ds is not None + + +############################################################################### +# Test cutline with zero-width sliver + + +def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_inner_ring(tmp_vsimem): + + geojson = { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [-101.5, 37], + [-101.4, 37], + [-101.4, 36.9], + [-101.5, 36.9], + [-101.5, 37], + ], + # The below ring has a zero-width sliver + [ + [-101.4311, 36.91909], + [-101.43106, 36.91913], + [-101.43111, 36.91908], + [-101.4311, 36.91909], + ], + # This one is OK + [ + [-101.49, 36.95], + [-101.48, 36.95], + [-101.48, 36.94], + [-101.49, 36.94], + [-101.49, 36.95], + ], + ] + ], + } + gdal.FileFromMemBuffer(tmp_vsimem / "cutline.geojson", json.dumps(geojson)) + src_ds = gdal.GetDriverByName("MEM").Create("", 100, 100) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + src_ds.SetSpatialRef(srs) + src_ds.SetGeoTransform([-101.6, 0.1, 0, 37.1, 0, -0.1]) + ds = gdal.Warp( + "", src_ds, format="MEM", cutlineDSName=tmp_vsimem / "cutline.geojson" + ) + assert ds is not None + + ############################################################################### # Test support for propagating coordinate epoch @@ -3412,7 +3501,7 @@ def test_gdalwarp_lib_sum_preserving_multiband(): ############################################################################### -def test_gdalwarp_lib_sum_preserving_accross_antimeridian(): +def test_gdalwarp_lib_sum_preserving_across_antimeridian(): src_ds = gdal.Translate( "", @@ -3944,3 +4033,10 @@ def test_gdalwarp_lib_to_projection_without_inverse_method(): 1218250.2778614, ], ) + + +def test_gdalwarp_lib_no_crash_on_none_dst(): + + ds1 = gdal.Open("../gcore/data/byte.tif") + with pytest.raises(ValueError): + gdal.Warp(None, ds1) diff --git a/autotest/utilities/test_ogrinfo_lib.py b/autotest/utilities/test_ogrinfo_lib.py index 1fdc10130dce..0cdcbc214fe3 100755 --- a/autotest/utilities/test_ogrinfo_lib.py +++ b/autotest/utilities/test_ogrinfo_lib.py @@ -637,7 +637,7 @@ def test_ogrinfo_lib_relationships(): ret = gdal.VectorInfo(ds) expected = """Relationship: composite_many_to_many Type: Composite - Related table type: feature + Related table type: features Cardinality: ManyToMany Left table name: table6 Right table name: table7 @@ -667,7 +667,7 @@ def test_ogrinfo_lib_json_relationships(): # print(ret["relationships"]["composite_many_to_many"]) assert ret["relationships"]["composite_many_to_many"] == { "type": "Composite", - "related_table_type": "feature", + "related_table_type": "features", "cardinality": "ManyToMany", "left_table_name": "table6", "right_table_name": "table7", diff --git a/ci/travis/conda/compile.sh b/ci/travis/conda/compile.sh index c200601633d0..dcb73e80a665 100755 --- a/ci/travis/conda/compile.sh +++ b/ci/travis/conda/compile.sh @@ -5,18 +5,24 @@ mkdir -p packages CONDA_PLAT="" if grep -q "windows" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="win" + ARCH="64" fi if grep -q "ubuntu" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="linux" + ARCH="64" fi -if grep -q "macos" <<< "$GHA_CI_PLATFORM"; then +if grep -q "macos-14" <<< "$GHA_CI_PLATFORM"; then CONDA_PLAT="osx" + ARCH="arm64" +elif grep -q "macos" <<< "$GHA_CI_PLATFORM"; then + CONDA_PLAT="osx" + ARCH="64" fi -conda build recipe --clobber-file recipe/recipe_clobber.yaml --output-folder packages -m ".ci_support/${CONDA_PLAT}_64_.yaml" -conda create -y -n test -c ./packages/${CONDA_PLAT}-64 python libgdal gdal +conda build recipe --clobber-file recipe/recipe_clobber.yaml --output-folder packages -m ".ci_support/${CONDA_PLAT}_${ARCH}_.yaml" +conda create -y -n test -c ./packages/${CONDA_PLAT}-${ARCH} python libgdal gdal conda deactivate conda activate test diff --git a/doc/images/logo-hobu.png b/doc/images/logo-hobu.png new file mode 100644 index 000000000000..42019ed56922 Binary files /dev/null and b/doc/images/logo-hobu.png differ diff --git a/doc/source/about_no_title.rst b/doc/source/about_no_title.rst index 4b9d5f68523e..4c6634af523f 100644 --- a/doc/source/about_no_title.rst +++ b/doc/source/about_no_title.rst @@ -1,13 +1,13 @@ .. include:: ./substitutions.rst -GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source :ref:`license` by the `Open Source Geospatial Foundation`_. As a library, it presents a single raster abstract data model and single vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line utilities for data translation and processing. The `NEWS`_ page describes the January 2024 GDAL/OGR 3.8.3 release. +GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source :ref:`license` by the `Open Source Geospatial Foundation`_. As a library, it presents a single raster abstract data model and single vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line utilities for data translation and processing. The `NEWS`_ page describes the February 2024 GDAL/OGR 3.8.4 release. .. image:: ../images/OSGeo_project.png :alt: OSGeo project :target: `Open Source Geospatial Foundation`_ .. _`Open Source Geospatial Foundation`: http://www.osgeo.org/ -.. _`NEWS`: https://github.com/OSGeo/gdal/blob/v3.8.3/NEWS.md +.. _`NEWS`: https://github.com/OSGeo/gdal/blob/v3.8.4/NEWS.md See :ref:`software_using_gdal` diff --git a/doc/source/api/python/osgeo.gdal.rst b/doc/source/api/python/osgeo.gdal.rst index f344667e2470..a3ac74b7b754 100644 --- a/doc/source/api/python/osgeo.gdal.rst +++ b/doc/source/api/python/osgeo.gdal.rst @@ -5,3 +5,4 @@ osgeo.gdal module :members: :undoc-members: :show-inheritance: + :exclude-members: thisown diff --git a/doc/source/community/service_providers.rst b/doc/source/community/service_providers.rst index 7a0208320e3b..bcf9cb77ecf1 100644 --- a/doc/source/community/service_providers.rst +++ b/doc/source/community/service_providers.rst @@ -55,6 +55,20 @@ the most out of it. to providing to its clients the best of its expertise around Open Source geospatial software - such as GDAL/OGR, PROJ, MapServer, and QGIS - and Open standards. +.. container:: service-provider + + |logo_hobu| + + .. container:: service-provider-description + + `Hobu, Inc.`_ (United States) is a `sam.gov + `__-registered company founded by Howard Butler with + more than twenty years experience supporting the creation, enhancement, + and management of geospatial open source geospatial software such as + `PDAL `__, GDAL/OGR, PROJ, and open standards such as + `GeoJSON `__, `PROJJSON + `__, and `Cloud + Optimized Point Cloud `__. Contributors ------------ @@ -181,3 +195,10 @@ The steps to add a company to the service providers list are outlined in the :re :class: img-logos :width: 100 px :target: `mundialis`_ + + +.. _`Hobu, Inc.`: https://hobu.co/ +.. |logo_hobu| image:: ../../images/logo-hobu.png + :class: img-logos + :width: 100 px + :target: `Hobu, Inc.`_ diff --git a/doc/source/development/building_from_source.rst b/doc/source/development/building_from_source.rst index c037301d0977..4ffce26053c9 100644 --- a/doc/source/development/building_from_source.rst +++ b/doc/source/development/building_from_source.rst @@ -415,7 +415,7 @@ CURL .. option:: CURL_LIBRARY_RELEASE - Path to a shared or static library file, such as ``libcurl.dll``, + Path to a shared or static library file, such as ``libcurl.so``, ``libcurl.lib``, or other name. .. option:: CURL_USE_STATIC_LIBS=ON/OFF @@ -632,7 +632,7 @@ If not found, an internal copy of libgeotiff can be used. .. option:: GEOTIFF_LIBRARY_RELEASE - Path to a shared or static library file, such as ``geotiff.dll``, + Path to a shared or static library file, such as ``libgeotiff.so``, ``geotiff.lib``, or other name. A similar variable ``GEOTIFF_LIBRARY_DEBUG`` can also be specified to a similar library for building Debug releases. @@ -1680,7 +1680,7 @@ PROJ .. option:: PROJ_LIBRARY_RELEASE - Path to a shared or static library file, such as ``proj.dll``, + Path to a shared or static library file, such as ``libproj.so``, ``proj.lib``, or other name. A similar variable ``PROJ_LIBRARY_DEBUG`` can also be specified to a similar library for building Debug releases. @@ -1778,7 +1778,7 @@ and the :ref:`sql_sqlite_dialect`. .. option:: SQLite3_LIBRARY - Path to a shared or static library file, such as ``sqlite3.dll``, + Path to a shared or static library file, such as ``libsqlite3.so``, ``sqlite3.lib`` or other name. .. option:: GDAL_USE_SQLITE3=ON/OFF @@ -1866,7 +1866,7 @@ If not found, an internal copy of libtiff can be used. .. option:: TIFF_LIBRARY_RELEASE - Path to a shared or static library file, such as ``tiff.dll``, + Path to a shared or static library file, such as ``libtiff.so``, ``tiff.lib``, or other name. A similar variable ``TIFF_LIBRARY_DEBUG`` can also be specified to a similar library for building Debug releases. diff --git a/doc/source/development/dev_documentation.rst b/doc/source/development/dev_documentation.rst index 6946670ea065..e23d8965d7bf 100644 --- a/doc/source/development/dev_documentation.rst +++ b/doc/source/development/dev_documentation.rst @@ -30,6 +30,26 @@ Once installed, running ``sphinx-autobuild -b html source build`` from the ``doc and serve it on a local web server at ``http://127.0.0.1:8000``. The pages served will be automatically refreshed as changes are made to underlying ``rst`` documentation files. +Python API documentation +------------------------ + +Sphinx uses the `autodoc `_ extension +to generate documentation for the Python API from Python function docstrings. +To be correctly parsed by ``autodoc``, docstrings should follow the `numpydoc Style guide `_. +Docstrings may be found in two locations. If the function was defined in Python +(i.e., using a ``%pythoncode`` SWIG directive), then the docstring must be +placed within the function definition. If the function is defined in C++ only, +then the docstring should be placed in a separate file +containing only docstrings (located in :source_file:`swig/include/python/docs`). +Sphinx loads the Python bindings when generating documentation, so for it to see any changes +the following steps must be completed: + +- rebuild the Python bindings from the build directory (``cmake --build . --target python_binding``) +- make the updated Python bindings visible to Python, either by installing them, or by running ``scripts/setdevenv.sh`` + from the build directory +- update the timestamp of the ``rst`` files associated with the page where the documentation appears (e.g., ``touch doc/source/api/python/osgeo.ogr.rst``) + + .. _rst_style: Sphinx RST Style guide diff --git a/doc/source/download.rst b/doc/source/download.rst index 686f2e0f72e8..8583de31abc6 100644 --- a/doc/source/download.rst +++ b/doc/source/download.rst @@ -13,15 +13,21 @@ Download Current Release ------------------------------------------------------------------------------ +* **2024-02-18** `gdal-3.8.4.tar.gz`_ `3.8.4 Release Notes`_ (`3.8.4 md5`_) + +.. _`3.8.4 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.4/NEWS.md +.. _`gdal-3.8.4.tar.gz`: https://github.com/OSGeo/gdal/releases/download/v3.8.4/gdal-3.8.4.tar.gz +.. _`3.8.4 md5`: https://github.com/OSGeo/gdal/releases/download/v3.8.4/gdal-3.8.4.tar.gz.md5 + +Past Releases +------------------------------------------------------------------------------ + * **2024-01-08** `gdal-3.8.3.tar.gz`_ `3.8.3 Release Notes`_ (`3.8.3 md5`_) .. _`3.8.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.3/NEWS.md .. _`gdal-3.8.3.tar.gz`: https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz .. _`3.8.3 md5`: https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz.md5 -Past Releases ------------------------------------------------------------------------------- - * **2023-12-20** `gdal-3.8.2.tar.gz`_ `3.8.2 Release Notes`_ (`3.8.2 md5`_) .. _`3.8.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.8.2/NEWS.md diff --git a/doc/source/drivers/raster/ceos.rst b/doc/source/drivers/raster/ceos.rst index 290a43e8b6ca..bdf5f7b32056 100644 --- a/doc/source/drivers/raster/ceos.rst +++ b/doc/source/drivers/raster/ceos.rst @@ -16,7 +16,7 @@ This driver is known to work with CEOS data produced by Spot Image, but will have problems with many other data sources. In particular, it will only work with eight bit unsigned data. -See the separate `SAR_CEOS <#SAR_CEOS>`__ driver for access to SAR CEOS +See the separate :ref:`raster.sar_ceos` driver for access to SAR CEOS data products. NOTE: Implemented as :source_file:`frmts/ceos/ceosdataset.cpp`. diff --git a/doc/source/drivers/raster/georaster.rst b/doc/source/drivers/raster/georaster.rst index 397e4a19ec6c..e9a31dee6b54 100644 --- a/doc/source/drivers/raster/georaster.rst +++ b/doc/source/drivers/raster/georaster.rst @@ -185,6 +185,102 @@ Creation Options generated. If :co:`GENPYRAMID` is not informed the resample method NN (nearest neighbor) will apply. +- .. co:: GENSTATS + :choices: TRUE, FALSE + :default: FALSE + + To generate statistics from the given rasters, set this value to TRUE. + Default value is FALSE. + This option must be present in order to generate the stats, otherwise, + other GENSTATS options are ignored. + + +- .. co:: GENSTATS_SAMPLINGFACTOR + :choices: + :default: 1 + + Defines the number of cells skipped in both row and column dimensions when + the statistics are computed. + For example, when setting this value to 4, one-sixteenth of the cells are sampled. + The higher the value, the less accurate the statistics are likely to be, + but the more quickly they will be computed. + Defaults to 1, which means that all cells are sampled. + +- .. co:: GENSTATS_SAMPLINGWINDOW + :choices: + + This parameter identifies the upper-left (row, column) and lower-right + (row, column) coordinates of a rectangular window, and raster space is assumed. + The intersection of the MBR (minimum bounding rectangle) of the + GeoRaster object in raster space is used for computing statistics. + When this value is not specified, statistics are computed for the entire raster. + +- .. co:: GENSTATS_HISTOGRAM + :choices: TRUE, FALSE + :default: FALSE + + When this value is set to TRUE, a histogram will be computed and stored. + Defaults to FALSE, so a histogram won't be generated. + +- .. co:: GENSTATS_LAYERNUMBERS + :choices: , - + + Defines the numbers of the layers for which to compute the statistics. + This can include numbers, ranges (indicated by hyphens), and commas + to separate any combination of those. + For example, '1,3-5,7', '1,3,6', '1-6'. + If this value is not specified, statistics will be computed for all + layers. + +- .. co:: GENSTATS_USEBIN + :choices: TRUE, FALSE + :default: TRUE + + Defaults to TRUE. + Specifies whether or not to use a provided bin function + (specified in :co:`GENSTATS_BINFUNCTION`). + When this value is set to TRUE, the bin function to be + used follows the following sequence: (1) the bin function + specified in :co:`GENSTATS_BINFUNCTION`. (2) the bin + function specified by the element in the + GeoRaster XML metadata. (3) a dynamically generated bin + function generated as follows: + Min and max are the actual min and max values of the raster + Numbins is defined as: + * celldepth = 1, numbins = 2. + * cellDepth = 2, numbins = 4. + * cellDepth = 4, numbins = 8. + * cellDepth >= 8, numbins = 256. + + When this value is set to FALSE, then the bin function + to use is the one generated dynamically as previously + described. + +- .. co:: GENSTATS_BINFUNCTION + :choices: + + An array whose element specify the bin type, total + number of bins, first bin number, minimum cell value, + and maximum cell value. Bin type must be linear (0). + When this value is not specified, and :co:`GENSTATS_USEBIN` + is TRUE, then the bin function to use is as follows: + + 1. A valid function defined in the GeoRaster metadata. + 2. The same bin function generated when :co:`GENSTATS_USEBIN` is FALSE. + +- .. co:: GENSTATS_NODATA + :choices: TRUE, FALSE + :default: FALSE + + Specifies whether or not to compare each cell values + with NODATA values defined in the metadata when computing + statistics. When set to TRUE, all pixels with a NODATA + value won't be considered. When set to FALSE, NODATA + pixels will be considered as regular pixels. + + A NODATA value is used for cells whose values are either not known + or meaningless + - .. co:: QUALITY :choices: 0-100 :default: 75 @@ -366,23 +462,28 @@ Loading: General use of GeoRaster ------------------------ -| GeoRaster can be used in any GDAL command line tool with all the - available options. Like a image subset extraction or re-project: -| % gdal_translate -of gtiff geor:scott/tiger@dbdemo,landsat,scene,id=54 - output.tif \\ +GeoRaster can be used in any GDAL command line tool with all the available options. +Like a image subset extraction or re-project: + +.. code-block:: bash + + % gdal_translate -of gtiff geor:scott/tiger@dbdemo,landsat,scene,id=54 output.tif \   -srcwin 0 0 800 600 - % gdalwarp -of png geor:scott/tiger@dbdemo,st_rdt_1,130 output.png - -t_srs EPSG:9000913 - Two different GeoRaster can be used as input and output on the same - operation: -| % gdal_translate -of georaster - geor:scott/tiger@dbdemo,landsat,scene,id=54 - geor:scott/tiger@proj1,projview,image -co INSERT="VALUES - (102, SDO_GEOR.INIT())" - Applications that use GDAL can theoretically read and write from - GeoRaster just like any other format but most of then are more - inclined to try to access files on the file system so one alternative - is to create VRT to represent the GeoRaster description, e.g.: -| % gdal_translate -of VRT geor:scott/tiger@dbdemo,landsat,scene,id=54 - view_54.vrt - % openenv view_54.vrt + % gdalwarp -of png geor:scott/tiger@dbdemo,st_rdt_1,130 output.png + -t_srs EPSG:9000913 + +Two different GeoRaster can be used as input and output on the same operation: + +.. code-block:: bash + + % gdal_translate -of georaster geor:scott/tiger@dbdemo,landsat,scene,id=54 \ + geor:scott/tiger@proj1,projview,image -co INSERT="VALUES (102, SDO_GEOR.INIT())" + +Applications that use GDAL can theoretically read and write from GeoRaster just like +any other format but most of then are more inclined to try to access files on the file +system so one alternative is to create VRT to represent the GeoRaster description, e.g.: + +.. code-block:: bash + + % gdal_translate -of VRT geor:scott/tiger@dbdemo,landsat,scene,id=54 view_54.vrt + % openenv view_54.vrt diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index 21c8501d1781..3353ffd07eb7 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -465,8 +465,9 @@ Creation Options * ``LERC_ZSTD`` is available when ``LERC`` and ``ZSTD`` are available. * ``JXL`` is for JPEG-XL, and is only available when using internal libtiff and building GDAL against - https://github.com/libjxl/libjxl . For GDAL < 3.6.0, JXL compression may only be used alongside - ``INTERLEAVE=PIXEL`` (the default) on datasets with 4 bands or less. + https://github.com/libjxl/libjxl . Supported data types are ``Byte``, ``UInt16`` and ``Float32`` only. + For GDAL < 3.6.0, JXL compression may only be used alongside ``INTERLEAVE=PIXEL`` (the default) on + datasets with 4 bands or less. * ``NONE`` is the default. diff --git a/doc/source/drivers/raster/hfa.rst b/doc/source/drivers/raster/hfa.rst index cd20cf172d52..2bc1a80c12a6 100644 --- a/doc/source/drivers/raster/hfa.rst +++ b/doc/source/drivers/raster/hfa.rst @@ -16,14 +16,15 @@ and c128. Compressed and missing tiles in Erdas files should be handled properly on read. Files between 2GiB and 4GiB in size should work on Windows NT, and may work on some Unix platforms. Files with external spill files -(needed for datasets larger than 2GiB) are also support for reading and +(needed for datasets larger than 2GiB) are also supported for reading and writing. -Metadata reading and writing is supported at the dataset level, and for -bands, but this is GDAL specific metadata - not metadata in an Imagine -recognized form. The metadata is stored in a table called GDAL_MetaData -with each column being a metadata item. The title is the key and the row -1 value is the value. +Metadata reading and writing is supported at the dataset and band level. +But this is GDAL specific metadata, not metadata in an Imagine recognized form. +The metadata is stored in a table called GDAL_MetaData which as many +columns as metadata items. +The title of the column is the ``key`` of the ``key``=``value`` metadata item pair +and the value of row 1 is ``value``. Driver capabilities ------------------- @@ -43,7 +44,7 @@ Erdas Imagine files can be created with any GDAL defined band type, including the complex types. Created files may have any number of bands. Pseudo-Color tables will be written if using the GDALDriver::CreateCopy() methodology. Most projections should be -supported though translation of unusual datums (other than WGS84, WGS72, +supported, though translation of unusual datums (other than WGS84, WGS72, NAD83, and NAD27) may be problematic. Creation Options: @@ -110,7 +111,7 @@ Creation Options: :choices: YES, NO :default: NO - Force use of ArcGIS PE String in file + Force the use of ESRI Projection Engine (PE) String in file instead of Imagine coordinate system format. In some cases this improves ArcGIS coordinate system compatibility. @@ -119,8 +120,9 @@ Creation Options: :default: NO :since: 3.7 - Disable use of ArcGIS PE String in - file. Default is NO (that is ArcGIS PE String may be written if needed). + Disable use of ESRI Projection Engine (PE) String in file. + The default value is NO, allowing the ESRI PE String + to be written if needed. Erdas Imagine supports external creation of overviews (with gdaladdo for instance). To force them to be created in an .rrd file (rather than @@ -180,7 +182,7 @@ by the HFA driver: The block size (tile width/height) used for overviews can be specified by setting this - configuration option to a power- of-two value between 32 and 2048. + configuration option to a power-of-two value between 32 and 2048. - .. config:: USE_SPILL :choices: YES, NO diff --git a/doc/source/drivers/raster/jp2ecw.rst b/doc/source/drivers/raster/jp2ecw.rst index 006748fa97e3..30f741ee574f 100644 --- a/doc/source/drivers/raster/jp2ecw.rst +++ b/doc/source/drivers/raster/jp2ecw.rst @@ -280,6 +280,15 @@ JP2ECW driver also arranges JP2 codestream to allow optimal access to power of two overviews. This is controlled with the creation option LEVELS." +Create support +-------------- + +While the driver advertises the Create() capability, contrary to most other +drivers that implement it, the implementation of RasterIO() and WriteBlock() +in the JP2ECW driver does not support arbitrary random writing. +Data must be written in the dataset from top to bottom, whole line(s) at a +time. + Configuration Options --------------------- diff --git a/doc/source/drivers/raster/ogcapi.rst b/doc/source/drivers/raster/ogcapi.rst index 09d84ae05136..04204af49f7e 100644 --- a/doc/source/drivers/raster/ogcapi.rst +++ b/doc/source/drivers/raster/ogcapi.rst @@ -81,16 +81,18 @@ The following open options are available: GeoJSON items otherwise. - .. oo:: IMAGE_FORMAT - :choices: AUTO, PNG, PNG_PREFERRED, JPEG, JPEG_PREFERRED + :choices: AUTO, PNG, PNG_PREFERRED, JPEG, JPEG_PREFERRED, GEOTIFF :default: AUTO Which format to use for pixel acquisition, for tiles or map API. - Defaults to AUTO, which means - that PNG will be used if available, and fallback to JPEG otherwise. - If specifying PNG or JPEG, they must be available, otherwise the driver will - return an error. If specifying the one of the PNG_PREFERRED or JPEG_PREFERRED - value, the specified format will be used if available, and the driver will - fallback to the other format otherwise. + AUTO - This is the default and specifies that PNG images will be checked first, + then JPEG and then any additional formats the server supports. + PNG_PREFERRED - Same as AUTO + JPEG_PREFERRED - Similar to AUTO, but the order is JPEG, PNG and then any additional + formats the server supports + JPEG - Use only JPEG images. If none are available then the driver will return an error + PNG - Use only PNG images. If none are available then the driver will return an error + GEOTIFF - Use only GEOTIFF images. If none are available then the driver will return an error - .. oo:: VECTOR_FORMAT :choices: AUTO, GEOJSON, GEOJSON_PREFERRED, MVT, MVT_PREFERRED diff --git a/doc/source/drivers/raster/pdf.rst b/doc/source/drivers/raster/pdf.rst index 3825c56e3fc2..bdc949b6550b 100644 --- a/doc/source/drivers/raster/pdf.rst +++ b/doc/source/drivers/raster/pdf.rst @@ -721,9 +721,15 @@ Only GDAL builds against static builds of PDFium have been tested. Building PDFium can be challenging, and particular builds must be used to work properly with GDAL. -With GDAL >= 3.8 +With GDAL >= 3.9 ++++++++++++++++ +The scripts in the ``__ +repository must be used to build a patched version of PDFium. + +With GDAL = 3.8 ++++++++++++++++ + The scripts in the ``__ repository must be used to build a patched version of PDFium. diff --git a/doc/source/drivers/raster/sar_ceos.rst b/doc/source/drivers/raster/sar_ceos.rst index a35033ed4dc0..31849e033a53 100644 --- a/doc/source/drivers/raster/sar_ceos.rst +++ b/doc/source/drivers/raster/sar_ceos.rst @@ -13,7 +13,7 @@ main imagery file. This driver works with most Radarsat, JERS-1 and ERS data products, including single look complex products; however, it is unlikely to work for -non-Radar CEOS products. The simpler `CEOS <#CEOS>`__ driver is often +non-Radar CEOS products. The simpler :ref:`raster.ceos` driver is often appropriate for these. This driver will attempt to read 15 lat/long GCPS by sampling the diff --git a/doc/source/drivers/raster/usgsdem.rst b/doc/source/drivers/raster/usgsdem.rst index 0559038d7e60..7157b2598588 100644 --- a/doc/source/drivers/raster/usgsdem.rst +++ b/doc/source/drivers/raster/usgsdem.rst @@ -10,7 +10,7 @@ USGSDEM -- USGS ASCII DEM (and CDED) GDAL includes support for reading USGS ASCII DEM files. This is the traditional format used by USGS before being replaced by SDTS, and is -the format used for CDED DEM data products from the Canada. Most popular +the format used for CDED DEM data products from Canada. Most popular variations on USGS DEM files should be supported, including correct recognition of coordinate system, and georeferenced positioning. diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index 8b3333304e81..1c860c439ef2 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -17,6 +17,11 @@ potentially applied as well as various kinds of metadata altered or added. VRT descriptions of datasets can be saved in an XML format normally given the extension .vrt. +Note .vrt files starting with + +- open with :ref:`ogrinfo`, etc. +- open with :ref:`gdalinfo`, etc. + The VRT format can also describe :ref:`gdal_vrttut_warped` and :ref:`gdal_vrttut_pansharpen` @@ -81,9 +86,13 @@ The following creations options are supported: .vrt Format ----------- -A `XML schema of the GDAL VRT format `_ +A `XML schema of the GDAL VRT format `_ is available. +Note, .vrt files starting with +- open with ogrinfo, etc. +- open with gdalinfo, etc. + Virtual files stored on disk are kept in an XML format with the following elements. @@ -185,7 +194,7 @@ The attributes for VRTRasterBand are: - **blockYSize** (optional, GDAL >= 3.3): block height. If not specified, defaults to the minimum of the raster height and 128. -This element may have Metadata, ColorInterp, NoDataValue, HideNoDataValue, ColorTable, GDALRasterAttributeTable, Description and MaskBand subelements as well as the various kinds of source elements such as SimpleSource, ComplexSource, AveragedSource, KernelFilteredSource and ArraySource. A raster band may have many "sources" indicating where the actual raster data should be fetched from, and how it should be mapped into the raster bands pixel space. +This element may have Metadata, ColorInterp, NoDataValue, HideNoDataValue, ColorTable, GDALRasterAttributeTable, Description and MaskBand subelements as well as the various kinds of source elements such as SimpleSource, ComplexSource, AveragedSource, NoDataFromMaskSource, KernelFilteredSource and ArraySource. A raster band may have many "sources" indicating where the actual raster data should be fetched from, and how it should be mapped into the raster bands pixel space. The allowed subelements for VRTRasterBand are : @@ -304,6 +313,8 @@ The allowed subelements for VRTRasterBand are : - **AveragedSource**: The AveragedSource is derived from the SimpleSource and shares the same properties except that it uses an averaging resampling instead of a nearest neighbour algorithm as in SimpleSource, when the size of the destination rectangle is not the same as the size of the source rectangle. Note: a more general mechanism to specify resampling algorithms can be used. See above paragraph about the 'resampling' attribute. +- **NoDataFromMaskSource**: (GDAL >= 3.9) The NoDataFromMaskSource is derived from the SimpleSource and shares the same properties except that it replaces the value of the source with the value of the NODATA child element when the value of the mask band of the source is less or equal to the MaskValueThreshold child element. + - **ComplexSource**: The ComplexSource_ is derived from the SimpleSource (so it shares the SourceFilename, SourceBand, SrcRect and DstRect elements), but it provides support to rescale and offset the range of the source values. Certain regions of the source can be masked by specifying the NODATA value, or starting with GDAL 3.3, with the true element. - **KernelFilteredSource**: The KernelFilteredSource_ is a pixel source derived from the Simple Source (so it shares the SourceFilename, SourceBand, SrcRect and DstRect elements, but it also passes the data through a simple filtering kernel specified with the Kernel element. @@ -498,6 +509,25 @@ For example, a Gaussian blur: +NoDataFromMaskSource +~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.9 + +The NoDataFromMaskSource is derived from the SimpleSource and shares the same properties except that it replaces the value of the source with the value of the NODATA child element when the value of the mask band of the source is less or equal to the MaskValueThreshold child element. +An optional RemappedValue element can be set to specify the value onto which valid pixels whose value is the one of NODATA should be remapped to. When RemappedValue is not explicitly specified, for Byte bands, if NODATA=255, it is implicitly set to 254, otherwise it is set to NODATA+1. + +.. code-block:: xml + + + in.tif + 1 + 128 + 0 + 1 + + + ArraySource ~~~~~~~~~~~ @@ -726,13 +756,13 @@ Except if (from top priority to lesser priority) : ------------------------------- So far we have described how to derive new virtual datasets from existing -files supports by GDAL. However, it is also common to need to utilize +files supported by GDAL. However, it is also common to need to utilize raw binary raster files for which the regular layout of the data is known but for which no format specific driver exists. This can be accomplished by writing a .vrt file describing the raw file. For example, the following .vrt describes a raw raster file containing -floating point complex pixels in a file called l2p3hhsso.img. The +floating point complex pixels in a file called *l2p3hhsso.img*. The image data starts from the first byte (ImageOffset=0). The byte offset between pixels is 8 (PixelOffset=8), the size of a CFloat32. The byte offset from the start of one line to the start of the next is 9376 bytes @@ -1727,6 +1757,7 @@ PanchroBand and SpectralBand elements must have at least a **SourceFilename** ch element to specify the name of the dataset. They may also have a **SourceBand** child element to specify the number of the band in the dataset (starting with 1). If not specify, the first band will be assumed. +**OpenOptions** can also be specified The SpectralBand element must generally have a **dstBand** attribute to specify the number of the output band (starting with 1) to which the input spectral band must be mapped. @@ -1748,18 +1779,30 @@ with panchromatic.tif. panchromatic.tif + + ALL_CPUS + 1 multispectral.tif + + ALL_CPUS + 1 multispectral.tif + + ALL_CPUS + 2 multispectral.tif + + ALL_CPUS + 3 diff --git a/doc/source/drivers/raster/xyz.rst b/doc/source/drivers/raster/xyz.rst index 0cb9c409b0a9..66c0d7d4c45d 100644 --- a/doc/source/drivers/raster/xyz.rst +++ b/doc/source/drivers/raster/xyz.rst @@ -14,7 +14,8 @@ the documentation of the :ref:`gdal_grid` utility). Those datasets are ASCII files with (at least) 3 columns, each line containing the X and Y coordinates of the center of the cell and the -value of the cell. +value of the cell. (Note the XYZ driver only uses the first band of +the dataset. I.e., columns beyond the third are ignored.) The spacing between each cell must be constant. diff --git a/doc/source/drivers/vector/gml.rst b/doc/source/drivers/vector/gml.rst index 4e8080845b1d..89eb17d83eb2 100644 --- a/doc/source/drivers/vector/gml.rst +++ b/doc/source/drivers/vector/gml.rst @@ -791,7 +791,7 @@ Syntax of .gfs files -------------------- A XML Schema for .gfs files can be found at -https://raw.githubusercontent.com/OSGeo/gdal/master/data/gfs.xsd . +https://raw.githubusercontent.com/OSGeo/gdal/master/ogr/ogrsf_frmts/gml/data/gfs.xsd . Let's consider the following test.gml file : diff --git a/doc/source/drivers/vector/gpkg.rst b/doc/source/drivers/vector/gpkg.rst index 8f9425c041cb..798192145868 100644 --- a/doc/source/drivers/vector/gpkg.rst +++ b/doc/source/drivers/vector/gpkg.rst @@ -168,15 +168,20 @@ Relationships .. versionadded:: 3.6 -Relationship retrieval is supported, respecting the OGC GeoPackage Related Tables Extension. -If the Related Tables Extension is not in use then relationships will be reported for tables -which utilize FOREIGN KEY constraints. +Many-to-many relationship retrieval is supported, respecting the OGC GeoPackage Related Tables Extension. +One-to-many relationships will also be reported for tables which utilize FOREIGN KEY constraints. Relationship creation, deletion and updating is supported since GDAL 3.7. Relationships can only be updated to change their base or related table fields, or the relationship related table type. It is not permissible to change the base or related table itself, or the mapping table details. If this is desired then a new relationship should be created instead. +Note that when a many-to-many relationship is created in a GeoPackage, GDAL will always +insert the mapping table into the gpkg_contents table. Formally this is not required +by the Related Tables Extension (instead, the table should only be listed in gpkgext_relations), +however failing to list the mapping table in gpkg_contents prevents it from being usable +in some other applications (e.g. ESRI software). + Dataset open options -------------------- diff --git a/doc/source/drivers/vector/pg.rst b/doc/source/drivers/vector/pg.rst index 055958a22572..490e1caa9708 100644 --- a/doc/source/drivers/vector/pg.rst +++ b/doc/source/drivers/vector/pg.rst @@ -19,6 +19,8 @@ instead use the :ref:`PostgreSQL SQL Dump driver `. You can find additional information on the driver in the :ref:`Advanced OGR PostgreSQL driver Information ` page. +Starting with GDAL 3.9, only PostgreSQL >= 9 and PostGIS >= 2 are supported. + Driver capabilities ------------------- @@ -81,7 +83,7 @@ and named views will be treated as layers. The driver also supports the `geography `__ -column type introduced in PostGIS 1.5. +column type. The driver also supports reading and writing the following non-linear geometry types :CIRCULARSTRING, COMPOUNDCURVE, @@ -228,7 +230,7 @@ Layer Creation Options :choices: geometry, geography, BYTEA, OID The GEOM_TYPE layer creation option can be set to one - of "geometry", "geography" (PostGIS >= 1.5), "BYTEA" or "OID" to + of "geometry", "geography", "BYTEA" or "OID" to force the type of geometry used for a table. For a PostGIS database, "geometry" is the default value. PostGIS "geography" assumes a geographic SRS (before PostGIS 2.2, it was even required to be EPSG:4326), but the @@ -266,7 +268,7 @@ Layer Creation Options :choices: 2, 3, XYM, XYZM Control the dimension of the layer. Important - to set to 2 for 2D layers with PostGIS 1.0+ as it has constraints on + to set to 2 for 2D layers as it has constraints on the geometry dimension during loading. - .. lco:: GEOMETRY_NAME diff --git a/doc/source/drivers/vector/vrt.rst b/doc/source/drivers/vector/vrt.rst index 93a2555d69ff..b1de74ac1050 100644 --- a/doc/source/drivers/vector/vrt.rst +++ b/doc/source/drivers/vector/vrt.rst @@ -544,3 +544,7 @@ Other Notes that is possible. For instance if the source is an RDBMS. You can turn off that feature by setting the *useSpatialSubquery* attribute of the GeometryField element to FALSE. +- .vrt files starting with + - open with ogrinfo, etc. + - open with gdalinfo, etc. + diff --git a/doc/source/index.rst b/doc/source/index.rst index 1f37d01ff378..31f6f8829ba7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -4,7 +4,10 @@ GDAL .. include:: ./about_no_title.rst -This documentation is also available as a `PDF file `_. +This documentation is also available as a `PDF file `_, +and `a .ZIP of individual HTML pages `_ for offline browsing. (The .ZIP also includes that .PDF.) + + .. toctree:: :maxdepth: 2 diff --git a/doc/source/programs/gdal2xyz.rst b/doc/source/programs/gdal2xyz.rst index a3dc4738e959..4c896769a625 100644 --- a/doc/source/programs/gdal2xyz.rst +++ b/doc/source/programs/gdal2xyz.rst @@ -22,14 +22,13 @@ Synopsis [-skipnodata] [-csv] [-srcnodata ] [-dstnodata ] - [] + Description ----------- The :program:`gdal2xyz` utility can be used to translate a raster file into xyz format. -`gdal2xyz` can be used as an alternative to `gdal_translate of=xyz`, but supporting other options, -for example: +`gdal2xyz` can be used as an alternative to `gdal_translate of=xyz`. Features include: * Select more then one band * Skip or replace nodata value @@ -101,5 +100,13 @@ Examples gdal2xyz -b 1 -b 2 -dstnodata 0 input.tif output.txt -To create a text file in `xyz` format from the input file `input.tif`, including the first and second bands, -while replacing the dataset nodata values with zeros. +To create a text file in `xyz` format from the input file `input.tif`. +The first columns, x and y, are the coordinates of the centers of each cell. +The remaining columns represent the first and second bands. +We also replace the dataset nodata values with zeros. + + +Caveats +------- + +gdal2xyz output values with a limited precision. Use ``gdal_translate -of XYZ`` if more precision is desired. diff --git a/doc/source/programs/gdal_grid.rst b/doc/source/programs/gdal_grid.rst index 91ac2c90683d..e4534a22ea4d 100644 --- a/doc/source/programs/gdal_grid.rst +++ b/doc/source/programs/gdal_grid.rst @@ -48,6 +48,9 @@ computer. .. include:: options/ot.rst +If not set then a default type is used, which might not be supported +by the relevant driver, causing a error. + .. include:: options/of.rst .. option:: -txe @@ -444,15 +447,27 @@ content: -This description specifies so called 2.5D geometry with three coordinates X, Y -and Z. Z value will be used for interpolation. Now you can use *dem.vrt* -with all OGR programs (start with :ref:`ogrinfo` to test that everything works -fine). The datasource will contain a single layer called *"dem"* filled -with point features constructed from values in CSV file. Using this technique -you can handle CSV files with more than three columns, switch columns, etc. +This description specifies so called 2.5D geometry with three coordinates +X, Y and Z. The Z value will be used for interpolation. Now you can +use *dem.vrt* with all OGR programs (start with :ref:`ogrinfo` to test that +everything works fine). The datasource will contain a single layer called +*"dem"* filled with point features constructed from values in the CSV file. +Using this technique you can handle CSV files with more than three +columns, switch columns, etc. OK, now the final step: + +.. code-block:: -If your CSV file does not contain column headers then it can be handled in the -following way: + gdal_grid dem.vrt demv.tif + +Or, if we do not wish to use a VRT file: + +.. code-block:: + + gdal_grid -l dem -oo X_POSSIBLE_NAMES=Easting \ + -oo Y_POSSIBLE_NAMES=Northing -zfield Elevation dem.csv dem.tif + +If your CSV file does not contain column headers then it can be handled +in the VRT file in the following way: .. code-block:: xml @@ -461,6 +476,27 @@ following way: The :ref:`vector.csv` description page contains details on CSV format supported by GDAL/OGR. +Creating multiband files +------------------------ + +Creating multiband files is not directly possible with gdal_grid. +One might use gdal_grid multiple times to create one band per file, +and then use :ref:`gdalbuildvrt` -separate and then :ref:`gdal_translate`: + +.. code-block:: bash + + gdal_grid ... 1.tif; gdal_grid ... 2.tif; gdal_grid ... 3.tif + gdalbuildvrt -separate 123.vrt 1.tif 2.tif 3.tif + gdal_translate 123.vrt 123.tif + +Or just :ref:`gdal_merge`, to combine the one-band files into a single one: + +.. code-block:: bash + + gdal_grid ... a.tif; gdal_grid ... b.tif; gdal_grid ... c.tif + gdal_merge.py -separate a.tif b.tif c.tif -o d.tif + + C API ----- @@ -475,7 +511,8 @@ Values to interpolate will be read from Z value of geometry record. :: - gdal_grid -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff + gdal_grid -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 \ + -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff The next command does the same thing as the previous one, but reads values to interpolate from the attribute field specified with **-zfield** option @@ -485,5 +522,7 @@ The :config:`GDAL_NUM_THREADS` is also set to parallelize the computation. :: - gdal_grid -zfield "Elevation" -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt dem.tiff --config GDAL_NUM_THREADS ALL_CPUS + gdal_grid -zfield "Elevation" -a invdist:power=2.0:smoothing=1.0 -txe 85000 89000 \ + -tye 894000 890000 -outsize 400 400 -of GTiff -ot Float64 -l dem dem.vrt \ + dem.tiff --config GDAL_NUM_THREADS ALL_CPUS diff --git a/doc/source/programs/gdal_rasterize.rst b/doc/source/programs/gdal_rasterize.rst index 71060b79ba11..32121529f2b1 100644 --- a/doc/source/programs/gdal_rasterize.rst +++ b/doc/source/programs/gdal_rasterize.rst @@ -6,7 +6,7 @@ gdal_rasterize .. only:: html - Burns vector geometries into a raster. + Burns vector geometries into a raster .. Index:: gdal_rasterize @@ -202,7 +202,7 @@ raster data is only supported since GDAL 2.1.0. The GDAL supported output file. Must support update mode access. This file will be created (or overwritten if it already exists). -The program create a new target raster image when any of the :option:`-of`, +The program creates a new target raster image when any of the :option:`-of`, :option:`-a_nodata`, :option:`-init`, :option:`-a_srs`, :option:`-co`, :option:`-te`, :option:`-tr`, :option:`-tap`, :option:`-ts`, or :option:`-ot` options are used. The resolution or size must be specified using the :option:`-tr` or :option:`-ts` option for all new @@ -216,13 +216,13 @@ This utility is also callable from C with :cpp:func:`GDALRasterize`. .. versionadded:: 2.1 -Example -------- +Examples +-------- The following would burn all polygons from mask.shp into the RGB TIFF file work.tif with the color red (RGB = 255,0,0). -:: +.. code-block:: gdal_rasterize -b 1 -b 2 -b 3 -burn 255 -burn 0 -burn 0 -l mask mask.shp work.tif @@ -230,7 +230,7 @@ file work.tif with the color red (RGB = 255,0,0). The following would burn all "class A" buildings into the output elevation file, pulling the top elevation from the ROOF_H attribute. -:: +.. code-block:: gdal_rasterize -a ROOF_H -where "class='A'" -l footprints footprints.shp city_dem.tif @@ -238,6 +238,6 @@ The following would burn all polygons from footprint.shp into a new 1000x1000 rgb TIFF as the color red. Note that :option:`-b` is not used; the order of the :option:`-burn` options determines the bands of the output raster. -:: +.. code-block:: gdal_rasterize -burn 255 -burn 0 -burn 0 -ot Byte -ts 1000 1000 -l footprints footprints.shp mask.tif diff --git a/doc/source/programs/gdal_translate.rst b/doc/source/programs/gdal_translate.rst index e55d4e43f681..8c13b82d5e56 100644 --- a/doc/source/programs/gdal_translate.rst +++ b/doc/source/programs/gdal_translate.rst @@ -443,7 +443,8 @@ To create a JPEG-compressed TIFF with internal mask from a RGBA dataset :: - gdal_translate rgba.tif withmask.tif -b 1 -b 2 -b 3 -mask 4 -co COMPRESS=JPEG -co PHOTOMETRIC=YCBCR --config GDAL_TIFF_INTERNAL_MASK YES + gdal_translate rgba.tif withmask.tif -b 1 -b 2 -b 3 -mask 4 -co COMPRESS=JPEG \ + -co PHOTOMETRIC=YCBCR --config GDAL_TIFF_INTERNAL_MASK YES To create a RGBA dataset from a RGB dataset with a mask diff --git a/doc/source/programs/gdalbuildvrt.rst b/doc/source/programs/gdalbuildvrt.rst index 253e1bcc462a..f3bf3eec0a11 100644 --- a/doc/source/programs/gdalbuildvrt.rst +++ b/doc/source/programs/gdalbuildvrt.rst @@ -24,6 +24,7 @@ Synopsis [-addalpha] [-hidenodata] [-srcnodata "[ ]..."] [-vrtnodata "[ ]..." [-ignore_srcmaskband] + [-nodata_max_mask_threshold ] [-a_srs ] [-r {nearest|bilinear|cubic|cubicspline|lanczos|average|mode}] [-oo =]... @@ -146,6 +147,14 @@ changed in later versions. not be taken into account, and in case of overlapping between sources, the last one will override previous ones in areas of overlap. +.. option:: -nodata_max_mask_threshold + + .. versionadded:: 3.9 + + Insert a source, which replaces the value of the source + with the value of :option:`-vrtnodata` (or 0 if not specified) when the value + of the mask band of the source is less or equal to the threshold. + .. option:: -b Select an input to be processed. Bands are numbered from 1. diff --git a/doc/source/programs/gdallocationinfo.rst b/doc/source/programs/gdallocationinfo.rst index 8814cced6f75..5f8d9df82970 100644 --- a/doc/source/programs/gdallocationinfo.rst +++ b/doc/source/programs/gdallocationinfo.rst @@ -94,15 +94,15 @@ pixel. Currently it reports: - The location of the pixel in pixel/line space. - The result of a LocationInfo metadata query against the datasource. - This is implement for VRT files which will report the + This is implemented for VRT files which will report the file(s) used to satisfy requests for that pixel, and by the - :ref:`raster.mbtiles` driver + :ref:`raster.mbtiles` driver. - The raster pixel value of that pixel for all or a subset of the bands. - The unscaled pixel value if a Scale and/or Offset apply to the band. The pixel selected is requested by x/y coordinate on the command line, or read from stdin. More than one coordinate pair can be supplied when reading -coordinates from stdin. By default pixel/line coordinates are expected. +coordinates from stdin. By default integer pixel/line coordinates are expected. However with use of the :option:`-geoloc`, :option:`-wgs84`, or :option:`-l_srs` switches it is possible to specify the location in other coordinate systems. diff --git a/doc/source/programs/gdaltindex.rst b/doc/source/programs/gdaltindex.rst index f54cf6d28e73..15cd905f4188 100644 --- a/doc/source/programs/gdaltindex.rst +++ b/doc/source/programs/gdaltindex.rst @@ -21,7 +21,7 @@ Synopsis [-f ] [-tileindex ] [-write_absolute_path] [-skip_different_projection] [-t_srs ] [-src_srs_name ] [-src_srs_format {AUTO|WKT|EPSG|PROJ}] - [-lyr_name ] + [-lyr_name ] [-lco =]... [-gti_filename ] [-tr ] [-te ] [-ot ] [-bandcount ] [-nodata [,...]] @@ -130,6 +130,12 @@ tileindex, or as input for the :ref:`GTI ` driver. Layer name to create/append to in the output tile index file. +.. option:: -lco = + + .. versionadded:: 3.9 + + Layer creation option (format specific) + .. option:: The name of the output file to create/append to. The default dataset will diff --git a/doc/source/programs/gdaltransform.rst b/doc/source/programs/gdaltransform.rst index 6411c2d6ea43..b91ef59fddbc 100644 --- a/doc/source/programs/gdaltransform.rst +++ b/doc/source/programs/gdaltransform.rst @@ -6,7 +6,7 @@ gdaltransform .. only:: html - Transforms coordinates. + Transforms coordinates .. Index:: gdaltransform @@ -16,11 +16,11 @@ Synopsis .. code-block:: gdaltransform [--help] [--help-general] - [-i] [-s_srs ] [-t_srs ] [-to >NAME>=]... + [-i] [-s_srs ] [-t_srs ] [-to =]... [-s_coord_epoch ] [-t_coord_epoch ] [-ct ] [-order ] [-tps] [-rpc] [-geoloc] [-gcp [elevation]]... [-output_xy] - [srcfile [dstfile]] + [ []] Description @@ -116,13 +116,15 @@ projection,including GCP-based transformations. .. option:: - File with source projection definition or GCP's. If - not given, source projection is read from the command-line :option:`-s_srs` - or :option:`-gcp` parameters + Raster dataset with source projection definition or GCPs. If + not given, source projection/GCPs are read from the command-line :option:`-s_srs` + or :option:`-gcp` parameters. + + Note that only the SRS and/or GCPs of this input file is taken into account, and not its pixel content. .. option:: - File with destination projection definition. + Raster dataset with destination projection definition. Coordinates are read as pairs, triples (for 3D,) or (since GDAL 3.0.0,) quadruplets (for X,Y,Z,time) of numbers per line from standard @@ -144,7 +146,7 @@ Reprojection Example Simple reprojection from one projected coordinate system to another: -:: +.. code-block:: bash gdaltransform -s_srs EPSG:28992 -t_srs EPSG:31370 177502 311865 @@ -152,9 +154,9 @@ Simple reprojection from one projected coordinate system to another: Produces the following output in meters in the "Belge 1972 / Belgian Lambert 72" projection: -:: +.. code-block:: bash - 244510.77404604 166154.532871342 -1046.79270555763 + 244296.724777415 165937.350217148 0 Image RPC Example +++++++++++++++++ @@ -165,14 +167,14 @@ used, the transformation is from output georeferenced (WGS84) coordinates back to image coordinates. -:: +.. code-block:: bash gdaltransform -i -rpc 06OCT20025052-P2AS-005553965230_01_P001.TIF 125.67206 39.85307 50 Produces this output measured in pixels and lines on the image: -:: +.. code-block:: bash 3499.49282422381 2910.83892848414 50 @@ -182,7 +184,7 @@ X,Y,Z,time transform 15-term time-dependent Helmert coordinate transformation from ITRF2000 to ITRF93 for a coordinate at epoch 2000.0 -:: +.. code-block:: bash gdaltransform -ct "+proj=pipeline +step +proj=unitconvert +xy_in=deg \ +xy_out=rad +step +proj=cart +step +proj=helmert +convention=position_vector \ @@ -192,8 +194,23 @@ for a coordinate at epoch 2000.0 +proj=unitconvert +xy_in=rad +xy_out=deg" 2 49 0 2000 -Produces this output measured in longitude degrees, latitude degrees and ellipsoid height in metre: +Produces this output measured in longitude degrees, latitude degrees and ellipsoid height in metres: -:: +.. code-block:: bash 2.0000005420366 49.0000003766711 -0.0222802283242345 + +Ground control points ++++++++++++++++++++++ + +Where is the address "370 S. 300 E." in Salt Lake City, given we know +some nearby corners' coordinates? + +.. code-block:: bash + + echo 300 -370 | gdaltransform \ + -gcp 0 -500 -111.89114803 40.75846686 \ + -gcp 0 0 -111.89114717 40.76932606 \ + -gcp 500 0 -111.87685039 40.76940631 + + -111.8825697384 40.761338402 0 diff --git a/doc/source/programs/gdalwarp.rst b/doc/source/programs/gdalwarp.rst index 463113ede342..38896dab9b99 100644 --- a/doc/source/programs/gdalwarp.rst +++ b/doc/source/programs/gdalwarp.rst @@ -71,7 +71,7 @@ with control information. Blue, Green, Red, NearInfraRed in an output dataset with bands ordered as Red, Green, Blue. - :: + .. code-block:: bash gdalwarp in_bgrn.tif out_rgb.tif -b 3 -b 2 -b 1 -overwrite @@ -84,7 +84,7 @@ with control information. is only useful when updating an existing dataset, e.g to warp one band at at time. - :: + .. code-block:: bash gdal_create -if in_red.tif -bands 3 out_rgb.tif gdalwarp in_red.tif out_rgb.tif -srcband 1 -dstband 1 @@ -579,13 +579,13 @@ less than 100% then you know things are IO bound. Otherwise they are CPU bound. The ``--debug`` option may also provide useful information. For instance, after running the following: -.. code-block:: +.. code-block:: bash gdalwarp --debug on abc.tif def.tif a message like the following will be output: -:: +.. code-block:: GDAL: 224 block reads on 32 block band 1 of utm.tif @@ -594,7 +594,7 @@ that 224 block reads were done, implying that lots of data was having to be re-read, presumably because of a limited IO cache. You will also see messages like: -:: +.. code-block:: GDAL: GDALWarpKernel()::GWKNearestNoMasksByte() Src=0,0,512x512 Dst=0,0,512x512 @@ -648,7 +648,7 @@ Examples - Basic transformation: -:: +.. code-block:: bash gdalwarp -t_srs EPSG:4326 input.tif output.tif @@ -657,7 +657,7 @@ Examples control points mapping the corners to lat/long could be warped to a UTM projection with a command like this: -:: +.. code-block:: bash gdalwarp -t_srs '+proj=utm +zone=11 +datum=WGS84' -overwrite raw_spot.tif utm11.tif @@ -667,19 +667,21 @@ Examples .. versionadded:: 2.2 -:: +.. code-block:: bash - gdalwarp -overwrite HDF4_SDS:ASTER_L1B:"pg-PR1B0000-2002031402_100_001":2 pg-PR1B0000-2002031402_100_001_2.tif + gdalwarp -overwrite HDF4_SDS:ASTER_L1B:"pg-PR1B0000-2002031402_100_001":2 \ + pg-PR1B0000-2002031402_100_001_2.tif - To apply a cutline on a un-georeferenced image and clip from pixel (220,60) to pixel (1160,690): -:: +.. code-block:: bash - gdalwarp -overwrite -to SRC_METHOD=NO_GEOTRANSFORM -to DST_METHOD=NO_GEOTRANSFORM -te 220 60 1160 690 -cutline cutline.csv in.png out.tif + gdalwarp -overwrite -to SRC_METHOD=NO_GEOTRANSFORM -to DST_METHOD=NO_GEOTRANSFORM \ + -te 220 60 1160 690 -cutline cutline.csv in.png out.tif where cutline.csv content is like: -:: +.. code-block:: id,WKT 1,"POLYGON((....))" @@ -688,7 +690,7 @@ where cutline.csv content is like: .. versionadded:: 2.2 -:: +.. code-block:: bash gdalwarp -overwrite in_dem.tif out_dem.tif -s_srs EPSG:4326+5773 -t_srs EPSG:4979 @@ -702,4 +704,10 @@ This utility is also callable from C with :cpp:func:`GDALWarp`. See also -------- -`Wiki page discussing options and behaviours of gdalwarp `_ +.. only:: not man + + `Wiki page discussing options and behaviours of gdalwarp `_ + +.. only:: man + + Wiki page discussing options and behaviours of gdalwarp: https://trac.osgeo.org/gdal/wiki/UserDocs/GdalWarp diff --git a/doc/source/programs/index.rst b/doc/source/programs/index.rst index 0514bbf4af60..af107ef12aa5 100644 --- a/doc/source/programs/index.rst +++ b/doc/source/programs/index.rst @@ -25,80 +25,81 @@ Raster programs :hidden: Common options - gdalinfo - gdal_translate - gdaladdo - gdalwarp - gdaltindex - gdalbuildvrt - gdal_contour - gdaldem - rgb2pct - pct2rgb - gdalattachpct - gdal_merge + + gdal-config gdal2tiles gdal2xyz - gdal_rasterize - gdaltransform - nearblack - gdal_retile + gdal_calc + gdal_contour + gdal_create + gdal_edit + gdal_fillnodata + gdal_footprint gdal_grid - gdal_proximity + gdal_merge + gdal_pansharpen gdal_polygonize + gdal_proximity + gdal_rasterize + gdal_retile gdal_sieve - gdal_fillnodata + gdal_translate + gdal_viewshed + gdaladdo + gdalattachpct + gdalbuildvrt + gdalcompare + gdaldem + gdalinfo gdallocationinfo - gdalsrsinfo - gdalmove - gdal_edit - gdal_calc - gdal_pansharpen - gdal-config gdalmanage - gdalcompare - gdal_viewshed - gdal_create - gdal_footprint + gdalmove + gdalsrsinfo + gdaltindex + gdaltransform + gdalwarp + nearblack + pct2rgb + rgb2pct .. only:: html - :ref:`Common options ` - - :ref:`gdalinfo`: Lists information about a raster dataset. - - :ref:`gdal_translate`: Converts raster data between different formats. - - :ref:`gdaladdo`: Builds or rebuilds overview images. - - :ref:`gdalwarp`: Image reprojection and warping utility. - - :ref:`gdaltindex`: Builds an OGR-supported dataset as a raster tileindex. - - :ref:`gdalbuildvrt`: Builds a VRT from a list of datasets. - - :ref:`gdal_contour`: Builds vector contour lines from a raster elevation model. - - :ref:`gdaldem`: Tools to analyze and visualize DEMs. - - :ref:`rgb2pct`: Convert a 24bit RGB image to 8bit paletted. - - :ref:`pct2rgb`: Convert an 8bit paletted image to 24bit RGB. - - :ref:`gdalattachpct`: Attach a color table to a raster file from an input file. - - :ref:`gdal_merge`: Mosaics a set of images. + - :ref:`gdal-config`: Determines various information about a GDAL installation. - :ref:`gdal2tiles`: Generates directory with TMS tiles, KMLs and simple web viewers. - :ref:`gdal2xyz`: Translates a raster file into xyz format. - - :ref:`gdal_rasterize`: Burns vector geometries into a raster. - - :ref:`gdaltransform`: Transforms coordinates. - - :ref:`nearblack`: Convert nearly black/white borders to black. - - :ref:`gdal_retile`: Retiles a set of tiles and/or build tiled pyramid levels. + - :ref:`gdal_calc`: Command line raster calculator with numpy syntax. + - :ref:`gdal_contour`: Builds vector contour lines from a raster elevation model. + - :ref:`gdal_create`: Create a raster file (without source dataset). + - :ref:`gdal_edit`: Edit in place various information of an existing GDAL dataset. + - :ref:`gdal_fillnodata`: Fill raster regions by interpolation from edges. + - :ref:`gdal_footprint`: Compute footprint of a raster. - :ref:`gdal_grid`: Creates regular grid from the scattered data. - - :ref:`gdal_proximity`: Produces a raster proximity map. + - :ref:`gdal_merge`: Mosaics a set of images. + - :ref:`gdal_pansharpen`: Perform a pansharpen operation. - :ref:`gdal_polygonize`: Produces a polygon feature layer from a raster. + - :ref:`gdal_proximity`: Produces a raster proximity map. + - :ref:`gdal_rasterize`: Burns vector geometries into a raster. + - :ref:`gdal_retile`: Retiles a set of tiles and/or build tiled pyramid levels. - :ref:`gdal_sieve`: Removes small raster polygons. - - :ref:`gdal_fillnodata`: Fill raster regions by interpolation from edges. + - :ref:`gdal_translate`: Converts raster data between different formats. + - :ref:`gdal_viewshed`: Compute a visibility mask for a raster. + - :ref:`gdaladdo`: Builds or rebuilds overview images. + - :ref:`gdalattachpct`: Attach a color table to a raster file from an input file. + - :ref:`gdalbuildvrt`: Builds a VRT from a list of datasets. + - :ref:`gdalcompare`: Compare two images. + - :ref:`gdaldem`: Tools to analyze and visualize DEMs. + - :ref:`gdalinfo`: Lists information about a raster dataset. - :ref:`gdallocationinfo`: Raster query tool - - :ref:`gdalsrsinfo`: Lists info about a given SRS in number of formats (WKT, PROJ.4, etc.) - - :ref:`gdalmove`: Transform georeferencing of raster file in place. - - :ref:`gdal_edit`: Edit in place various information of an existing GDAL dataset. - - :ref:`gdal_calc`: Command line raster calculator with numpy syntax. - - :ref:`gdal_pansharpen`: Perform a pansharpen operation. - - :ref:`gdal-config`: Determines various information about a GDAL installation. - :ref:`gdalmanage`: Identify, delete, rename and copy raster data files. - - :ref:`gdalcompare`: Compare two images. - - :ref:`gdal_viewshed`: Compute a visibility mask for a raster. - - :ref:`gdal_create`: Create a raster file (without source dataset). - - :ref:`gdal_footprint`: Compute footprint of a raster. + - :ref:`gdalmove`: Transform georeferencing of raster file in place. + - :ref:`gdalsrsinfo`: Lists info about a given SRS in number of formats (WKT, PROJ.4, etc.) + - :ref:`gdaltindex`: Builds an OGR-supported dataset as a raster tileindex. + - :ref:`gdaltransform`: Transforms coordinates. + - :ref:`gdalwarp`: Image reprojection and warping utility. + - :ref:`nearblack`: Convert nearly black/white borders to black. + - :ref:`pct2rgb`: Convert an 8bit paletted image to 24bit RGB. + - :ref:`rgb2pct`: Convert a 24bit RGB image to 8bit paletted. Multidimensional Raster programs -------------------------------- diff --git a/doc/source/programs/ogrinfo.rst b/doc/source/programs/ogrinfo.rst index 221b3bb9a6a4..cd7cbb8c1dcf 100644 --- a/doc/source/programs/ogrinfo.rst +++ b/doc/source/programs/ogrinfo.rst @@ -44,7 +44,7 @@ edit data. .. option:: -json Display the output in json format, conforming to the - `ogrinfo.schema.json `__ + `ogrinfo_output.schema.json `__ schema. .. versionadded:: 3.7 diff --git a/doc/source/programs/ogrlineref.rst b/doc/source/programs/ogrlineref.rst index 744fd4219bf0..b3969b1d1875 100644 --- a/doc/source/programs/ogrlineref.rst +++ b/doc/source/programs/ogrlineref.rst @@ -6,7 +6,7 @@ ogrlineref .. only:: html - Create linear reference and provide some calculations using it. + Create linear reference and provide some calculations using it .. Index:: ogrlineref @@ -42,8 +42,8 @@ The :program:`ogrlineref` program can be used to: - return the portion of the path according to the "linear referenced" begin and end distances -The :program:`ogrlineref` creates a linear reference - a file containing -a segments of special length (e.g. 1 km in reference units) and gets coordinates, +The :program:`ogrlineref` utility creates a linear reference - a file containing +segments of a certain length (e.g. 1 km in reference units.) The user can get coordinates, linear referenced distances or sublines (subpaths) from this file. The utility does not require the ``M`` or ``Z`` components in the geometry. The results can be stored in any OGR supported format. diff --git a/doc/source/software_using_gdal.rst b/doc/source/software_using_gdal.rst index 7a2150ae827f..f25df05bdb7a 100644 --- a/doc/source/software_using_gdal.rst +++ b/doc/source/software_using_gdal.rst @@ -72,6 +72,7 @@ Proprietary license / Other - `3D DEM Viewer `_ from MS MacroSystem. - `Cadcorp SIS: `_ A Windows GIS with a GDAL and OGR plugins. +- `Carmenta Engine `_ (previously known as SpatialAce): A GIS Rapid Application Development environment - `CARTO `_ A cloud mapping platform to analyze and visualize geospatial data. - `Cartographica `_ Macintosh GIS package. - `CatchmentSIM `_ A Windows terrain analysis model for hydrologic applications. @@ -113,7 +114,6 @@ Proprietary license / Other - `SkylineGlobe `_ The Skyline suite of interactive applications allows you to build, view, query and analyze customized, virtual 3D landscapes. - `SpacEyes3D `_ 3D visualization software for cartographic data. - `Spatial Manager `_ A product suite designed designed to manage spatial data in a simple, fast and inexpensive way. Uses GDAL to import/export data. -- `Carmenta Engine `_ (previously known as SpatialAce): A GIS Rapid Application Development environment - `TacitView `_ An imagery visualization and exploitation package for military intelligence. - `TatukGIS `_ Desktop GIS mapping and data editing application. - `Team Awareness Kit `_ Suite of georeferenced imagery and situational awareness tools developed for military planning and execution, now available for civilian use. diff --git a/doc/source/sponsors/index.rst b/doc/source/sponsors/index.rst index bd17ad8a0f44..059f558eea7d 100644 --- a/doc/source/sponsors/index.rst +++ b/doc/source/sponsors/index.rst @@ -131,6 +131,10 @@ the health of the project: `Regrid `__ + .. container:: horizontal-logo + + `Satelligence `__ + .. container:: horizontal-logo `Space Intelligence `__ diff --git a/doc/source/tutorials/gdal_grid_tut.rst b/doc/source/tutorials/gdal_grid_tut.rst index 61549bfcf1d8..f71720c15f0f 100644 --- a/doc/source/tutorials/gdal_grid_tut.rst +++ b/doc/source/tutorials/gdal_grid_tut.rst @@ -34,7 +34,8 @@ That is what GDAL Grid API is about. It helps you to interpolate your data There are two ways of using this interface. Programmatically it is available through the :cpp:func:`GDALGridCreate` C function; for end users there is a -:ref:`gdal_grid` utility. The rest of this document discusses details on algorithms +:ref:`gdal_grid` utility (which see for an up-to-date list of all current features). +The rest of this document discusses details on algorithms and their parameters implemented in GDAL Grid API. Interpolation of the Scattered Data diff --git a/doc/source/user/sql_sqlite_dialect.rst b/doc/source/user/sql_sqlite_dialect.rst index c482622667bb..f1c6d8fd8259 100644 --- a/doc/source/user/sql_sqlite_dialect.rst +++ b/doc/source/user/sql_sqlite_dialect.rst @@ -343,11 +343,11 @@ returns: :: OGRFeature(SELECT):0 - POINT (2.342878767069653 48.85661793020374) + POINT (2.34287687375113 48.856622357411) .. code-block:: shell - ogrinfo cities.csv -dialect sqlite -sql "SELECT *, ogr_geocode(city, 'country') AS country, ST_Centroid(ogr_geocode(city)) FROM cities" + ogrinfo cities.csv -dialect sqlite -sql "SELECT *, ogr_geocode(city, 'country_code') AS country_code, ST_Centroid(ogr_geocode(city)) FROM cities" returns: @@ -357,57 +357,30 @@ returns: :: OGRFeature(SELECT):0 - id (Real) = 1 - city (String) = Paris - country (String) = France métropolitaine - POINT (2.342878767069653 48.85661793020374) + city (String) = Paris + country_code (String) = fr + POINT (2.34287687375113 48.856622357411) OGRFeature(SELECT):1 - id (Real) = 2 - city (String) = London - country (String) = United Kingdom - POINT (-0.109369427546499 51.500506667319407) + city (String) = London + country_code (String) = gb + POINT (-0.109415723431508 51.5004964757441) OGRFeature(SELECT):2 - id (Real) = 3 - city (String) = Rennes - country (String) = France métropolitaine - POINT (-1.68185153381778 48.111663929761093) + city (String) = Rennes + country_code (String) = fr + POINT (-1.68185479486048 48.1116771631195) OGRFeature(SELECT):3 - id (Real) = 4 - city (String) = Strasbourg - country (String) = France métropolitaine - POINT (7.767762859150757 48.571233274141846) + city (String) = New York + country_code (String) = us + POINT (-73.9388908443975 40.6632061220125) OGRFeature(SELECT):4 - id (Real) = 5 - city (String) = New York - country (String) = United States of America - POINT (-73.938140243499049 40.663799577449979) - - OGRFeature(SELECT):5 - id (Real) = 6 - city (String) = Berlin - country (String) = Deutschland - POINT (13.402306623451983 52.501470321410636) - - OGRFeature(SELECT):6 - id (Real) = 7 - city (String) = Beijing - POINT (116.391195 39.9064702) - - OGRFeature(SELECT):7 - id (Real) = 8 - city (String) = Brasilia - country (String) = Brasil - POINT (-52.830435216371839 -10.828214867369699) - - OGRFeature(SELECT):8 - id (Real) = 9 - city (String) = Moscow - country (String) = Российская Федерация - POINT (37.367988106866868 55.556208255649558) + city (String) = Beijing + country_code (String) = cn + POINT (116.3912972 39.9057136) + .. highlight:: sql diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst index 1e36919d9922..dbd62e56ea5c 100644 --- a/doc/source/user/virtual_file_systems.rst +++ b/doc/source/user/virtual_file_systems.rst @@ -390,7 +390,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following - proxy=value - proxyauth=value - proxyuserpwd=value -- pc_url_signing=yes/no: whether to use the URL signing mechanism of Microsoft Planetary Computer (https://planetarycomputer.microsoft.com/docs/concepts/sas/). (GDAL >= 3.5.2) +- pc_url_signing=yes/no: whether to use the URL signing mechanism of Microsoft Planetary Computer (https://planetarycomputer.microsoft.com/docs/concepts/sas/). (GDAL >= 3.5.2). Note that starting with GDAL 3.9, this may also be set with the path-specific option ( cf :cpp:func:`VSISetPathSpecificOption`) ``VSICURL_PC_URL_SIGNING`` set to ``YES``. - pc_collection=name: name of the collection of the dataset for Planetary Computer URL signing. Only used when pc_url_signing=yes. (GDAL >= 3.5.2) Partial downloads (requires the HTTP server to support random reading) are done with a 16 KB granularity by default. Starting with GDAL 2.3, the chunk size can be configured with the :config:`CPL_VSIL_CURL_CHUNK_SIZE` configuration option, with a value in bytes. If the driver detects sequential reading, it will progressively increase the chunk size up to 128 times :config:`CPL_VSIL_CURL_CHUNK_SIZE` (so 2 MB by default) to improve download performance. diff --git a/docker/README.md b/docker/README.md index 1d8362c4eaea..0e22b44ab15c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -98,11 +98,11 @@ If you are getting a ``: arena 0 background thread creation failed (1) # Images of releases -Tagged images of recent past releases are available. The last ones (at time of writing) are for GDAL 3.8.3 and PROJ 9.3.1, for linux/amd64 and linux/arm64: -* ghcr.io/osgeo/gdal:alpine-small-3.8.3 -* ghcr.io/osgeo/gdal:alpine-normal-3.8.3 -* ghcr.io/osgeo/gdal:ubuntu-small-3.8.3 -* ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 +Tagged images of recent past releases are available. The last ones (at time of writing) are for GDAL 3.8.4 and PROJ 9.3.1, for linux/amd64 and linux/arm64: +* ghcr.io/osgeo/gdal:alpine-small-3.8.4 +* ghcr.io/osgeo/gdal:alpine-normal-3.8.4 +* ghcr.io/osgeo/gdal:ubuntu-small-3.8.4 +* ghcr.io/osgeo/gdal:ubuntu-full-3.8.4 ## Multi-arch Images diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile index ee94d2ad288f..21ce77bc742f 100644 --- a/docker/ubuntu-full/Dockerfile +++ b/docker/ubuntu-full/Dockerfile @@ -197,12 +197,12 @@ RUN . /buildscripts/bh-set-envvars.sh \ ARG WITH_PDFIUM=yes RUN if echo "$WITH_PDFIUM" | grep -Eiq "^(y(es)?|1|true)$" ; then ( \ - wget -q https://github.com/rouault/pdfium_build_gdal_3_8/releases/download/pdfium_5952_v1/install-ubuntu2004-rev5952.tar.gz \ - && tar -xzf install-ubuntu2004-rev5952.tar.gz \ + wget -q https://github.com/rouault/pdfium_build_gdal_3_9/releases/download/pdfium_6309_v1/install-ubuntu2004-rev6309.tar.gz \ + && tar -xzf install-ubuntu2004-rev6309.tar.gz \ && chown -R root:root install \ && mv install/lib/* /usr/lib/ \ && mv install/include/* /usr/include/ \ - && rm -rf install-ubuntu2004-rev5952.tar.gz install \ + && rm -rf install-ubuntu2004-rev6309.tar.gz install \ && apt-get update -y \ && apt-get install -y --fix-missing --no-install-recommends liblcms2-dev${APT_ARCH_SUFFIX} \ && rm -rf /var/lib/apt/lists/* \ @@ -243,7 +243,8 @@ RUN . /buildscripts/bh-set-envvars.sh \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libparquet-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-acero-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt-get update -y \ && apt-get install -y --fix-missing --no-install-recommends rsync ccache \ diff --git a/frmts/dimap/dimapdataset.cpp b/frmts/dimap/dimapdataset.cpp index 3247eb952d20..93c16a96a790 100644 --- a/frmts/dimap/dimapdataset.cpp +++ b/frmts/dimap/dimapdataset.cpp @@ -1515,6 +1515,8 @@ int DIMAPDataset::ReadImageInformation2() "", "Processing_Information.Product_Settings.Geometric_Settings", "GEOMETRIC_", + "Processing_Information.Product_Settings.Radiometric_Settings", + "RADIOMETRIC_", "Quality_Assessment.Imaging_Quality_Measurement", "CLOUDCOVER_", nullptr, diff --git a/frmts/ecw/ecwcreatecopy.cpp b/frmts/ecw/ecwcreatecopy.cpp index ed0ac5b0f266..cb9b8738b066 100644 --- a/frmts/ecw/ecwcreatecopy.cpp +++ b/frmts/ecw/ecwcreatecopy.cpp @@ -1032,10 +1032,33 @@ CPLErr GDALECWCompressor::Initialize( const char *pszGMLJP2V2Def = CSLFetchNameValue(papszOptions, "GMLJP2V2_DEF"); if (pszGMLJP2V2Def != nullptr) + { WriteJP2Box(oJP2MD.CreateGMLJP2V2(nXSize, nYSize, pszGMLJP2V2Def, poSrcDS)); + } else - WriteJP2Box(oJP2MD.CreateGMLJP2(nXSize, nYSize)); + { + if (!poSRS || poSRS->IsEmpty() || + GDALJP2Metadata::IsSRSCompatible(poSRS)) + { + WriteJP2Box(oJP2MD.CreateGMLJP2(nXSize, nYSize)); + } + else if (CSLFetchNameValue(papszOptions, "GMLJP2")) + { + CPLError(CE_Warning, CPLE_AppDefined, + "GMLJP2 box was explicitly required but " + "cannot be written due " + "to lack of georeferencing and/or unsupported " + "georeferencing " + "for GMLJP2"); + } + else + { + CPLDebug( + "JP2ECW", + "Cannot write GMLJP2 box due to unsupported SRS"); + } + } } if (CPLFetchBool(papszOptions, "GeoJP2", true)) WriteJP2Box(oJP2MD.CreateJP2GeoTIFF()); diff --git a/frmts/ers/ersdataset.cpp b/frmts/ers/ersdataset.cpp index 919d64f7d4b4..d83ed19b4251 100644 --- a/frmts/ers/ersdataset.cpp +++ b/frmts/ers/ersdataset.cpp @@ -881,17 +881,18 @@ int ERSProxyRasterBand::GetOverviewCount() GDALDataset *ERSDataset::Open(GDALOpenInfo *poOpenInfo) { + if (!Identify(poOpenInfo) || poOpenInfo->fpL == nullptr) + return nullptr; + + int &nRecLevel = GetRecLevel(); // cppcheck-suppress knownConditionTrueFalse - if (GetRecLevel()) + if (nRecLevel) { CPLError(CE_Failure, CPLE_AppDefined, "Attempt at recursively opening ERS dataset"); return nullptr; } - if (!Identify(poOpenInfo) || poOpenInfo->fpL == nullptr) - return nullptr; - /* -------------------------------------------------------------------- */ /* Ingest the file as a tree of header nodes. */ /* -------------------------------------------------------------------- */ @@ -1020,7 +1021,6 @@ GDALDataset *ERSDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ if (EQUAL(poHeader->Find("DataSetType", ""), "Translated")) { - int &nRecLevel = GetRecLevel(); nRecLevel++; poDS->poDepFile = GDALDataset::FromHandle( GDALOpen(osDataFilePath, poOpenInfo->eAccess)); diff --git a/frmts/georaster/georaster_dataset.cpp b/frmts/georaster/georaster_dataset.cpp index 453a1a9566d4..2328d645e7e6 100644 --- a/frmts/georaster/georaster_dataset.cpp +++ b/frmts/georaster/georaster_dataset.cpp @@ -789,6 +789,21 @@ char **GeoRasterDataset::GetFileList() return papszFileList; } +static bool ParseCommaSeparatedString(const char *str, double pfValues[], + int nMaxValues) +{ + int nValues = 0; + char **papszTokens = CSLTokenizeString2(str, ",", 0); + + for (int i = 0; papszTokens && papszTokens[i] && nValues < nMaxValues; i++) + { + pfValues[nValues++] = CPLAtof(papszTokens[i]); + } + + CSLDestroy(papszTokens); + return nValues == nMaxValues; +} + // --------------------------------------------------------------------------- // Create() // --------------------------------------------------------------------------- @@ -1252,6 +1267,103 @@ GDALDataset *GeoRasterDataset::Create(const char *pszFilename, int nXSize, poGRD->poGeoRaster->nPyramidLevels = atoi(pszFetched); } + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS"); + + if (pszFetched != nullptr) + { + poGRD->poGeoRaster->bGenStats = EQUAL(pszFetched, "TRUE"); + } + + bool bGenStatsOptionsUsed = false; + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_SAMPLINGFACTOR"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + poGRD->poGeoRaster->nGenStatsSamplingFactor = atoi(pszFetched); + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_SAMPLINGWINDOW"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + + // Sampling window contains 4 double values + const int nSize = 4; + if (!ParseCommaSeparatedString( + pszFetched, poGRD->poGeoRaster->dfGenStatsSamplingWindow, + nSize)) + { + CPLError(CE_Failure, CPLE_IllegalArg, + "Wrong comma separated string for sampling window (%s)", + pszFetched); + delete poGRD; + return nullptr; + } + + poGRD->poGeoRaster->bGenStatsUseSamplingWindow = true; + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_HISTOGRAM"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + poGRD->poGeoRaster->bGenStatsHistogram = EQUAL(pszFetched, "TRUE"); + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_LAYERNUMBERS"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + poGRD->poGeoRaster->sGenStatsLayerNumbers = pszFetched; + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_USEBIN"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + poGRD->poGeoRaster->bGenStatsUseBin = EQUAL(pszFetched, "TRUE"); + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_BINFUNCTION"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + const int nSize = 5; + if (!ParseCommaSeparatedString( + pszFetched, poGRD->poGeoRaster->dfGenStatsBinFunction, nSize)) + { + CPLError(CE_Failure, CPLE_IllegalArg, + "Wrong comma separated string for bin function (%s)", + pszFetched); + delete poGRD; + return nullptr; + } + } + + pszFetched = CSLFetchNameValue(papszOptions, "GENSTATS_NODATA"); + + if (pszFetched != nullptr) + { + bGenStatsOptionsUsed = true; + poGRD->poGeoRaster->bGenStatsNodata = EQUAL(pszFetched, "TRUE"); + } + + if (bGenStatsOptionsUsed && !poGRD->poGeoRaster->bGenStats) + { + CPLError( + CE_Warning, CPLE_AppDefined, + "Some GENSTATS* options were used but GENSTATS is not set. " + "Statistics won't be computed, please set GENSTATS option to true " + "if you want to generate statistics"); + } + // ------------------------------------------------------------------- // Return a new Dataset // ------------------------------------------------------------------- @@ -2421,7 +2533,7 @@ void GeoRasterDataset::SetSubdatasets(GeoRasterWrapper *poGRW) papszSubdatasets = CSLSetNameValue( papszSubdatasets, CPLSPrintf("SUBDATASET_%d_DESC", nCount), - CPLSPrintf("%s.Table=%s", szOwner, szTable)); + CPLSPrintf("Table=%s.%s", szOwner, szTable)); nCount++; } while (poStmt->Fetch()); diff --git a/frmts/georaster/georaster_priv.h b/frmts/georaster/georaster_priv.h index 4812d10994f1..c1eba12b16ab 100644 --- a/frmts/georaster/georaster_priv.h +++ b/frmts/georaster/georaster_priv.h @@ -391,6 +391,9 @@ class GeoRasterWrapper char *GetVAT(int nBand); bool GeneratePyramid(int nLevels, const char *pszResampling, bool bInternal = false); + bool GenerateStatistics(int nSamplingFactor, double *pdfSamplingWindow, + bool bHistogram, const char *pszLayerNumbers, + bool bUseBin, double *pdfBinFunction, bool bNodata); void DeletePyramid(); void PrepareToOverwrite(); bool InitializeMask(int nLevel, int nBlockColumns, int nBlockRows, @@ -465,6 +468,16 @@ class GeoRasterWrapper bool bHasBitmapMask; bool bUniqueFound; + bool bGenStats; + int nGenStatsSamplingFactor; + bool bGenStatsUseSamplingWindow; + double dfGenStatsSamplingWindow[4]; + bool bGenStatsHistogram; + CPLString sGenStatsLayerNumbers; + bool bGenStatsUseBin; + double dfGenStatsBinFunction[5]; + bool bGenStatsNodata; + int eModelCoordLocation; unsigned int anULTCoordinate[3]; diff --git a/frmts/georaster/georaster_wrapper.cpp b/frmts/georaster/georaster_wrapper.cpp index a0d4a7b1a17c..cdb9dbd8c9cb 100644 --- a/frmts/georaster/georaster_wrapper.cpp +++ b/frmts/georaster/georaster_wrapper.cpp @@ -43,7 +43,8 @@ // --------------------------------------------------------------------------- GeoRasterWrapper::GeoRasterWrapper() - : sPyramidResampling("NN"), sCompressionType("NONE"), sInterleaving("BSQ") + : sPyramidResampling("NN"), sCompressionType("NONE"), sInterleaving("BSQ"), + bGenStatsUseSamplingWindow(false), sGenStatsLayerNumbers("") { nRasterId = -1; phMetadata = nullptr; @@ -115,6 +116,20 @@ GeoRasterWrapper::GeoRasterWrapper() pasGCPList = nullptr; nGCPCount = 0; bFlushGCP = false; + bGenStats = false; + nGenStatsSamplingFactor = 1; + bGenStatsHistogram = false; + bGenStatsUseBin = true; + bGenStatsNodata = false; + dfGenStatsBinFunction[0] = 0.0; + dfGenStatsBinFunction[1] = 0.0; + dfGenStatsBinFunction[2] = 0.0; + dfGenStatsBinFunction[3] = 0.0; + dfGenStatsBinFunction[4] = 0.0; + dfGenStatsSamplingWindow[0] = 0.0; + dfGenStatsSamplingWindow[1] = 0.0; + dfGenStatsSamplingWindow[2] = 0.0; + dfGenStatsSamplingWindow[3] = 0.0; } // --------------------------------------------------------------------------- @@ -543,6 +558,75 @@ GeoRasterWrapper *GeoRasterWrapper::Open(const char *pszStringId, bool bUpdate) return poGRW; } +static bool ValidateInsertExpression(const CPLString &sInsertStatement) +{ + bool bInQuotes = false; + const size_t nStrLength = sInsertStatement.length(); + + for (size_t nPos = 0; nPos < nStrLength; ++nPos) + { + const char cCurrentChar = sInsertStatement[nPos]; + + if (cCurrentChar == '\'') + { + if (bInQuotes) + { + if (nPos + 1 < nStrLength && sInsertStatement[nPos + 1] == '\'') + { + ++nPos; + } + else + { + bInQuotes = false; + } + } + else + { + bInQuotes = true; + } + } + else if (!bInQuotes) + { + if (cCurrentChar == ';') + { + return false; + } + else if (nPos < nStrLength - 1) + { + const char cNextChar = sInsertStatement[nPos + 1]; + const bool bIsInvalid = + (cCurrentChar == '-' && cNextChar == '-') || + (cCurrentChar == '/' && cNextChar == '/') || + (cCurrentChar == '/' && cNextChar == '*') || + (cCurrentChar == '*' && cNextChar == '/'); + + if (bIsInvalid) + { + return false; + } + } + } + } + // Quotes must be properly closed, if not, this variable will be true + // thus having an unclosed string + return !bInQuotes; +} + +static bool ValidateDescriptionExpression(const CPLString &sInsertStatement) +{ + const char *rgpszInvalidChars[] = {";", "--", "/*", "*/", "//"}; + + for (const char *pszInvalidChar : rgpszInvalidChars) + { + if (sInsertStatement.find(pszInvalidChar) != std::string::npos) + { + return false; + } + } + + return true; +} + // --------------------------------------------------------------------------- // Create() // --------------------------------------------------------------------------- @@ -601,6 +685,12 @@ bool GeoRasterWrapper::Create(char *pszDescription, char *pszInsert, if (pszDescription) { + if (!ValidateDescriptionExpression(pszDescription)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "DESCRIPTION expression contains invalid values."); + return false; + } snprintf(szDescription, sizeof(szDescription), "%s", pszDescription); } @@ -617,6 +707,14 @@ bool GeoRasterWrapper::Create(char *pszDescription, char *pszInsert, if (pszInsert) { sValues = pszInsert; + sValues.Trim(); // Remove spaces on the left and on the right + + if (!ValidateInsertExpression(sValues)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "INSERT expression contains invalid values."); + return false; + } if (pszInsert[0] == '(' && sValues.ifind("VALUES") == std::string::npos) @@ -3618,6 +3716,22 @@ bool GeoRasterWrapper::FlushMetadata() } } + if (bGenStats) + { + if (GenerateStatistics(nGenStatsSamplingFactor, + dfGenStatsSamplingWindow, bGenStatsHistogram, + sGenStatsLayerNumbers.c_str(), bGenStatsUseBin, + dfGenStatsBinFunction, bGenStatsNodata)) + { + CPLDebug("GEOR", "Generated statistics successfully."); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Error generating statistics!"); + } + } + return true; } @@ -3775,6 +3889,89 @@ void GeoRasterWrapper::DeletePyramid() delete poStmt; } +// --------------------------------------------------------------------------- +// GenerateStatistics() +// --------------------------------------------------------------------------- +bool GeoRasterWrapper::GenerateStatistics(int nSamplingFactor, + double *pdfSamplingWindow, + bool bHistogram, + const char *pszLayerNumbers, + bool bUseBin, double *pdfBinFunction, + bool bNodata) +{ + // Length is 6 because it's the length of string FALSE. + char szHistogram[] = "FALSE"; + char szNodata[] = "FALSE"; + char szUseBin[] = "FALSE"; + char szUseWin[] = "FALSE"; + if (bHistogram) + { + strncpy(szHistogram, "TRUE", 5); + } + if (bNodata) + { + strncpy(szNodata, "TRUE", 5); + } + if (bUseBin) + { + strncpy(szUseBin, "TRUE", 5); + } + if (bGenStatsUseSamplingWindow) + { + strncpy(szUseWin, "TRUE", 5); + } + + char szLayerNumbers[OWTEXT]; + snprintf(szLayerNumbers, sizeof(szLayerNumbers), "%s", pszLayerNumbers); + + OWStatement *poStmt = poConnection->CreateStatement( + CPLSPrintf("DECLARE\n" + " gr sdo_georaster;\n" + " swin SDO_NUMBER_ARRAY := SDO_NUMBER_ARRAY(:1, :2, :3," + " :4);\n" + " binfunc SDO_NUMBER_ARRAY := SDO_NUMBER_ARRAY(:5, :6, :7," + " :8, :9);\n" + " res VARCHAR2(5);\n" + "BEGIN\n" + " IF :usewin = 'FALSE' THEN\n" + " swin := NULL;\n" + " END IF;" + " IF :usebin = 'FALSE' THEN\n" + " binfunc := NULL;\n" + " END IF;" + " SELECT t.%s INTO gr FROM %s t WHERE %s FOR UPDATE;\n" + " res := sdo_geor.generateStatistics(gr, " + "'samplingFactor='||:samplingfactor, swin,\n" + " :histogram, :layernums, :usebin, binfunc, :nodata);\n" + " UPDATE %s t SET t.%s = gr WHERE %s;\n" + " COMMIT;\n" + "END;\n", + sColumn.c_str(), sTable.c_str(), sWhere.c_str(), + sTable.c_str(), sColumn.c_str(), sWhere.c_str())); + + // Bind sampling window + for (int i = 0; i < 4; i++) + { + poStmt->Bind(&pdfSamplingWindow[i]); + } + // Bind bin function + for (int i = 0; i < 5; i++) + { + poStmt->Bind(&pdfBinFunction[i]); + } + poStmt->BindName(":samplingfactor", &nSamplingFactor); + poStmt->BindName(":layernums", szLayerNumbers); + poStmt->BindName(":histogram", szHistogram); + poStmt->BindName(":nodata", szNodata); + poStmt->BindName(":usebin", szUseBin); + poStmt->BindName(":usewin", szUseWin); + + bool bResult = poStmt->Execute(); + delete poStmt; + + return bResult; +} + // --------------------------------------------------------------------------- // CreateBitmapMask() // --------------------------------------------------------------------------- diff --git a/frmts/georaster/georasterdrivercore.cpp b/frmts/georaster/georasterdrivercore.cpp index 2acafdc91a8f..206358081f6e 100644 --- a/frmts/georaster/georasterdrivercore.cpp +++ b/frmts/georaster/georasterdrivercore.cpp @@ -100,6 +100,33 @@ void GEORDriverSetCommonMetadata(GDALDriver *poDriver) " " " " "