diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 8fb672e12dab..1ba740adefa9 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -219,6 +219,8 @@ jobs: export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib $GITHUB_WORKSPACE/install-gdal/bin/gdalinfo --version PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 -c "from osgeo import gdal;print(gdal.VersionInfo(None))" + # Test fix for https://github.com/conda-forge/gdal-feedstock/issues/995 + PYTHONWARNINGS="error" PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 -c "from osgeo import gdal" PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 $GITHUB_WORKSPACE/scripts/check_doc.py - name: CMake with rpath run: | @@ -418,7 +420,6 @@ jobs: - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: activate-environment: gdalenv - miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true auto-update-conda: true @@ -433,7 +434,7 @@ jobs: cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \ sqlite tiledb zstd cryptopp cgal doxygen librttopo libkml openssl xz \ openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \ - arrow-cpp pyarrow libaec libheif libavif cmake + arrow-cpp pyarrow libaec libheif libavif cmake fsspec - name: Check CMake version shell: bash -l {0} run: | @@ -526,7 +527,6 @@ jobs: - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: activate-environment: gdalenv - miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true auto-update-conda: true diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 47bbd31b54cf..59df41621efd 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -52,7 +52,6 @@ jobs: - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: - #miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true channels: conda-forge diff --git a/.github/workflows/coverity_scan/build.sh b/.github/workflows/coverity_scan/build.sh index 3becfb98f422..596bbbc100a9 100755 --- a/.github/workflows/coverity_scan/build.sh +++ b/.github/workflows/coverity_scan/build.sh @@ -22,10 +22,33 @@ cmake ${GDAL_SOURCE_DIR:=..} \ /tmp/cov-analysis-linux64/bin/cov-build --dir cov-int make "-j$(nproc)" tar czf cov-int.tgz cov-int -curl \ - --form token=$COVERITY_SCAN_TOKEN \ - --form email=$COVERITY_SCAN_EMAIL \ - --form file=@cov-int.tgz \ - --form version=master \ - --form description="`git rev-parse --abbrev-ref HEAD` `git rev-parse --short HEAD`" \ - https://scan.coverity.com/builds?project=GDAL + +apt-get update -y +apt-get install -y jq + +## Below from https://scan.coverity.com/projects/gdal/builds/new + +# Initialize a build. Fetch a cloud upload url. +curl -X POST \ + -d version=master \ + -d description="$(git rev-parse --abbrev-ref HEAD) $(git rev-parse --short HEAD)" \ + -d token=$COVERITY_SCAN_TOKEN \ + -d email=$COVERITY_SCAN_EMAIL \ + -d file_name="cov-int.tgz" \ + https://scan.coverity.com/projects/749/builds/init \ + | tee response + +# Store response data to use in later stages. +upload_url=$(jq -r '.url' response) +build_id=$(jq -r '.build_id' response) + +# Upload the tarball to the Cloud. +curl -X PUT \ + --header 'Content-Type: application/json' \ + --upload-file cov-int.tgz \ + $upload_url + +# Trigger the build on Scan. +curl -X PUT \ + -d token=$COVERITY_SCAN_TOKEN \ + https://scan.coverity.com/projects/749/builds/$build_id/enqueue diff --git a/.github/workflows/fedora_rawhide/build.sh b/.github/workflows/fedora_rawhide/build.sh index b4d22a77b2b9..423a817451cc 100755 --- a/.github/workflows/fedora_rawhide/build.sh +++ b/.github/workflows/fedora_rawhide/build.sh @@ -10,7 +10,17 @@ cmake ${GDAL_SOURCE_DIR:=..} \ -DCMAKE_CXX_FLAGS="-std=c++20 -Werror -O1 -D_FORTIFY_SOURCE=2" \ -DCMAKE_SHARED_LINKER_FLAGS="-lstdc++" \ -DUSE_CCACHE=ON \ + -DEMBED_RESOURCE_FILES=ON \ -DCMAKE_INSTALL_PREFIX=/usr \ -DWERROR_DEV_FLAG="-Werror=dev" make -j$(nproc) make -j$(nproc) install DESTDIR=/tmp/install-gdal + +ctest -V -j $(nproc) + +rm -rf data +cmake ${GDAL_SOURCE_DIR:=..} \ + -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON +rm -rf /tmp/install-gdal +make -j$(nproc) +make -j$(nproc) install DESTDIR=/tmp/install-gdal diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 19db499f741f..360ef0687397 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -264,6 +264,10 @@ jobs: # FIXME the default BUILD_CMD here isn't working...we get an error # about the quotes not matching. - name: Build + env: + TRAVIS: yes + TRAVIS_BRANCH: ${{ matrix.travis_branch }} + BUILD_NAME: ${{ matrix.travis_branch }} run: | if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"; then BUILD_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.build_script }}" @@ -271,11 +275,20 @@ jobs: BUILD_CMD="sh -c 'cmake .. && make -j$(nproc)'" fi + # For cache + mkdir -p .gdal + mkdir -p build-${{ matrix.id }} docker run --name gdal-build \ --rm \ + -e CI \ + -e GITHUB_WORKFLOW \ + -e TRAVIS \ + -e TRAVIS_BRANCH \ + -e BUILD_NAME \ -e "GDAL_SOURCE_DIR=$(pwd)" \ -u $(id -u ${USER}):$(id -g ${USER}) \ + -v $(pwd)/.gdal:/.gdal:rw \ -v $(pwd):$(pwd):rw \ -v ${{ github.workspace }}/.ccache:/.ccache:rw \ --workdir $(pwd)/build-${{ matrix.id }} \ @@ -322,7 +335,7 @@ jobs: fi # For cache - mkdir .gdal + mkdir -p .gdal docker run \ -e CI \ diff --git a/.github/workflows/ubuntu_24.04/Dockerfile.ci b/.github/workflows/ubuntu_24.04/Dockerfile.ci index c6c830fca30c..771f18c983d4 100644 --- a/.github/workflows/ubuntu_24.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_24.04/Dockerfile.ci @@ -158,3 +158,25 @@ RUN python3 -m pip install -U --break-system-packages -r /tmp/requirements.txt # cfchecker requires udunits2 RUN apt-get install -y --allow-unauthenticated libudunits2-0 libudunits2-data RUN python3 -m pip install --break-system-packages cfchecker + +RUN python3 -m pip install --break-system-packages fsspec + +# Manually install ADBC packages from Ubuntu 22.04 as there are no 24.04 packages at time of writing. +RUN curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake-dev_14-1_amd64.deb + +# Install libduckdb +RUN curl -LO -fsS https://github.com/duckdb/duckdb/releases/download/v1.1.2/libduckdb-linux-amd64.zip \ + && unzip libduckdb-linux-amd64.zip libduckdb.so \ + && mv libduckdb.so /usr/lib/x86_64-linux-gnu \ + && rm -f libduckdb-linux-amd64.zip diff --git a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt index 2be3770e4ce0..06b013069a74 100644 --- a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt +++ b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt @@ -145,7 +145,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda JPEGXL -raster- (rwv): JPEG-XL (*.jxl) GPKG -raster,vector- (rw+vs): GeoPackage (*.gpkg, *.gpkg.zip) SQLite -raster,vector- (rw+v): SQLite / Spatialite / RasterLite2 (*.sqlite, *.db) - OpenFileGDB -raster,vector- (rw+v): ESRI FileGDB (*.gdb) + OpenFileGDB -raster,vector- (rw+v): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb) CAD -raster,vector- (rovs): AutoCAD Driver (*.dwg) PLSCENES -raster,vector- (ro): Planet Labs Scenes API NGW -raster,vector- (rw+s): NextGIS Web diff --git a/.github/workflows/ubuntu_24.04/expected_ogrinfo_formats.txt b/.github/workflows/ubuntu_24.04/expected_ogrinfo_formats.txt index b365a54281a2..ef95d09f9d76 100644 --- a/.github/workflows/ubuntu_24.04/expected_ogrinfo_formats.txt +++ b/.github/workflows/ubuntu_24.04/expected_ogrinfo_formats.txt @@ -42,7 +42,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda PostgreSQL -vector- (rw+): PostgreSQL/PostGIS MySQL -vector- (rw+): MySQL OCI -vector- (rw+): Oracle Spatial - OpenFileGDB -raster,vector- (rw+v): ESRI FileGDB (*.gdb) + OpenFileGDB -raster,vector- (rw+v): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb) FileGDB -vector- (rw+): ESRI FileGDB (*.gdb) DXF -vector- (rw+v): AutoCAD DXF (*.dxf) CAD -raster,vector- (rovs): AutoCAD Driver (*.dwg) @@ -82,6 +82,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda JSONFG -vector- (rw+v): OGC Features and Geometries JSON (*.json) MiraMonVector -vector- (rw+v): MiraMon Vectors (.pol, .arc, .pnt) (*.pol, *.arc, *.pnt) XODR -vector- (ro): OpenDRIVE - Open Dynamic Road Information for Vehicle Environment (*.xodr) + ADBC -vector- (ro): Arrow Database Connectivity TIGER -vector- (rov): U.S. Census TIGER/Line AVCBin -vector- (rov): Arc/Info Binary Coverage AVCE00 -vector- (rov): Arc/Info E00 (ASCII) Coverage (*.e00) diff --git a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt index 44fdb6efd66a..cccc2db769f6 100644 --- a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt +++ b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt @@ -144,7 +144,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda STACIT -raster- (rovs): Spatio-Temporal Asset Catalog Items NSIDCbin -raster- (rov): NSIDC Sea Ice Concentrations binary (.bin) (*.bin) GPKG -raster,vector- (rw+vs): GeoPackage (*.gpkg, *.gpkg.zip) - OpenFileGDB -raster,vector- (rw+v): ESRI FileGDB (*.gdb) + OpenFileGDB -raster,vector- (rw+v): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb) CAD -raster,vector- (rovs): AutoCAD Driver (*.dwg) PLSCENES -raster,vector- (ro): Planet Labs Scenes API NGW -raster,vector- (rw+s): NextGIS Web diff --git a/.github/workflows/windows_conda_expected_ogrinfo_formats.txt b/.github/workflows/windows_conda_expected_ogrinfo_formats.txt index e9de701fc9d7..908f8769c49c 100644 --- a/.github/workflows/windows_conda_expected_ogrinfo_formats.txt +++ b/.github/workflows/windows_conda_expected_ogrinfo_formats.txt @@ -39,7 +39,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda PGeo -vector- (ro): ESRI Personal GeoDatabase (*.mdb) MSSQLSpatial -vector- (rw+): Microsoft SQL Server Spatial Database (BCP) PostgreSQL -vector- (rw+): PostgreSQL/PostGIS - OpenFileGDB -raster,vector- (rw+v): ESRI FileGDB (*.gdb) + OpenFileGDB -raster,vector- (rw+v): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb) DXF -vector- (rw+v): AutoCAD DXF (*.dxf) CAD -raster,vector- (rovs): AutoCAD Driver (*.dwg) FlatGeobuf -vector- (rw+v): FlatGeobuf (*.fgb) @@ -77,6 +77,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda PMTiles -vector- (rw+v): ProtoMap Tiles (*.pmtiles) JSONFG -vector- (rw+v): OGC Features and Geometries JSON (*.json) MiraMonVector -vector- (rw+v): MiraMon Vectors (.pol, .arc, .pnt) (*.pol, *.arc, *.pnt) + ADBC -vector- (ro): Arrow Database Connectivity TIGER -vector- (rov): U.S. Census TIGER/Line AVCBin -vector- (rov): Arc/Info Binary Coverage AVCE00 -vector- (rov): Arc/Info E00 (ASCII) Coverage (*.e00) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0215f92638f..2fb810e2ea38 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,8 @@ repos: - id: black exclude: > (?x)^( - swig/python/osgeo/| + swig/python/osgeo/__init__.py| + swig/python/osgeo/gdalnumeric.py| autotest/ogr/data/ ) - repo: https://github.com/timothycrosley/isort @@ -14,7 +15,8 @@ repos: - id: isort exclude: > (?x)^( - swig/python/osgeo/| + swig/python/osgeo/__init__.py| + swig/python/osgeo/gdalnumeric.py| autotest/ogr/data/ ) - repo: https://github.com/pycqa/flake8 @@ -23,7 +25,8 @@ repos: - id: flake8 exclude: > (?x)^( - swig/python/osgeo/| + swig/python/osgeo/__init__.py| + swig/python/osgeo/gdalnumeric.py| examples/| autotest/ogr/data/ ) @@ -54,6 +57,7 @@ repos: frmts/grib/degrib/degrib| frmts/grib/degrib/g2clib| port/utf8.h| + ogr/ogrsf_frmts/adbc/ogr_adbc_internal.h| ogr/ogrsf_frmts/cad/libopencad/| ogr/ogrsf_frmts/geojson/libjson/| ogr/ogrsf_frmts/flatgeobuf/flatbuffers/| diff --git a/HOWTO-RELEASE b/HOWTO-RELEASE index afb1e184452f..684d3b589afd 100644 --- a/HOWTO-RELEASE +++ b/HOWTO-RELEASE @@ -29,29 +29,29 @@ Prerequisites: Process : -1) "make completion" to regenerate scripts/gdal-bash-completion.sh - if new command line switches have been added. scripts/completionFinder.py - must also be edited before if new utilities/scripts are added/removed. +1) "make completion" to regenerate scripts/gdal-bash-completion.sh + if new command line switches have been added. scripts/completionFinder.py + must also be edited before if new utilities/scripts are added/removed. -2.1) Update the release date, and number information in gcore/gdal_version.h.in - (*NOT* gdal_version.h which is a generated file) - Note: the format of GDAL_RELEASE_DATE should be YYYYMMDD. +2) Update the release date, and number information in gcore/gdal_version.h.in + (*NOT* gdal_version.h which is a generated file) + Note: the format of GDAL_RELEASE_DATE should be YYYYMMDD. -2.2) Update two instances of year in CITATION file to the current year. +3) Update two instances of year in CITATION file to the current year. -2.3) Update "version" and "date-released" in CITATION.cff (Note: the DOI does - *not* need to be updated as it is a generic one) +4) Update "version" and "date-released" in CITATION.cff (Note: the DOI does + *not* need to be updated as it is a generic one) -3) Update the VERSION file (if not already done) +5) Update the VERSION file (if not already done) -3.1) Update the version information in the following files: - - ./swig/python/gdal-utils/osgeo_utils/__init__.py (gdal-utils python package) - - ./swig/python/README.rst (libgdal) +6) Update the version information in the following files: + - ./swig/python/gdal-utils/osgeo_utils/__init__.py (gdal-utils python package) + - ./swig/python/README.rst (libgdal) -4) Update the GDAL_SOVERSION number at top of gdal.cmake according to the +7) Update the GDAL_SOVERSION number at top of gdal.cmake according to the directions given in the comment preceding it. -5) Prepare release overview in the NEWS.md file. For example, to get all changes +8) Prepare release overview in the NEWS.md file. For example, to get all changes from v3.4.0 to current HEAD git log --reverse -v v3.4.0..HEAD . ":(exclude)autotest" ":(exclude)doc" ":(exclude).github" @@ -62,23 +62,28 @@ Process : - for bugfixes releases, forward port additions of NEWS.md to master -6) If this is a feature release (e.g 3.1), prepare a branch. +9) Update doc/source/about_no_title.rst, doc/source/download.rst and doc/source/download_past.rst + to advertise the new release and link to the release notes. Do that in a dedicated commit. + + (It is necessary to do that at that stage, so that the ReadTheDocs "stable" branch, based on the latest tag, proposes to download it) + +10) If this is a feature release (e.g 3.1), prepare a branch. git checkout master git pull origin master git checkout -b release/3.1 git push origin release/3.1 -7) Tag the release with a RC suffix: +11) Tag the release with a RC suffix: git checkout release/3.1 git pull origin release/3.1 git tag -a -m "Create tag v3.1.0RC1" v3.1.0RC1 git push origin v3.1.0RC1 -8) Prepare archives +12) Prepare archives -8.0) Ensure you have the following prerequisites (beyond normal build deps): +12.0) Ensure you have the following prerequisites (beyond normal build deps): If make is not GNU make, e.g., export MAKE=gmake @@ -97,7 +102,7 @@ Process : md5sum (GNU version) in path (not a POSIX requirement) -8.1) Create the source distributions using the mkgdaldist.sh script. +12.1) Create the source distributions using the mkgdaldist.sh script. The argument should be the version number (i.e. 1.4.2). As our process involves doing betas or RCs, use the -rc option so that the filenames include this information (after promotion to official release, filename renaming will have @@ -118,7 +123,7 @@ Process : a previous release (diff -Nur gdal-3.0.1 gdal-3.0.2). This is more easily doable for a bugfix release. -8.2) Create a snapshot of the documentation (only for feature releases) +12.2) Create a snapshot of the documentation (only for feature releases) 1. Refresh https://download.osgeo.org/gdal/for_doc/javadoc.zip @@ -128,7 +133,7 @@ Process : 3. ./build_doc_snapshot 310 This generates gdal310doc.zip -8.3) Publish the resulting files on download.osgeo.org, +12.3) Publish the resulting files on download.osgeo.org, in /osgeo/download/gdal/X.Y.Z (where X.Y.Z is the version number) with ~/.ssh/config containing: @@ -137,23 +142,23 @@ Process : ProxyCommand ssh rouault@download.osgeo.org -W $(sed -e "s/^osgeo7-//;s/$/.lxd/" <<< "%h"):%p -9) Announce the release candidate availability to gdal-dev@ - Example: https://lists.osgeo.org/pipermail/gdal-dev/2019-June/050509.html +13) Announce the release candidate availability to gdal-dev@ + Example: https://lists.osgeo.org/pipermail/gdal-dev/2019-June/050509.html -10) If new RC is needed, update gcore/gdal_version.h.in with the new date +14) If new RC is needed, update gcore/gdal_version.h.in with the new date and update NEWS,and go to 7) If no new RC is needed, after a few days, raise a motion to gdal-dev@ for approval -11) Once the vote has positively completed, +15) Once the vote has positively completed, -11.1) Tag the release as final: +15.1) Tag the release as final: git checkout v3.1.0RC1 git tag -a -m "Create tag v3.1.0" v3.1.0 git push origin v3.1.0 -11.2) Log on download.osgeo.org, go to /osgeo/download/gdal/X.Y.Z +15.2) Log on download.osgeo.org, go to /osgeo/download/gdal/X.Y.Z Remove the RC suffixes for the final RC, like: mv gdal-3.8.3rc1.tar.xz gdal-3.8.3.tar.xz @@ -176,22 +181,18 @@ Process : Check that everything is fine: md5sum -c *.md5 -11.3) In /osgeo/download/gdal, add a symlink from X.Y.Z to CURRENT (except for stable releases in a "old" branch). +15.3) In /osgeo/download/gdal, add a symlink from X.Y.Z to CURRENT (except for stable releases in a "old" branch). % ln -sf X.Y.Z CURRENT -12) (Removed) - -13) Update doc/source/about_no_title.rst and doc/source/download.rst to advertise the new release and link to the release notes - -14) Update GitHub to close the release milestone. +16) Update GitHub to close the release milestone. Then create a new milestone for the next release. -15) Upload the new Python bindings to Pypi (requires upload rights to +17) Upload the new Python bindings to Pypi (requires upload rights to the GDAL package by one of the current owners : HowardB/FrankW/EvenR) ( procedure taken from http://peterdowns.com/posts/first-time-with-pypi.html ) -15.1) Prerequisite: +17.1) Prerequisite: a) Install twine https://pypi.org/project/twine/ b) Create a $HOME/.pypirc file : @@ -210,7 +211,7 @@ the GDAL package by one of the current owners : HowardB/FrankW/EvenR) username: yourlogin password: yourpassword -15.2) create gdal sdist and upload to pypi: +17.2) create gdal sdist and upload to pypi: a) Create a build directory if not already done. Go to it and run cmake @@ -227,7 +228,7 @@ the GDAL package by one of the current owners : HowardB/FrankW/EvenR) f) For real : twine upload dist/gdal-*.gz -15.3) create gdal-utils wheel and upload to pypi: +17.3) create gdal-utils wheel and upload to pypi: a) cd $BUILD_DIR/swig/python/gdal-utils @@ -242,7 +243,7 @@ the GDAL package by one of the current owners : HowardB/FrankW/EvenR) d) For real : twine upload dist/gdal_utils-*.whl -16) Generate signed maven artifacts with GPG for Java bindings. +18) Generate signed maven artifacts with GPG for Java bindings. This step is required in order to deploy the maven artifacts to the central Maven repository. Before this step can proceed you must set up a signing key as described here: @@ -272,7 +273,7 @@ the GDAL package by one of the current owners : HowardB/FrankW/EvenR) that contains all the maven artifacts with signatures. This file is what will be uploaded to maven central. See the next step. -17) Deploy maven artifacts to Maven central. +19) Deploy maven artifacts to Maven central. NOTE: Before you can deploy to maven central you must set up an account in Sonatype JIRA. That can be done here: @@ -329,14 +330,14 @@ e) Click the "Release" button and that is it! The release should be available in - gdalXYZ.zip.md5 - gdalautotest-X.Y.Z.zip -19) Regenerate Docker images +20) Regenerate Docker images e.g ./docker/build-all.sh --with-multi-arch --release --tag 3.3.1 --gdal v3.3.1 --proj 8.1.0 --platform linux/arm64,linux/amd64 --push and update docker/README.md with the latest release -20) Announce release to : +21) Announce release to : - major release: gdal-dev@lists.osgeo.org, gdal-announce@lists.osgeo.org, news_item@osgeo.org. - bugfix release: gdal-dev@lists.osgeo.org, gdal-announce@lists.osgeo.org @@ -348,10 +349,12 @@ Note: gdal-announce@ is moderated. Make sure that your email address is approved ('mod' tick disabled in https://lists.osgeo.org/mailman/admin/gdal-announce/members), or your message manually approved, with an administrator of the list. -21) For a feature release: after it has been done, in the master branch, update +22) For a feature release: after it has been done, in the master branch, update the VERSION to the next one and in gcore/gdal_version.h.in, update GDAL_VERSION_MAJOR/_MINOR, GDAL_RELEASE_NAME (with a dev suffix, like "3.7.0dev"), and set GDAL_RELEASE_DATE to a date like {YEAR}9999. Update swig/python/gdal-utils/osgeo_utils/__init__.py to something like (3, 6, 99, 0) if master is 3.7.0dev +23) For bugfixes releases, forward port to master changes done in + doc/source/about_no_title.rst, doc/source/download.rst and doc/source/download_past.rst diff --git a/NEWS.md b/NEWS.md index dd957241a787..4cb6db444f56 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,5 @@ # GDAL/OGR 3.10.0 Release Notes -FIXME: Preliminary notes up to commit d8112b785c1a2bedfc11849779bd148d55b10915 - GDAL/OGR 3.10.0 is a feature release. Those notes include changes since GDAL 3.9.0, but not already included in a GDAL 3.9.x bugfix release. @@ -132,6 +130,10 @@ See [MIGRATION_GUIDE.TXT](https://github.com/OSGeo/gdal/blob/release/3.10/MIGRAT the first level (#10167) * GDALWarpResolveWorkingDataType(): ignore srcNoDataValue if it doesn't fit into the data type of the source band +* Warper: fix shifted/aliased result with Lanczos resampling when XSCALE < 1 or + YSCALE < 1 (#11042) +* Warper: optimize speed of Lanczos resampling in Byte case without any + masking/alpha * Warper: make sure to check that angular unit is degree for heuristics related to geographic CRS (#10975) @@ -139,6 +141,7 @@ See [MIGRATION_GUIDE.TXT](https://github.com/OSGeo/gdal/blob/release/3.10/MIGRAT * gdalinfo: add -nonodata and -nomask options * gdalinfo/ogrinfo: make --formats -json work (#10878) +* gdalbuildvrt: add a -co switch (#11062) * gdal_translate: use GDALIsValueExactAs<> to check range of [u]int64 nodata * gdal_viewshed: multi-threaded optimization (#9991) * gdal_viewshed: support observers to left, right, above and below of input @@ -168,6 +171,10 @@ Multi-driver changes: Derived driver: * make it optional +DIMAP driver: + * for PNEO products, use the new color interpretations for the NIR, RedEdge + and DeepBlue/Coastal bands + DTED driver: * added metadata items Security Control and Security Handling (#10173) @@ -210,6 +217,8 @@ GTiff driver: * use main dataset GetSiblingFiles() for overviews * SRS writer: do not use EPSG:4326 if angular unit is not degree * make driver optional (but must be explicitly disabled) + * tif_jxl: writer: propagate DISTANCE/QUALITY setting to extra channel + * fix memory leak when trying to open COG in update mode * Internal libtiff: resync with upstream HEIF driver: @@ -240,11 +249,20 @@ JP2KAK driver: * add Cycc=yes/no creation option to set YCbCr/YUV color space and set it to YES by default (#10623) +JP2Lura driver: + * fix Identify() method + * planned for removal in GDAL 3.11 + JPEG driver: * make sure not to expose DNG_UniqueCameraModel/DNG_CameraSerialNumber if the corresponding EXIF tags are set at the same value * add support for reading Pix4DMapper CRS from XMP +JPEGXL driver: + * writer: propagate DISTANCE/QUALITY setting to extra channel (#11095) + * writer: allow a minimum DISTANCE of 0.01, max of 25 and revise QUALITY to + DISTANCE formula (#11095) + KMLSuperOverlay driver: * recognize with forming a rectangle (#10629) @@ -356,9 +374,12 @@ XYZ driver: * add COLUMN_ORDER open option Zarr driver: + * Zarr V2 creation: fix bug when creating dataset with partial blocks and need + to re-read them in the writing process when compression is involved (#11016) * Allow int64 attributes (#10065) - * SerializeNumericNoData(): use CPLJSonObject::Add(uint64_t) to avoid potential - undefined behavior casts + * SerializeNumericNoData(): use CPLJSonObject::Add(uint64_t) to avoid + potential undefined behavior casts + * Create(): remove created files / directories if an error occurs (#11023) ## OGR 3.10.0 @@ -381,6 +402,8 @@ Zarr driver: * SQLite/GPKG: Add ST_Length(geom, use_ellipsoid) * GetNextArrowArray() generic implementation: avoid calling VSI_MALLOC_ALIGNED_AUTO_VERBOSE() with a zero size +* Arrow reading: generic code path (as used by GeoJSON): fix mis-handling of + timezones * OGRFeature: optimizations while accessing field count * OGRFeature: SetXXX() methods: more informative warning messages reporting field name and value for out-of-range values @@ -414,6 +437,7 @@ Zarr driver: when possible * Add OGRWKBTransform() for in-place coordinate transformation of WKB geometries * OGR_GreatCircle_ API: do not hardcode Earth radius +* ogr_core.h: suppress warning when building with -Wpedantic for C < 23 (#2322) ### OGRSpatialReference @@ -421,6 +445,8 @@ Zarr driver: expected axis order * OGRSpatialReference::EPSGTreatsAsLatLong()/EPSGTreatsAsNorthingEasting(): remove the check on the EPSG authority +* Add OSRGetAuthorityListFromDatabase() to get the list of CRS authorities used + in the PROJ database. ### Utilities @@ -433,6 +459,7 @@ Zarr driver: transformation * ogr2ogr: optim: call GetArrowStream() only once on source layer when using Arrow interface +* ogr2ogr: fix -explodecollections on empty geometries (#11067) * validate_gpkg.py: make it robust to CURRENT_TIMESTAMP instead of 'now' ### Vector drivers @@ -448,6 +475,7 @@ Arrow ecosystem: * Arrow/Parquet/generic arrow: add write support for arrow.json extension * Add a Arrow VSI file system (for libarrow >= 16.0) allowing to use GDAL VSI file systems as libarrow compatible file systems. + * Add (minimum) support for libarrow 18.0.0 Arrow driver * add read support for StringView and BinaryView (but not in OGR generic Arrow @@ -473,6 +501,10 @@ ESRIJSON driver: * make it able to parse response of some CadastralSpecialServices APIs (#9996) * use 'alias' field member to set OGR alternative field name +FileGDB/OpenFileGDB drivers: + * update (and unify) list of reserved keywords that can't be used for column + and table names (#11094) + JSONFG driver: * avoid Polyhedron/Prism geometry instantiation during initial scan @@ -488,6 +520,14 @@ GML driver: * XSD parser: fix to resolve schema imports using open option USE_SCHEMA_IMPORT (#10500) * make it buildable as plugin if NAS driver is explicitly disabled + * add a GML_DOWNLOAD_SCHEMA config option matching the DOWNLOAD_SCHEMA open + option (and deprecate undocumented GML_DOWNLOAD_WFS_SCHEMA) + +GPKG driver: + * prevent from creating field with same name, or with the name of the geometry + field + * CreateField(): check we are not reaching the max number of fields + * SQLite/GPKG: turn on SQLite 'PRAGMA secure_delete=1' by default HANA driver: * Add support for REAL_VECTOR type (#10499) @@ -520,6 +560,10 @@ OAPIF driver: * make it buildable as plugin (independently of WFS driver) * add a DATETIME open option (#10909) +OCI driver: + * OCI: use TIMESTAMP(3) and tweak NLS_TIME[STAMP][_TZ]_FORMAT to accept + milliseconds (#11057) + ODBC driver: * add GDAL_DMD_LONGNAME @@ -541,6 +585,10 @@ Parquet driver: * dataset mode: detect bbox geometry column when opening current Overture Maps * dataset mode: make sure all files are closed before closing the GDALDataset +PDF driver: + * reader: fixes to handle recursive resources, /OC property attached to a + XObjet and an empty UTF-16 layer name (#11034) + PostgreSQL driver: * OGR_PG_SKIP_CONFLICTS: optionally insert with ON CONFLICT DO NOTHING (#10156) * avoid error when the original search_path is empty @@ -566,6 +614,7 @@ WFS driver: ## SWIG bindings * Python/Java: replace sprintf() with snprintf() to avoid warnings on OSX +* fix memleak in gdal.GetConfigOptions() ### Python bindings @@ -582,6 +631,8 @@ WFS driver: * make MDArray.Write(array_of_strings) work with a 0-d string variable * Avoid linear scan in gdal_array.NumericTypeCodeToGDALTypeCode (#10694) * Dataset.Close(): invalidate children before closing dataset +* __init__.py: remove calls to warnings.simplefilter() (#11140) +* fix compatibility issue with SWIG 4.3.0 and PYTHONWARNINGS=error ### Java bindings diff --git a/README.md b/README.md index 40abb5b270ac..b76a3c4f35a8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ GDAL - Geospatial Data Abstraction Library [![Build Status](https://github.com/OSGeo/gdal/actions/workflows/conda.yml/badge.svg)](https://github.com/osgeo/gdal/actions?query=workflow%3A%22Conda%22+branch%3Amaster) [![Build Status](https://scan.coverity.com/projects/749/badge.svg?flat=1)](https://scan.coverity.com/projects/gdal) [![Documentation build Status](https://github.com/OSGeo/gdal/workflows/Docs/badge.svg)](https://github.com/osgeo/gdal/actions?query=workflow%3A%22Docs%22+branch%3Amaster) -[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/gdal.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:gdal) +[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/gdal.svg)](https://issues.oss-fuzz.com/issues?q=status:open%20gdal) [![Coverage Status](https://coveralls.io/repos/github/OSGeo/gdal/badge.svg?branch=master)](https://coveralls.io/github/OSGeo/gdal?branch=master) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8250/badge)](https://www.bestpractices.dev/projects/8250) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/OSGeo/gdal/badge)](https://securityscorecards.dev/viewer/?uri=github.com/OSGeo/gdal) @@ -26,7 +26,6 @@ GDAL is an open source MIT licensed translator library for raster and vector geo * GIT repository: https://github.com/OSGeo/gdal * Bug tracker: https://github.com/OSGeo/gdal/issues * Download: https://download.osgeo.org/gdal -* Wiki: https://trac.osgeo.org/gdal - Various user and developer contributed documentation and hints * Mailing list: https://lists.osgeo.org/mailman/listinfo/gdal-dev [//]: # (numfocus-fiscal-sponsor-attribution) diff --git a/alg/gdalwarpkernel.cpp b/alg/gdalwarpkernel.cpp index 52c58d0d219b..9f07096d01d7 100644 --- a/alg/gdalwarpkernel.cpp +++ b/alg/gdalwarpkernel.cpp @@ -2849,11 +2849,14 @@ static bool GWKBilinearResampleNoMasks4SampleT(const GDALWarpKernel *poWK, // or http://en.wikipedia.org/wiki/Cubic_Hermite_spline : CINTx(p_1,p0,p1,p2) // http://en.wikipedia.org/wiki/Bicubic_interpolation: matrix notation -// TODO(schwehr): Use an inline function. -#define CubicConvolution(distance1, distance2, distance3, f0, f1, f2, f3) \ - (f1 + 0.5 * (distance1 * (f2 - f0) + \ - distance2 * (2.0 * f0 - 5.0 * f1 + 4.0 * f2 - f3) + \ - distance3 * (3.0 * (f1 - f2) + f3 - f0))) +template +static inline T CubicConvolution(T distance1, T distance2, T distance3, T f0, + T f1, T f2, T f3) +{ + return (f1 + T(0.5) * (distance1 * (f2 - f0) + + distance2 * (2 * f0 - 5 * f1 + 4 * f2 - f3) + + distance3 * (3 * (f1 - f2) + f3 - f0))); +} /************************************************************************/ /* GWKCubicComputeWeights() */ @@ -2861,19 +2864,18 @@ static bool GWKBilinearResampleNoMasks4SampleT(const GDALWarpKernel *poWK, // adfCoeffs[2] = 1.0 - (adfCoeffs[0] + adfCoeffs[1] - adfCoeffs[3]); -// TODO(schwehr): Use an inline function. -#define GWKCubicComputeWeights(dfX_, adfCoeffs) \ - { \ - const double dfX = dfX_; \ - const double dfHalfX = 0.5 * dfX; \ - const double dfThreeX = 3.0 * dfX; \ - const double dfHalfX2 = dfHalfX * dfX; \ - \ - adfCoeffs[0] = dfHalfX * (-1 + dfX * (2 - dfX)); \ - adfCoeffs[1] = 1 + dfHalfX2 * (-5 + dfThreeX); \ - adfCoeffs[2] = dfHalfX * (1 + dfX * (4 - dfThreeX)); \ - adfCoeffs[3] = dfHalfX2 * (-1 + dfX); \ - } +template +static inline void GWKCubicComputeWeights(T x, T coeffs[4]) +{ + const T halfX = T(0.5) * x; + const T threeX = T(3.0) * x; + const T halfX2 = halfX * x; + + coeffs[0] = halfX * (-1 + x * (2 - x)); + coeffs[1] = 1 + halfX2 * (-5 + threeX); + coeffs[2] = halfX * (1 + x * (4 - threeX)); + coeffs[3] = halfX2 * (-1 + x); +} // TODO(schwehr): Use an inline function. #define CONVOL4(v1, v2) \ @@ -2969,10 +2971,7 @@ static bool GWKCubicResample4Sample(const GDALWarpKernel *poWK, int iBand, return true; } -// We do not define USE_SSE_CUBIC_IMPL since in practice, it gives zero -// perf benefit. - -#if defined(USE_SSE_CUBIC_IMPL) && (defined(__x86_64) || defined(_M_X64)) +#if defined(__x86_64) || defined(_M_X64) /************************************************************************/ /* XMMLoad4Values() */ @@ -2985,7 +2984,7 @@ static CPL_INLINE __m128 XMMLoad4Values(const GByte *ptr) { unsigned int i; memcpy(&i, ptr, 4); - __m128i xmm_i = _mm_cvtsi32_si128(s); + __m128i xmm_i = _mm_cvtsi32_si128(i); // Zero extend 4 packed unsigned 8-bit integers in a to packed // 32-bit integers. #if __SSE4_1__ @@ -3001,7 +3000,7 @@ static CPL_INLINE __m128 XMMLoad4Values(const GUInt16 *ptr) { GUInt64 i; memcpy(&i, ptr, 8); - __m128i xmm_i = _mm_cvtsi64_si128(s); + __m128i xmm_i = _mm_cvtsi64_si128(i); // Zero extend 4 packed unsigned 16-bit integers in a to packed // 32-bit integers. #if __SSE4_1__ @@ -3038,7 +3037,7 @@ static CPL_INLINE float XMMHorizontalAdd(__m128 v) } #endif -#endif // defined(USE_SSE_CUBIC_IMPL) && (defined(__x86_64) || defined(_M_X64)) +#endif // (defined(__x86_64) || defined(_M_X64)) /************************************************************************/ /* GWKCubicResampleSrcMaskIsDensity4SampleRealT() */ @@ -3046,6 +3045,8 @@ static CPL_INLINE float XMMHorizontalAdd(__m128 v) // Note: if USE_SSE_CUBIC_IMPL, only instantiate that for Byte and UInt16, // because there are a few assumptions above those types. +// We do not define USE_SSE_CUBIC_IMPL since in practice, it gives zero +// perf benefit. template static CPL_INLINE bool GWKCubicResampleSrcMaskIsDensity4SampleRealT( @@ -4360,14 +4361,76 @@ static bool GWKResampleOptimizedLanczos(const GDALWarpKernel *poWK, int iBand, return true; } +/************************************************************************/ +/* GWKComputeWeights() */ +/************************************************************************/ + +static void GWKComputeWeights(GDALResampleAlg eResample, int iMin, int iMax, + double dfDeltaX, double dfXScale, int jMin, + int jMax, double dfDeltaY, double dfYScale, + double *padfWeightsHorizontal, + double *padfWeightsVertical, double &dfInvWeights) +{ + + const FilterFuncType pfnGetWeight = apfGWKFilter[eResample]; + CPLAssert(pfnGetWeight); + const FilterFunc4ValuesType pfnGetWeight4Values = + apfGWKFilter4Values[eResample]; + CPLAssert(pfnGetWeight4Values); + + int i = iMin; // Used after for. + int iC = 0; // Used after for. + double dfAccumulatorWeightHorizontal = 0.0; + for (; i + 2 < iMax; i += 4, iC += 4) + { + padfWeightsHorizontal[iC] = (i - dfDeltaX) * dfXScale; + padfWeightsHorizontal[iC + 1] = padfWeightsHorizontal[iC] + dfXScale; + padfWeightsHorizontal[iC + 2] = + padfWeightsHorizontal[iC + 1] + dfXScale; + padfWeightsHorizontal[iC + 3] = + padfWeightsHorizontal[iC + 2] + dfXScale; + dfAccumulatorWeightHorizontal += + pfnGetWeight4Values(padfWeightsHorizontal + iC); + } + for (; i <= iMax; ++i, ++iC) + { + const double dfWeight = pfnGetWeight((i - dfDeltaX) * dfXScale); + padfWeightsHorizontal[iC] = dfWeight; + dfAccumulatorWeightHorizontal += dfWeight; + } + + int j = jMin; // Used after for. + int jC = 0; // Used after for. + double dfAccumulatorWeightVertical = 0.0; + for (; j + 2 < jMax; j += 4, jC += 4) + { + padfWeightsVertical[jC] = (j - dfDeltaY) * dfYScale; + padfWeightsVertical[jC + 1] = padfWeightsVertical[jC] + dfYScale; + padfWeightsVertical[jC + 2] = padfWeightsVertical[jC + 1] + dfYScale; + padfWeightsVertical[jC + 3] = padfWeightsVertical[jC + 2] + dfYScale; + dfAccumulatorWeightVertical += + pfnGetWeight4Values(padfWeightsVertical + jC); + } + for (; j <= jMax; ++j, ++jC) + { + const double dfWeight = pfnGetWeight((j - dfDeltaY) * dfYScale); + padfWeightsVertical[jC] = dfWeight; + dfAccumulatorWeightVertical += dfWeight; + } + + dfInvWeights = + 1. / (dfAccumulatorWeightHorizontal * dfAccumulatorWeightVertical); +} + /************************************************************************/ /* GWKResampleNoMasksT() */ /************************************************************************/ template -static bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, - double dfSrcX, double dfSrcY, T *pValue, - double *padfWeight) +static bool +GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, + double dfSrcY, T *pValue, double *padfWeightsHorizontal, + double *padfWeightsVertical, double &dfInvWeights) { // Commonly used; save locally. @@ -4392,52 +4455,33 @@ static bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, const double dfDeltaX = dfSrcX - 0.5 - iSrcX; const double dfDeltaY = dfSrcY - 0.5 - iSrcY; - const FilterFuncType pfnGetWeight = apfGWKFilter[poWK->eResample]; - CPLAssert(pfnGetWeight); - const FilterFunc4ValuesType pfnGetWeight4Values = - apfGWKFilter4Values[poWK->eResample]; - CPLAssert(pfnGetWeight4Values); - const double dfXScale = std::min(poWK->dfXScale, 1.0); const double dfYScale = std::min(poWK->dfYScale, 1.0); - // Loop over all rows in the kernel. - double dfAccumulatorWeightHorizontal = 0.0; - double dfAccumulatorWeightVertical = 0.0; - int iMin = 1 - nXRadius; if (iSrcX + iMin < 0) iMin = -iSrcX; int iMax = nXRadius; if (iSrcX + iMax >= nSrcXSize - 1) iMax = nSrcXSize - 1 - iSrcX; - int i = iMin; // Used after for. - int iC = 0; // Used after for. - for (; i + 2 < iMax; i += 4, iC += 4) - { - padfWeight[iC] = (i - dfDeltaX) * dfXScale; - padfWeight[iC + 1] = padfWeight[iC] + dfXScale; - padfWeight[iC + 2] = padfWeight[iC + 1] + dfXScale; - padfWeight[iC + 3] = padfWeight[iC + 2] + dfXScale; - dfAccumulatorWeightHorizontal += pfnGetWeight4Values(padfWeight + iC); - } - for (; i <= iMax; ++i, ++iC) - { - const double dfWeight = pfnGetWeight((i - dfDeltaX) * dfXScale); - padfWeight[iC] = dfWeight; - dfAccumulatorWeightHorizontal += dfWeight; - } - int j = 1 - nYRadius; - if (iSrcY + j < 0) - j = -iSrcY; + int jMin = 1 - nYRadius; + if (iSrcY + jMin < 0) + jMin = -iSrcY; int jMax = nYRadius; if (iSrcY + jMax >= nSrcYSize - 1) jMax = nSrcYSize - 1 - iSrcY; - double dfAccumulator = 0.0; + if (iBand == 0) + { + GWKComputeWeights(poWK->eResample, iMin, iMax, dfDeltaX, dfXScale, jMin, + jMax, dfDeltaY, dfYScale, padfWeightsHorizontal, + padfWeightsVertical, dfInvWeights); + } - for (; j <= jMax; ++j) + // Loop over all rows in the kernel. + double dfAccumulator = 0.0; + for (int jC = 0, j = jMin; j <= jMax; ++j, ++jC) { const GPtrDiff_t iSampJ = iSrcOffset + static_cast(j) * nSrcXSize; @@ -4445,42 +4489,41 @@ static bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, // Loop over all pixels in the row. double dfAccumulatorLocal = 0.0; double dfAccumulatorLocal2 = 0.0; - iC = 0; - i = iMin; + int iC = 0; + int i = iMin; // Process by chunk of 4 cols. for (; i + 2 < iMax; i += 4, iC += 4) { // Retrieve the pixel & accumulate. - dfAccumulatorLocal += pSrcBand[i + iSampJ] * padfWeight[iC]; - dfAccumulatorLocal += pSrcBand[i + 1 + iSampJ] * padfWeight[iC + 1]; + dfAccumulatorLocal += + pSrcBand[i + iSampJ] * padfWeightsHorizontal[iC]; + dfAccumulatorLocal += + pSrcBand[i + 1 + iSampJ] * padfWeightsHorizontal[iC + 1]; dfAccumulatorLocal2 += - pSrcBand[i + 2 + iSampJ] * padfWeight[iC + 2]; + pSrcBand[i + 2 + iSampJ] * padfWeightsHorizontal[iC + 2]; dfAccumulatorLocal2 += - pSrcBand[i + 3 + iSampJ] * padfWeight[iC + 3]; + pSrcBand[i + 3 + iSampJ] * padfWeightsHorizontal[iC + 3]; } dfAccumulatorLocal += dfAccumulatorLocal2; if (i < iMax) { - dfAccumulatorLocal += pSrcBand[i + iSampJ] * padfWeight[iC]; - dfAccumulatorLocal += pSrcBand[i + 1 + iSampJ] * padfWeight[iC + 1]; + dfAccumulatorLocal += + pSrcBand[i + iSampJ] * padfWeightsHorizontal[iC]; + dfAccumulatorLocal += + pSrcBand[i + 1 + iSampJ] * padfWeightsHorizontal[iC + 1]; i += 2; iC += 2; } if (i == iMax) { - dfAccumulatorLocal += pSrcBand[i + iSampJ] * padfWeight[iC]; + dfAccumulatorLocal += + pSrcBand[i + iSampJ] * padfWeightsHorizontal[iC]; } - // Calculate the Y weight. - const double dfWeight = pfnGetWeight((j - dfDeltaY) * dfYScale); - dfAccumulator += dfWeight * dfAccumulatorLocal; - dfAccumulatorWeightVertical += dfWeight; + dfAccumulator += padfWeightsVertical[jC] * dfAccumulatorLocal; } - const double dfAccumulatorWeight = - dfAccumulatorWeightHorizontal * dfAccumulatorWeightVertical; - - *pValue = GWKClampValueT(dfAccumulator / dfAccumulatorWeight); + *pValue = GWKClampValueT(dfAccumulator * dfInvWeights); return true; } @@ -4496,7 +4539,9 @@ static bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, template static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, T *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { // Commonly used; save locally. const int nSrcXSize = poWK->nSrcXSize; @@ -4518,60 +4563,42 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, const T *pSrcBand = reinterpret_cast(poWK->papabySrcImage[iBand]); - const FilterFuncType pfnGetWeight = apfGWKFilter[poWK->eResample]; - CPLAssert(pfnGetWeight); - const FilterFunc4ValuesType pfnGetWeight4Values = - apfGWKFilter4Values[poWK->eResample]; - CPLAssert(pfnGetWeight4Values); - const double dfDeltaX = dfSrcX - 0.5 - iSrcX; const double dfDeltaY = dfSrcY - 0.5 - iSrcY; const double dfXScale = std::min(poWK->dfXScale, 1.0); const double dfYScale = std::min(poWK->dfYScale, 1.0); - // Loop over all rows in the kernel. - double dfAccumulatorWeightHorizontal = 0.0; - double dfAccumulatorWeightVertical = 0.0; - double dfAccumulator = 0.0; - int iMin = 1 - nXRadius; if (iSrcX + iMin < 0) iMin = -iSrcX; int iMax = nXRadius; if (iSrcX + iMax >= nSrcXSize - 1) iMax = nSrcXSize - 1 - iSrcX; - int i, iC; - for (iC = 0, i = iMin; i + 2 < iMax; i += 4, iC += 4) - { - padfWeight[iC] = (i - dfDeltaX) * dfXScale; - padfWeight[iC + 1] = padfWeight[iC] + dfXScale; - padfWeight[iC + 2] = padfWeight[iC + 1] + dfXScale; - padfWeight[iC + 3] = padfWeight[iC + 2] + dfXScale; - dfAccumulatorWeightHorizontal += pfnGetWeight4Values(padfWeight + iC); - } - for (; i <= iMax; ++i, ++iC) - { - double dfWeight = pfnGetWeight((i - dfDeltaX) * dfXScale); - padfWeight[iC] = dfWeight; - dfAccumulatorWeightHorizontal += dfWeight; - } - int j = 1 - nYRadius; - if (iSrcY + j < 0) - j = -iSrcY; + int jMin = 1 - nYRadius; + if (iSrcY + jMin < 0) + jMin = -iSrcY; int jMax = nYRadius; if (iSrcY + jMax >= nSrcYSize - 1) jMax = nSrcYSize - 1 - iSrcY; - // Process by chunk of 4 rows. - for (; j + 2 < jMax; j += 4) + if (iBand == 0) { - const GPtrDiff_t iSampJ = - iSrcOffset + static_cast(j) * nSrcXSize; + GWKComputeWeights(poWK->eResample, iMin, iMax, dfDeltaX, dfXScale, jMin, + jMax, dfDeltaY, dfYScale, padfWeightsHorizontal, + padfWeightsVertical, dfInvWeights); + } + GPtrDiff_t iSampJ = iSrcOffset + static_cast(jMin) * nSrcXSize; + // Process by chunk of 4 rows. + int jC = 0; + int j = jMin; + double dfAccumulator = 0.0; + for (; j + 2 < jMax; j += 4, iSampJ += 4 * nSrcXSize, jC += 4) + { // Loop over all pixels in the row. - iC = 0; - i = iMin; + int iC = 0; + int i = iMin; // Process by chunk of 4 cols. XMMReg4Double v_acc_1 = XMMReg4Double::Zero(); XMMReg4Double v_acc_2 = XMMReg4Double::Zero(); @@ -4590,7 +4617,7 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, XMMReg4Double::Load4Val(pSrcBand + i + iSampJ + 3 * nSrcXSize); XMMReg4Double v_padfWeight = - XMMReg4Double::Load4Val(padfWeight + iC); + XMMReg4Double::Load4Val(padfWeightsHorizontal + iC); v_acc_1 += v_pixels_1 * v_padfWeight; v_acc_2 += v_pixels_2 * v_padfWeight; @@ -4610,7 +4637,7 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, XMMReg2Double::Load2Val(pSrcBand + i + iSampJ + 3 * nSrcXSize); XMMReg2Double v_padfWeight = - XMMReg2Double::Load2Val(padfWeight + iC); + XMMReg2Double::Load2Val(padfWeightsHorizontal + iC); v_acc_1.AddToLow(v_pixels_1 * v_padfWeight); v_acc_2.AddToLow(v_pixels_2 * v_padfWeight); @@ -4628,40 +4655,29 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, if (i == iMax) { - dfAccumulatorLocal_1 += - static_cast(pSrcBand[i + iSampJ]) * padfWeight[iC]; + dfAccumulatorLocal_1 += static_cast(pSrcBand[i + iSampJ]) * + padfWeightsHorizontal[iC]; dfAccumulatorLocal_2 += static_cast(pSrcBand[i + iSampJ + nSrcXSize]) * - padfWeight[iC]; + padfWeightsHorizontal[iC]; dfAccumulatorLocal_3 += static_cast(pSrcBand[i + iSampJ + 2 * nSrcXSize]) * - padfWeight[iC]; + padfWeightsHorizontal[iC]; dfAccumulatorLocal_4 += static_cast(pSrcBand[i + iSampJ + 3 * nSrcXSize]) * - padfWeight[iC]; + padfWeightsHorizontal[iC]; } - // Calculate the Y weight. - const double dfWeight0 = (j - dfDeltaY) * dfYScale; - const double dfWeight1 = dfWeight0 + dfYScale; - const double dfWeight2 = dfWeight1 + dfYScale; - const double dfWeight3 = dfWeight2 + dfYScale; - double adfWeight[4] = {dfWeight0, dfWeight1, dfWeight2, dfWeight3}; - - dfAccumulatorWeightVertical += pfnGetWeight4Values(adfWeight); - dfAccumulator += adfWeight[0] * dfAccumulatorLocal_1; - dfAccumulator += adfWeight[1] * dfAccumulatorLocal_2; - dfAccumulator += adfWeight[2] * dfAccumulatorLocal_3; - dfAccumulator += adfWeight[3] * dfAccumulatorLocal_4; + dfAccumulator += padfWeightsVertical[jC] * dfAccumulatorLocal_1; + dfAccumulator += padfWeightsVertical[jC + 1] * dfAccumulatorLocal_2; + dfAccumulator += padfWeightsVertical[jC + 2] * dfAccumulatorLocal_3; + dfAccumulator += padfWeightsVertical[jC + 3] * dfAccumulatorLocal_4; } - for (; j <= jMax; ++j) + for (; j <= jMax; ++j, iSampJ += nSrcXSize, ++jC) { - const GPtrDiff_t iSampJ = - iSrcOffset + static_cast(j) * nSrcXSize; - // Loop over all pixels in the row. - iC = 0; - i = iMin; + int iC = 0; + int i = iMin; // Process by chunk of 4 cols. XMMReg4Double v_acc = XMMReg4Double::Zero(); for (; i + 2 < iMax; i += 4, iC += 4) @@ -4670,7 +4686,7 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, XMMReg4Double v_pixels = XMMReg4Double::Load4Val(pSrcBand + i + iSampJ); XMMReg4Double v_padfWeight = - XMMReg4Double::Load4Val(padfWeight + iC); + XMMReg4Double::Load4Val(padfWeightsHorizontal + iC); v_acc += v_pixels * v_padfWeight; } @@ -4679,27 +4695,23 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, if (i < iMax) { - dfAccumulatorLocal += pSrcBand[i + iSampJ] * padfWeight[iC]; - dfAccumulatorLocal += pSrcBand[i + 1 + iSampJ] * padfWeight[iC + 1]; + dfAccumulatorLocal += + pSrcBand[i + iSampJ] * padfWeightsHorizontal[iC]; + dfAccumulatorLocal += + pSrcBand[i + 1 + iSampJ] * padfWeightsHorizontal[iC + 1]; i += 2; iC += 2; } if (i == iMax) { - dfAccumulatorLocal += - static_cast(pSrcBand[i + iSampJ]) * padfWeight[iC]; + dfAccumulatorLocal += static_cast(pSrcBand[i + iSampJ]) * + padfWeightsHorizontal[iC]; } - // Calculate the Y weight. - double dfWeight = pfnGetWeight((j - dfDeltaY) * dfYScale); - dfAccumulator += dfWeight * dfAccumulatorLocal; - dfAccumulatorWeightVertical += dfWeight; + dfAccumulator += padfWeightsVertical[jC] * dfAccumulatorLocal; } - const double dfAccumulatorWeight = - dfAccumulatorWeightHorizontal * dfAccumulatorWeightVertical; - - *pValue = GWKClampValueT(dfAccumulator / dfAccumulatorWeight); + *pValue = GWKClampValueT(dfAccumulator * dfInvWeights); return true; } @@ -4711,10 +4723,13 @@ static bool GWKResampleNoMasks_SSE2_T(const GDALWarpKernel *poWK, int iBand, template <> bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, GByte *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { return GWKResampleNoMasks_SSE2_T(poWK, iBand, dfSrcX, dfSrcY, pValue, - padfWeight); + padfWeightsHorizontal, padfWeightsVertical, + dfInvWeights); } /************************************************************************/ @@ -4724,10 +4739,13 @@ bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, template <> bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, GInt16 *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { return GWKResampleNoMasks_SSE2_T(poWK, iBand, dfSrcX, dfSrcY, pValue, - padfWeight); + padfWeightsHorizontal, padfWeightsVertical, + dfInvWeights); } /************************************************************************/ @@ -4737,10 +4755,13 @@ bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, template <> bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, GUInt16 *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { return GWKResampleNoMasks_SSE2_T(poWK, iBand, dfSrcX, dfSrcY, pValue, - padfWeight); + padfWeightsHorizontal, padfWeightsVertical, + dfInvWeights); } /************************************************************************/ @@ -4750,10 +4771,13 @@ bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, template <> bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, float *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { return GWKResampleNoMasks_SSE2_T(poWK, iBand, dfSrcX, dfSrcY, pValue, - padfWeight); + padfWeightsHorizontal, padfWeightsVertical, + dfInvWeights); } #ifdef INSTANTIATE_FLOAT64_SSE2_IMPL @@ -4765,10 +4789,13 @@ bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, template <> bool GWKResampleNoMasksT(const GDALWarpKernel *poWK, int iBand, double dfSrcX, double dfSrcY, double *pValue, - double *padfWeight) + double *padfWeightsHorizontal, + double *padfWeightsVertical, + double &dfInvWeights) { return GWKResampleNoMasks_SSE2_T(poWK, iBand, dfSrcX, dfSrcY, pValue, - padfWeight); + padfWeightsHorizontal, padfWeightsVertical, + dfInvWeights); } #endif /* INSTANTIATE_FLOAT64_SSE2_IMPL */ @@ -6087,6 +6114,141 @@ static CPLErr GWKRealCase(GDALWarpKernel *poWK) return GWKRun(poWK, "GWKRealCase", GWKRealCaseThread); } +/************************************************************************/ +/* GWKCubicResampleNoMasks4MultiBandT() */ +/************************************************************************/ + +/* We restrict to 64bit processors because they are guaranteed to have SSE2 */ +/* and enough SSE registries */ +#if defined(__x86_64) || defined(_M_X64) + +static inline float Convolute4x4(const __m128 row0, const __m128 row1, + const __m128 row2, const __m128 row3, + const __m128 weightsXY0, + const __m128 weightsXY1, + const __m128 weightsXY2, + const __m128 weightsXY3) +{ + return XMMHorizontalAdd(_mm_add_ps( + _mm_add_ps(_mm_mul_ps(row0, weightsXY0), _mm_mul_ps(row1, weightsXY1)), + _mm_add_ps(_mm_mul_ps(row2, weightsXY2), + _mm_mul_ps(row3, weightsXY3)))); +} + +template +static void GWKCubicResampleNoMasks4MultiBandT(const GDALWarpKernel *poWK, + double dfSrcX, double dfSrcY, + const GPtrDiff_t iDstOffset) +{ + const double dfSrcXShifted = dfSrcX - 0.5; + const int iSrcX = static_cast(dfSrcXShifted); + const double dfSrcYShifted = dfSrcY - 0.5; + const int iSrcY = static_cast(dfSrcYShifted); + const GPtrDiff_t iSrcOffset = + iSrcX + static_cast(iSrcY) * poWK->nSrcXSize; + + // Get the bilinear interpolation at the image borders. + if (iSrcX - 1 < 0 || iSrcX + 2 >= poWK->nSrcXSize || iSrcY - 1 < 0 || + iSrcY + 2 >= poWK->nSrcYSize) + { + for (int iBand = 0; iBand < poWK->nBands; iBand++) + { + T value; + GWKBilinearResampleNoMasks4SampleT(poWK, iBand, dfSrcX, dfSrcY, + &value); + reinterpret_cast(poWK->papabyDstImage[iBand])[iDstOffset] = + value; + } + } + else + { + const float fDeltaX = static_cast(dfSrcXShifted) - iSrcX; + const float fDeltaY = static_cast(dfSrcYShifted) - iSrcY; + + float afCoeffsX[4]; + float afCoeffsY[4]; + GWKCubicComputeWeights(fDeltaX, afCoeffsX); + GWKCubicComputeWeights(fDeltaY, afCoeffsY); + const auto weightsX = _mm_loadu_ps(afCoeffsX); + const auto weightsXY0 = + _mm_mul_ps(_mm_load1_ps(&afCoeffsY[0]), weightsX); + const auto weightsXY1 = + _mm_mul_ps(_mm_load1_ps(&afCoeffsY[1]), weightsX); + const auto weightsXY2 = + _mm_mul_ps(_mm_load1_ps(&afCoeffsY[2]), weightsX); + const auto weightsXY3 = + _mm_mul_ps(_mm_load1_ps(&afCoeffsY[3]), weightsX); + + const GPtrDiff_t iOffset = iSrcOffset - poWK->nSrcXSize - 1; + + int iBand = 0; + // Process 2 bands at a time + for (; iBand + 1 < poWK->nBands; iBand += 2) + { + const T *CPL_RESTRICT pBand0 = + reinterpret_cast(poWK->papabySrcImage[iBand]); + const auto row0_0 = XMMLoad4Values(pBand0 + iOffset); + const auto row1_0 = + XMMLoad4Values(pBand0 + iOffset + poWK->nSrcXSize); + const auto row2_0 = + XMMLoad4Values(pBand0 + iOffset + 2 * poWK->nSrcXSize); + const auto row3_0 = + XMMLoad4Values(pBand0 + iOffset + 3 * poWK->nSrcXSize); + + const T *CPL_RESTRICT pBand1 = + reinterpret_cast(poWK->papabySrcImage[iBand + 1]); + const auto row0_1 = XMMLoad4Values(pBand1 + iOffset); + const auto row1_1 = + XMMLoad4Values(pBand1 + iOffset + poWK->nSrcXSize); + const auto row2_1 = + XMMLoad4Values(pBand1 + iOffset + 2 * poWK->nSrcXSize); + const auto row3_1 = + XMMLoad4Values(pBand1 + iOffset + 3 * poWK->nSrcXSize); + + const float fValue_0 = + Convolute4x4(row0_0, row1_0, row2_0, row3_0, weightsXY0, + weightsXY1, weightsXY2, weightsXY3); + + const float fValue_1 = + Convolute4x4(row0_1, row1_1, row2_1, row3_1, weightsXY0, + weightsXY1, weightsXY2, weightsXY3); + + T *CPL_RESTRICT pDstBand0 = + reinterpret_cast(poWK->papabyDstImage[iBand]); + pDstBand0[iDstOffset] = GWKClampValueT(fValue_0); + + T *CPL_RESTRICT pDstBand1 = + reinterpret_cast(poWK->papabyDstImage[iBand + 1]); + pDstBand1[iDstOffset] = GWKClampValueT(fValue_1); + } + if (iBand < poWK->nBands) + { + const T *CPL_RESTRICT pBand0 = + reinterpret_cast(poWK->papabySrcImage[iBand]); + const auto row0 = XMMLoad4Values(pBand0 + iOffset); + const auto row1 = + XMMLoad4Values(pBand0 + iOffset + poWK->nSrcXSize); + const auto row2 = + XMMLoad4Values(pBand0 + iOffset + 2 * poWK->nSrcXSize); + const auto row3 = + XMMLoad4Values(pBand0 + iOffset + 3 * poWK->nSrcXSize); + + const float fValue = + Convolute4x4(row0, row1, row2, row3, weightsXY0, weightsXY1, + weightsXY2, weightsXY3); + + T *CPL_RESTRICT pDstBand = + reinterpret_cast(poWK->papabyDstImage[iBand]); + pDstBand[iDstOffset] = GWKClampValueT(fValue); + } + } + + if (poWK->pafDstDensity) + poWK->pafDstDensity[iDstOffset] = 1.0f; +} + +#endif // defined(__x86_64) || defined(_M_X64) + /************************************************************************/ /* GWKResampleNoMasksOrDstDensityOnlyThreadInternal() */ /************************************************************************/ @@ -6125,8 +6287,10 @@ static void GWKResampleNoMasksOrDstDensityOnlyThreadInternal(void *pData) int *pabSuccess = static_cast(CPLMalloc(sizeof(int) * nDstXSize)); const int nXRadius = poWK->nXRadius; - double *padfWeight = + double *padfWeightsX = static_cast(CPLCalloc(1 + nXRadius * 2, sizeof(double))); + double *padfWeightsY = static_cast( + CPLCalloc(1 + poWK->nYRadius * 2, sizeof(double))); const double dfSrcCoordPrecision = CPLAtof(CSLFetchNameValueDef( poWK->papszWarpOptions, "SRC_COORD_PRECISION", "0")); const double dfErrorThreshold = CPLAtof( @@ -6189,6 +6353,23 @@ static void GWKResampleNoMasksOrDstDensityOnlyThreadInternal(void *pData) const GPtrDiff_t iDstOffset = iDstX + static_cast(iDstY) * nDstXSize; +#if defined(__x86_64) || defined(_M_X64) + if constexpr (bUse4SamplesFormula && eResample == GRA_Cubic && + (std::is_same::value || + std::is_same::value)) + { + if (poWK->nBands > 1 && !poWK->bApplyVerticalShift) + { + GWKCubicResampleNoMasks4MultiBandT( + poWK, padfX[iDstX] - poWK->nSrcXOff, + padfY[iDstX] - poWK->nSrcYOff, iDstOffset); + + continue; + } + } +#endif // defined(__x86_64) || defined(_M_X64) + + [[maybe_unused]] double dfInvWeights = 0; for (int iBand = 0; iBand < poWK->nBands; iBand++) { T value = 0; @@ -6212,7 +6393,8 @@ static void GWKResampleNoMasksOrDstDensityOnlyThreadInternal(void *pData) { GWKResampleNoMasksT( poWK, iBand, padfX[iDstX] - poWK->nSrcXOff, - padfY[iDstX] - poWK->nSrcYOff, &value, padfWeight); + padfY[iDstX] - poWK->nSrcYOff, &value, padfWeightsX, + padfWeightsY, dfInvWeights); } if (poWK->bApplyVerticalShift) @@ -6250,7 +6432,8 @@ static void GWKResampleNoMasksOrDstDensityOnlyThreadInternal(void *pData) CPLFree(padfY); CPLFree(padfZ); CPLFree(pabSuccess); - CPLFree(padfWeight); + CPLFree(padfWeightsX); + CPLFree(padfWeightsY); } template diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 90634aaf4d6c..9e9920db5b57 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -165,7 +165,6 @@ if (BUILD_APPS) add_executable(gdaltorture EXCLUDE_FROM_ALL gdaltorture.cpp) add_executable(gdal2ogr EXCLUDE_FROM_ALL gdal2ogr.c) add_executable(gdalasyncread EXCLUDE_FROM_ALL gdalasyncread.cpp) - add_executable(gdalwarpsimple EXCLUDE_FROM_ALL gdalwarpsimple.c) add_executable(multireadtest EXCLUDE_FROM_ALL multireadtest.cpp) if(NOT MSVC AND CMAKE_THREAD_LIBS_INIT) target_link_libraries(multireadtest PRIVATE ${CMAKE_THREAD_LIBS_INIT}) @@ -180,7 +179,6 @@ if (BUILD_APPS) gdalasyncread gdaltorture gdalflattenmask - gdalwarpsimple multireadtest test_ogrsf testreprojmulti) diff --git a/apps/gdal_footprint_lib.cpp b/apps/gdal_footprint_lib.cpp index 978c67d6ec81..a647517bf3a7 100644 --- a/apps/gdal_footprint_lib.cpp +++ b/apps/gdal_footprint_lib.cpp @@ -1028,7 +1028,7 @@ static bool GDALFootprintProcess(GDALDataset *poSrcDS, OGRLayer *poDstLayer, CPLAssert(poGeom); if (poGeom->getGeometryType() == wkbPolygon) { - poMP->addGeometryDirectly(poGeom.release()); + poMP->addGeometry(std::move(poGeom)); } } poMemLayer = std::make_unique("", nullptr, wkbUnknown); @@ -1095,7 +1095,7 @@ static bool GDALFootprintProcess(GDALDataset *poSrcDS, OGRLayer *poDstLayer, } } if (!poNewPoly->IsEmpty()) - poMP->addGeometryDirectly(poNewPoly.release()); + poMP->addGeometry(std::move(poNewPoly)); } poGeom = std::move(poMP); } diff --git a/apps/gdal_rasterize_lib.cpp b/apps/gdal_rasterize_lib.cpp index 6b6981438133..2d456ffc460e 100644 --- a/apps/gdal_rasterize_lib.cpp +++ b/apps/gdal_rasterize_lib.cpp @@ -366,7 +366,7 @@ static void InvertGeometries(GDALDatasetH hDstDS, double adfGeoTransform[6] = {}; GDALGetGeoTransform(hDstDS, adfGeoTransform); - OGRLinearRing *poUniverseRing = new OGRLinearRing(); + auto poUniverseRing = std::make_unique(); poUniverseRing->addPoint( adfGeoTransform[0] + -2 * adfGeoTransform[1] + -2 * adfGeoTransform[2], @@ -391,9 +391,9 @@ static void InvertGeometries(GDALDatasetH hDstDS, adfGeoTransform[0] + -2 * adfGeoTransform[1] + -2 * adfGeoTransform[2], adfGeoTransform[3] + -2 * adfGeoTransform[4] + -2 * adfGeoTransform[5]); - OGRPolygon *poUniversePoly = new OGRPolygon(); - poUniversePoly->addRingDirectly(poUniverseRing); - poInvertMP->addGeometryDirectly(poUniversePoly); + auto poUniversePoly = std::make_unique(); + poUniversePoly->addRing(std::move(poUniverseRing)); + poInvertMP->addGeometry(std::move(poUniversePoly)); bool bFoundNonPoly = false; // If we have GEOS, use it to "subtract" each polygon from the universe @@ -434,6 +434,9 @@ static void InvertGeometries(GDALDatasetH hDstDS, return; } + OGRPolygon &hUniversePoly = + *poInvertMP->getGeometryRef(poInvertMP->getNumGeometries() - 1); + /* -------------------------------------------------------------------- */ /* If we don't have GEOS, add outer rings of polygons as inner */ /* rings of poUniversePoly and inner rings as sub-polygons. Note */ @@ -460,15 +463,18 @@ static void InvertGeometries(GDALDatasetH hDstDS, } const auto ProcessPoly = - [poUniversePoly, poInvertMP](OGRPolygon *poPoly) + [&hUniversePoly, poInvertMP](OGRPolygon *poPoly) { for (int i = poPoly->getNumInteriorRings() - 1; i >= 0; --i) { - auto poNewPoly = new OGRPolygon(); - poNewPoly->addRingDirectly(poPoly->stealInteriorRing(i)); - poInvertMP->addGeometryDirectly(poNewPoly); + auto poNewPoly = std::make_unique(); + std::unique_ptr poRing( + poPoly->stealInteriorRing(i)); + poNewPoly->addRing(std::move(poRing)); + poInvertMP->addGeometry(std::move(poNewPoly)); } - poUniversePoly->addRingDirectly(poPoly->stealExteriorRing()); + std::unique_ptr poShell(poPoly->stealExteriorRing()); + hUniversePoly.addRing(std::move(poShell)); }; if (eGType == wkbPolygon) diff --git a/apps/gdalenhance.cpp b/apps/gdalenhance.cpp index 95a4d3bf0d1f..d5a72fac2634 100644 --- a/apps/gdalenhance.cpp +++ b/apps/gdalenhance.cpp @@ -339,6 +339,13 @@ MAIN_START(argc, argv) } } + if (padfScaleMin == nullptr || padfScaleMax == nullptr) + { + fprintf(stderr, "-equalize or -config filename command line options " + "must be specified.\n"); + Usage(); + } + /* -------------------------------------------------------------------- */ /* If there is no destination, just report the scaling values */ /* and luts. */ @@ -352,9 +359,8 @@ MAIN_START(argc, argv) for (iBand = 0; iBand < nBandCount; iBand++) { fprintf(fpConfig, "%d:Band ", iBand + 1); - if (padfScaleMin != nullptr) - fprintf(fpConfig, "%g:ScaleMin %g:ScaleMax ", - padfScaleMin[iBand], padfScaleMax[iBand]); + fprintf(fpConfig, "%g:ScaleMin %g:ScaleMax ", padfScaleMin[iBand], + padfScaleMax[iBand]); if (papanLUTs) { @@ -372,13 +378,6 @@ MAIN_START(argc, argv) exit(0); } - if (padfScaleMin == nullptr || padfScaleMax == nullptr) - { - fprintf(stderr, "-equalize or -config filename command line options " - "must be specified.\n"); - exit(1); - } - /* ==================================================================== */ /* Create a virtual dataset. */ /* ==================================================================== */ diff --git a/apps/gdallocationinfo.cpp b/apps/gdallocationinfo.cpp index b5c3c42117da..bcd6e5225141 100644 --- a/apps/gdallocationinfo.cpp +++ b/apps/gdallocationinfo.cpp @@ -26,12 +26,6 @@ #include -#ifdef _WIN32 -#include -#else -#include -#endif - /************************************************************************/ /* GetSRSAsWKT */ /************************************************************************/ @@ -300,7 +294,7 @@ MAIN_START(argc, argv) if (std::isnan(dfGeoX)) { // Is it an interactive terminal ? - if (isatty(static_cast(fileno(stdin)))) + if (CPLIsInteractive(stdin)) { if (!osSourceSRS.empty()) { diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 9b429ec7fe97..11b3a42c735b 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -1269,7 +1269,7 @@ GDALDatasetH GDALTileIndex(const char *pszDest, int nSrcCount, auto poRing = std::make_unique(); for (int k = 0; k < 5; k++) poRing->addPoint(adfX[k], adfY[k]); - poPoly->addRingDirectly(poRing.release()); + poPoly->addRing(std::move(poRing)); poFeature->SetGeometryDirectly(poPoly.release()); if (poLayer->CreateFeature(poFeature.get()) != OGRERR_NONE) diff --git a/apps/gdaltransform.cpp b/apps/gdaltransform.cpp index 6455a060c5d5..7b77da3dd10e 100644 --- a/apps/gdaltransform.cpp +++ b/apps/gdaltransform.cpp @@ -30,12 +30,6 @@ #include "ogr_srs_api.h" #include "commonutils.h" -#ifdef _WIN32 -#include -#else -#include -#endif - /************************************************************************/ /* Usage() */ /************************************************************************/ @@ -359,7 +353,7 @@ MAIN_START(argc, argv) if (!bCoordOnCommandLine) { // Is it an interactive terminal ? - if (isatty(static_cast(fileno(stdin)))) + if (CPLIsInteractive(stdin)) { if (pszSrcFilename != nullptr) { diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 093e5aae7183..737a161f96b7 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -2552,8 +2552,8 @@ static GDALDatasetH GDALWarpDirect(const char *pszDest, GDALDatasetH hDstDS, { CPLString osMsg; osMsg.Printf("Processing %s [%d/%d]", - GDALGetDescription(pahSrcDS[iSrc]), iSrc + 1, - nSrcCount); + CPLGetFilename(GDALGetDescription(pahSrcDS[iSrc])), + iSrc + 1, nSrcCount); return pfnExternalProgress((iSrc + dfComplete) / nSrcCount, osMsg.c_str(), pExternalProgressData); } @@ -3427,7 +3427,7 @@ static CPLErr LoadCutline(const std::string &osCutlineDSNameOrWKT, OGRwkbGeometryType eType = wkbFlatten(poGeom->getGeometryType()); if (eType == wkbPolygon) - poMultiPolygon->addGeometryDirectly(poGeom.release()); + poMultiPolygon->addGeometry(std::move(poGeom)); else if (eType == wkbMultiPolygon) { for (const auto *poSubGeom : poGeom->toMultiPolygon()) @@ -5401,7 +5401,7 @@ static CPLErr TransformCutlineToSource(GDALDataset *poSrcDS, { const double dfCutlineBlendDist = CPLAtof(CSLFetchNameValueDef( *ppapszWarpOptions, "CUTLINE_BLEND_DIST", "0")); - OGRLinearRing *poRing = new OGRLinearRing(); + auto poRing = std::make_unique(); poRing->addPoint(-dfCutlineBlendDist, -dfCutlineBlendDist); poRing->addPoint(-dfCutlineBlendDist, dfCutlineBlendDist + poSrcDS->GetRasterYSize()); @@ -5411,7 +5411,7 @@ static CPLErr TransformCutlineToSource(GDALDataset *poSrcDS, -dfCutlineBlendDist); poRing->addPoint(-dfCutlineBlendDist, -dfCutlineBlendDist); OGRPolygon oSrcDSFootprint; - oSrcDSFootprint.addRingDirectly(poRing); + oSrcDSFootprint.addRing(std::move(poRing)); OGREnvelope sSrcDSEnvelope; oSrcDSFootprint.getEnvelope(&sSrcDSEnvelope); OGREnvelope sCutlineEnvelope; diff --git a/apps/gdalwarpsimple.c b/apps/gdalwarpsimple.c deleted file mode 100644 index 781b9a03bbfd..000000000000 --- a/apps/gdalwarpsimple.c +++ /dev/null @@ -1,451 +0,0 @@ -/****************************************************************************** - * - * Project: Mapinfo Image Warper - * Purpose: Commandline program for doing a variety of image warps, including - * image reprojection. - * Author: Frank Warmerdam - * - ****************************************************************************** - * Copyright (c) 2002, i3 - information integration and imaging - * Fort Collin, CO - * - * SPDX-License-Identifier: MIT - ****************************************************************************/ - -#include "gdal_alg.h" -#include "cpl_string.h" -#include "ogr_srs_api.h" - -static GDALDatasetH GDALWarpCreateOutput(GDALDatasetH hSrcDS, - const char *pszFilename, - const char *pszFormat, - const char *pszSourceSRS, - const char *pszTargetSRS, int nOrder, - char **papszCreateOptions); - -static double dfMinX = 0.0, dfMinY = 0.0, dfMaxX = 0.0, dfMaxY = 0.0; -static double dfXRes = 0.0, dfYRes = 0.0; -static int nForcePixels = 0, nForceLines = 0; - -/************************************************************************/ -/* Usage() */ -/************************************************************************/ - -static void Usage() - -{ - printf("Usage: gdalwarpsimple [--version] [--formats]\n" - " [-s_srs ] [-t_srs ] [-order ] [-et " - "]\n" - " [-te ] [-tr ] [-ts " - " ]\n" - " [-of ] [-co =]... \n"); - exit(1); -} - -/************************************************************************/ -/* SanitizeSRS */ -/************************************************************************/ - -char *SanitizeSRS(const char *pszUserInput) - -{ - OGRSpatialReferenceH hSRS; - char *pszResult = NULL; - - CPLErrorReset(); - - hSRS = OSRNewSpatialReference(NULL); - if (OSRSetFromUserInput(hSRS, pszUserInput) == OGRERR_NONE) - OSRExportToWkt(hSRS, &pszResult); - else - { - CPLError(CE_Failure, CPLE_AppDefined, - "Translating source or target SRS failed:\n%s", pszUserInput); - exit(1); - } - - OSRDestroySpatialReference(hSRS); - - return pszResult; -} - -/************************************************************************/ -/* main() */ -/************************************************************************/ - -int main(int argc, char **argv) - -{ - GDALDatasetH hSrcDS, hDstDS; - const char *pszFormat = "GTiff"; - char *pszTargetSRS = NULL; - char *pszSourceSRS = NULL; - const char *pszSrcFilename = NULL, *pszDstFilename = NULL; - int bCreateOutput = FALSE, i, nOrder = 0; - void *hTransformArg, *hGenImgProjArg = NULL, *hApproxArg = NULL; - char **papszWarpOptions = NULL; - double dfErrorThreshold = 0.125; - GDALTransformerFunc pfnTransformer = NULL; - char **papszCreateOptions = NULL; - - GDALAllRegister(); - - /* -------------------------------------------------------------------- */ - /* Parse arguments. */ - /* -------------------------------------------------------------------- */ - for (i = 1; i < argc; i++) - { - if (EQUAL(argv[i], "--version")) - { - printf("%s\n", GDALVersionInfo("--version")); - exit(0); - } - else if (EQUAL(argv[i], "--formats")) - { - int iDr; - - printf("Supported Formats:\n"); - for (iDr = 0; iDr < GDALGetDriverCount(); iDr++) - { - GDALDriverH hDriver = GDALGetDriver(iDr); - - printf(" %s: %s\n", GDALGetDriverShortName(hDriver), - GDALGetDriverLongName(hDriver)); - } - - exit(0); - } - else if (EQUAL(argv[i], "-co") && i < argc - 1) - { - papszCreateOptions = CSLAddString(papszCreateOptions, argv[++i]); - bCreateOutput = TRUE; - } - else if (EQUAL(argv[i], "-of") && i < argc - 1) - { - pszFormat = argv[++i]; - bCreateOutput = TRUE; - } - else if (EQUAL(argv[i], "-t_srs") && i < argc - 1) - { - pszTargetSRS = SanitizeSRS(argv[++i]); - } - else if (EQUAL(argv[i], "-s_srs") && i < argc - 1) - { - pszSourceSRS = SanitizeSRS(argv[++i]); - } - else if (EQUAL(argv[i], "-order") && i < argc - 1) - { - nOrder = atoi(argv[++i]); - } - else if (EQUAL(argv[i], "-et") && i < argc - 1) - { - dfErrorThreshold = CPLAtof(argv[++i]); - } - else if (EQUAL(argv[i], "-tr") && i < argc - 2) - { - dfXRes = CPLAtof(argv[++i]); - dfYRes = fabs(CPLAtof(argv[++i])); - bCreateOutput = TRUE; - } - else if (EQUAL(argv[i], "-ts") && i < argc - 2) - { - nForcePixels = atoi(argv[++i]); - nForceLines = atoi(argv[++i]); - bCreateOutput = TRUE; - } - else if (EQUAL(argv[i], "-te") && i < argc - 4) - { - dfMinX = CPLAtof(argv[++i]); - dfMinY = CPLAtof(argv[++i]); - dfMaxX = CPLAtof(argv[++i]); - dfMaxY = CPLAtof(argv[++i]); - bCreateOutput = TRUE; - } - else if (argv[i][0] == '-') - Usage(); - else if (pszSrcFilename == NULL) - pszSrcFilename = argv[i]; - else if (pszDstFilename == NULL) - pszDstFilename = argv[i]; - else - Usage(); - } - - if (pszDstFilename == NULL) - Usage(); - - /* -------------------------------------------------------------------- */ - /* Open source dataset. */ - /* -------------------------------------------------------------------- */ - hSrcDS = GDALOpen(pszSrcFilename, GA_ReadOnly); - - if (hSrcDS == NULL) - exit(2); - - /* -------------------------------------------------------------------- */ - /* Check that there's at least one raster band */ - /* -------------------------------------------------------------------- */ - if (GDALGetRasterCount(hSrcDS) == 0) - { - fprintf(stderr, "Input file %s has no raster bands.\n", pszSrcFilename); - exit(2); - } - - if (pszSourceSRS == NULL) - { - if (GDALGetProjectionRef(hSrcDS) != NULL && - strlen(GDALGetProjectionRef(hSrcDS)) > 0) - pszSourceSRS = CPLStrdup(GDALGetProjectionRef(hSrcDS)); - - else if (GDALGetGCPProjection(hSrcDS) != NULL && - strlen(GDALGetGCPProjection(hSrcDS)) > 0 && - GDALGetGCPCount(hSrcDS) > 1) - pszSourceSRS = CPLStrdup(GDALGetGCPProjection(hSrcDS)); - else - pszSourceSRS = CPLStrdup(""); - } - - if (pszTargetSRS == NULL) - pszTargetSRS = CPLStrdup(pszSourceSRS); - - /* -------------------------------------------------------------------- */ - /* Does the output dataset already exist? */ - /* -------------------------------------------------------------------- */ - CPLPushErrorHandler(CPLQuietErrorHandler); - hDstDS = GDALOpen(pszDstFilename, GA_Update); - CPLPopErrorHandler(); - - if (hDstDS != NULL && bCreateOutput) - { - fprintf( - stderr, - "Output dataset %s exists,\n" - "but some commandline options were provided indicating a new " - "dataset\n" - "should be created. Please delete existing dataset and run again.", - pszDstFilename); - exit(1); - } - - /* -------------------------------------------------------------------- */ - /* If not, we need to create it. */ - /* -------------------------------------------------------------------- */ - if (hDstDS == NULL) - { - hDstDS = GDALWarpCreateOutput(hSrcDS, pszDstFilename, pszFormat, - pszSourceSRS, pszTargetSRS, nOrder, - papszCreateOptions); - papszWarpOptions = CSLSetNameValue(papszWarpOptions, "INIT", "0"); - CSLDestroy(papszCreateOptions); - papszCreateOptions = NULL; - } - - if (hDstDS == NULL) - exit(1); - - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - hTransformArg = hGenImgProjArg = GDALCreateGenImgProjTransformer( - hSrcDS, pszSourceSRS, hDstDS, pszTargetSRS, TRUE, 1000.0, nOrder); - - if (hTransformArg == NULL) - exit(1); - - pfnTransformer = GDALGenImgProjTransform; - - /* -------------------------------------------------------------------- */ - /* Warp the transformer with a linear approximator unless the */ - /* acceptable error is zero. */ - /* -------------------------------------------------------------------- */ - if (dfErrorThreshold != 0.0) - { - hTransformArg = hApproxArg = GDALCreateApproxTransformer( - GDALGenImgProjTransform, hGenImgProjArg, dfErrorThreshold); - pfnTransformer = GDALApproxTransform; - } - - /* -------------------------------------------------------------------- */ - /* Now actually invoke the warper to do the work. */ - /* -------------------------------------------------------------------- */ - GDALSimpleImageWarp(hSrcDS, hDstDS, 0, NULL, pfnTransformer, hTransformArg, - GDALTermProgress, NULL, papszWarpOptions); - - CSLDestroy(papszWarpOptions); - - if (hApproxArg != NULL) - GDALDestroyApproxTransformer(hApproxArg); - - if (hGenImgProjArg != NULL) - GDALDestroyGenImgProjTransformer(hGenImgProjArg); - - /* -------------------------------------------------------------------- */ - /* Cleanup. */ - /* -------------------------------------------------------------------- */ - GDALClose(hDstDS); - GDALClose(hSrcDS); - - GDALDumpOpenDatasets(stderr); - - GDALDestroyDriverManager(); - - exit(0); -} - -/************************************************************************/ -/* GDALWarpCreateOutput() */ -/* */ -/* Create the output file based on various commandline options, */ -/* and the input file. */ -/************************************************************************/ - -static GDALDatasetH GDALWarpCreateOutput(GDALDatasetH hSrcDS, - const char *pszFilename, - const char *pszFormat, - const char *pszSourceSRS, - const char *pszTargetSRS, int nOrder, - char **papszCreateOptions) - -{ - GDALDriverH hDriver; - GDALDatasetH hDstDS; - void *hTransformArg; - double adfDstGeoTransform[6]; - int nPixels = 0, nLines = 0; - GDALColorTableH hCT; - - /* -------------------------------------------------------------------- */ - /* Find the output driver. */ - /* -------------------------------------------------------------------- */ - hDriver = GDALGetDriverByName(pszFormat); - if (hDriver == NULL || - GDALGetMetadataItem(hDriver, GDAL_DCAP_CREATE, NULL) == NULL) - { - int iDr; - - printf("Output driver `%s' not recognised or does not support\n", - pszFormat); - printf("direct output file creation. The following format drivers are " - "configured\n" - "and support direct output:\n"); - - for (iDr = 0; iDr < GDALGetDriverCount(); iDr++) - { - GDALDriverH hDriver = GDALGetDriver(iDr); - - if (GDALGetMetadataItem(hDriver, GDAL_DCAP_CREATE, NULL) != NULL) - { - printf(" %s: %s\n", GDALGetDriverShortName(hDriver), - GDALGetDriverLongName(hDriver)); - } - } - printf("\n"); - exit(1); - } - - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - hTransformArg = GDALCreateGenImgProjTransformer( - hSrcDS, pszSourceSRS, NULL, pszTargetSRS, TRUE, 1000.0, nOrder); - - if (hTransformArg == NULL) - return NULL; - - /* -------------------------------------------------------------------- */ - /* Get approximate output definition. */ - /* -------------------------------------------------------------------- */ - if (GDALSuggestedWarpOutput(hSrcDS, GDALGenImgProjTransform, hTransformArg, - adfDstGeoTransform, &nPixels, - &nLines) != CE_None) - return NULL; - - GDALDestroyGenImgProjTransformer(hTransformArg); - - /* -------------------------------------------------------------------- */ - /* Did the user override some parameters? */ - /* -------------------------------------------------------------------- */ - if (dfXRes != 0.0 && dfYRes != 0.0) - { - CPLAssert(nPixels == 0 && nLines == 0); - if (dfMinX == 0.0 && dfMinY == 0.0 && dfMaxX == 0.0 && dfMaxY == 0.0) - { - dfMinX = adfDstGeoTransform[0]; - dfMaxX = adfDstGeoTransform[0] + adfDstGeoTransform[1] * nPixels; - dfMaxY = adfDstGeoTransform[3]; - dfMinY = adfDstGeoTransform[3] + adfDstGeoTransform[5] * nLines; - } - - nPixels = (int)((dfMaxX - dfMinX + (dfXRes / 2.0)) / dfXRes); - nLines = (int)((dfMaxY - dfMinY + (dfYRes / 2.0)) / dfYRes); - adfDstGeoTransform[0] = dfMinX; - adfDstGeoTransform[3] = dfMaxY; - adfDstGeoTransform[1] = dfXRes; - adfDstGeoTransform[5] = -dfYRes; - } - - else if (nForcePixels != 0 && nForceLines != 0) - { - if (dfMinX == 0.0 && dfMinY == 0.0 && dfMaxX == 0.0 && dfMaxY == 0.0) - { - dfMinX = adfDstGeoTransform[0]; - dfMaxX = adfDstGeoTransform[0] + adfDstGeoTransform[1] * nPixels; - dfMaxY = adfDstGeoTransform[3]; - dfMinY = adfDstGeoTransform[3] + adfDstGeoTransform[5] * nLines; - } - - dfXRes = (dfMaxX - dfMinX) / nForcePixels; - dfYRes = (dfMaxY - dfMinY) / nForceLines; - - adfDstGeoTransform[0] = dfMinX; - adfDstGeoTransform[3] = dfMaxY; - adfDstGeoTransform[1] = dfXRes; - adfDstGeoTransform[5] = -dfYRes; - - nPixels = nForcePixels; - nLines = nForceLines; - } - - else if (dfMinX != 0.0 || dfMinY != 0.0 || dfMaxX != 0.0 || dfMaxY != 0.0) - { - dfXRes = adfDstGeoTransform[1]; - dfYRes = fabs(adfDstGeoTransform[5]); - - nPixels = (int)((dfMaxX - dfMinX + (dfXRes / 2.0)) / dfXRes); - nLines = (int)((dfMaxY - dfMinY + (dfYRes / 2.0)) / dfYRes); - - adfDstGeoTransform[0] = dfMinX; - adfDstGeoTransform[3] = dfMaxY; - } - - /* -------------------------------------------------------------------- */ - /* Create the output file. */ - /* -------------------------------------------------------------------- */ - printf("Creating output file is that %dP x %dL.\n", nPixels, nLines); - - hDstDS = GDALCreate(hDriver, pszFilename, nPixels, nLines, - GDALGetRasterCount(hSrcDS), - GDALGetRasterDataType(GDALGetRasterBand(hSrcDS, 1)), - papszCreateOptions); - - if (hDstDS == NULL) - return NULL; - - /* -------------------------------------------------------------------- */ - /* Write out the projection definition. */ - /* -------------------------------------------------------------------- */ - GDALSetProjection(hDstDS, pszTargetSRS); - GDALSetGeoTransform(hDstDS, adfDstGeoTransform); - - /* -------------------------------------------------------------------- */ - /* Copy the color table, if required. */ - /* -------------------------------------------------------------------- */ - hCT = GDALGetRasterColorTable(GDALGetRasterBand(hSrcDS, 1)); - if (hCT != NULL) - GDALSetRasterColorTable(GDALGetRasterBand(hDstDS, 1), hCT); - - return hDstDS; -} diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 97e1568f8218..563430c9ae51 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -688,13 +688,13 @@ static std::unique_ptr LoadGeometry(const std::string &osDS, "should be manually inspected.", poFeat->GetFID(), osDS.c_str()); - oGC.addGeometryDirectly(poValid.release()); + oGC.addGeometry(std::move(poValid)); } else { CPLError(CE_Failure, CPLE_AppDefined, "Geometry of feature " CPL_FRMT_GIB " of %s " - "is invalid, and could not been made valid.", + "is invalid, and could not be made valid.", poFeat->GetFID(), osDS.c_str()); oGC.empty(); break; @@ -702,7 +702,7 @@ static std::unique_ptr LoadGeometry(const std::string &osDS, } else { - oGC.addGeometryDirectly(poSrcGeom.release()); + oGC.addGeometry(std::move(poSrcGeom)); } } } diff --git a/apps/ogrdissolve.cpp b/apps/ogrdissolve.cpp deleted file mode 100644 index 233c884920cc..000000000000 --- a/apps/ogrdissolve.cpp +++ /dev/null @@ -1,1211 +0,0 @@ -/****************************************************************************** - * - * Project: OpenGIS Simple Features Reference Implementation - * Purpose: Allow a user to dissolve geometries based on an attribute. - * Author: Howard Butler, hobu.inc@gmail.com - * - ****************************************************************************** - * Copyright (c) 2007, Howard Butler - * - * SPDX-License-Identifier: MIT - ****************************************************************************/ - -#include "ogrsf_frmts.h" -#include "ogr_p.h" -#include "cpl_conv.h" -#include "cpl_string.h" -#include "ogr_api.h" -#include "commonutils.h" -#include -#include - -static void Usage(); - -static int DissolveLayer(OGRDataSource *poSrcDS, OGRLayer *poSrcLayer, - OGRDataSource *poDstDS, char **papszLSCO, - const char *pszNewLayerName, int bTransform, - OGRSpatialReference *poOutputSRS, - OGRSpatialReference *poSourceSRS, - char **papszSelFields, int bAppend, int eGType, - int bOverwrite); - -static int bSkipFailures = FALSE; -static int nGroupTransactions = 200; -static int bPreserveFID = FALSE; -static int nFIDToFetch = OGRNullFID; - -typedef std::multimap StringGeometryMMap; -typedef std::map StringGeometryColMap; -typedef std::map StringGeometryMap; -typedef std::list GeometriesList; - -/************************************************************************/ -/* main() */ -/************************************************************************/ - -MAIN_START(nArgc, papszArgv) - -{ - const char *pszFormat = "ESRI Shapefile"; - const char *pszDataSource = NULL; - const char *pszDestDataSource = NULL; - char **papszLayers = NULL; - char **papszDSCO = NULL, **papszLCO = NULL; - int bTransform = FALSE; - int bAppend = FALSE, bUpdate = FALSE, bOverwrite = FALSE; - const char *pszOutputSRSDef = NULL; - const char *pszSourceSRSDef = NULL; - OGRSpatialReference *poOutputSRS = NULL; - OGRSpatialReference *poSourceSRS = NULL; - const char *pszNewLayerName = NULL; - const char *pszWHERE = NULL; - OGRGeometry *poSpatialFilter = NULL; - const char *pszSelect; - char **papszSelFields = NULL; - const char *pszSQLStatement = NULL; - int eGType = -2; - - /* -------------------------------------------------------------------- */ - /* Register format(s). */ - /* -------------------------------------------------------------------- */ - OGRRegisterAll(); - - /* -------------------------------------------------------------------- */ - /* Processing command line arguments. */ - /* -------------------------------------------------------------------- */ - nArgc = OGRGeneralCmdLineProcessor(nArgc, &papszArgv, 0); - - if (nArgc < 1) - exit(-nArgc); - - for (int iArg = 1; iArg < nArgc; iArg++) - { - if (EQUAL(papszArgv[iArg], "--help")) - { - Usage(); - } - if ((EQUAL(papszArgv[iArg], "-f") || EQUAL(papszArgv[iArg], "-of")) && - iArg < nArgc - 1) - { - pszFormat = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-dsco") && iArg < nArgc - 1) - { - papszDSCO = CSLAddString(papszDSCO, papszArgv[++iArg]); - } - else if (EQUAL(papszArgv[iArg], "-lco") && iArg < nArgc - 1) - { - papszLCO = CSLAddString(papszLCO, papszArgv[++iArg]); - } - else if (EQUAL(papszArgv[iArg], "-preserve_fid")) - { - bPreserveFID = TRUE; - } - else if (STARTS_WITH_CI(papszArgv[iArg], "-skip")) - { - bSkipFailures = TRUE; - } - else if (EQUAL(papszArgv[iArg], "-append")) - { - bAppend = TRUE; - } - else if (EQUAL(papszArgv[iArg], "-overwrite")) - { - bOverwrite = TRUE; - } - else if (EQUAL(papszArgv[iArg], "-update")) - { - bUpdate = TRUE; - } - else if (EQUAL(papszArgv[iArg], "-fid") && papszArgv[iArg + 1] != NULL) - { - nFIDToFetch = atoi(papszArgv[++iArg]); - } - else if (EQUAL(papszArgv[iArg], "-sql") && papszArgv[iArg + 1] != NULL) - { - pszSQLStatement = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-nln") && iArg < nArgc - 1) - { - pszNewLayerName = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-nlt") && iArg < nArgc - 1) - { - int bIs3D = FALSE; - CPLString osGeomName = papszArgv[iArg + 1]; - if (strlen(papszArgv[iArg + 1]) > 3 && - STARTS_WITH_CI(papszArgv[iArg + 1] + - strlen(papszArgv[iArg + 1]) - 3, - "25D")) - { - bIs3D = TRUE; - osGeomName.resize(osGeomName.size() - 3); - } - else if (strlen(papszArgv[iArg + 1]) > 1 && - STARTS_WITH_CI(papszArgv[iArg + 1] + - strlen(papszArgv[iArg + 1]) - 1, - "Z")) - { - bIs3D = TRUE; - osGeomName.pop_back(); - } - if (EQUAL(osGeomName, "NONE")) - eGType = wkbNone; - else if (EQUAL(osGeomName, "GEOMETRY")) - eGType = wkbUnknown; - else - { - eGType = OGRFromOGCGeomType(osGeomName); - if (eGType == wkbUnknown) - { - fprintf(stderr, "-nlt %s: type not recognised.\n", - papszArgv[iArg + 1]); - exit(1); - } - } - if (eGType != wkbNone && bIs3D) - eGType = wkbSetZ((OGRwkbGeometryType)eGType); - iArg++; - } - else if (EQUAL(papszArgv[iArg], "-tg") && iArg < nArgc - 1) - { - nGroupTransactions = atoi(papszArgv[++iArg]); - } - else if (EQUAL(papszArgv[iArg], "-s_srs") && iArg < nArgc - 1) - { - pszSourceSRSDef = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-a_srs") && iArg < nArgc - 1) - { - pszOutputSRSDef = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-t_srs") && iArg < nArgc - 1) - { - pszOutputSRSDef = papszArgv[++iArg]; - bTransform = TRUE; - } - else if (EQUAL(papszArgv[iArg], "-spat") && - papszArgv[iArg + 1] != NULL && papszArgv[iArg + 2] != NULL && - papszArgv[iArg + 3] != NULL && papszArgv[iArg + 4] != NULL) - { - OGRLinearRing oRing; - - oRing.addPoint(CPLAtof(papszArgv[iArg + 1]), - CPLAtof(papszArgv[iArg + 2])); - oRing.addPoint(CPLAtof(papszArgv[iArg + 1]), - CPLAtof(papszArgv[iArg + 4])); - oRing.addPoint(CPLAtof(papszArgv[iArg + 3]), - CPLAtof(papszArgv[iArg + 4])); - oRing.addPoint(CPLAtof(papszArgv[iArg + 3]), - CPLAtof(papszArgv[iArg + 2])); - oRing.addPoint(CPLAtof(papszArgv[iArg + 1]), - CPLAtof(papszArgv[iArg + 2])); - - poSpatialFilter = new OGRPolygon(); - poSpatialFilter->toPolygon()->addRing(&oRing); - iArg += 4; - } - else if (EQUAL(papszArgv[iArg], "-where") && - papszArgv[iArg + 1] != NULL) - { - pszWHERE = papszArgv[++iArg]; - } - else if (EQUAL(papszArgv[iArg], "-select") && - papszArgv[iArg + 1] != NULL) - { - pszSelect = papszArgv[++iArg]; - papszSelFields = - CSLTokenizeStringComplex(pszSelect, " ,", FALSE, FALSE); - } - else if (papszArgv[iArg][0] == '-') - { - Usage(); - } - else if (pszDestDataSource == NULL) - pszDestDataSource = papszArgv[iArg]; - else if (pszDataSource == NULL) - pszDataSource = papszArgv[iArg]; - else - papszLayers = CSLAddString(papszLayers, papszArgv[iArg]); - } - - if (pszDataSource == NULL) - Usage(); - - /* -------------------------------------------------------------------- */ - /* Open data source. */ - /* -------------------------------------------------------------------- */ - OGRDataSource *poDS; - - poDS = OGRSFDriverRegistrar::Open(pszDataSource, FALSE); - - /* -------------------------------------------------------------------- */ - /* Report failure */ - /* -------------------------------------------------------------------- */ - if (poDS == NULL) - { - OGRSFDriverRegistrar *poR = OGRSFDriverRegistrar::GetRegistrar(); - - printf("FAILURE:\n" - "Unable to open datasource `%s' with the following drivers.\n", - pszDataSource); - - for (int iDriver = 0; iDriver < poR->GetDriverCount(); iDriver++) - { - printf(" -> %s\n", poR->GetDriver(iDriver)->GetName()); - } - - exit(1); - } - - /* -------------------------------------------------------------------- */ - /* Try opening the output datasource as an existing, writable */ - /* -------------------------------------------------------------------- */ - OGRDataSource *poODS; - - if (bUpdate) - { - poODS = OGRSFDriverRegistrar::Open(pszDestDataSource, TRUE); - if (poODS == NULL) - { - printf("FAILURE:\n" - "Unable to open existing output datasource `%s'.\n", - pszDestDataSource); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Find the output driver. */ - /* -------------------------------------------------------------------- */ - else - { - OGRSFDriverRegistrar *poR = OGRSFDriverRegistrar::GetRegistrar(); - OGRSFDriver *poDriver = NULL; - int iDriver; - - for (iDriver = 0; iDriver < poR->GetDriverCount() && poDriver == NULL; - iDriver++) - { - if (EQUAL(poR->GetDriver(iDriver)->GetName(), pszFormat)) - { - poDriver = poR->GetDriver(iDriver); - } - } - - if (poDriver == NULL) - { - printf("Unable to find driver `%s'.\n", pszFormat); - printf("The following drivers are available:\n"); - - for (iDriver = 0; iDriver < poR->GetDriverCount(); iDriver++) - { - printf(" -> `%s'\n", poR->GetDriver(iDriver)->GetName()); - } - exit(1); - } - - if (!poDriver->TestCapability(ODrCCreateDataSource)) - { - printf("%s driver does not support data source creation.\n", - pszFormat); - exit(1); - } - - /* -------------------------------------------------------------------- - */ - /* Create the output data source. */ - /* -------------------------------------------------------------------- - */ - poODS = poDriver->CreateDataSource(pszDestDataSource, papszDSCO); - if (poODS == NULL) - { - printf("%s driver failed to create %s\n", pszFormat, - pszDestDataSource); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Parse the output SRS definition if possible. */ - /* -------------------------------------------------------------------- */ - if (pszOutputSRSDef != NULL) - { - poOutputSRS = new OGRSpatialReference(); - poOutputSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (poOutputSRS->SetFromUserInput(pszOutputSRSDef) != OGRERR_NONE) - { - printf("Failed to process SRS definition: %s\n", pszOutputSRSDef); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Parse the source SRS definition if possible. */ - /* -------------------------------------------------------------------- */ - if (pszSourceSRSDef != NULL) - { - poSourceSRS = new OGRSpatialReference(); - poSourceSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (poSourceSRS->SetFromUserInput(pszSourceSRSDef) != OGRERR_NONE) - { - printf("Failed to process SRS definition: %s\n", pszSourceSRSDef); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Special case for -sql clause. No source layers required. */ - /* -------------------------------------------------------------------- */ - if (pszSQLStatement != NULL) - { - OGRLayer *poResultSet; - - if (pszWHERE != NULL) - printf("-where clause ignored in combination with -sql.\n"); - if (CSLCount(papszLayers) > 0) - printf("layer names ignored in combination with -sql.\n"); - - poResultSet = poDS->ExecuteSQL(pszSQLStatement, poSpatialFilter, NULL); - - if (poResultSet != NULL) - { - if (!DissolveLayer(poDS, poResultSet, poODS, papszLCO, - pszNewLayerName, bTransform, poOutputSRS, - poSourceSRS, papszSelFields, bAppend, eGType, - bOverwrite)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Terminating translation prematurely after failed\n" - "translation from sql statement."); - - exit(1); - } - poDS->ReleaseResultSet(poResultSet); - } - } - - /* -------------------------------------------------------------------- */ - /* Process each data source layer. */ - /* -------------------------------------------------------------------- */ - for (int iLayer = 0; - pszSQLStatement == NULL && iLayer < poDS->GetLayerCount(); iLayer++) - { - OGRLayer *poLayer = poDS->GetLayer(iLayer); - - if (poLayer == NULL) - { - printf("FAILURE: Couldn't fetch advertised layer %d!\n", iLayer); - exit(1); - } - - if (CSLCount(papszLayers) == 0 || - CSLFindString(papszLayers, poLayer->GetLayerDefn()->GetName()) != - -1) - { - if (pszWHERE != NULL) - poLayer->SetAttributeFilter(pszWHERE); - - if (poSpatialFilter != NULL) - poLayer->SetSpatialFilter(poSpatialFilter); - - if (!DissolveLayer(poDS, poLayer, poODS, papszLCO, pszNewLayerName, - bTransform, poOutputSRS, poSourceSRS, - papszSelFields, bAppend, eGType, bOverwrite) && - !bSkipFailures) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Terminating translation prematurely after failed\n" - "translation of layer %s\n", - poLayer->GetLayerDefn()->GetName()); - - exit(1); - } - } - } - - /* -------------------------------------------------------------------- */ - /* Close down. */ - /* -------------------------------------------------------------------- */ - OGRSpatialReference::DestroySpatialReference(poOutputSRS); - OGRSpatialReference::DestroySpatialReference(poSourceSRS); - OGRDataSource::DestroyDataSource(poODS); - OGRDataSource::DestroyDataSource(poDS); - - CSLDestroy(papszSelFields); - CSLDestroy(papszArgv); - CSLDestroy(papszLayers); - CSLDestroy(papszDSCO); - CSLDestroy(papszLCO); - - OGRCleanupAll(); - -#ifdef DBMALLOC - malloc_dump(1); -#endif - - return 0; -} - -MAIN_END - -/************************************************************************/ -/* Usage() */ -/************************************************************************/ - -static void Usage() - -{ - OGRSFDriverRegistrar *poR = OGRSFDriverRegistrar::GetRegistrar(); - - printf("Usage: ogr2ogr [--help] [--help-general]\n" - " [-skipfailures] [-append] [-update]\n" - " [-select field_list] [-where restricted_where] \n" - " [-sql ] \n" - " [-spat xmin ymin xmax ymax] [-preserve_fid] [-fid " - "FID]\n" - " [-a_srs srs_def] [-t_srs srs_def] [-s_srs srs_def]\n" - " [-f format_name] [-overwrite] [[-dsco NAME=VALUE] " - "...]\n" - " dst_datasource_name src_datasource_name\n" - " [-lco NAME=VALUE] [-nln name] [-nlt type] [layer " - "[layer ...]]\n" - "\n" - " -f format_name: output file format name, possible values are:\n"); - - for (int iDriver = 0; iDriver < poR->GetDriverCount(); iDriver++) - { - OGRSFDriver *poDriver = poR->GetDriver(iDriver); - - if (poDriver->TestCapability(ODrCCreateDataSource)) - printf(" -f \"%s\"\n", poDriver->GetName()); - } - - printf( - " -append: Append to existing layer instead of creating new if it " - "exists\n" - " -overwrite: delete the output layer and recreate it empty\n" - " -update: Open existing output datasource in update mode\n" - " -select field_list: Comma-delimited list of fields from input layer " - "to\n" - " copy to the new layer (defaults to all)\n" - " -where restricted_where: Attribute query (like SQL WHERE)\n" - " -sql statement: Execute given SQL statement and save result.\n" - " -skipfailures: skip features or layers that fail to convert\n" - " -spat xmin ymin xmax ymax: spatial query extents\n" - " -dsco NAME=VALUE: Dataset creation option (format specific)\n" - " -lco NAME=VALUE: Layer creation option (format specific)\n" - " -nln name: Assign an alternate name to the new layer\n" - " -nlt type: Force a geometry type for new layer. One of NONE, " - "GEOMETRY,\n" - " POINT, LINESTRING, POLYGON, GEOMETRYCOLLECTION, MULTIPOINT, " - "MULTILINE,\n" - " MULTIPOLYGON, or MULTILINESTRING. Add \"25D\" for 3D layers.\n" - " Default is type of source layer.\n"); - - printf(" -a_srs srs_def: Assign an output SRS\n" - " -t_srs srs_def: Reproject/transform to this SRS on output\n" - " -s_srs srs_def: Override source SRS\n" - "\n" - " Srs_def can be a full WKT definition (hard to escape properly),\n" - " or a well known definition (i.e. EPSG:4326) or a file with a WKT\n" - " definition.\n"); - - exit(1); -} - -StringGeometryMap *CollectGeometries(OGRLayer *poSrcLayer, - const char **papszFields) -{ - - /* -------------------------------------------------------------------- */ - /* CollectGeometries returns a dictionary where the keys are the */ - /* values in the fields that the user has selected and the values */ - /* are a GeometryCollection of all of the geometries for records */ - /* with that value. */ - /* -------------------------------------------------------------------- */ - - StringGeometryMMap poGeometriesMap; - - poSrcLayer->ResetReading(); - - int iField; - /* -------------------------------------------------------------------- */ - /* Get all of the features and put them in a multi map. This may */ - /* include values for which the selected fields is NULL. */ - /* -------------------------------------------------------------------- */ - - for (auto &poFeature : poSrcLayer) - { - CPLString poKey(""); - - for (iField = 0; papszFields[iField] != NULL; iField++) - { - int nField = poFeature->GetFieldIndex(papszFields[iField]); - poKey = poKey + poFeature->GetFieldAsString(nField); - } - - if (poFeature->GetGeometryRef()->IsValid()) - { - poGeometriesMap.insert( - std::make_pair(CPLString(poKey), poFeature->GetGeometryRef())); - } - else - { - CPLDebug("CollectGeometries", - "Geometry was invalid not adding!!!!"); - } - } - - /* -------------------------------------------------------------------- */ - /* Loop through our features map and get a unique list of field */ - /* values. This could be done using something other than a map */ - /* of course, but it was convenient. */ - /* -------------------------------------------------------------------- */ - - typedef std::map StringIntMap; - StringIntMap::const_iterator ipos; - StringIntMap poFieldsmap; - - StringGeometryMMap::const_iterator pos; - - for (pos = poGeometriesMap.begin(); pos != poGeometriesMap.end(); ++pos) - { - - /* we currently throw out any null field values at this time */ - // if (!(pos->first.empty())) { - poFieldsmap[CPLString(pos->first.c_str())] = 1; - // } - } - - /* -------------------------------------------------------------------- */ - /* Make a new map of GeometryCollection for each value in the */ - /* poFieldsmap. This is a 1:1 relationship, and all of the */ - /* geometries for a given field are all put into the same */ - /* GeometryCollection. After we build the poCollections, we will */ - /* use GEOS' buffer(0) trick to have GEOS perform the segmentation */ - /* -------------------------------------------------------------------- */ - - StringGeometryColMap poCollections; - - CPLDebug("CollectGeometries", "Field map size: %d", poFieldsmap.size()); - - for (ipos = poFieldsmap.begin(); ipos != poFieldsmap.end(); ++ipos) - { - - CPLString fid = ipos->first; - CPLDebug("CollectGeometries", "First %s Second %d", ipos->first.c_str(), - ipos->second); - - OGRGeometryCollection *geom = new OGRGeometryCollection; - - for (pos = poGeometriesMap.lower_bound(fid); - pos != poGeometriesMap.upper_bound(fid); ++pos) - { - geom->addGeometry(pos->second); - } - poCollections.insert(std::make_pair(fid, geom)); - } - - CPLDebug("CollectGeometries", "Geo map size: %d", poCollections.size()); - - /* -------------------------------------------------------------------- */ - /* Loop through our poCollections map and buffer(0) each */ - /* GeometryCollection. GEOS will collapse the geometries down */ - /* -------------------------------------------------------------------- */ - - StringGeometryMap *buffers = new StringGeometryMap; - - StringGeometryColMap::const_iterator collections_i; - - for (collections_i = poCollections.begin(); - collections_i != poCollections.end(); ++collections_i) - { - CPLDebug("CollectGeometries", "poCollections Geometry size %d", - collections_i->second->getNumGeometries()); - OGRGeometry *buffer = collections_i->second->Buffer(0); - buffers->insert(std::make_pair(collections_i->first, buffer)); - } - - for (collections_i = poCollections.begin(); - collections_i != poCollections.end(); ++collections_i) - { - delete collections_i->second; - } - - return buffers; -} - -GeometriesList *FlattenGeometries(GeometriesList *input) -{ - - GeometriesList::const_iterator geometry_i; - GeometriesList *output = new GeometriesList; - - CPLDebug("CollectGeometries", - "Input geometries in FlattenGeometries size: %d", input->size()); - for (geometry_i = input->begin(); geometry_i != input->end(); ++geometry_i) - { - - OGRGeometry *buffer = (*geometry_i); - // int nGeometries = buffer->getNumGeometries(); - OGRwkbGeometryType iGType = buffer->getGeometryType(); - - if (iGType == wkbPolygon) - { - output->push_back(buffer); - CPLDebug("CollectGeometries", - "Collapsing wkbPolygon geometries......"); - } - if (iGType == wkbMultiPolygon) - { - OGRMultiPolygon *geom = buffer->toMultiPolygon(); - for (int i = 0; i < geom->getNumGeometries(); i++) - { - OGRPolygon *g = geom->getGeometryRef(i)->toPolygon(); - output->push_back((OGRGeometry *)g); - } - - CPLDebug("CollectGeometries", - "Collapsing wkbMultiPolygon geometries......"); - } - if (iGType == wkbGeometryCollection) - { - OGRGeometryCollection *geom = buffer->toGeometryCollection(); - GeometriesList *collection = new GeometriesList; - GeometriesList::const_iterator g_i; - for (int i = 0; i < geom->getNumGeometries(); i++) - { - OGRGeometry *g = geom->getGeometryRef(i); - collection->push_back(g); - } - GeometriesList *collapsed = FlattenGeometries(collection); - for (g_i = collapsed->begin(); g_i != collapsed->end(); g_i++) - { - output->push_back((OGRGeometry *)(*g_i)); - CPLDebug("CollectGeometries", - "Collapsing wkbGeometryCollection geometries."); - } - } - // CPLDebug( "CollectGeometries", - // "Buffered Geometry size %d", - // nGeometries); - } - - return output; -} - -/************************************************************************/ -/* DissolveLayer() */ -/************************************************************************/ - -static int DissolveLayer(OGRDataSource *poSrcDS, OGRLayer *poSrcLayer, - OGRDataSource *poDstDS, char **papszLCO, - const char *pszNewLayerName, int bTransform, - OGRSpatialReference *poOutputSRS, - OGRSpatialReference *poSourceSRS, - char **papszSelFields, int bAppend, int eGType, - int bOverwrite) - -{ - - OGRLayer *poDstLayer; - OGRFeatureDefn *poFDefn; - OGRErr eErr; - int bForceToPolygon = FALSE; - int bForceToMultiPolygon = FALSE; - - if (pszNewLayerName == NULL) - pszNewLayerName = poSrcLayer->GetLayerDefn()->GetName(); - - if (wkbFlatten(eGType) == wkbPolygon) - bForceToPolygon = TRUE; - else if (wkbFlatten(eGType) == wkbMultiPolygon) - bForceToMultiPolygon = TRUE; - - /* -------------------------------------------------------------------- */ - /* Setup coordinate transformation if we need it. */ - /* -------------------------------------------------------------------- */ - OGRCoordinateTransformation *poCT = NULL; - - if (bTransform) - { - if (poSourceSRS == NULL) - poSourceSRS = poSrcLayer->GetSpatialRef(); - - if (poSourceSRS == NULL) - { - printf("Can't transform coordinates, source layer has no\n" - "coordinate system. Use -s_srs to set one.\n"); - exit(1); - } - - CPLAssert(NULL != poSourceSRS); - CPLAssert(NULL != poOutputSRS); - - poCT = OGRCreateCoordinateTransformation(poSourceSRS, poOutputSRS); - if (poCT == NULL) - { - char *pszWKT = NULL; - - printf("Failed to create coordinate transformation between the\n" - "following coordinate systems. This may be because they\n" - "are not transformable, or because projection services\n" - "(PROJ.4 DLL/.so) could not be loaded.\n"); - - poSourceSRS->exportToPrettyWkt(&pszWKT, FALSE); - printf("Source:\n%s\n", pszWKT); - - poOutputSRS->exportToPrettyWkt(&pszWKT, FALSE); - printf("Target:\n%s\n", pszWKT); - exit(1); - } - } - - /* -------------------------------------------------------------------- */ - /* Get other info. */ - /* -------------------------------------------------------------------- */ - poFDefn = poSrcLayer->GetLayerDefn(); - - if (poOutputSRS == NULL) - poOutputSRS = poSrcLayer->GetSpatialRef(); - - /* -------------------------------------------------------------------- */ - /* Find the layer. */ - /* -------------------------------------------------------------------- */ - int iLayer = -1; - poDstLayer = NULL; - - for (iLayer = 0; iLayer < poDstDS->GetLayerCount(); iLayer++) - { - OGRLayer *poLayer = poDstDS->GetLayer(iLayer); - - if (poLayer != NULL && - EQUAL(poLayer->GetLayerDefn()->GetName(), pszNewLayerName)) - { - poDstLayer = poLayer; - break; - } - } - - /* -------------------------------------------------------------------- */ - /* If the user requested overwrite, and we have the layer in */ - /* question we need to delete it now so it will get recreated */ - /* (overwritten). */ - /* -------------------------------------------------------------------- */ - if (poDstLayer != NULL && bOverwrite) - { - if (poDstDS->DeleteLayer(iLayer) != OGRERR_NONE) - { - fprintf(stderr, "DeleteLayer() failed when overwrite requested.\n"); - return FALSE; - } - poDstLayer = NULL; - } - - /* -------------------------------------------------------------------- */ - /* If the layer does not exist, then create it. */ - /* -------------------------------------------------------------------- */ - if (poDstLayer == NULL) - { - if (eGType == -2) - eGType = poFDefn->GetGeomType(); - - if (!poDstDS->TestCapability(ODsCCreateLayer)) - { - fprintf( - stderr, - "Layer %s not found, and CreateLayer not supported by driver.", - pszNewLayerName); - return FALSE; - } - - CPLErrorReset(); - - poDstLayer = poDstDS->CreateLayer(pszNewLayerName, poOutputSRS, - (OGRwkbGeometryType)eGType, papszLCO); - - if (poDstLayer == NULL) - return FALSE; - - bAppend = FALSE; - } - - /* -------------------------------------------------------------------- */ - /* Otherwise we will append to it, if append was requested. */ - /* -------------------------------------------------------------------- */ - else if (!bAppend) - { - printf("FAILED: Layer %s already exists, and -append not specified.\n" - " Consider using -append, or -overwrite.\n", - pszNewLayerName); - return FALSE; - } - - /* -------------------------------------------------------------------- */ - /* Add fields. Default to copy all field. */ - /* If only a subset of all fields requested, then output only */ - /* the selected fields, and in the order that they were */ - /* selected. */ - /* -------------------------------------------------------------------- */ - int iField; - - if (papszSelFields && !bAppend) - { - for (iField = 0; papszSelFields[iField] != NULL; iField++) - { - int iSrcField = poFDefn->GetFieldIndex(papszSelFields[iField]); - if (iSrcField >= 0) - poDstLayer->CreateField(poFDefn->GetFieldDefn(iSrcField)); - else - { - printf("Field '%s' not found in source layer.\n", - papszSelFields[iField]); - if (!bSkipFailures) - return FALSE; - } - } - } - else if (!bAppend) - { - for (iField = 0; iField < poFDefn->GetFieldCount(); iField++) - poDstLayer->CreateField(poFDefn->GetFieldDefn(iField)); - } - - /* -------------------------------------------------------------------- */ - /* Transfer features. */ - /* -------------------------------------------------------------------- */ - OGRFeature *poFeature; - int nFeaturesInTransaction = 0; - - poSrcLayer->ResetReading(); - - if (nGroupTransactions) - poDstLayer->StartTransaction(); - - StringGeometryMap *buffers = - CollectGeometries(poSrcLayer, (const char **)papszSelFields); - - StringGeometryMap::const_iterator buffers_i; - GeometriesList *input = new GeometriesList; - - CPLDebug("CollectGeometries", "Buffers size: %d", buffers->size()); - for (buffers_i = buffers->begin(); buffers_i != buffers->end(); ++buffers_i) - { - input->push_back(buffers_i->second); - } - GeometriesList *geometries = FlattenGeometries(input); - - GeometriesList::const_iterator g_i; - for (g_i = geometries->begin(); g_i != geometries->end(); g_i++) - { - OGRFeature *feature = new OGRFeature(poFDefn); - feature->SetGeometry((*g_i)); - feature->SetField("TAXDIST", "fid"); - poDstLayer->CreateFeature(feature); - } - - if (nGroupTransactions) - poDstLayer->CommitTransaction(); - - // getGeometryType - // if( pszNewLayerName == NULL ) - // pszNewLayerName = poSrcLayer->GetLayerDefn()->GetName(); - // - // if( wkbFlatten(eGType) == wkbPolygon ) - // bForceToPolygon = TRUE; - // else if( wkbFlatten(eGType) == wkbMultiPolygon ) - // bForceToMultiPolygon = TRUE; - // - // /* -------------------------------------------------------------------- - // */ - // /* Setup coordinate transformation if we need it. */ - // /* -------------------------------------------------------------------- - // */ - // OGRCoordinateTransformation *poCT = NULL; - // - // if( bTransform ) - // { - // if( poSourceSRS == NULL ) - // poSourceSRS = poSrcLayer->GetSpatialRef(); - // - // if( poSourceSRS == NULL ) - // { - // printf( "Can't transform coordinates, source layer has no\n" - // "coordinate system. Use -s_srs to set one.\n" ); - // exit( 1 ); - // } - // - // CPLAssert( NULL != poSourceSRS ); - // CPLAssert( NULL != poOutputSRS ); - // - // poCT = OGRCreateCoordinateTransformation( poSourceSRS, - // poOutputSRS ); if( poCT == NULL ) - // { - // char *pszWKT = NULL; - // - // printf("Failed to create coordinate transformation between - // the\n" - // "following coordinate systems. This may be because - // they\n" "are not transformable, or because projection - // services\n" - // "(PROJ.4 DLL/.so) could not be loaded.\n" ); - // - // poSourceSRS->exportToPrettyWkt( &pszWKT, FALSE ); - // printf( "Source:\n%s\n", pszWKT ); - // - // poOutputSRS->exportToPrettyWkt( &pszWKT, FALSE ); - // printf( "Target:\n%s\n", pszWKT ); - // exit( 1 ); - // } - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* Get other info. */ - // /* -------------------------------------------------------------------- - // */ - // poFDefn = poSrcLayer->GetLayerDefn(); - // - // if( poOutputSRS == NULL ) - // poOutputSRS = poSrcLayer->GetSpatialRef(); - // - // /* -------------------------------------------------------------------- - // */ - // /* Find the layer. */ - // /* -------------------------------------------------------------------- - // */ - // int iLayer = -1; - // poDstLayer = NULL; - // - // for( iLayer = 0; iLayer < poDstDS->GetLayerCount(); iLayer++ ) - // { - // OGRLayer *poLayer = poDstDS->GetLayer(iLayer); - // - // if( poLayer != NULL - // && EQUAL(poLayer->GetLayerDefn()->GetName(),pszNewLayerName) - // ) - // { - // poDstLayer = poLayer; - // break; - // } - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* If the user requested overwrite, and we have the layer in */ - // /* question we need to delete it now so it will get recreated */ - // /* (overwritten). */ - // /* -------------------------------------------------------------------- - // */ - // if( poDstLayer != NULL && bOverwrite ) - // { - // if( poDstDS->DeleteLayer( iLayer ) != OGRERR_NONE ) - // { - // fprintf( stderr, - // "DeleteLayer() failed when overwrite requested.\n" - // ); - // return FALSE; - // } - // poDstLayer = NULL; - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* If the layer does not exist, then create it. */ - // /* -------------------------------------------------------------------- - // */ - // if( poDstLayer == NULL ) - // { - // if( eGType == -2 ) - // eGType = poFDefn->GetGeomType(); - // - // if( !poDstDS->TestCapability( ODsCCreateLayer ) ) - // { - // fprintf( stderr, - // "Layer %s not found, and CreateLayer not supported by - // driver.", - // pszNewLayerName ); - // return FALSE; - // } - // - // CPLErrorReset(); - // - // poDstLayer = poDstDS->CreateLayer( pszNewLayerName, poOutputSRS, - // (OGRwkbGeometryType) eGType, - // papszLCO ); - // - // if( poDstLayer == NULL ) - // return FALSE; - // - // bAppend = FALSE; - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* Otherwise we will append to it, if append was requested. */ - // /* -------------------------------------------------------------------- - // */ - // else if( !bAppend ) - // { - // printf( "FAILED: Layer %s already exists, and -append not - // specified.\n" - // " Consider using -append, or -overwrite.\n", - // pszNewLayerName ); - // return FALSE; - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* Add fields. Default to copy all field. */ - // /* If only a subset of all fields requested, then output only */ - // /* the selected fields, and in the order that they were */ - // /* selected. */ - // /* -------------------------------------------------------------------- - // */ - // int iField; - // - // if (papszSelFields && !bAppend ) - // { - // for( iField=0; papszSelFields[iField] != NULL; iField++) - // { - // int iSrcField = - // poFDefn->GetFieldIndex(papszSelFields[iField]); if (iSrcField - // >= 0) - // poDstLayer->CreateField( poFDefn->GetFieldDefn(iSrcField) - // ); - // else - // { - // printf( "Field '%s' not found in source layer.\n", - // papszSelFields[iField] ); - // if( !bSkipFailures ) - // return FALSE; - // } - // } - // } - // else if( !bAppend ) - // { - // for( iField = 0; iField < poFDefn->GetFieldCount(); iField++ ) - // poDstLayer->CreateField( poFDefn->GetFieldDefn(iField) ); - // } - // - // /* -------------------------------------------------------------------- - // */ - // /* Transfer features. */ - // /* -------------------------------------------------------------------- - // */ - // OGRFeature *poFeature; - // int nFeaturesInTransaction = 0; - // - // poSrcLayer->ResetReading(); - // - // if( nGroupTransactions ) - // poDstLayer->StartTransaction(); - // - // while( true ) - // { - // OGRFeature *poDstFeature = NULL; - // - // if( nFIDToFetch != OGRNullFID ) - // { - // // Only fetch feature on first pass. - // if( nFeaturesInTransaction == 0 ) - // poFeature = poSrcLayer->GetFeature(nFIDToFetch); - // else - // poFeature = NULL; - // } - // else - // poFeature = poSrcLayer->GetNextFeature(); - // - // if( poFeature == NULL ) - // break; - // - // if( ++nFeaturesInTransaction == nGroupTransactions ) - // { - // poDstLayer->CommitTransaction(); - // poDstLayer->StartTransaction(); - // nFeaturesInTransaction = 0; - // } - // - // CPLErrorReset(); - // poFDefn = poSrcLayer->GetLayerDefn(); - // - // if( poDstFeature->SetFrom( poFeature, TRUE ) != OGRERR_NONE ) - // { - // if( nGroupTransactions ) - // poDstLayer->CommitTransaction(); - // - // CPLError( CE_Failure, CPLE_AppDefined, - // "Unable to translate feature %d from layer %s.\n", - // poFeature->GetFID(), poFDefn->GetName() ); - // - // OGRFeature::DestroyFeature( poFeature ); - // OGRFeature::DestroyFeature( poDstFeature ); - // return FALSE; - // } - // - // if( bPreserveFID ) - // poDstFeature->SetFID( poFeature->GetFID() ); - // - // if( poCT && poDstFeature->GetGeometryRef() != NULL ) - // { - // eErr = poDstFeature->GetGeometryRef()->transform( poCT ); - // if( eErr != OGRERR_NONE ) - // { - // if( nGroupTransactions ) - // poDstLayer->CommitTransaction(); - // - // printf( "Failed to transform feature %d.\n", - // static_cast(poFeature->GetFID()) ); - // if( !bSkipFailures ) - // { - // OGRFeature::DestroyFeature( poFeature ); - // OGRFeature::DestroyFeature( poDstFeature ); - // return FALSE; - // } - // } - // } - // - // if( poDstFeature->GetGeometryRef() != NULL && bForceToPolygon ) - // { - // poDstFeature->SetGeometryDirectly( - // OGRGeometryFactory::forceToPolygon( - // poDstFeature->StealGeometry() ) ); - // } - // - // if( poDstFeature->GetGeometryRef() != NULL && - // bForceToMultiPolygon ) - // { - // poDstFeature->SetGeometryDirectly( - // OGRGeometryFactory::forceToMultiPolygon( - // poDstFeature->StealGeometry() ) ); - // } - // - // OGRFeature::DestroyFeature( poFeature ); - // - // CPLErrorReset(); - // if( poDstLayer->CreateFeature( poDstFeature ) != OGRERR_NONE - // && !bSkipFailures ) - // { - // if( nGroupTransactions ) - // poDstLayer->RollbackTransaction(); - // - // OGRFeature::DestroyFeature( poDstFeature ); - // return FALSE; - // } - // - // OGRFeature::DestroyFeature( poDstFeature ); - // } - // - // if( nGroupTransactions ) - // poDstLayer->CommitTransaction(); - // - // /* -------------------------------------------------------------------- - // */ - // /* Cleaning */ - // /* -------------------------------------------------------------------- - // */ - // delete poCT; - - return TRUE; -} diff --git a/apps/test_ogrsf.cpp b/apps/test_ogrsf.cpp index 4ead11298eb2..0b4e72b6e9d4 100644 --- a/apps/test_ogrsf.cpp +++ b/apps/test_ogrsf.cpp @@ -1655,6 +1655,7 @@ static int TestOGRLayerFeatureCount(GDALDataset *poDS, OGRLayer *poLayer, } delete poFeat; + const auto nFCEndOfIter = LOG_ACTION(poLayer->GetFeatureCount()); if (nFC != nClaimedFC) { bRet = FALSE; @@ -1662,12 +1663,12 @@ static int TestOGRLayerFeatureCount(GDALDataset *poDS, OGRLayer *poLayer, " doesn't match actual, " CPL_FRMT_GIB ".\n", nClaimedFC, nFC); } - else if (nFC != LOG_ACTION(poLayer->GetFeatureCount())) + else if (nFC != nFCEndOfIter) { bRet = FALSE; printf("ERROR: Feature count at end of layer, " CPL_FRMT_GIB ", differs from at start, " CPL_FRMT_GIB ".\n", - poLayer->GetFeatureCount(), nFC); + nFCEndOfIter, nFC); } else if (bVerbose) printf("INFO: Feature count verified.\n"); @@ -4187,8 +4188,7 @@ static int TestLayerGetArrowStream(OGRLayer *poLayer) { if (array.length != 0) { - bRet = false; - printf("ERROR: get_next() return an array with length != 0 " + printf("WARNING: get_next() return an array with length != 0 " "after end of iteration\n"); } if (array.release) diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index 90e2c33445ad..ba43804e23b4 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -1777,6 +1777,16 @@ TEST_F(test_cpl, CPLParseMemorySize) EXPECT_GT(nValue, 100 * 1024 * 1024); EXPECT_TRUE(bUnitSpecified); + result = CPLParseMemorySize("0", &nValue, &bUnitSpecified); + EXPECT_EQ(result, CE_None); + EXPECT_EQ(nValue, 0); + EXPECT_FALSE(bUnitSpecified); + + result = CPLParseMemorySize("0MB", &nValue, &bUnitSpecified); + EXPECT_EQ(result, CE_None); + EXPECT_EQ(nValue, 0); + EXPECT_TRUE(bUnitSpecified); + result = CPLParseMemorySize(" 802 ", &nValue, &bUnitSpecified); EXPECT_EQ(result, CE_None); EXPECT_EQ(nValue, 802); diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index 5ae28fcd95f7..9f2de42a3150 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -4394,4 +4394,67 @@ TEST_F(test_ogr, OGRPolygon_addRingDirectly) ASSERT_EQ(p.addRingDirectly(lr.release()), OGRERR_NONE); } +TEST_F(test_ogr, OGRFeature_SetGeometry) +{ + OGRFeatureDefn *poFeatureDefn = new OGRFeatureDefn(); + poFeatureDefn->Reference(); + + OGRFeature oFeat(poFeatureDefn); + std::unique_ptr poGeom; + OGRGeometry *poTmpGeom; + ASSERT_EQ( + OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, &poTmpGeom), + OGRERR_NONE); + poGeom.reset(poTmpGeom); + ASSERT_EQ(oFeat.SetGeometry(std::move(poGeom)), OGRERR_NONE); + EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getX(), 3); + EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getY(), 7); + + // set it again to make sure previous feature geometry is freed + ASSERT_EQ( + OGRGeometryFactory::createFromWkt("POINT (2 8)", nullptr, &poTmpGeom), + OGRERR_NONE); + poGeom.reset(poTmpGeom); + ASSERT_EQ(oFeat.SetGeometry(std::move(poGeom)), OGRERR_NONE); + EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getX(), 2); + EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getY(), 8); + + poFeatureDefn->Release(); +} + +TEST_F(test_ogr, OGRFeature_SetGeomField) +{ + OGRFeatureDefn *poFeatureDefn = new OGRFeatureDefn(); + poFeatureDefn->Reference(); + + OGRGeomFieldDefn oGeomField("second", wkbPoint); + poFeatureDefn->AddGeomFieldDefn(&oGeomField); + + OGRFeature oFeat(poFeatureDefn); + + // failure + { + std::unique_ptr poGeom; + OGRGeometry *poTmpGeom; + ASSERT_EQ(OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, + &poTmpGeom), + OGRERR_NONE); + poGeom.reset(poTmpGeom); + EXPECT_EQ(oFeat.SetGeomField(13, std::move(poGeom)), OGRERR_FAILURE); + } + + // success + { + std::unique_ptr poGeom; + OGRGeometry *poTmpGeom; + ASSERT_EQ(OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, + &poTmpGeom), + OGRERR_NONE); + poGeom.reset(poTmpGeom); + EXPECT_EQ(oFeat.SetGeomField(1, std::move(poGeom)), OGRERR_NONE); + } + + poFeatureDefn->Release(); +} + } // namespace diff --git a/autotest/gcore/basic_test.py b/autotest/gcore/basic_test.py index cb389656c94d..3b399d820925 100755 --- a/autotest/gcore/basic_test.py +++ b/autotest/gcore/basic_test.py @@ -176,17 +176,20 @@ def test_basic_test_8(): license_text.startswith("GDAL/OGR is released under the MIT license") or "GDAL/OGR Licensing" in license_text ) - - # Use a subprocess to avoid the cached license text - env = os.environ.copy() - env["GDAL_DATA"] = "tmp" - with open("tmp/LICENSE.TXT", "wt") as f: - f.write("fake_license") - license_text = subprocess.check_output( - [sys.executable, "basic_test_subprocess.py"], env=env - ).decode("utf-8") - os.unlink("tmp/LICENSE.TXT") - assert license_text.startswith("fake_license") + if "EMBED_RESOURCE_FILES=YES" in gdal.VersionInfo("BUILD_INFO"): + assert len(license_text) > 1000 + + if "USE_ONLY_EMBEDDED_RESOURCE_FILES=YES" not in gdal.VersionInfo("BUILD_INFO"): + # Use a subprocess to avoid the cached license text + env = os.environ.copy() + env["GDAL_DATA"] = "tmp" + with open("tmp/LICENSE.TXT", "wt") as f: + f.write("fake_license") + license_text = subprocess.check_output( + [sys.executable, "basic_test_subprocess.py"], env=env + ).decode("utf-8") + os.unlink("tmp/LICENSE.TXT") + assert license_text.startswith("fake_license") ############################################################################### diff --git a/autotest/gcore/misc.py b/autotest/gcore/misc.py index cca79ca34e23..18d1f4c806c0 100755 --- a/autotest/gcore/misc.py +++ b/autotest/gcore/misc.py @@ -281,61 +281,20 @@ def misc_6_internal(datatype, nBands, setDriversDone): drv = gdal.GetDriver(i) md = drv.GetMetadata() if ("DCAP_CREATECOPY" in md or "DCAP_CREATE" in md) and "DCAP_RASTER" in md: - # print ('drv = %s, nBands = %d, datatype = %s' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype))) - - skip = False - # FIXME: A few cases that crashes and should be investigated - if drv.ShortName == "JPEG2000": - if (nBands == 2 or nBands >= 5) or not ( - datatype == gdal.GDT_Byte - or datatype == gdal.GDT_Int16 - or datatype == gdal.GDT_UInt16 - ): - skip = True - if skip is False: - dirname = "tmp/tmp/tmp_%s_%d_%s" % ( - drv.ShortName, - nBands, - gdal.GetDataTypeName(datatype), - ) - try: - os.mkdir(dirname) - except OSError: - try: - os.stat(dirname) - # Hum the directory already exists... Not expected, but let's try to go on - except OSError: - reason = ( - "Cannot create %s before drv = %s, nBands = %d, datatype = %s" - % ( - dirname, - drv.ShortName, - nBands, - gdal.GetDataTypeName(datatype), - ) - ) - pytest.fail(reason) - - filename = get_filename(drv, dirname) - - dst_ds = drv.CreateCopy(filename, ds) - has_succeeded = dst_ds is not None - if dst_ds: - # check that domain == None doesn't crash - dst_ds.GetMetadata(None) - dst_ds.GetMetadataItem("", None) - dst_ds = None - - size = 0 - stat = gdal.VSIStatL(filename) - if stat is not None: - size = stat.size - + dirname = "tmp/tmp/tmp_%s_%d_%s" % ( + drv.ShortName, + nBands, + gdal.GetDataTypeName(datatype), + ) + try: + os.mkdir(dirname) + except OSError: try: - shutil.rmtree(dirname) + os.stat(dirname) + # Hum the directory already exists... Not expected, but let's try to go on except OSError: reason = ( - "Cannot remove %s after drv = %s, nBands = %d, datatype = %s" + "Cannot create %s before drv = %s, nBands = %d, datatype = %s" % ( dirname, drv.ShortName, @@ -345,114 +304,142 @@ def misc_6_internal(datatype, nBands, setDriversDone): ) pytest.fail(reason) - if has_succeeded and drv.ShortName not in setDriversDone and nBands > 0: - setDriversDone.add(drv.ShortName) + filename = get_filename(drv, dirname) - # The first list of drivers fail to detect short writing - # The second one is because they are verbose in stderr - if ( - "DCAP_VIRTUALIO" in md - and size != 0 - and drv.ShortName - not in [ - "JPEG2000", - "KMLSUPEROVERLAY", - "HF2", - "ZMap", - "DDS", - "TileDB", - ] - and drv.ShortName not in ["GIF", "JP2ECW", "JP2Lura"] - ): + dst_ds = drv.CreateCopy(filename, ds) + has_succeeded = dst_ds is not None + if dst_ds: + # check that domain == None doesn't crash + dst_ds.GetMetadata(None) + dst_ds.GetMetadataItem("", None) + dst_ds = None - for j in range(10): - truncated_size = (size * j) / 10 - vsimem_filename = ( - "/vsimem/test_truncate/||maxlength=%d||" - % truncated_size - ) + get_filename(drv, "")[1:] - # print('drv = %s, nBands = %d, datatype = %s, truncated_size = %d' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype), truncated_size)) - dst_ds = drv.CreateCopy(vsimem_filename, ds) - error_detected = False - if dst_ds is None: - error_detected = True - else: - gdal.ErrorReset() - dst_ds = None - if gdal.GetLastErrorMsg() != "": - error_detected = True - if not error_detected: - msg = ( - "write error not detected with with drv = %s, nBands = %d, datatype = %s, truncated_size = %d" - % ( - drv.ShortName, - nBands, - gdal.GetDataTypeName(datatype), - truncated_size, - ) - ) - print(msg) + size = 0 + stat = gdal.VSIStatL(filename) + if stat is not None: + size = stat.size - fl = gdal.ReadDirRecursive("/vsimem/test_truncate") - if fl is not None: - for myf in fl: - gdal.Unlink("/vsimem/test_truncate/" + myf) - fl = gdal.ReadDirRecursive("/vsimem/test_truncate") - if fl is not None: - print(fl) - - if drv.ShortName not in [ - "ECW", - "JP2ECW", - "VRT", - "XPM", + try: + shutil.rmtree(dirname) + except OSError: + reason = ( + "Cannot remove %s after drv = %s, nBands = %d, datatype = %s" + % ( + dirname, + drv.ShortName, + nBands, + gdal.GetDataTypeName(datatype), + ) + ) + pytest.fail(reason) + + if has_succeeded and drv.ShortName not in setDriversDone and nBands > 0: + setDriversDone.add(drv.ShortName) + + # The first list of drivers fail to detect short writing + # The second one is because they are verbose in stderr + if ( + "DCAP_VIRTUALIO" in md + and size != 0 + and drv.ShortName + not in [ "JPEG2000", - "FIT", - "RST", - "INGR", - "USGSDEM", "KMLSUPEROVERLAY", - "GMT", - ]: - dst_ds = drv.CreateCopy( - filename, ds, callback=misc_6_interrupt_callback_class().cbk - ) - if dst_ds is not None: - dst_ds = None - - try: - shutil.rmtree(dirname) - except OSError: - pass + "HF2", + "ZMap", + "DDS", + "TileDB", + ] + and drv.ShortName not in ["GIF", "JP2ECW", "JP2Lura"] + ): - pytest.fail( - "interruption did not work with drv = %s, nBands = %d, datatype = %s" + for j in range(10): + truncated_size = (size * j) / 10 + vsimem_filename = ( + "/vsimem/test_truncate/||maxlength=%d||" % truncated_size + ) + get_filename(drv, "")[1:] + # print('drv = %s, nBands = %d, datatype = %s, truncated_size = %d' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype), truncated_size)) + dst_ds = drv.CreateCopy(vsimem_filename, ds) + error_detected = False + if dst_ds is None: + error_detected = True + else: + gdal.ErrorReset() + dst_ds = None + if gdal.GetLastErrorMsg() != "": + error_detected = True + if not error_detected: + msg = ( + "write error not detected with with drv = %s, nBands = %d, datatype = %s, truncated_size = %d" % ( drv.ShortName, nBands, gdal.GetDataTypeName(datatype), + truncated_size, ) ) + print(msg) + fl = gdal.ReadDirRecursive("/vsimem/test_truncate") + if fl is not None: + for myf in fl: + gdal.Unlink("/vsimem/test_truncate/" + myf) + fl = gdal.ReadDirRecursive("/vsimem/test_truncate") + if fl is not None: + print(fl) + + if drv.ShortName not in [ + "ECW", + "JP2ECW", + "VRT", + "XPM", + "JPEG2000", + "FIT", + "RST", + "INGR", + "USGSDEM", + "KMLSUPEROVERLAY", + "GMT", + ]: + dst_ds = drv.CreateCopy( + filename, ds, callback=misc_6_interrupt_callback_class().cbk + ) + if dst_ds is not None: dst_ds = None try: shutil.rmtree(dirname) except OSError: pass - try: - os.mkdir(dirname) - except OSError: - reason = ( - "Cannot create %s before drv = %s, nBands = %d, datatype = %s" - % ( - dirname, - drv.ShortName, - nBands, - gdal.GetDataTypeName(datatype), - ) + + pytest.fail( + "interruption did not work with drv = %s, nBands = %d, datatype = %s" + % ( + drv.ShortName, + nBands, + gdal.GetDataTypeName(datatype), ) - pytest.fail(reason) + ) + + dst_ds = None + + try: + shutil.rmtree(dirname) + except OSError: + pass + try: + os.mkdir(dirname) + except OSError: + reason = ( + "Cannot create %s before drv = %s, nBands = %d, datatype = %s" + % ( + dirname, + drv.ShortName, + nBands, + gdal.GetDataTypeName(datatype), + ) + ) + pytest.fail(reason) ds = None diff --git a/autotest/gcore/test_gdal_fsspec.py b/autotest/gcore/test_gdal_fsspec.py new file mode 100644 index 000000000000..38deb8da07be --- /dev/null +++ b/autotest/gcore/test_gdal_fsspec.py @@ -0,0 +1,225 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: Test gdal_fsspec module +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 20124, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import pytest + +from osgeo import gdal + +fsspec = pytest.importorskip("fsspec") +pytest.importorskip("fsspec.spec") + +from osgeo import gdal_fsspec # NOQA + + +def test_gdal_fsspec_open_read(): + + with fsspec.open("gdalvsi://data/byte.tif") as f: + assert len(f.read()) == gdal.VSIStatL("data/byte.tif").size + + +def test_gdal_fsspec_info_file(): + + fs = fsspec.filesystem("gdalvsi") + info = fs.info("data/byte.tif") + assert "mtime" in info + del info["mtime"] + assert (info["mode"] & 32768) != 0 + del info["mode"] + assert info == { + "name": "data/byte.tif", + "size": 736, + "type": "file", + } + + +def test_gdal_fsspec_info_dir(): + + fs = fsspec.filesystem("gdalvsi") + info = fs.info("data") + assert (info["mode"] & 16384) != 0 + del info["mode"] + assert info == { + "name": "data", + "size": 0, + "type": "directory", + } + + +def test_gdal_fsspec_info_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.info("/i/do/not/exist") + + +def test_gdal_fsspec_ls(): + + fs = fsspec.filesystem("gdalvsi") + ret = fs.ls("data") + assert len(ret) > 2 + item_of_interest = None + for item in ret: + if item["name"] == "data/byte.tif": + item_of_interest = item + break + assert item_of_interest + assert "mtime" in item_of_interest + del item_of_interest["mtime"] + assert item_of_interest == { + "name": "data/byte.tif", + "size": 736, + "type": "file", + } + + +def test_gdal_fsspec_ls_file(): + + fs = fsspec.filesystem("gdalvsi") + ret = fs.ls("data/byte.tif") + assert ret == ["data/byte.tif"] + + +def test_gdal_fsspec_ls_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.ls("gdalvsi://i/do/not/exist") + + +def test_gdal_fsspec_modified(): + + fs = fsspec.filesystem("gdalvsi") + modified = fs.modified("data/byte.tif") + assert modified is not None + import datetime + + assert isinstance(modified, datetime.datetime) + + +def test_gdal_fsspec_modified_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.modified("gdalvsi://i/do/not/exist") + + +def test_gdal_fsspec_rm(): + + with fsspec.open("gdalvsi:///vsimem/foo.bin", "wb") as f: + f.write(b"""bar""") + fs = fsspec.filesystem("gdalvsi") + fs.info("/vsimem/foo.bin") + fs.rm("/vsimem/foo.bin") + with pytest.raises(FileNotFoundError): + fs.info("/vsimem/foo.bin") + + +def test_gdal_fsspec_rm_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.rm("/vsimem/foo.bin") + + +def test_gdal_fsspec_copy(): + + with fsspec.open("gdalvsi:///vsimem/foo.bin", "wb") as f: + f.write(b"""bar""") + fs = fsspec.filesystem("gdalvsi") + fs.copy("/vsimem/foo.bin", "/vsimem/bar.bin") + assert fs.info("/vsimem/bar.bin")["size"] == 3 + assert fs.info("/vsimem/foo.bin")["size"] == 3 + fs.rm("/vsimem/foo.bin") + fs.rm("/vsimem/bar.bin") + + +def test_gdal_fsspec_copy_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.copy("/vsimem/foo.bin", "/vsimem/bar.bin") + + +def test_gdal_fsspec_mv(): + + with fsspec.open("gdalvsi:///vsimem/foo.bin", "wb") as f: + f.write(b"""bar""") + fs = fsspec.filesystem("gdalvsi") + fs.mv("/vsimem/foo.bin", "/vsimem/bar.bin") + assert fs.info("/vsimem/bar.bin")["size"] == 3 + with pytest.raises(FileNotFoundError): + fs.info("/vsimem/foo.bin") + fs.rm("/vsimem/bar.bin") + + +def test_gdal_fsspec_mv_error(): + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(FileNotFoundError): + fs.mv("/vsimem/foo.bin", "/bar.bin") + + +def test_gdal_fsspec_mkdir(tmp_path): + + fs = fsspec.filesystem("gdalvsi") + + my_path = str(tmp_path) + "/my_dir" + + fs.mkdir(my_path) + assert fs.info(my_path)["type"] == "directory" + with pytest.raises(FileExistsError): + fs.mkdir(my_path) + fs.rmdir(my_path) + + fs.mkdir(my_path + "/my_subdir") + assert fs.info(my_path)["type"] == "directory" + assert fs.info(my_path + "/my_subdir")["type"] == "directory" + fs.rmdir(my_path + "/my_subdir") + fs.rmdir(my_path) + with pytest.raises(FileNotFoundError): + fs.info(my_path) + + fs = fsspec.filesystem("gdalvsi") + with pytest.raises(Exception): + fs.mkdir(my_path + "/my_subdir", create_parents=False) + with pytest.raises(FileNotFoundError): + fs.info(my_path) + + +def test_gdal_fsspec_makedirs(tmp_path): + + fs = fsspec.filesystem("gdalvsi") + + my_path = str(tmp_path) + "/my_dir" + fs.makedirs(my_path) + assert fs.info(my_path)["type"] == "directory" + with pytest.raises(FileExistsError): + fs.makedirs(my_path) + fs.makedirs(my_path, exist_ok=True) + fs.rmdir(my_path) + + +def test_gdal_fsspec_usable_by_pyarrow_dataset(tmp_vsimem): + + ds = pytest.importorskip("pyarrow.dataset") + + tmp_vsimem_file = str(tmp_vsimem / "tmp.parquet") + gdal.FileFromMemBuffer( + tmp_vsimem_file, open("../ogr/data/parquet/test.parquet", "rb").read() + ) + + fs_vsimem = fsspec.filesystem("gdalvsi") + + assert ds.dataset(tmp_vsimem_file, filesystem=fs_vsimem) is not None + + assert ds.dataset(str(tmp_vsimem), filesystem=fs_vsimem) is not None diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 88f3caac637f..baab96cfff06 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -9749,6 +9749,41 @@ def test_tiff_write_jpegxl_alpha_distance_zero(): gdal.Unlink(filename) +############################################################################### + + +@pytest.mark.require_creation_option("GTiff", "JXL_ALPHA_DISTANCE") +def test_tiff_write_jpegxl_five_bands_lossy(tmp_vsimem): + + outfilename = str(tmp_vsimem / "test_tiff_write_jpegxl_five_bands_lossy.tif") + gdal.Translate( + outfilename, + "data/byte.tif", + options="-co COMPRESS=JXL -co JXL_LOSSLESS=NO -co JXL_DISTANCE=3 -b 1 -b 1 -b 1 -b 1 -b 1", + ) + ds = gdal.Open(outfilename) + assert ds.GetRasterBand(3).Checksum() not in (0, 4672) + assert ds.GetRasterBand(4).Checksum() not in (0, 4672) + assert ds.GetRasterBand(5).Checksum() not in (0, 4672) + + +############################################################################### + + +@pytest.mark.require_creation_option("GTiff", "JXL_ALPHA_DISTANCE") +def test_tiff_write_jpegxl_five_bands_lossless(tmp_vsimem): + + outfilename = str(tmp_vsimem / "test_tiff_write_jpegxl_five_bands_lossy.tif") + gdal.Translate( + outfilename, + "data/byte.tif", + options="-co COMPRESS=JXL -co JXL_LOSSLESS=YES -b 1 -b 1 -b 1 -b 1 -b 1", + ) + ds = gdal.Open(outfilename) + for i in range(5): + assert ds.GetRasterBand(i + 1).Checksum() == 4672 + + ############################################################################### # Test creating overviews with NaN nodata diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index 25d610133971..12b69d9eaca7 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -237,16 +237,16 @@ def test_vsifile_4(): # Test vsicache -@pytest.mark.parametrize("cache_size", ("0", "65536", None)) -def test_vsifile_5(cache_size): +@pytest.mark.parametrize("cache_size", ("0", "64kb", None)) +def test_vsifile_5(tmp_path, cache_size): - fp = gdal.VSIFOpenL("tmp/vsifile_5.bin", "wb") + fp = gdal.VSIFOpenL(tmp_path / "vsifile_5.bin", "wb") ref_data = "".join(["%08X" % i for i in range(5 * 32768)]) gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp) gdal.VSIFCloseL(fp) with gdal.config_options({"VSI_CACHE": "YES", "VSI_CACHE_SIZE": cache_size}): - fp = gdal.VSIFOpenL("tmp/vsifile_5.bin", "rb") + fp = gdal.VSIFOpenL(tmp_path / "vsifile_5.bin", "rb") gdal.VSIFSeekL(fp, 50000, 0) if gdal.VSIFTellL(fp) != 50000: @@ -277,8 +277,6 @@ def test_vsifile_5(cache_size): gdal.VSIFCloseL(fp) - gdal.Unlink("tmp/vsifile_5.bin") - ############################################################################### # Test vsicache an read errors (https://github.com/qgis/QGIS/issues/45293) diff --git a/autotest/gdrivers/jpegxl.py b/autotest/gdrivers/jpegxl.py index 4983e012b119..55ee592a9cc2 100755 --- a/autotest/gdrivers/jpegxl.py +++ b/autotest/gdrivers/jpegxl.py @@ -127,7 +127,7 @@ def test_jpegxl_rgba_distance(): @pytest.mark.parametrize( - "quality,equivalent_distance", [(100, 0), (90, 1), (10, 12.65)] + "quality,equivalent_distance", [(100, 0), (10, 15.266666666666667)] ) def test_jpegxl_rgba_quality(quality, equivalent_distance): diff --git a/autotest/ogr/data/duckdb/poly.duckdb b/autotest/ogr/data/duckdb/poly.duckdb new file mode 100644 index 000000000000..71322dc9c328 Binary files /dev/null and b/autotest/ogr/data/duckdb/poly.duckdb differ diff --git a/autotest/ogr/ogr_adbc.py b/autotest/ogr/ogr_adbc.py new file mode 100755 index 000000000000..edaf0e937140 --- /dev/null +++ b/autotest/ogr/ogr_adbc.py @@ -0,0 +1,351 @@ +#!/usr/bin/env pytest +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Test read functionality for OGR ADBC driver. +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import gdaltest +import pytest + +from osgeo import gdal, ogr + + +def _has_adbc_driver_manager(): + drv = gdal.GetDriverByName("ADBC") + return drv and drv.GetMetadataItem("HAS_ADBC_DRIVER_MANAGER") + + +pytestmark = [ + pytest.mark.require_driver("ADBC"), + pytest.mark.skipif( + not _has_adbc_driver_manager(), + reason="ADBC driver built without AdbcDriverManager", + ), +] + +############################################################################### + + +def _has_sqlite_driver(): + import ctypes + + try: + return ctypes.cdll.LoadLibrary("libadbc_driver_sqlite.so") is not None + except Exception: + return False + + +############################################################################### + + +def test_ogr_adbc_driver_open_option(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "ADBC:", gdal.OF_VECTOR, open_options=["ADBC_DRIVER=adbc_driver_sqlite"] + ) as ds: + assert ds.GetLayerCount() == 0 + with ds.ExecuteSQL("SELECT sqlite_version()") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f + assert f.GetField(0).startswith("3.") + + +############################################################################### + + +def test_ogr_adbc_invalid_driver(): + + with pytest.raises(Exception): + gdal.OpenEx( + "ADBC:", gdal.OF_VECTOR, open_options=["ADBC_DRIVER=invalid_driver"] + ) + + +############################################################################### + + +def test_ogr_adbc_invalid_dataset(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with pytest.raises(Exception): + gdal.OpenEx( + "ADBC:/i/do/not/exist.db", + gdal.OF_VECTOR, + open_options=["ADBC_DRIVER=adbc_driver_sqlite"], + ) + + +############################################################################### + + +def test_ogr_adbc_sqlite3(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "data/sqlite/poly_spatialite.sqlite", gdal.OF_VECTOR, allowed_drivers=["ADBC"] + ) as ds: + assert ds.GetLayerCount() == 13 + assert ds.GetLayer(-1) is None + assert ds.GetLayer(ds.GetLayerCount()) is None + lyr = ds.GetLayer(0) + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + +############################################################################### + + +def test_ogr_adbc_sql_open_option(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "ADBC:data/sqlite/poly_spatialite.sqlite", + gdal.OF_VECTOR, + open_options=["SQL=SELECT * FROM poly"], + ) as ds: + assert ds.GetLayerCount() == 1 + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 10 + + +############################################################################### + + +def test_ogr_adbc_invalid_sql(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with pytest.raises(Exception): + gdal.OpenEx( + "ADBC:data/sqlite/poly_spatialite.sqlite", + gdal.OF_VECTOR, + open_options=["SQL=SELECT * FROM"], + ) + + +############################################################################### + + +def test_ogr_adbc_generic_open_option(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "ADBC:", + gdal.OF_VECTOR, + open_options=[ + "ADBC_DRIVER=adbc_driver_sqlite", + "ADBC_OPTION_uri=data/sqlite/poly_spatialite.sqlite", + ], + ) as ds: + assert ds.GetLayerCount() == 13 + + +############################################################################### + + +def test_ogr_adbc_execute_sql(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "data/sqlite/poly_spatialite.sqlite", + gdal.OF_VECTOR, + open_options=["SQL="], + allowed_drivers=["ADBC"], + ) as ds: + assert ds.GetLayerCount() == 0 + with ds.ExecuteSQL("SELECT * FROM poly") as lyr: + assert lyr.GetFeatureCount() == 10 + + +############################################################################### + + +def _has_libduckdb(): + import ctypes + + try: + return ctypes.cdll.LoadLibrary("libduckdb.so") is not None + except Exception: + return False + + +############################################################################### + + +def test_ogr_adbc_duckdb_parquet(): + + if not _has_libduckdb(): + pytest.skip("libduckdb.so missing") + + with gdal.OpenEx( + "data/parquet/partitioned_flat/part.0.parquet", + gdal.OF_VECTOR, + allowed_drivers=["ADBC"], + ) as ds: + assert ds.GetLayerCount() == 1 + lyr = ds.GetLayer(0) + assert lyr.TestCapability(ogr.OLCFastFeatureCount) + assert lyr.GetFeatureCount() == 3 + + +############################################################################### + + +def test_ogr_adbc_duckdb_parquet_with_sql_open_option(): + + if not _has_libduckdb(): + pytest.skip("libduckdb.so missing") + + with gdal.OpenEx( + "data/parquet/partitioned_flat/part.0.parquet", + gdal.OF_VECTOR, + allowed_drivers=["ADBC"], + open_options=["SQL=SELECT * FROM part.0 ORDER BY one DESC LIMIT 2"], + ) as ds: + assert ds.GetLayerCount() == 1 + lyr = ds.GetLayer(0) + assert lyr.TestCapability(ogr.OLCFastFeatureCount) == 0 + assert lyr.GetFeatureCount() == 2 + + +############################################################################### +# Run test_ogrsf on a SQLite3 database + + +def test_ogr_adbc_test_ogrsf_sqlite3(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + import test_cli_utilities + + if test_cli_utilities.get_test_ogrsf_path() is None: + pytest.skip() + + ret = gdaltest.runexternal( + test_cli_utilities.get_test_ogrsf_path() + + " -ro ADBC:data/sqlite/first_geometry_null.db" + ) + + assert "INFO" in ret + assert "ERROR" not in ret + + +############################################################################### +# Run test_ogrsf on a single Parquet file + + +def test_ogr_adbc_test_ogrsf_parquet(): + + if not _has_libduckdb(): + pytest.skip("libduckdb.so missing") + + import test_cli_utilities + + if test_cli_utilities.get_test_ogrsf_path() is None: + pytest.skip() + + ret = gdaltest.runexternal( + test_cli_utilities.get_test_ogrsf_path() + + " -ro ADBC:data/parquet/partitioned_flat/part.0.parquet" + ) + + assert "INFO" in ret + assert "ERROR" not in ret + + +############################################################################### +# Run test_ogrsf on a partitioned Parquet dataset + + +def test_ogr_adbc_test_ogrsf_parquet_filename_with_glob(): + + if not _has_libduckdb(): + pytest.skip("libduckdb.so missing") + + import test_cli_utilities + + if test_cli_utilities.get_test_ogrsf_path() is None: + pytest.skip() + + ret = gdaltest.runexternal( + test_cli_utilities.get_test_ogrsf_path() + + " -ro ADBC:data/parquet/partitioned_flat/*.parquet" + ) + + assert "INFO" in ret + assert "ERROR" not in ret + + +############################################################################### +# Run test_ogrsf on a DuckDB dataset + + +def test_ogr_adbc_test_ogrsf_duckdb(): + + if not _has_libduckdb(): + pytest.skip("libduckdb.so missing") + + import test_cli_utilities + + if test_cli_utilities.get_test_ogrsf_path() is None: + pytest.skip() + + ret = gdaltest.runexternal( + test_cli_utilities.get_test_ogrsf_path() + " -ro ADBC:data/duckdb/poly.duckdb" + ) + + assert "INFO" in ret + assert "ERROR" not in ret + + +############################################################################### + + +def test_ogr_adbc_layer_list(): + + if not _has_sqlite_driver(): + pytest.skip("adbc_driver_sqlite missing") + + with gdal.OpenEx( + "data/sqlite/poly_spatialite.sqlite", gdal.OF_VECTOR, allowed_drivers=["ADBC"] + ) as ds: + assert ds.GetLayerCount() == 13 + assert ds.GetLayerByName("table_list") + assert ds.GetLayerCount() == 14 + # Re-issue GetLayerByName() to check it has been added to the list + # of known layers and is no instantiated twice. + lyr = ds.GetLayerByName("table_list") + assert lyr + assert ds.GetLayerCount() == 14 + assert lyr.GetFeatureCount() == 14 + found = False + for f in lyr: + if ( + f["catalog_name"] == "main" + and f["table_name"] == "spatial_ref_sys" + and f["table_type"] == "table" + ): + found = True + assert found diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index 822a46334459..3abdd951a3d4 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -10540,3 +10540,49 @@ def test_gpkg_create_more_than_2000_fields(tmp_vsimem): with pytest.raises(Exception, match="Limit of 2000 columns reached"): lyr.CreateField(ogr.FieldDefn("foo")) assert lyr.GetLayerDefn().GetFieldCount() == 2000 - 2 + + +############################################################################### +# Test that secure_delete is turned on + + +@gdaltest.enable_exceptions() +def test_gpkg_secure_delete(tmp_vsimem): + + filename = str(tmp_vsimem / "secure_delete.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + + with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == 1 + + lyr = ds.CreateLayer("test") + lyr.CreateField(ogr.FieldDefn("foo")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "very_secret" + lyr.CreateFeature(f) + + f = gdal.VSIFOpenL(filename, "rb") + data = gdal.VSIFReadL(1, 100000, f) + gdal.VSIFCloseL(f) + assert b"very_secret" in data + + with ogr.Open(filename, update=1) as ds: + + with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == 1 + + lyr = ds.GetLayer(0) + lyr.DeleteFeature(1) + + f = gdal.VSIFOpenL(filename, "rb") + data = gdal.VSIFReadL(1, 100000, f) + gdal.VSIFCloseL(f) + assert b"very_secret" not in data + + with gdaltest.config_option("OGR_SQLITE_PRAGMA", "secure_delete=0"): + with ogr.Open(filename, update=1) as ds: + with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == 0 diff --git a/autotest/ogr/ogr_osm.py b/autotest/ogr/ogr_osm.py index 65c141a12399..70f664fc5b28 100755 --- a/autotest/ogr/ogr_osm.py +++ b/autotest/ogr/ogr_osm.py @@ -921,6 +921,13 @@ def test_ogr_osm_tags_json_special_characters(): def test_ogr_osmconf_ini(): + if "EMBED_RESOURCE_FILES=YES" in gdal.VersionInfo( + "BUILD_INFO" + ) or "USE_ONLY_EMBEDDED_RESOURCE_FILES=YES" in gdal.VersionInfo("BUILD_INFO"): + pytest.skip( + "Test cannot work with EMBED_RESOURCE_FILES=YES/USE_ONLY_EMBEDDED_RESOURCE_FILES=YES" + ) + import configparser with ogr.Open("data/osm/test_json.pbf") as ds: diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index 4dba5ef4d41f..bf2581e52d8b 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -102,7 +102,9 @@ def get_lyr(): with pytest.raises(Exception): lyr.GetName() - assert get_lyr().GetFeatureCount() == 10 + # This leaks memory + if not gdaltest.is_travis_branch("sanitize"): + assert get_lyr().GetFeatureCount() == 10 # Check that we can actually remove the files (i.e. references on dataset have been dropped) os.unlink(tmp_path / "test_ogr_sql_execute_sql.shp") diff --git a/autotest/ogr/ogr_sxf.py b/autotest/ogr/ogr_sxf.py index 8cff70753320..b769f8f34199 100755 --- a/autotest/ogr/ogr_sxf.py +++ b/autotest/ogr/ogr_sxf.py @@ -71,11 +71,11 @@ def test_ogr_sxf_2(): # Open SXF datasource with custom RSC file. -def test_ogr_sxf_3(): +def test_ogr_sxf_3(tmp_path): lyr_names = ["SYSTEM", "Not_Classified"] - sxf_name = "tmp/test_ogr_sxf_3.sxf" - rsc_name = "tmp/test_ogr_sxf_3.rsc" + sxf_name = str(tmp_path / "test_ogr_sxf_3.sxf") + rsc_name = str(tmp_path / "test_ogr_sxf_3.rsc") fake_rsc = open(rsc_name, "w") fake_rsc.close() shutil.copy("data/sxf/100_test.sxf", sxf_name) diff --git a/autotest/osr/osr_basic.py b/autotest/osr/osr_basic.py index c3b59c8a454d..c14da1d0b007 100755 --- a/autotest/osr/osr_basic.py +++ b/autotest/osr/osr_basic.py @@ -2478,3 +2478,13 @@ def test_osr_basic_export_wkt_utm_south(): j = json.loads(srs.ExportToPROJJSON()) assert j["conversion"]["id"]["code"] == 16101 + + +############################################################################### + + +def test_osr_basic_GetAuthorityListFromDatabase(): + + ret = osr.GetAuthorityListFromDatabase() + assert "EPSG" in ret + assert "PROJ" in ret diff --git a/autotest/pymod/gdaltest.py b/autotest/pymod/gdaltest.py index 417a0768181e..e02254740f9d 100755 --- a/autotest/pymod/gdaltest.py +++ b/autotest/pymod/gdaltest.py @@ -2102,87 +2102,8 @@ def reopen(ds, update=False, open_options=None): ) -# VSIFile helper class - - -class VSIFile: - def __init__(self, path, mode, encoding="utf-8"): - self._path = path - self._mode = mode - - self._binary = "b" in mode - self._encoding = encoding - - self._fp = gdal.VSIFOpenExL(self._path, self._mode, True) - if self._fp is None: - raise OSError(gdal.VSIGetLastErrorMsg()) - - self._closed = False - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __iter__(self): - return self - - def __next__(self): - line = gdal.CPLReadLineL(self._fp) - if line is None: - raise StopIteration - if self._binary: - return line.encode() - return line - - def close(self): - if self._closed: - return - - self._closed = True - gdal.VSIFCloseL(self._fp) - - def read(self, size=-1): - if size == -1: - pos = self.tell() - self.seek(0, 2) - size = self.tell() - self.seek(pos) - - raw = gdal.VSIFReadL(1, size, self._fp) - - if self._binary: - return bytes(raw) - else: - return raw.decode(self._encoding) - - def write(self, x): - - if self._binary: - assert type(x) in (bytes, bytearray, memoryview) - else: - assert type(x) is str - x = x.encode(self._encoding) - - planned_write = len(x) - actual_write = gdal.VSIFWriteL(x, 1, planned_write, self._fp) - - if planned_write != actual_write: - raise OSError( - f"Expected to write {planned_write} bytes but {actual_write} were written" - ) - - def seek(self, offset, whence=0): - if gdal.VSIFSeekL(self._fp, offset, whence) != 0: - raise OSError(gdal.VSIGetLastErrorMsg()) - - def tell(self): - return gdal.VSIFTellL(self._fp) - - def vsi_open(path, mode="r"): - return VSIFile(path, mode) + return gdal.VSIFile(path, mode) def vrt_has_open_support(): diff --git a/autotest/pymod/test_cli_utilities.py b/autotest/pymod/test_cli_utilities.py index 58669d86c839..57864cab5ed0 100755 --- a/autotest/pymod/test_cli_utilities.py +++ b/autotest/pymod/test_cli_utilities.py @@ -208,6 +208,14 @@ def get_gdaldem_path(): # +def get_gdalenhance_path(): + return get_cli_utility_path("gdalenhance") + + +############################################################################### +# + + def get_gdal_rasterize_path(): return get_cli_utility_path("gdal_rasterize") diff --git a/autotest/pyscripts/test_gdalmove.py b/autotest/pyscripts/test_gdalmove.py index b8e65a63de1f..84080916d47f 100755 --- a/autotest/pyscripts/test_gdalmove.py +++ b/autotest/pyscripts/test_gdalmove.py @@ -14,6 +14,7 @@ import shutil +import gdaltest import pytest import test_py_scripts @@ -36,6 +37,9 @@ def script_path(): def test_gdalmove_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalmove", "--help" ) @@ -47,6 +51,9 @@ def test_gdalmove_help(script_path): def test_gdalmove_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalmove", "--version" ) diff --git a/autotest/utilities/test_gdalenhance.py b/autotest/utilities/test_gdalenhance.py new file mode 100644 index 000000000000..c31ada4e6558 --- /dev/null +++ b/autotest/utilities/test_gdalenhance.py @@ -0,0 +1,107 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# +# Project: GDAL/OGR Test Suite +# Purpose: gdalenhance testing +# Author: Daniel Baston +# +############################################################################### +# Copyright (c) 2024, ISciences LLC +# +# SPDX-License-Identifier: MIT +############################################################################### + +import gdaltest +import pytest +import test_cli_utilities + +from osgeo import gdal + +pytestmark = pytest.mark.skipif( + test_cli_utilities.get_gdalenhance_path() is None, + reason="gdalenhance not available", +) + + +@pytest.fixture() +def gdalenhance_path(): + return test_cli_utilities.get_gdalenhance_path() + + +############################################################################### +# Output a lookup table, then apply it to the image + + +def test_gdalenhance_output_histogram(gdalenhance_path, tmp_path): + + out, err = gdaltest.runexternal_out_and_err( + f"{gdalenhance_path} -quiet -equalize ../gcore/data/rgbsmall.tif" + ) + + assert not err + + lines = out.strip().split("\n") + assert len(lines) == 3 + + assert lines[0].startswith("1:Band ") + assert lines[1].startswith("2:Band ") + assert lines[2].startswith("3:Band ") + + lut_fname = tmp_path / "lut.txt" + + with open(lut_fname, "w") as outfile: + for line in lines: + outfile.write(line.strip()) + outfile.write("\n") + + enhanced_fname = tmp_path / "out.tif" + + out, err = gdaltest.runexternal_out_and_err( + f"{gdalenhance_path} -quiet -config {lut_fname} ../gcore/data/rgbsmall.tif {enhanced_fname}" + ) + + assert not err + + assert enhanced_fname.exists() + + +############################################################################### +# Write a new image directly + + +def test_gdalenhance_output_image(gdalenhance_path, tmp_path): + + infile = "../gcore/data/rgbsmall.tif" + outfile = tmp_path / "out.tif" + + out, err = gdaltest.runexternal_out_and_err( + f"{gdalenhance_path} -quiet -equalize -co COMPRESS=DEFLATE {infile} {outfile}" + ) + + assert not err + + with gdal.Open(infile) as src, gdal.Open(outfile) as dst: + assert src.RasterCount == dst.RasterCount + assert src.RasterXSize == dst.RasterXSize + assert src.RasterYSize == dst.RasterYSize + + # check that -co was honored + assert dst.GetMetadata("IMAGE_STRUCTURE")["COMPRESSION"] == "DEFLATE" + + +############################################################################### +# Usage printed with invalid arguments + + +def test_gdalenhance_invalid_usage(gdalenhance_path, tmp_path): + + infile = "../gcore/data/rgbsmall.tif" + outfile = tmp_path / "out.tif" + + out, err = gdaltest.runexternal_out_and_err( + f"{gdalenhance_path} -quiet {infile} {outfile}" + ) + + assert "ret code = 1" in err + assert "Usage" in out diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index fbfc2391540a..b3d0363ca362 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4343,3 +4343,80 @@ def test_gdalwarp_lib_src_is_geog_arc_second(): assert out_ds.RasterXSize == 5464 assert out_ds.RasterYSize == 5464 assert out_ds.GetRasterBand(1).Checksum() == 31856 + + +############################################################################### +# Test GWKCubicResampleNoMasks4MultiBandT() + + +def test_gdalwarp_lib_cubic_multiband_byte_4sample_optim(): + + src_ds = gdal.Open("../gdrivers/data/small_world.tif") + + # RGB only + out_ds = gdal.Warp( + "", + src_ds, + options="-f MEM -tr 0.9 0.9 -te -10 40.1 8.9 59 -r cubic", + ) + assert out_ds.RasterXSize == 21 + assert out_ds.RasterYSize == 21 + assert [out_ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 4785, + 4689, + 5007, + ] + + # With dest alpha + out_ds = gdal.Warp( + "", + src_ds, + options="-f MEM -tr 0.9 0.9 -te -10 40.1 8.9 59 -r cubic -dstalpha", + ) + assert out_ds.RasterXSize == 21 + assert out_ds.RasterYSize == 21 + assert [out_ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 4785, + 4689, + 5007, + ] + assert out_ds.GetRasterBand(4).ComputeRasterMinMax() == (255, 255) + + # Test edge effects + # (slightly change the target resolution so that the nearest approximation + # doesn't kick in) + out_ds = gdal.Warp( + "", + src_ds, + options="-f MEM -r cubic -tr 0.9000001 0.9000001 -wo XSCALE=1 -wo YSCALE=1", + ) + assert out_ds.RasterXSize == 400 + assert out_ds.RasterYSize == 200 + assert out_ds.ReadRaster() == src_ds.ReadRaster() + + +############################################################################### +# Test GWKCubicResampleNoMasks4MultiBandT() + + +def test_gdalwarp_lib_cubic_multiband_uint16_4sample_optim(): + + src_ds = gdal.Open("../gdrivers/data/small_world.tif") + src_ds = gdal.Translate( + "", src_ds, options="-f MEM -ot UInt16 -scale 0 255 0 65535" + ) + + # RGB only + out_ds = gdal.Warp( + "", + src_ds, + options="-f MEM -tr 0.9 0.9 -te -10 40.1 8.9 59 -r cubic", + ) + out_ds = gdal.Translate("", out_ds, options="-f MEM -ot Byte -scale 0 65535 0 255") + assert out_ds.RasterXSize == 21 + assert out_ds.RasterYSize == 21 + assert [out_ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 4785, + 4689, + 5007, + ] diff --git a/ci/travis/conda/libgdal-adbc.patch b/ci/travis/conda/libgdal-adbc.patch new file mode 100644 index 000000000000..23f5daac0ca4 --- /dev/null +++ b/ci/travis/conda/libgdal-adbc.patch @@ -0,0 +1,78 @@ +diff --git a/recipe/build.sh b/recipe/build.sh +index b5329fd..8bfa9ef 100644 +--- a/recipe/build.sh ++++ b/recipe/build.sh +@@ -48,6 +48,7 @@ cmake -G "Unix Makefiles" \ + -DGDAL_ENABLE_DRIVER_NETCDF=OFF \ + -DGDAL_ENABLE_DRIVER_HDF4=OFF \ + -DGDAL_ENABLE_DRIVER_HDF5=OFF \ ++ -DOGR_ENABLE_DRIVER_ADBC=OFF \ + -DOGR_REGISTER_DRIVER_ARROW_FOR_LATER_PLUGIN=ON \ + -DOGR_REGISTER_DRIVER_PARQUET_FOR_LATER_PLUGIN=ON \ + -DGDAL_REGISTER_DRIVER_JP2OPENJPEG_FOR_LATER_PLUGIN=ON \ +@@ -62,6 +63,7 @@ cmake -G "Unix Makefiles" \ + -DGDAL_REGISTER_DRIVER_NETCDF_FOR_LATER_PLUGIN=ON \ + -DGDAL_REGISTER_DRIVER_HDF4_FOR_LATER_PLUGIN=ON \ + -DGDAL_REGISTER_DRIVER_HDF5_FOR_LATER_PLUGIN=ON \ ++ -DOGR_REGISTER_DRIVER_ADBC_FOR_LATER_PLUGIN=ON \ + -DOGR_DRIVER_ARROW_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-arrow-parquet'" \ + -DOGR_DRIVER_PARQUET_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-arrow-parquet'" \ + -DGDAL_DRIVER_JP2OPENJPEG_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-jp2openjpeg'" \ +@@ -77,6 +79,7 @@ cmake -G "Unix Makefiles" \ + -DGDAL_DRIVER_HDF4_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-hdf4'" \ + -DGDAL_DRIVER_HDF5_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-hdf5'" \ + -DGDAL_ENABLE_HDF5_GLOBAL_LOCK:BOOL=ON \ ++ -DOGR_DRIVER_ADBC_PLUGIN_INSTALLATION_MESSAGE="You may install it with 'conda install -c conda-forge libgdal-adbc'" \ + -DBUILD_PYTHON_BINDINGS:BOOL=OFF \ + -DBUILD_JAVA_BINDINGS:BOOL=OFF \ + -DBUILD_CSHARP_BINDINGS:BOOL=OFF \ +diff --git a/recipe/meta.yaml b/recipe/meta.yaml +index 067eacd..12aab43 100644 +--- a/recipe/meta.yaml ++++ b/recipe/meta.yaml +@@ -580,6 +580,45 @@ outputs: + license: MIT + license_file: LICENSE.TXT + ++ - name: libgdal-adbc ++ script: build_plugin.sh # [unix] ++ script: build_plugin.bat # [win] ++ build: ++ # FIXME: libadbc-driver-manager is not available currently on Windows ++ skip: true # [win] ++ ignore_run_exports_from: ++ - expat ++ script_env: ++ - GDAL_PLUGIN_NAME=ADBC ++ - GDAL_PLUGIN_TYPE=ogr ++ - GDAL_PLUGIN_DEPS=-DGDAL_USE_ADBCDRIVERMANAGER=ON ++ requirements: ++ build: ++ - cmake ++ - {{ compiler('c') }} ++ - {{ stdlib("c") }} ++ - {{ compiler('cxx') }} ++ - pkg-config # [not win] ++ - make # [unix] ++ - ninja # [win] ++ host: ++ - {{ pin_subpackage('libgdal-core', exact=True) }} ++ - libkml-devel ++ - expat ++ - libadbc-driver-manager ++ run: ++ - libgdal-core >={{ ".".join(version.split(".")[:2]) }} ++ test: ++ commands: ++ - test -f ${PREFIX}/lib/gdalplugins/${GDAL_PLUGIN_TYPE}_${GDAL_PLUGIN_NAME}${SHLIB_EXT} # [unix] ++ - if not exist %LIBRARY_LIB%\gdalplugins\%GDAL_PLUGIN_TYPE%_%GDAL_PLUGIN_NAME%.dll exit 1 # [win] ++ - ogrinfo --format ${GDAL_PLUGIN_NAME} # [unix] ++ - ogrinfo --format %GDAL_PLUGIN_NAME% # [win] ++ about: ++ summary: Vector driver ADBC for the Geospatial Data Abstraction Library (GDAL) ++ license: MIT ++ license_file: LICENSE.TXT ++ + - name: libgdal + build: + run_exports: diff --git a/ci/travis/conda/setup.sh b/ci/travis/conda/setup.sh index 15dcff7011f5..eb08b2531a5c 100755 --- a/ci/travis/conda/setup.sh +++ b/ci/travis/conda/setup.sh @@ -19,6 +19,8 @@ git clone https://github.com/conda-forge/gdal-feedstock.git cd gdal-feedstock +patch -p1 < ../ci/travis/conda/libgdal-adbc.patch + cat > recipe/recipe_clobber.yaml <) + else() + target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) + endif() +endfunction() diff --git a/doc/build/html_extra/robots.txt b/doc/build/html_extra/robots.txt deleted file mode 100644 index 1b0f0fd8b207..000000000000 --- a/doc/build/html_extra/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -User-agent: * -Allow: /en/latest/ -Disallow: /en/ diff --git a/doc/build_doc_snapshot.sh b/doc/build_doc_snapshot.sh index 4d6805ccc1c9..1a3de3701892 100755 --- a/doc/build_doc_snapshot.sh +++ b/doc/build_doc_snapshot.sh @@ -13,7 +13,9 @@ rm -f .doxygen_up_to_date rm -rf build/html rm -rf build/latex make html -make latexpdf + +python3 -m sphinx -T -b latex -d build/doctrees -D language=en source build/latex +(cd build/latex && (latexmk -r latexmkrc -pdf -f -dvi- -ps- -jobname=gdal -interaction=nonstopmode || test -f gdal.pdf)) rm -rf "${TMPDIR}" mkdir ${TMPDIR} diff --git a/doc/images/programs/gdalenhance_little_spruce.jpg b/doc/images/programs/gdalenhance_little_spruce.jpg new file mode 100644 index 000000000000..5815f7d3956e Binary files /dev/null and b/doc/images/programs/gdalenhance_little_spruce.jpg differ diff --git a/doc/images/programs/gdalenhance_little_spruce_enhanced.jpg b/doc/images/programs/gdalenhance_little_spruce_enhanced.jpg new file mode 100644 index 000000000000..3804af6ac38b Binary files /dev/null and b/doc/images/programs/gdalenhance_little_spruce_enhanced.jpg differ diff --git a/doc/requirements.txt b/doc/requirements.txt index 539d5630643e..f8a3ca41ccd7 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,6 @@ # This file may be used to create an environment using: # $ pip install --upgrade -r +fsspec numpy sphinx breathe diff --git a/doc/source/about_no_title.rst b/doc/source/about_no_title.rst index 0f0364b0efea..c60c858cce3e 100644 --- a/doc/source/about_no_title.rst +++ b/doc/source/about_no_title.rst @@ -1,5 +1,13 @@ GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source :ref:`license` by the `Open Source Geospatial Foundation`_. As a library, it presents a single raster abstract data model and single vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line utilities for data translation and processing. The `NEWS`_ page describes the October 2024 GDAL/OGR 3.9.3 release. +.. note:: + + The GDAL project is currently soliciting feedback to help focus + :ref:`GDAL Sponsorship Program ` activities. + We would highly appreciate you fill in the `survey `__ that will + provide guidance about priorities for the program's resources (open until November 11th, 2024). + + .. only:: html |offline-download| diff --git a/doc/source/api/python/general.rst b/doc/source/api/python/general.rst index 7c7cb58a19ab..a77c90f3844c 100644 --- a/doc/source/api/python/general.rst +++ b/doc/source/api/python/general.rst @@ -95,6 +95,26 @@ Error Handling File Management --------------- +osgeo.gdal_fsspec module +++++++++++++++++++++++++ + +.. automodule:: osgeo.gdal_fsspec + :members: + :undoc-members: + :show-inheritance: + :noindex: + +osgeo.gdal.VSIFile class +++++++++++++++++++++++++ + +.. autoclass:: osgeo.gdal.VSIFile + :members: + :undoc-members: + :noindex: + +Low level functions ++++++++++++++++++++ + .. autofunction:: osgeo.gdal.CloseDir .. autofunction:: osgeo.gdal.CopyFile diff --git a/doc/source/api/python/osgeo.gdal_fsspec.rst b/doc/source/api/python/osgeo.gdal_fsspec.rst new file mode 100644 index 000000000000..2b1bbe35b138 --- /dev/null +++ b/doc/source/api/python/osgeo.gdal_fsspec.rst @@ -0,0 +1,12 @@ +.. + The documentation displayed on this page is automatically generated from + Python docstrings. See https://gdal.org/development/dev_documentation.html + for information on updating this content. + +osgeo.gdal_fsspec module +======================== + +.. automodule:: osgeo.gdal_fsspec + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/python/osgeo.rst b/doc/source/api/python/osgeo.rst index 4585e139a2e3..2b504daa35f7 100644 --- a/doc/source/api/python/osgeo.rst +++ b/doc/source/api/python/osgeo.rst @@ -12,6 +12,7 @@ Submodules osgeo.gdal osgeo.gdal_array + osgeo.gdal_fsspec osgeo.gdalconst osgeo.gnm osgeo.ogr diff --git a/doc/source/api/vector_c_api.rst b/doc/source/api/vector_c_api.rst index ea70e352227c..65dc4074b266 100644 --- a/doc/source/api/vector_c_api.rst +++ b/doc/source/api/vector_c_api.rst @@ -14,3 +14,6 @@ ogr_core.h and ogr_api.h: Vector C API .. doxygenfile:: ogr_api.h :project: api + +.. doxygenfile:: gdal_adbc.h + :project: api diff --git a/doc/source/development/building_from_source.rst b/doc/source/development/building_from_source.rst index adaa100d5bb8..8ca9f74d05f8 100644 --- a/doc/source/development/building_from_source.rst +++ b/doc/source/development/building_from_source.rst @@ -224,6 +224,36 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory. `CMAKE_SKIP_INSTALL_RPATH `__ variable is not set. +Resource files embedding +++++++++++++++++++++++++ + +Starting with GDAL 3.11, if a C23-compatible compiler is used, such as +clang >= 19 or GCC >= 15, it is possible to embed resource files inside +the GDAL library, without relying on resource files to be available on the file +system (such resource files are located through an hard-coded +path at build time in ``${CMAKE_INSTALL_PREFIX}/share/gdal``, or at run-time +through the :config:`GDAL_DATA` configuration option). + +The following CMake options control that behavior: + +.. option:: EMBED_RESOURCE_FILES=ON/OFF + + .. versionadded:: 3.11 + + Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON + for static library builds (BUILD_SHARED_LIBS=OFF). + When ON, resource files needed by GDAL will be embedded into the GDAL library + and/or relevant plugins. + +.. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF + + .. versionadded:: 3.11 + + Even if EMBED_RESOURCE_FILES=ON, GDAL will still try to locate resource + files on the file system by default , and fallback to the embedded version if + not found. By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt + at locating resource files on the file system is made. Default is OFF. + CMake package dependent options +++++++++++++++++++++++++++++++ diff --git a/doc/source/development/rfc/index.rst b/doc/source/development/rfc/index.rst index 73f24bcf634e..fc7714e67617 100644 --- a/doc/source/development/rfc/index.rst +++ b/doc/source/development/rfc/index.rst @@ -107,3 +107,4 @@ RFC list rfc98_build_requirements_gdal_3_9 rfc99_geometry_coordinate_precision rfc101_raster_dataset_threadsafety + rfc102_embedded_resources diff --git a/doc/source/development/rfc/rfc102_embedded_resources.rst b/doc/source/development/rfc/rfc102_embedded_resources.rst new file mode 100644 index 000000000000..ea41f2ec985b --- /dev/null +++ b/doc/source/development/rfc/rfc102_embedded_resources.rst @@ -0,0 +1,165 @@ +.. _rfc-102: + +=================================================================== +RFC 102: Embedding resource files into libgdal +=================================================================== + +============== ============================================= +Author: Even Rouault +Contact: even.rouault @ spatialys.com +Started: 2024-Oct-01 +Status: Adopted, implemented +Target: GDAL 3.11 +============== ============================================= + +Summary +------- + +This RFC uses C23 ``#embed`` pre-processor directive, when available, +to be able to optionally embed GDAL resource files directly into libgdal. + +A similar `PROJ RFC-8 `__ has been +submitted for PROJ to embed its :file:`proj.db` and :file:`proj.ini` files. + +Motivation +---------- + +Some parts of GDAL core, mostly drivers, require external resource files located +in the filesystem. Locating these resource files is difficult for use cases where +the GDAL binaries are relocated during installation time. +One such case could be the GDAL embedded in Rasterio or Fiona binary wheels where :config:`GDAL_DATA` must be set to the directory of the resource files. +Web-assembly (WASM) use cases come also to mind as users of GDAL builds where +resources are directly included in libgdal. + +Technical solution +------------------ + +The C23 standard includes a `#embed "filename" `__ +pre-processor directive that ingests the specified filename and returns its +content as tokens that can be stored in a unsigned char or char array. + +Getting the content of a file into a variable is as simple as the following +(which also demonstrates adding a nul-terminating character when this is needed): + +.. code-block:: c + + static const char szPDS4Template[] = { + #embed "data/pds4_template.xml" + , '\0'}; + +Compiler support +---------------- + +Support for that directive is still very new. clang 19.1 is the +first compiler which has a release including it, and has an efficient +implementation of it, able to embed very large files with minimum RAM and CPU +usage. + +The development version of GCC 15 also supports it, but in a non-optimized way +for now. i.e. trying to include large files, of several tens of megabytes could +cause significant compilation time, but without impact on runtime. This is not +an issue for GDAL use cases, and there is intent from GCC developers to improve +this in the future. + +Embedding PROJ's :file:`proj.db` of size 9.1 MB with GCC 15dev at time of writing takes +18 seconds and 1.7 GB RAM, compared to 0.4 second and 400 MB RAM for clang 19, +which is still reasonable (Generating :file:`proj.db` itself from its source .sql files +takes one minute on the same system). + +There is no timeline for Visual Studio C/C++ at time of writing (it has been +`requested by users `__) + +To be noted that currently clang 19.1 only supports ``#embed`` in .c files, not +C++ ones (the C++ standard has not yet adopted this feature). So embedding +resources must be done in a .c file, which is obviously not a problem since +we can easily export symbols/functions from a .c file to be available by C++. + +New CMake options +----------------- + +Resources will only be embedded if the new ``EMBED_RESOURCE_FILES`` CMake option +is set to ``ON``. This option will default to ``ON`` for static library builds +and if `C23 ``#embed`` is detected to be available. Users might also turn it to ON for +shared library builds. A CMake error is emitted if the option is turned on but +the compiler lacks support for it. + +A complementary CMake option ``USE_ONLY_EMBEDDED_RESOURCE_FILES`` will also +be added. It will default to ``OFF``. When set to ON, GDAL will not try to +locate resource files in the GDAL_DATA directory burnt at build time into libgdal +(``${install_prefix}/share/gdal``), or by the :config:`GDAL_DATA` configuration option. + +Said otherwise, if ``EMBED_RESOURCE_FILES=ON`` but ``USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF``, +GDAL will first try to locate resource files from the file system, and +fallback to the embedded version if not found. + +The resource files will still be installed in ``${install_prefix}/share/gdal``, +unless ``USE_ONLY_EMBEDDED_RESOURCE_FILES`` is set to ON. + +Impacted code +------------- + +- gcore: embedding LICENSE.TXT, and tms_*.json files +- frmts/grib: embedding GRIB2 CSV files +- frmts/hdf5: embedding bag_template.xml +- frmts/nitf: embedding nitf_spec.xml +- frmts/pdf: embedding pdf_composition.xml +- frmts/pds: embedding pds4_template.xml and vicar.json +- ogr/ogrsf_frmts/dgn: embedding seed_2d.dgn and seed_3d.dgn +- ogr/ogrsf_frmts/dxf: embedding header.dxf and leader.dxf +- ogr/ogrsf_frmts/gml: embedding .gfs files and gml_registry.xml +- ogr/ogrsf_frmts/gmlas: embedding gmlasconf.xml +- ogr/ogrsf_frmts/miramon: embedding MM_m_idofic.csv +- ogr/ogrsf_frmts/osm: embedding osm_conf.ini +- ogr/ogrsf_frmts/plscenes: embedding plscenesconf.json +- ogr/ogrsf_frmts/s57: embedding s57*.csv files +- ogr/ogrsf_frmts/sxf: embedding default.rsc +- ogr/ogrsf_frmts/vdv: embedding vdv452.xml + +Considered alternatives +----------------------- + +Including resource files into libraries has been a long-wished feature of C/C++. +Different workarounds have emerged over the years, such as the use of the +``od -x`` utility, GNU ``ld`` linker ``-b`` mode, or CMake-based solutions such +as https://jonathanhamberg.com/post/cmake-file-embedding/ + +We could potentially use the later to address non-C23 capable compilers, but +we have chosen not to do that, for the sake of implementation simplicity. And, +if considering using the CMake trick as the only solution, we should note that +C23 #embed has the potential for better compile time, as demonstrated by clang +implementation. + +Backward compatibility +---------------------- + +Fully backwards compatible. + +C23 is not required, unless EMBED_RESOURCE_FILES is enabled in GDAL. + +Documentation +------------- + +The 2 new CMake variables will be documented. + +Testing +------- + +The existing fedora:rawhide continuous integration target, which has now clang +19.1 available, will be modified to test the effect of the new variables. + +Local builds using GCC 15dev builds of https://jwakely.github.io/pkg-gcc-latest/ +have also be successfully done during the development of the candidate implementation + +Related issues and PRs +---------------------- + +- https://github.com/OSGeo/gdal/issues/10780 + +- `GDAL candidate implementation `__ + +- `PROJ RFC-8 Embedding resource files into libproj `__ + +Voting history +-------------- + ++1 from PSC members JukkaR, JavierJS, KurtS, HowardB and EvenR diff --git a/doc/source/download.rst b/doc/source/download.rst index cde867e88ea6..ca336557fdc3 100644 --- a/doc/source/download.rst +++ b/doc/source/download.rst @@ -27,7 +27,13 @@ Current Release Past Releases ............. -Links to :ref:`download_past` are also available. +.. only:: html + + Links to :ref:`download_past` are also available. + +.. only:: not html + + Links to `past releases `__ are also available. .. _source: diff --git a/doc/source/download_past.rst b/doc/source/download_past.rst index 1e4b3b54ed12..f732485793ef 100644 --- a/doc/source/download_past.rst +++ b/doc/source/download_past.rst @@ -283,279 +283,279 @@ Past Releases .. _`2.4.2 md5`: https://download.osgeo.org/gdal/2.4.2/gdal-2.4.2.tar.gz.md5 -* **2019-05** `gdal-3.0.0.tar.gz`_ `3.0.0 Release Notes`_ (`3.0.0 md5`)_ +* **2019-05** `gdal-3.0.0.tar.gz`_ `3.0.0 Release Notes`_ (`3.0.0 md5`_) .. _`gdal-3.0.0.tar.gz`: http://download.osgeo.org/gdal/3.0.0/gdal-3.0.0.tar.gz .. _`3.0.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.0.0/gdal/NEWS .. _`3.0.0 md5`: http://download.osgeo.org/gdal/3.0.0/gdal-3.0.0.tar.gz.md5 -* **2019-03** `gdal-2.4.1.tar.gz`_ `2.4.1 Release Notes`_ (`2.4.1 md5`)_ +* **2019-03** `gdal-2.4.1.tar.gz`_ `2.4.1 Release Notes`_ (`2.4.1 md5`_) .. _`gdal-2.4.1.tar.gz`: http://download.osgeo.org/gdal/2.4.1/gdal-2.4.1.tar.gz .. _`2.4.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.4.1/gdal/NEWS .. _`2.4.1 md5`: http://download.osgeo.org/gdal/2.4.1/gdal-2.4.1.tar.gz.md5 -* **2018-12** `gdal-2.4.0.tar.gz`_ `2.4.0 Release Notes`_ (`2.4.0 md5`)_ +* **2018-12** `gdal-2.4.0.tar.gz`_ `2.4.0 Release Notes`_ (`2.4.0 md5`_) .. _`gdal-2.4.0.tar.gz`: http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz .. _`2.4.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.4.0/gdal/NEWS .. _`2.4.0 md5`: http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz.md5 -* **2019-05** `gdal-3.0.0.tar.gz`_ `3.0.0 Release Notes`_ (`3.0.0 md5`)_ +* **2019-05** `gdal-3.0.0.tar.gz`_ `3.0.0 Release Notes`_ (`3.0.0 md5`_) .. _`gdal-3.0.0.tar.gz`: http://download.osgeo.org/gdal/3.0.0/gdal-3.0.0.tar.gz .. _`3.0.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v3.0.0/gdal/NEWS .. _`3.0.0 md5`: http://download.osgeo.org/gdal/3.0.0/gdal-3.0.0.tar.gz.md5 -* **2019-03** `gdal-2.4.1.tar.gz`_ `2.4.1 Release Notes`_ (`2.4.1 md5`)_ +* **2019-03** `gdal-2.4.1.tar.gz`_ `2.4.1 Release Notes`_ (`2.4.1 md5`_) .. _`gdal-2.4.1.tar.gz`: http://download.osgeo.org/gdal/2.4.1/gdal-2.4.1.tar.gz .. _`2.4.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.4.1/gdal/NEWS .. _`2.4.1 md5`: http://download.osgeo.org/gdal/2.4.1/gdal-2.4.1.tar.gz.md5 -* **2018-12** `gdal-2.4.0.tar.gz`_ `2.4.0 Release Notes`_ (`2.4.0 md5`)_ +* **2018-12** `gdal-2.4.0.tar.gz`_ `2.4.0 Release Notes`_ (`2.4.0 md5`_) .. _`gdal-2.4.0.tar.gz`: http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz .. _`2.4.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.4.0/gdal/NEWS .. _`2.4.0 md5`: http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz.md5 -* **2018-12** `gdal-2.3.3.tar.gz`_ `2.3.3 Release Notes`_ (`2.3.3 md5`)_ +* **2018-12** `gdal-2.3.3.tar.gz`_ `2.3.3 Release Notes`_ (`2.3.3 md5`_) .. _`gdal-2.3.3.tar.gz`: http://download.osgeo.org/gdal/2.3.3/gdal-2.3.3.tar.gz .. _`2.3.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.3.3/gdal/NEWS .. _`2.3.3 md5`: http://download.osgeo.org/gdal/2.3.3/gdal-2.3.3.tar.gz.md5 -* **2018-09** `gdal-2.3.2.tar.gz`_ `2.3.2 Release Notes`_ (`2.3.2 md5`)_ +* **2018-09** `gdal-2.3.2.tar.gz`_ `2.3.2 Release Notes`_ (`2.3.2 md5`_) .. _`gdal-2.3.2.tar.gz`: http://download.osgeo.org/gdal/2.3.2/gdal-2.3.2.tar.gz .. _`2.3.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.3.2/gdal/NEWS .. _`2.3.2 md5`: http://download.osgeo.org/gdal/2.3.2/gdal-2.3.2.tar.gz.md5 -* **2018-06** `gdal-2.3.1.tar.gz`_ `2.3.1 Release Notes`_ (`2.3.1 md5`)_ +* **2018-06** `gdal-2.3.1.tar.gz`_ `2.3.1 Release Notes`_ (`2.3.1 md5`_) .. _`gdal-2.3.1.tar.gz`: http://download.osgeo.org/gdal/2.3.1/gdal-2.3.1.tar.gz .. _`2.3.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.3.1/gdal/NEWS .. _`2.3.1 md5`: http://download.osgeo.org/gdal/2.3.1/gdal-2.3.1.tar.gz.md5 -* **2018-05** `gdal-2.3.0.tar.gz`_ `2.3.0 Release Notes`_ (`2.3.0 md5`)_ +* **2018-05** `gdal-2.3.0.tar.gz`_ `2.3.0 Release Notes`_ (`2.3.0 md5`_) .. _`gdal-2.3.0.tar.gz`: http://download.osgeo.org/gdal/2.3.0/gdal-2.3.0.tar.gz .. _`2.3.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.3.0/gdal/NEWS .. _`2.3.0 md5`: http://download.osgeo.org/gdal/2.3.0/gdal-2.3.0.tar.gz.md5 -* **2018-03** `gdal-2.2.4.tar.gz`_ `2.2.4 Release Notes`_ (`2.2.4 md5`)_ +* **2018-03** `gdal-2.2.4.tar.gz`_ `2.2.4 Release Notes`_ (`2.2.4 md5`_) .. _`gdal-2.2.4.tar.gz`: http://download.osgeo.org/gdal/2.2.4/gdal-2.2.4.tar.gz .. _`2.2.4 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.2.4/gdal/NEWS .. _`2.2.4 md5`: http://download.osgeo.org/gdal/2.2.4/gdal-2.2.4.tar.gz.md5 -* **2017-11** `gdal-2.2.3.tar.gz`_ `2.2.3 Release Notes`_ (`2.2.3 md5`)_ +* **2017-11** `gdal-2.2.3.tar.gz`_ `2.2.3 Release Notes`_ (`2.2.3 md5`_) .. _`gdal-2.2.3.tar.gz`: http://download.osgeo.org/gdal/2.2.3/gdal-2.2.3.tar.gz .. _`2.2.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.2.3/gdal/NEWS .. _`2.2.3 md5`: http://download.osgeo.org/gdal/2.2.3/gdal-2.2.3.tar.gz.md5 -* **2017-09** `gdal-2.2.2.tar.gz`_ `2.2.2 Release Notes`_ (`2.2.2 md5`)_ +* **2017-09** `gdal-2.2.2.tar.gz`_ `2.2.2 Release Notes`_ (`2.2.2 md5`_) .. _`gdal-2.2.2.tar.gz`: http://download.osgeo.org/gdal/2.2.2/gdal-2.2.2.tar.gz .. _`2.2.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.2.2/gdal/NEWS .. _`2.2.2 md5`: http://download.osgeo.org/gdal/2.2.2/gdal-2.2.2.tar.gz.md5 -* **2017-06** `gdal-2.2.1.tar.gz`_ `2.2.1 Release Notes`_ (`2.2.1 md5`)_ +* **2017-06** `gdal-2.2.1.tar.gz`_ `2.2.1 Release Notes`_ (`2.2.1 md5`_) .. _`gdal-2.2.1.tar.gz`: http://download.osgeo.org/gdal/2.2.1/gdal-2.2.1.tar.gz .. _`2.2.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.2.1/gdal/NEWS .. _`2.2.1 md5`: http://download.osgeo.org/gdal/2.2.1/gdal-2.2.1.tar.gz.md5 -* **2017-06** `gdal-2.1.4.tar.gz`_ `2.1.4 Release Notes`_ (`2.1.4 md5`)_ +* **2017-06** `gdal-2.1.4.tar.gz`_ `2.1.4 Release Notes`_ (`2.1.4 md5`_) .. _`gdal-2.1.4.tar.gz`: http://download.osgeo.org/gdal/2.1.4/gdal-2.1.4.tar.gz .. _`2.1.4 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.1.4/gdal/NEWS .. _`2.1.4 md5`: http://download.osgeo.org/gdal/2.1.4/gdal-2.1.4.tar.gz.md5 -* **2017-04** `gdal-2.2.0.tar.gz`_ `2.2.0 Release Notes`_ (`2.2.0 md5`)_ +* **2017-04** `gdal-2.2.0.tar.gz`_ `2.2.0 Release Notes`_ (`2.2.0 md5`_) .. _`gdal-2.2.0.tar.gz`: http://download.osgeo.org/gdal/2.2.0/gdal-2.2.0.tar.gz .. _`2.2.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.2.0/gdal/NEWS .. _`2.2.0 md5`: http://download.osgeo.org/gdal/2.2.0/gdal-2.2.0.tar.gz.md5 -* **2017-01** `gdal-2.1.3.tar.gz`_ `2.1.3 Release Notes`_ (`2.1.3 md5`)_ +* **2017-01** `gdal-2.1.3.tar.gz`_ `2.1.3 Release Notes`_ (`2.1.3 md5`_) .. _`gdal-2.1.3.tar.gz`: http://download.osgeo.org/gdal/2.1.3/gdal-2.1.3.tar.gz .. _`2.1.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.1.3/gdal/NEWS .. _`2.1.3 md5`: http://download.osgeo.org/gdal/2.1.3/gdal-2.1.3.tar.gz.md5 -* **2016-10** `gdal-2.1.2.tar.gz`_ `2.1.2 Release Notes`_ (`2.1.2 md5`)_ +* **2016-10** `gdal-2.1.2.tar.gz`_ `2.1.2 Release Notes`_ (`2.1.2 md5`_) .. _`gdal-2.1.2.tar.gz`: http://download.osgeo.org/gdal/2.1.2/gdal-2.1.2.tar.gz .. _`2.1.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.1.2/gdal/NEWS .. _`2.1.2 md5`: http://download.osgeo.org/gdal/2.1.2/gdal-2.1.2.tar.gz.md5 -* **2016-07** `gdal-2.1.1.tar.gz`_ `2.1.1 Release Notes`_ (`2.1.1 md5`)_ +* **2016-07** `gdal-2.1.1.tar.gz`_ `2.1.1 Release Notes`_ (`2.1.1 md5`_) .. _`gdal-2.1.1.tar.gz`: http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz .. _`2.1.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.1.1/gdal/NEWS .. _`2.1.1 md5`: http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz.md5 -* **2016-07** `gdal-2.0.3.tar.gz`_ `2.0.3 Release Notes`_ (`2.0.3 md5`)_ +* **2016-07** `gdal-2.0.3.tar.gz`_ `2.0.3 Release Notes`_ (`2.0.3 md5`_) .. _`gdal-2.0.3.tar.gz`: http://download.osgeo.org/gdal/2.0.3/gdal-2.0.3.tar.gz .. _`2.0.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.0.3/gdal/NEWS .. _`2.0.3 md5`: http://download.osgeo.org/gdal/2.0.3/gdal-2.0.3.tar.gz.md5 -* **2016-07** `gdal-1.11.5.tar.gz`_ `1.11.5 Release Notes`_ (`1.11.5 md5`)_ +* **2016-07** `gdal-1.11.5.tar.gz`_ `1.11.5 Release Notes`_ (`1.11.5 md5`_) .. _`gdal-1.11.5.tar.gz`: http://download.osgeo.org/gdal/1.11.5/gdal-1.11.5.tar.gz .. _`1.11.5 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.5/gdal/NEWS .. _`1.11.5 md5`: http://download.osgeo.org/gdal/1.11.5/gdal-1.11.5.tar.gz.md5 -* **2016-04** `gdal-2.1.0.tar.gz`_ `2.1.0 Release Notes`_ (`2.1.0 md5`)_ +* **2016-04** `gdal-2.1.0.tar.gz`_ `2.1.0 Release Notes`_ (`2.1.0 md5`_) .. _`gdal-2.1.0.tar.gz`: http://download.osgeo.org/gdal/2.1.0/gdal-2.1.0.tar.gz .. _`2.1.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.1.0/gdal/NEWS .. _`2.1.0 md5`: http://download.osgeo.org/gdal/2.1.0/gdal-2.1.0.tar.gz.md5 -* **2016-01** `gdal-2.0.2.tar.gz`_ `2.0.2 Release Notes`_ (`2.0.2 md5`)_ +* **2016-01** `gdal-2.0.2.tar.gz`_ `2.0.2 Release Notes`_ (`2.0.2 md5`_) .. _`gdal-2.0.2.tar.gz`: http://download.osgeo.org/gdal/2.0.2/gdal-2.0.2.tar.gz .. _`2.0.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.0.2/gdal/NEWS .. _`2.0.2 md5`: http://download.osgeo.org/gdal/2.0.2/gdal-2.0.2.tar.gz.md5 -* **2016-01** `gdal-1.11.4.tar.gz`_ `1.11.4 Release Notes`_ (`1.11.4 md5`)_ +* **2016-01** `gdal-1.11.4.tar.gz`_ `1.11.4 Release Notes`_ (`1.11.4 md5`_) .. _`gdal-1.11.4.tar.gz`: http://download.osgeo.org/gdal/1.11.4/gdal-1.11.4.tar.gz .. _`1.11.4 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.4/gdal/NEWS .. _`1.11.4 md5`: http://download.osgeo.org/gdal/1.11.4/gdal-1.11.4.tar.gz.md5 -* **2015-09** `gdal-2.0.1.tar.gz`_ `2.0.1 Release Notes`_ (`2.0.1 md5`)_ +* **2015-09** `gdal-2.0.1.tar.gz`_ `2.0.1 Release Notes`_ (`2.0.1 md5`_) .. _`gdal-2.0.1.tar.gz`: http://download.osgeo.org/gdal/2.0.1/gdal-2.0.1.tar.gz .. _`2.0.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.0.1/gdal/NEWS .. _`2.0.1 md5`: http://download.osgeo.org/gdal/2.0.1/gdal-2.0.1.tar.gz.md5 -* **2015-09** `gdal-1.11.3.tar.gz`_ `1.11.3 Release Notes`_ (`1.11.3 md5`)_ +* **2015-09** `gdal-1.11.3.tar.gz`_ `1.11.3 Release Notes`_ (`1.11.3 md5`_) .. _`gdal-1.11.3.tar.gz`: http://download.osgeo.org/gdal/1.11.3/gdal-1.11.3.tar.gz .. _`1.11.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.3/gdal/NEWS .. _`1.11.3 md5`: http://download.osgeo.org/gdal/1.11.3/gdal-1.11.3.tar.gz.md5 -* **2015-06** `gdal-2.0.0.tar.gz`_ `2.0.0 Release Notes`_ (`2.0.0 md5`)_ +* **2015-06** `gdal-2.0.0.tar.gz`_ `2.0.0 Release Notes`_ (`2.0.0 md5`_) .. _`gdal-2.0.0.tar.gz`: http://download.osgeo.org/gdal/2.0.0/gdal-2.0.0.tar.gz .. _`2.0.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v2.0.0/gdal/NEWS .. _`2.0.0 md5`: http://download.osgeo.org/gdal/2.0.0/gdal-2.0.0.tar.gz.md5 -* **2015-02** `gdal-1.11.2.tar.gz`_ `1.11.2 Release Notes`_ (`1.11.2 md5`)_ +* **2015-02** `gdal-1.11.2.tar.gz`_ `1.11.2 Release Notes`_ (`1.11.2 md5`_) .. _`gdal-1.11.2.tar.gz`: http://download.osgeo.org/gdal/1.11.2/gdal-1.11.2.tar.gz .. _`1.11.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.2/gdal/NEWS .. _`1.11.2 md5`: http://download.osgeo.org/gdal/1.11.2/gdal-1.11.2.tar.gz.md5 -* **2014-09** `gdal-1.11.1.tar.gz`_ `1.11.1 Release Notes`_ (`1.11.1 md5`)_ +* **2014-09** `gdal-1.11.1.tar.gz`_ `1.11.1 Release Notes`_ (`1.11.1 md5`_) .. _`gdal-1.11.1.tar.gz`: http://download.osgeo.org/gdal/1.11.1/gdal-1.11.1.tar.gz .. _`1.11.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.1/gdal/NEWS .. _`1.11.1 md5`: http://download.osgeo.org/gdal/1.11.1/gdal-1.11.1.tar.gz.md5 -* **2014-04** `gdal-1.11.0.tar.gz`_ `1.11.0 Release Notes`_ (`1.11.0 md5`)_ +* **2014-04** `gdal-1.11.0.tar.gz`_ `1.11.0 Release Notes`_ (`1.11.0 md5`_) .. _`gdal-1.11.0.tar.gz`: http://download.osgeo.org/gdal/1.11.0/gdal-1.11.0.tar.gz .. _`1.11.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.11.0/gdal/NEWS .. _`1.11.0 md5`: http://download.osgeo.org/gdal/1.11.0/gdal-1.11.0.tar.gz.md5 -* **2013-08** `gdal-1.10.1.tar.gz`_ `1.10.1 Release Notes`_ (`1.10.1 md5`)_ +* **2013-08** `gdal-1.10.1.tar.gz`_ `1.10.1 Release Notes`_ (`1.10.1 md5`_) .. _`gdal-1.10.1.tar.gz`: http://download.osgeo.org/gdal/1.10.1/gdal-1.10.1.tar.gz .. _`1.10.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.10.1/gdal/NEWS .. _`1.10.1 md5`: http://download.osgeo.org/gdal/1.10.1/gdal-1.10.1.tar.gz.md5 -* **2013-04** `gdal-1.10.0.tar.gz`_ `1.10.0 Release Notes`_ (`1.10.0 md5`)_ +* **2013-04** `gdal-1.10.0.tar.gz`_ `1.10.0 Release Notes`_ (`1.10.0 md5`_) .. _`gdal-1.10.0.tar.gz`: http://download.osgeo.org/gdal/1.10.0/gdal-1.10.0.tar.gz .. _`1.10.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.10.0/gdal/NEWS .. _`1.10.0 md5`: http://download.osgeo.org/gdal/1.10.0/gdal-1.10.0.tar.gz.md5 -* **2012-10** `gdal-1.9.2.tar.gz`_ `1.9.2 Release Notes`_ (`1.9.2 md5`)_ +* **2012-10** `gdal-1.9.2.tar.gz`_ `1.9.2 Release Notes`_ (`1.9.2 md5`_) .. _`gdal-1.9.2.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.2.tar.gz .. _`1.9.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.9.2/gdal/NEWS .. _`1.9.2 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.2.tar.gz.md5 -* **2012-05** `gdal-1.9.1.tar.gz`_ `1.9.1 Release Notes`_ (`1.9.1 md5`)_ +* **2012-05** `gdal-1.9.1.tar.gz`_ `1.9.1 Release Notes`_ (`1.9.1 md5`_) .. _`gdal-1.9.1.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.1.tar.gz .. _`1.9.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.9.1/gdal/NEWS .. _`1.9.1 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.1.tar.gz.md5 -* **2011-12** `gdal-1.9.0.tar.gz`_ `1.9.0 Release Notes`_ (`1.9.0 md5`)_ +* **2011-12** `gdal-1.9.0.tar.gz`_ `1.9.0 Release Notes`_ (`1.9.0 md5`_) .. _`gdal-1.9.0.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.0.tar.gz .. _`1.9.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.9.0/gdal/NEWS .. _`1.9.0 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.9.0.tar.gz.md5 -* **2011-07** `gdal-1.8.1.tar.gz`_ `1.8.1 Release Notes`_ (`1.8.1 md5`)_ +* **2011-07** `gdal-1.8.1.tar.gz`_ `1.8.1 Release Notes`_ (`1.8.1 md5`_) .. _`gdal-1.8.1.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.8.1.tar.gz .. _`1.8.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.8.1/gdal/NEWS .. _`1.8.1 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.8.1.tar.gz.md5 -* **2011-01** `gdal-1.8.0.tar.gz`_ `1.8.0 Release Notes`_ (`1.8.0 md5`)_ +* **2011-01** `gdal-1.8.0.tar.gz`_ `1.8.0 Release Notes`_ (`1.8.0 md5`_) .. _`gdal-1.8.0.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.8.0.tar.gz .. _`1.8.0 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.8.0/gdal/NEWS .. _`1.8.0 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.8.0.tar.gz.md5 -* **2010-11** `gdal-1.7.3.tar.gz`_ `1.7.3 Release Notes`_ (`1.7.3 md5`)_ +* **2010-11** `gdal-1.7.3.tar.gz`_ `1.7.3 Release Notes`_ (`1.7.3 md5`_) .. _`gdal-1.7.3.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.7.3.tar.gz .. _`1.7.3 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.7.3/gdal/NEWS .. _`1.7.3 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.7.3.tar.gz.md5 -* **2010-04** `gdal-1.7.2.tar.gz`_ `1.7.2 Release Notes`_ (`1.7.2 md5`)_ +* **2010-04** `gdal-1.7.2.tar.gz`_ `1.7.2 Release Notes`_ (`1.7.2 md5`_) .. _`gdal-1.7.2.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.7.2.tar.gz .. _`1.7.2 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.7.2/gdal/NEWS .. _`1.7.2 md5`: http://download.osgeo.org/gdal/old_releases/gdal-1.7.2.tar.gz.md5 -* **2010-02** `gdal-1.7.1.tar.gz`_ `1.7.1 Release Notes`_ (`1.7.1 md5`)_ +* **2010-02** `gdal-1.7.1.tar.gz`_ `1.7.1 Release Notes`_ (`1.7.1 md5`_) .. _`gdal-1.7.1.tar.gz`: http://download.osgeo.org/gdal/old_releases/gdal-1.7.1.tar.gz .. _`1.7.1 Release Notes`: https://github.com/OSGeo/gdal/blob/v1.7.1/gdal/NEWS diff --git a/doc/source/drivers/raster/cog.rst b/doc/source/drivers/raster/cog.rst index c7a50573c960..c46bac1f5703 100644 --- a/doc/source/drivers/raster/cog.rst +++ b/doc/source/drivers/raster/cog.rst @@ -125,7 +125,7 @@ General creation options The higher, the smaller file and slower compression time. - .. co:: JXL_DISTANCE - :choices: 0.1-15 + :choices: 0.01-25 :default: 1.0 Distance level for lossy JPEG-XL compression. @@ -137,7 +137,7 @@ General creation options The recommended range is [0.5,3]. - .. co:: JXL_ALPHA_DISTANCE - :choices: -1, 0, 0.1-15 + :choices: -1, 0, 0.01-25 :default: -1 :since: 3.7 diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index 62119b32c9df..f86a9fef8546 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -635,7 +635,7 @@ This driver supports the following creation options: The higher, the smaller file and slower compression time. - .. co:: JXL_DISTANCE - :choices: [0.1-15] + :choices: [0.01-25] :default: 1.0 Distance level for lossy JPEG-XL compression. @@ -647,7 +647,7 @@ This driver supports the following creation options: The recommended range is [0.5,3]. - .. co:: JXL_ALPHA_DISTANCE - :choices: -1,0,[0.1-15] + :choices: -1,0,[0.01-25] :default: -1 :since: 3.7 diff --git a/doc/source/drivers/raster/jpegxl.rst b/doc/source/drivers/raster/jpegxl.rst index f1453995c310..4ad829843a4d 100644 --- a/doc/source/drivers/raster/jpegxl.rst +++ b/doc/source/drivers/raster/jpegxl.rst @@ -108,7 +108,7 @@ The following creation options are available: The higher, the smaller file and slower compression time. - .. co:: DISTANCE - :choices: 0.1-15 + :choices: 0.01-25 :default: 1.0 Distance level for lossy JPEG-XL compression. @@ -120,7 +120,7 @@ The following creation options are available: The recommended range is [0.5,3]. - .. co:: ALPHA_DISTANCE - :choices: -1, 0, 0.1-15 + :choices: -1, 0, 0.01-25 :default: -1.0 :since: 3.7 diff --git a/doc/source/drivers/vector/adbc.rst b/doc/source/drivers/vector/adbc.rst new file mode 100644 index 000000000000..e8a3659d2715 --- /dev/null +++ b/doc/source/drivers/vector/adbc.rst @@ -0,0 +1,129 @@ +.. _vector.adbc: + +ADBC -- Arrow Database Connectivity +=================================== + +.. versionadded:: 3.11 + +.. shortname:: ADBC + +ADBC is a set of APIs and libraries for Arrow-native access to database. + +This driver has 2 modes: + +- either it has been built against the ``adbc-driver-manager`` library. In that + case, it can directly be used to connect to available ADBC drivers, and expose + content as classic OGR features, or as a ArrowArrayStream. + In that mode the driver metadata exposes the ``HAS_ADBC_DRIVER_MANAGER`` + metadata item. +- or it has not, in which case applications embedding GDAL must use + :cpp:func:`GDALSetAdbcLoadDriverOverride` as detailed in a below paragraph. + Note that use of that function can also be done even if the driver has been built + against the ``adbc-driver-manager`` library. + +Consult the `installation instruction `__ +for the various ADBC drivers. At time of writing, there are drivers for +SQLite3, PostgreSQL, Snowflake, BigQuery, DuckDB, Flight SQL, etc. + +The driver is read-only, and there is no support for spatial data currently. + +Connection string +----------------- + +Several connection strings are supported: + +- ``ADBC:{some_uri}``, together with the ``ADBC_DRIVER`` open option. +- a SQLite3 database filename, if the ``adbc_driver_sqlite`` is available. +- a DuckDB database filename, if the :file:`libduckdb.so`, :file:`libduckdb.dylib` + or :file:`duckdb.dll` is available (and it is in a system location, or can be + located through LD_LIBRARY_PATH on Linux, DYLD_LIBRARY_PATH on MacOSX or PATH on Windows). +- a Parquet database filename, if the :file:`libduckdb.so`, :file:`libduckdb.dylib` + or :file:`duckdb.dll` is available (and it is in a system location, or can be + located through LD_LIBRARY_PATH on Linux, DYLD_LIBRARY_PATH on MacOSX or PATH on Windows). +- a PostgreSQL URI starting with ``postgresql://``, if the ``adbc_driver_postgresql`` is available. + +Note: if present, the :ref:`vector.sqlite`, :ref:`vector.gpkg` or +:ref:`vector.parquet` drivers are registered before the ADBC driver, so they will +be used in priority when available. The ``ADBC:`` prefix or the ``-if ADBC`` +switch of :program:`ogrinfo` or :program:`ogr2ogr` can be used to use the ADBC +driver instead. + +Dataset open options +-------------------- + +|about-open-options| +The following open options are supported: + +- .. oo:: ADBC_DRIVER + :choices: + + ADBC driver name. Examples: ``adbc_driver_sqlite``, ``adbc_driver_postgresql``, + ``adbc_driver_bigquery``, ``adbc_driver_snowflake`` or a path to the + DuckDB shared library. + +- .. oo:: SQL + :choices: + + A SQL-like statement recognized by the driver, used to create a result + layer from the dataset. + +- .. oo:: ADBC_OPTION_xxx + :choices: + + Custom ADBC option to pass to AdbcDatabaseSetOption(). Options are + driver specific. + For example ``ADBC_OPTION_uri=some_value`` to pass the ``uri`` option. + +"table_list" special layer +-------------------------- + +For PostgreSQL, SQLite3, DuckDB and Parquet datasets, the driver automatically +instantiates OGR layers from available tables. +For other databases, the user must explicit provide a SQL open option or issue +a :cpp:func:`GDALDataset::ExecuteSQL` request. +To facilitate that process, a special OGR ``table_list`` layer can be queried +through :cpp:func:`GDALDataset::GetLayerByName` (or as the layer name with +:program:`ogrinfo`). +It returns for each table a OGR feature with the following fields (some +potentially unset or with an empty string): ``catalog_name``, ``schema_name``, +``table_name``, ``table_type``. + +Custom driver entry point +------------------------- + +A custom driver entry point can be specified by applications by calling +:cpp:func:`GDALSetAdbcLoadDriverOverride` (defined in header :file:`gdal_adbc.h`) +before using the driver. The specified init function will be used by the +GDAL ADBC driver as a way of locating and loading the ADBC driver if GDAL was +not built with ADBC Driver Manager support or if an embedding application has +an updated or augmented collection of drivers available. + +Examples +-------- + +- Assuming :file:`libduckdb.so`, :file:`libduckdb.dylib` or :file:`duckdb.dll` + is available (and it is in a system location, or can be located through + LD_LIBRARY_PATH on Linux, DYLD_LIBRARY_PATH on MacOSX or PATH on Windows). + + Convert a Parquet file to GeoPackage: + + :: + + ogr2ogr out.gpkg in.parquet + + +- Assuming :file:`libduckdb.so`, :file:`libduckdb.dylib` or :file:`duckdb.dll` + is available (and it is in a system location, or can be located through + LD_LIBRARY_PATH on Linux, DYLD_LIBRARY_PATH on MacOSX or PATH on Windows). + + Convert a DuckDB database to GeoPackage: + + :: + + ogr2ogr out.gpkg in.duckdb + + +See Also +-------- + +`ADBC: Arrow Database Connectivity `__ diff --git a/doc/source/drivers/vector/gpkg.rst b/doc/source/drivers/vector/gpkg.rst index 77c8c5364dc7..634ee61fd310 100644 --- a/doc/source/drivers/vector/gpkg.rst +++ b/doc/source/drivers/vector/gpkg.rst @@ -809,6 +809,18 @@ Examples -sql "SELECT poly.id, other.foo FROM poly JOIN other_schema.other USING (id)" \ -oo PRELUDE_STATEMENTS="ATTACH DATABASE 'other.gpkg' AS other_schema" +Secure deletion +--------------- + +Depending on how SQLite3 is built, `secure deletion `__ +might or might not be enabled. +Starting with GDAL 3.10, secure deletion is always enabled, unless +``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA` +configuration option. +Note that secure deletion does not recover potential lost space, so running +a `VACUUM `__ query is recommended to fully +optimized a database that has been subject to updates or deletions. + See Also -------- diff --git a/doc/source/drivers/vector/index.rst b/doc/source/drivers/vector/index.rst index 0ac42608c26a..859bba292217 100644 --- a/doc/source/drivers/vector/index.rst +++ b/doc/source/drivers/vector/index.rst @@ -22,6 +22,7 @@ Vector drivers :maxdepth: 1 :hidden: + adbc amigocloud arrow avcbin diff --git a/doc/source/drivers/vector/sqlite.rst b/doc/source/drivers/vector/sqlite.rst index 0c005056843d..7242910c69c5 100644 --- a/doc/source/drivers/vector/sqlite.rst +++ b/doc/source/drivers/vector/sqlite.rst @@ -556,6 +556,17 @@ and optimize it. ogrinfo db.sqlite -sql "VACUUM" +Secure deletion +--------------- + +Depending on how SQLite3 is built, `secure deletion `__ +might or might not be enabled. +Starting with GDAL 3.10, secure deletion is always enabled, unless +``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA` +configuration option. +Note that secure deletion does not recover potential lost space, so running +a `VACUUM `__ query is recommended to fully +optimized a database that has been subject to updates or deletions. Example ------- diff --git a/doc/source/programs/gdalenhance.rst b/doc/source/programs/gdalenhance.rst new file mode 100644 index 000000000000..72662177209a --- /dev/null +++ b/doc/source/programs/gdalenhance.rst @@ -0,0 +1,94 @@ +.. _gdalenhance: + +================================================================================ +gdalenhance +================================================================================ + +.. only:: html + + Enhance the contrast of raster images using histogram equalization. + +.. Index:: gdalenhance + +Synopsis +-------- + +.. code-block:: + + gdalenhance [--help-general] + [-of format] [-co "NAME=VALUE"]* + [-ot {Byte/Int16/UInt16/UInt32/Int32/Float32/Float64/ + CInt16/CInt32/CFloat32/CFloat64}] + [-equalize] + [-config filename] + [] + +Description +----------- + +The :program:`gdalenhance` utility enhances the contrast of raster images using +histogram equalization. It can either generate a LUT (look-up-table) that can +later be applied to enhance contrast, or it can write the enhanced image +directly. + +.. figure:: ../../images/programs/gdalenhance_little_spruce.jpg + + Original image. + +.. figure:: ../../images/programs/gdalenhance_little_spruce_enhanced.jpg + + Image after histogram equalization using :program:`gdalenhance`. + + +.. program:: gdalenhance + +.. include:: options/help_and_help_general.rst + +.. option:: -of format + + Select the output format. The default is GeoTIFF (GTiff). + +.. include:: options/co.rst + +.. include:: options/ot.rst + +.. option:: -equalize + + Get source image histogram and compute equalization LUTs from + it. + +.. option:: -config + + Apply a specified LUT to enhance the contrast of the image. + The number of lines in the LUT file should be equal to the number of bands + of the images. + Example of LUT file: + + .. code-block:: + + 1:Band -0.5:ScaleMin 255.5:ScaleMax 0 0 8 16 16 17 17 17 17 18 18 18 18 18 18 19 19 19 19 20 20 20 21 21 21 22 22 22 23 23 24 24 25 25 26 27 27 28 29 30 30 31 32 33 34 35 36 37 38 39 40 42 43 44 46 47 48 50 51 53 55 56 58 60 62 63 65 67 69 71 73 75 77 79 81 83 85 87 89 91 93 95 97 99 101 103 105 107 109 111 113 115 117 119 121 123 125 127 129 131 133 135 137 138 140 142 144 146 148 150 152 153 155 157 159 161 162 164 166 168 169 171 173 174 176 178 179 181 182 184 186 187 189 190 192 193 195 196 198 199 200 202 203 205 206 207 208 210 211 212 213 214 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 234 235 236 237 238 239 239 240 241 241 242 243 244 244 245 245 246 246 247 247 248 248 249 249 250 250 250 251 251 251 251 251 252 252 252 252 252 252 252 252 252 252 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 254 254 254 254 254 254 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 + 2:Band -0.5:ScaleMin 255.5:ScaleMax 0 0 8 16 16 16 17 17 17 17 17 17 17 17 17 18 18 18 18 18 18 19 19 19 19 19 20 20 20 20 21 21 22 22 23 23 24 24 25 26 27 27 28 29 30 31 32 33 34 35 36 37 38 39 41 42 43 45 46 47 49 51 52 54 55 57 59 60 62 64 66 67 69 71 73 75 77 79 81 83 85 87 89 91 93 95 97 99 101 103 105 108 110 112 114 116 118 120 122 124 126 128 130 132 134 136 138 140 142 144 146 148 150 152 154 156 157 159 161 163 165 166 168 170 172 173 175 177 179 180 182 183 185 187 188 190 191 193 194 196 197 199 200 202 203 204 206 207 208 209 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 230 231 232 233 234 235 235 236 237 238 239 239 240 241 241 242 243 243 244 245 245 246 246 247 247 248 248 248 249 249 250 250 250 250 251 251 251 251 252 252 252 252 252 252 252 252 252 252 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 254 254 254 254 254 254 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 + 3:Band -0.5:ScaleMin 255.5:ScaleMax 0 0 9 17 17 18 18 18 18 18 19 19 19 19 19 19 20 20 20 20 20 20 21 21 21 21 21 21 22 22 22 22 22 23 23 23 24 24 24 25 25 26 26 27 28 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 50 51 53 54 56 57 59 60 62 63 65 67 69 70 72 74 76 78 80 82 84 86 88 90 92 94 97 99 101 103 105 107 109 111 113 115 116 118 120 122 124 126 127 129 131 133 134 136 138 139 141 143 144 146 147 149 151 152 154 155 157 159 160 162 163 165 167 168 170 171 173 175 176 178 179 181 182 184 186 187 189 190 192 193 195 197 198 200 201 203 204 206 207 209 210 212 213 214 216 217 218 219 220 222 223 224 225 226 227 228 229 230 231 232 233 233 234 235 236 237 238 239 239 240 241 242 243 243 244 245 246 246 247 247 248 248 249 249 250 250 251 251 251 251 252 252 252 252 252 252 252 252 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 253 254 254 255 255 255 255 255 255 2550 + +Example +-------- + +.. code-block:: + + gdalenhance -equalize rgb.tif rgb_equalize_enhance.tif + +Apply equalization histogram to enhance the contrast of the image. + + +.. code-block:: + + gdalenhance -equalize rgb.tif + +Write an equalization LUT to the console. + + +.. code-block:: + + gdalenhance -config enhance_config rgb.tif rgb_custom_enhance.tif + +Apply a custom LUT (look up-table) to enhance the contrast of the image. diff --git a/doc/source/programs/index.rst b/doc/source/programs/index.rst index af107ef12aa5..a6ab1e221b56 100644 --- a/doc/source/programs/index.rst +++ b/doc/source/programs/index.rst @@ -50,6 +50,7 @@ Raster programs gdalbuildvrt gdalcompare gdaldem + gdalenhance gdalinfo gdallocationinfo gdalmanage @@ -89,6 +90,7 @@ Raster programs - :ref:`gdalbuildvrt`: Builds a VRT from a list of datasets. - :ref:`gdalcompare`: Compare two images. - :ref:`gdaldem`: Tools to analyze and visualize DEMs. + - :ref:`gdalenhance`: Enhance an image with LUT-based contrast enhancement - :ref:`gdalinfo`: Lists information about a raster dataset. - :ref:`gdallocationinfo`: Raster query tool - :ref:`gdalmanage`: Identify, delete, rename and copy raster data files. diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index 77d3e4e14e6c..f60cc6b6a0c0 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -19,6 +19,7 @@ acl ACL actinia ActiveX +adbc addalpha addCurve addfields @@ -369,6 +370,7 @@ cid CInt circularstrings ClampRequests +clang ClassificationCode Clayers CleanTimeout @@ -1018,6 +1020,7 @@ GDALThreadLocalDatasetCache gdaltindex gdaltransform gdalvirtualmem +gdalvsi gdalwarp gdalwmscache gdb @@ -1353,6 +1356,7 @@ IdentificationTolerance identificator Identificator IDisposable +idofic Idrisi iDriver idx @@ -1782,6 +1786,7 @@ minY MinZ Mipmaps MiraD +miramon mis mitab mkdir @@ -2409,6 +2414,7 @@ Placemark plaintext Plessis plmosaic +plscenes plscenesconf pM pnBufferSize @@ -2841,6 +2847,7 @@ RPF rpr RRaster rrd +rsc rsiz rst rsync @@ -3142,6 +3149,7 @@ swi Swif swiftclient swq +sxf sym symlinked syntaxes @@ -3413,6 +3421,7 @@ vcpkg vct vcvars vdc +vdv vecror VectorInfo VectorInfoOptions @@ -3455,6 +3464,7 @@ vsiaz vsicached vsicrypt vsicurl +VSIFile VSIFOpen vsigs vsigz diff --git a/doc/source/sponsors/index.rst b/doc/source/sponsors/index.rst index 8be481f461a8..0cdc1c2bdb65 100644 --- a/doc/source/sponsors/index.rst +++ b/doc/source/sponsors/index.rst @@ -251,6 +251,8 @@ Related resources - `Sustainable GDAL Sponsorship Prospectus`_. - :ref:`Sponsoring frequently asked questions (FAQ) `. +- :ref:`rfc-80` +- :ref:`rfc-83` .. Source of the PDF is at https://docs.google.com/document/d/1yhMWeI_LgEXPUkngqOitqcKfp7ov6WsS41v5ulz-kd0/edit# diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index 425d0292371a..3c9c4b7d8eaa 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -222,7 +222,10 @@ Performance and caching :program:`gdalwarp`. If its value is small (less than 100000), it is assumed to be measured in megabytes, otherwise in bytes. Alternatively, the value can be set to "X%" to mean X% - of the usable physical RAM. Note that this value is only consulted the first + of the usable physical RAM. + Since GDAL 3.11, the value of :config:`GDAL_CACHEMAX` may specify the + units directly (e.g., "500MB", "2GB"). + Note that this value is only consulted the first time the cache size is requested. To change this value programmatically during operation of the program it is better to use :cpp:func:`GDALSetCacheMax` (always in bytes) or or @@ -318,6 +321,9 @@ Performance and caching Set the size of the VSI cache. Be wary of large values for ``VSI_CACHE_SIZE`` when opening VRT datasources containing many source rasters, as this is a per-file cache. + Since GDAL 3.11, the value of ``VSI_CACHE_SIZE`` may be specified using + memory units (e.g., "25 MB"). + Driver management ^^^^^^^^^^^^^^^^^ @@ -421,7 +427,7 @@ General options option. - .. config:: CPL_VSIL_DEFLATE_CHUNK_SIZE - :default: 1 M + :default: 1M - .. config:: GDAL_DISABLE_CPLLOCALEC :choices: YES, NO @@ -610,7 +616,8 @@ Networking options :since: 2.3 Size of global least-recently-used (LRU) cache shared among all downloaded - content. + content. Value is assumed to represent bytes unless memory units are + specified (since GDAL 3.11). - .. config:: CPL_VSIL_CURL_USE_HEAD :choices: YES, NO @@ -658,6 +665,9 @@ Networking options :choices: :since: 2.3 + Value is assumed to represent bytes unless memory units are + specified (since GDAL 3.11). + - .. config:: GDAL_INGESTED_BYTES_AT_OPEN :since: 2.3 diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile index a79a377d36ba..e0747eb3d227 100644 --- a/docker/ubuntu-full/Dockerfile +++ b/docker/ubuntu-full/Dockerfile @@ -395,6 +395,24 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -V libarrow-dataset-dev${APT_ARCH_SUFFIX}=${ARROW_VERSION} \ && rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +# Manually install ADBC packages from Ubuntu 22.04 as there are no 24.04 packages at time of writing. +RUN . /buildscripts/bh-set-envvars.sh \ + && if test "${GCC_ARCH}" = "x86_64"; then \ + curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake-dev_14-1_amd64.deb \ + && rm -f libadbc*.deb; \ + fi + ARG WITH_DEBUG_SYMBOLS=no # Refresh grids @@ -483,6 +501,31 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && apt-get install -y -V libparquet${ARROW_SOVERSION} \ && apt-get install -y -V libarrow-dataset${ARROW_SOVERSION} +# Manually install ADBC packages from Ubuntu 22.04 as there are no 24.04 packages at time of writing. +RUN if test "$(uname -p)" = "x86_64"; then \ + curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake102_14-1_amd64.deb \ + && curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-snowflake-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-manager-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-sqlite-dev_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake102_14-1_amd64.deb \ + && dpkg -i libadbc-driver-snowflake-dev_14-1_amd64.deb \ + && rm -f libadbc*.deb; \ + fi + +# Install libduckdb +RUN if test "$(uname -p)" = "x86_64"; then \ + curl -LO -fsS https://github.com/duckdb/duckdb/releases/download/v1.1.2/libduckdb-linux-amd64.zip \ + && unzip libduckdb-linux-amd64.zip libduckdb.so \ + && mv libduckdb.so /usr/lib/x86_64-linux-gnu \ + && rm -f libduckdb-linux-amd64.zip; \ + fi + # Attempt to order layers starting with less frequently varying ones COPY --from=builder /build_thirdparty/usr/ /usr/ diff --git a/docker/util.sh b/docker/util.sh index ad037f937d5a..7c604794c7f5 100755 --- a/docker/util.sh +++ b/docker/util.sh @@ -357,18 +357,10 @@ EOF BASE_IMAGE=$(grep "ARG BASE_IMAGE=" ${SCRIPT_DIR}/Dockerfile | sed "s/ARG BASE_IMAGE=//") echo "Fetching digest for ${BASE_IMAGE} ${ARCH_PLATFORMS}..." ARCH_PLATFORM_ARCH=$(echo ${ARCH_PLATFORMS} | sed "s/linux\///") - - # Below no longer works with recent Ubuntu images with Docker client < 23.0.0 - # Cf https://github.com/moby/moby/issues/44898 - #TARGET_BASE_IMAGE_DIGEST=$(docker manifest inspect ${BASE_IMAGE} | jq --raw-output '.manifests[] | (if .platform.architecture == "'${ARCH_PLATFORM_ARCH}'" then .digest else empty end)') - # So workaround the issue by pulling explicitly the image - docker pull arm64v8/${BASE_IMAGE} - TARGET_BASE_IMAGE_DIGEST=$(docker inspect arm64v8/${BASE_IMAGE} | jq --raw-output '.[0].Id') - - echo "${TARGET_BASE_IMAGE_DIGEST}" + TARGET_BASE_IMAGE_DIGEST=$(docker manifest inspect ${BASE_IMAGE} | jq --raw-output '.manifests[] | (if .platform.architecture == "'${ARCH_PLATFORM_ARCH}'" then .digest else empty end)') + docker pull "${BASE_IMAGE}@${TARGET_BASE_IMAGE_DIGEST}" BUILD_ARGS+=("--build-arg" "TARGET_ARCH=${ARCH_PLATFORM_ARCH}") - #BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${BASE_IMAGE}@${TARGET_BASE_IMAGE_DIGEST}") - BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=arm64v8/${BASE_IMAGE}") + BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${BASE_IMAGE}@${TARGET_BASE_IMAGE_DIGEST}") # echo "${BUILD_ARGS[@]}" fi elif test "${DOCKER_BUILDX}" != "buildx" -a \( "${IMAGE_NAME}" = "osgeo/gdal:alpine-small-latest" -o "${IMAGE_NAME}" = "osgeo/gdal:alpine-normal-latest" \); then diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 26a45cfc5372..eb7b3f216dd1 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -193,8 +193,6 @@ include(jp2kak/driver_declaration.cmake) gdal_dependent_format(jpipkak "JPIP Streaming" "GDAL_USE_KDU") # Luratech SDK gdal_dependent_format(jp2lura "JPEG-2000 (based on Luratech)" "GDAL_USE_LURATECH") -# ESRI ArcSDE C API SDK -gdal_dependent_format(sde "ESRI ArcSDE Raster" "HAVE_SDE") # LizardTech's decoding software development kit (DSDK) include(mrsid/driver_declaration.cmake) include(georaster/driver_declaration.cmake) diff --git a/frmts/drivers.ini b/frmts/drivers.ini index 2b8c78828b7f..ec0ade0b2fff 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -266,6 +266,7 @@ PMTiles JSONFG MiraMonVector XODR +ADBC # Put TIGER and AVCBIN at end since they need poOpenInfo->GetSiblingFiles() Tiger diff --git a/frmts/ecw/ecwdataset.cpp b/frmts/ecw/ecwdataset.cpp index 3fe05508f654..933c07eb4149 100644 --- a/frmts/ecw/ecwdataset.cpp +++ b/frmts/ecw/ecwdataset.cpp @@ -82,7 +82,7 @@ ECWRasterBand::ECWRasterBand(ECWDataset *poDSIn, int nBandIn, int iOverviewIn, { nBlockXSize = static_cast(nTileWidth); } - nBlockXSize = std::min(nBlockXSize, nRasterXSize); + nBlockXSize = MIN(nBlockXSize, nRasterXSize); UINT32 nTileHeight = 0; poDSIn->poFileView->GetParameter( @@ -91,7 +91,7 @@ ECWRasterBand::ECWRasterBand(ECWDataset *poDSIn, int nBandIn, int iOverviewIn, { nBlockYSize = static_cast(nTileHeight); } - nBlockYSize = std::min(nBlockYSize, nRasterYSize); + nBlockYSize = MIN(nBlockYSize, nRasterYSize); } #endif diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index 21d90de23ccb..c95f94ffb19b 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -304,6 +304,9 @@ void CPL_STDCALL GDALAllRegister() #if defined(DEFERRED_XODR_DRIVER) DeclareDeferredOGRXODRPlugin(); #endif +#if defined(DEFERRED_ADBC_DRIVER) + DeclareDeferredOGRADBCPlugin(); +#endif // AutoLoadDrivers is a no-op if compiled with GDAL_NO_AUTOLOAD defined. poDriverManager->AutoLoadDrivers(); diff --git a/frmts/grib/CMakeLists.txt b/frmts/grib/CMakeLists.txt index 01f58c263e0a..b047f95ccfde 100644 --- a/frmts/grib/CMakeLists.txt +++ b/frmts/grib/CMakeLists.txt @@ -59,77 +59,86 @@ endif() gdal_standard_includes(gdal_GRIB) -set(GDAL_DATA_FILES - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_versions.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_center.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_process.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_subcenter.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_13.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_14.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_15.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_16.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_17.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_18.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_190.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_191.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_19.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_20.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_21.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_3.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_4.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_5.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_6.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_7.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_191.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_3.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_4.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_3.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_4.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_5.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_6.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_3.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_4.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_5.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_6.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_0.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_10.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_1.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_2.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_3.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_4.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_5.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_6.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_7.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_8.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_9.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_Canada.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_HPC.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_index.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_MRMS.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_NCEP.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_NDFD.csv - ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_5.csv -) -set_property( - TARGET ${GDAL_LIB_TARGET_NAME} - APPEND - PROPERTY RESOURCE "${GDAL_DATA_FILES}") +if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES) + set(GDAL_DATA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_versions.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_center.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_process.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_subcenter.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_13.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_14.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_15.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_16.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_17.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_18.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_190.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_191.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_19.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_20.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_21.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_3.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_4.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_5.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_6.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_0_7.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_191.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_3.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_10_4.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_1_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_20_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_3.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_4.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_5.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_2_6.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_3.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_4.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_5.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_3_6.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_0.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_10.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_1.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_2.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_3.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_4.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_5.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_6.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_7.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_8.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_4_9.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_Canada.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_HPC.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_index.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_MRMS.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_NCEP.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_2_local_NDFD.csv + ${CMAKE_CURRENT_SOURCE_DIR}/data/grib2_table_4_5.csv + ) + set_property( + TARGET ${GDAL_LIB_TARGET_NAME} + APPEND + PROPERTY RESOURCE "${GDAL_DATA_FILES}") +endif() + +if (EMBED_RESOURCE_FILES) + add_driver_embedded_resources(gdal_GRIB GDAL_ENABLE_DRIVER_GRIB_PLUGIN degrib/degrib/embedded_resources.c) +endif() +if (USE_ONLY_EMBEDDED_RESOURCE_FILES) + target_compile_definitions(gdal_GRIB PRIVATE USE_ONLY_EMBEDDED_RESOURCE_FILES) +endif() target_include_directories( gdal_GRIB PRIVATE $ diff --git a/frmts/grib/degrib/degrib/embedded_resources.c b/frmts/grib/degrib/degrib/embedded_resources.c new file mode 100644 index 000000000000..fe986e0f0558 --- /dev/null +++ b/frmts/grib/degrib/degrib/embedded_resources.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +#include "embedded_resources.h" + +#include "embedded_resources_gen1.c" + +const char *GRIBGetCSVFileContent(const char* pszFilename) +{ + static const struct + { + const char* pszFilename; + const char* pszContent; + } asResources[] = { +#include "embedded_resources_gen2.c" + }; + for( size_t i = 0; i < sizeof(asResources) / sizeof(asResources[0]); ++i ) + { + if( strcmp(pszFilename, asResources[i].pszFilename) == 0 ) + return asResources[i].pszContent; + } + return NULL; +} diff --git a/frmts/grib/degrib/degrib/embedded_resources.h b/frmts/grib/degrib/degrib/embedded_resources.h new file mode 100644 index 000000000000..2ec6d8a778fd --- /dev/null +++ b/frmts/grib/degrib/degrib/embedded_resources.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +#include "cpl_port.h" + +CPL_C_START + +const char *GRIBGetCSVFileContent(const char* pszFilename); + +CPL_C_END diff --git a/frmts/grib/degrib/degrib/embedded_resources_gen1.c b/frmts/grib/degrib/degrib/embedded_resources_gen1.c new file mode 100644 index 000000000000..cb854bede085 --- /dev/null +++ b/frmts/grib/degrib/degrib/embedded_resources_gen1.c @@ -0,0 +1,326 @@ +// File generated by generate_embedded_resources.py. DO NOT EDIT + +static const char grib2_center_csv [] = { + #embed "../../data/grib2_center.csv" + , 0 +}; + +static const char grib2_process_csv [] = { + #embed "../../data/grib2_process.csv" + , 0 +}; + +static const char grib2_subcenter_csv [] = { + #embed "../../data/grib2_subcenter.csv" + , 0 +}; + +static const char grib2_table_4_2_0_0_csv [] = { + #embed "../../data/grib2_table_4_2_0_0.csv" + , 0 +}; + +static const char grib2_table_4_2_0_1_csv [] = { + #embed "../../data/grib2_table_4_2_0_1.csv" + , 0 +}; + +static const char grib2_table_4_2_0_13_csv [] = { + #embed "../../data/grib2_table_4_2_0_13.csv" + , 0 +}; + +static const char grib2_table_4_2_0_14_csv [] = { + #embed "../../data/grib2_table_4_2_0_14.csv" + , 0 +}; + +static const char grib2_table_4_2_0_15_csv [] = { + #embed "../../data/grib2_table_4_2_0_15.csv" + , 0 +}; + +static const char grib2_table_4_2_0_16_csv [] = { + #embed "../../data/grib2_table_4_2_0_16.csv" + , 0 +}; + +static const char grib2_table_4_2_0_17_csv [] = { + #embed "../../data/grib2_table_4_2_0_17.csv" + , 0 +}; + +static const char grib2_table_4_2_0_18_csv [] = { + #embed "../../data/grib2_table_4_2_0_18.csv" + , 0 +}; + +static const char grib2_table_4_2_0_19_csv [] = { + #embed "../../data/grib2_table_4_2_0_19.csv" + , 0 +}; + +static const char grib2_table_4_2_0_190_csv [] = { + #embed "../../data/grib2_table_4_2_0_190.csv" + , 0 +}; + +static const char grib2_table_4_2_0_191_csv [] = { + #embed "../../data/grib2_table_4_2_0_191.csv" + , 0 +}; + +static const char grib2_table_4_2_0_2_csv [] = { + #embed "../../data/grib2_table_4_2_0_2.csv" + , 0 +}; + +static const char grib2_table_4_2_0_20_csv [] = { + #embed "../../data/grib2_table_4_2_0_20.csv" + , 0 +}; + +static const char grib2_table_4_2_0_21_csv [] = { + #embed "../../data/grib2_table_4_2_0_21.csv" + , 0 +}; + +static const char grib2_table_4_2_0_3_csv [] = { + #embed "../../data/grib2_table_4_2_0_3.csv" + , 0 +}; + +static const char grib2_table_4_2_0_4_csv [] = { + #embed "../../data/grib2_table_4_2_0_4.csv" + , 0 +}; + +static const char grib2_table_4_2_0_5_csv [] = { + #embed "../../data/grib2_table_4_2_0_5.csv" + , 0 +}; + +static const char grib2_table_4_2_0_6_csv [] = { + #embed "../../data/grib2_table_4_2_0_6.csv" + , 0 +}; + +static const char grib2_table_4_2_0_7_csv [] = { + #embed "../../data/grib2_table_4_2_0_7.csv" + , 0 +}; + +static const char grib2_table_4_2_10_0_csv [] = { + #embed "../../data/grib2_table_4_2_10_0.csv" + , 0 +}; + +static const char grib2_table_4_2_10_1_csv [] = { + #embed "../../data/grib2_table_4_2_10_1.csv" + , 0 +}; + +static const char grib2_table_4_2_10_191_csv [] = { + #embed "../../data/grib2_table_4_2_10_191.csv" + , 0 +}; + +static const char grib2_table_4_2_10_2_csv [] = { + #embed "../../data/grib2_table_4_2_10_2.csv" + , 0 +}; + +static const char grib2_table_4_2_10_3_csv [] = { + #embed "../../data/grib2_table_4_2_10_3.csv" + , 0 +}; + +static const char grib2_table_4_2_10_4_csv [] = { + #embed "../../data/grib2_table_4_2_10_4.csv" + , 0 +}; + +static const char grib2_table_4_2_1_0_csv [] = { + #embed "../../data/grib2_table_4_2_1_0.csv" + , 0 +}; + +static const char grib2_table_4_2_1_1_csv [] = { + #embed "../../data/grib2_table_4_2_1_1.csv" + , 0 +}; + +static const char grib2_table_4_2_1_2_csv [] = { + #embed "../../data/grib2_table_4_2_1_2.csv" + , 0 +}; + +static const char grib2_table_4_2_20_0_csv [] = { + #embed "../../data/grib2_table_4_2_20_0.csv" + , 0 +}; + +static const char grib2_table_4_2_20_1_csv [] = { + #embed "../../data/grib2_table_4_2_20_1.csv" + , 0 +}; + +static const char grib2_table_4_2_20_2_csv [] = { + #embed "../../data/grib2_table_4_2_20_2.csv" + , 0 +}; + +static const char grib2_table_4_2_2_0_csv [] = { + #embed "../../data/grib2_table_4_2_2_0.csv" + , 0 +}; + +static const char grib2_table_4_2_2_3_csv [] = { + #embed "../../data/grib2_table_4_2_2_3.csv" + , 0 +}; + +static const char grib2_table_4_2_2_4_csv [] = { + #embed "../../data/grib2_table_4_2_2_4.csv" + , 0 +}; + +static const char grib2_table_4_2_2_5_csv [] = { + #embed "../../data/grib2_table_4_2_2_5.csv" + , 0 +}; + +static const char grib2_table_4_2_2_6_csv [] = { + #embed "../../data/grib2_table_4_2_2_6.csv" + , 0 +}; + +static const char grib2_table_4_2_3_0_csv [] = { + #embed "../../data/grib2_table_4_2_3_0.csv" + , 0 +}; + +static const char grib2_table_4_2_3_1_csv [] = { + #embed "../../data/grib2_table_4_2_3_1.csv" + , 0 +}; + +static const char grib2_table_4_2_3_2_csv [] = { + #embed "../../data/grib2_table_4_2_3_2.csv" + , 0 +}; + +static const char grib2_table_4_2_3_3_csv [] = { + #embed "../../data/grib2_table_4_2_3_3.csv" + , 0 +}; + +static const char grib2_table_4_2_3_4_csv [] = { + #embed "../../data/grib2_table_4_2_3_4.csv" + , 0 +}; + +static const char grib2_table_4_2_3_5_csv [] = { + #embed "../../data/grib2_table_4_2_3_5.csv" + , 0 +}; + +static const char grib2_table_4_2_3_6_csv [] = { + #embed "../../data/grib2_table_4_2_3_6.csv" + , 0 +}; + +static const char grib2_table_4_2_4_0_csv [] = { + #embed "../../data/grib2_table_4_2_4_0.csv" + , 0 +}; + +static const char grib2_table_4_2_4_1_csv [] = { + #embed "../../data/grib2_table_4_2_4_1.csv" + , 0 +}; + +static const char grib2_table_4_2_4_10_csv [] = { + #embed "../../data/grib2_table_4_2_4_10.csv" + , 0 +}; + +static const char grib2_table_4_2_4_2_csv [] = { + #embed "../../data/grib2_table_4_2_4_2.csv" + , 0 +}; + +static const char grib2_table_4_2_4_3_csv [] = { + #embed "../../data/grib2_table_4_2_4_3.csv" + , 0 +}; + +static const char grib2_table_4_2_4_4_csv [] = { + #embed "../../data/grib2_table_4_2_4_4.csv" + , 0 +}; + +static const char grib2_table_4_2_4_5_csv [] = { + #embed "../../data/grib2_table_4_2_4_5.csv" + , 0 +}; + +static const char grib2_table_4_2_4_6_csv [] = { + #embed "../../data/grib2_table_4_2_4_6.csv" + , 0 +}; + +static const char grib2_table_4_2_4_7_csv [] = { + #embed "../../data/grib2_table_4_2_4_7.csv" + , 0 +}; + +static const char grib2_table_4_2_4_8_csv [] = { + #embed "../../data/grib2_table_4_2_4_8.csv" + , 0 +}; + +static const char grib2_table_4_2_4_9_csv [] = { + #embed "../../data/grib2_table_4_2_4_9.csv" + , 0 +}; + +static const char grib2_table_4_2_local_Canada_csv [] = { + #embed "../../data/grib2_table_4_2_local_Canada.csv" + , 0 +}; + +static const char grib2_table_4_2_local_HPC_csv [] = { + #embed "../../data/grib2_table_4_2_local_HPC.csv" + , 0 +}; + +static const char grib2_table_4_2_local_MRMS_csv [] = { + #embed "../../data/grib2_table_4_2_local_MRMS.csv" + , 0 +}; + +static const char grib2_table_4_2_local_NCEP_csv [] = { + #embed "../../data/grib2_table_4_2_local_NCEP.csv" + , 0 +}; + +static const char grib2_table_4_2_local_NDFD_csv [] = { + #embed "../../data/grib2_table_4_2_local_NDFD.csv" + , 0 +}; + +static const char grib2_table_4_2_local_index_csv [] = { + #embed "../../data/grib2_table_4_2_local_index.csv" + , 0 +}; + +static const char grib2_table_4_5_csv [] = { + #embed "../../data/grib2_table_4_5.csv" + , 0 +}; + +static const char grib2_table_versions_csv [] = { + #embed "../../data/grib2_table_versions.csv" + , 0 +}; diff --git a/frmts/grib/degrib/degrib/embedded_resources_gen2.c b/frmts/grib/degrib/degrib/embedded_resources_gen2.c new file mode 100644 index 000000000000..c43ff93fb854 --- /dev/null +++ b/frmts/grib/degrib/degrib/embedded_resources_gen2.c @@ -0,0 +1,66 @@ +// File generated by generate_embedded_resources.py. DO NOT EDIT +{ "grib2_center.csv", grib2_center_csv }, +{ "grib2_process.csv", grib2_process_csv }, +{ "grib2_subcenter.csv", grib2_subcenter_csv }, +{ "grib2_table_4_2_0_0.csv", grib2_table_4_2_0_0_csv }, +{ "grib2_table_4_2_0_1.csv", grib2_table_4_2_0_1_csv }, +{ "grib2_table_4_2_0_13.csv", grib2_table_4_2_0_13_csv }, +{ "grib2_table_4_2_0_14.csv", grib2_table_4_2_0_14_csv }, +{ "grib2_table_4_2_0_15.csv", grib2_table_4_2_0_15_csv }, +{ "grib2_table_4_2_0_16.csv", grib2_table_4_2_0_16_csv }, +{ "grib2_table_4_2_0_17.csv", grib2_table_4_2_0_17_csv }, +{ "grib2_table_4_2_0_18.csv", grib2_table_4_2_0_18_csv }, +{ "grib2_table_4_2_0_19.csv", grib2_table_4_2_0_19_csv }, +{ "grib2_table_4_2_0_190.csv", grib2_table_4_2_0_190_csv }, +{ "grib2_table_4_2_0_191.csv", grib2_table_4_2_0_191_csv }, +{ "grib2_table_4_2_0_2.csv", grib2_table_4_2_0_2_csv }, +{ "grib2_table_4_2_0_20.csv", grib2_table_4_2_0_20_csv }, +{ "grib2_table_4_2_0_21.csv", grib2_table_4_2_0_21_csv }, +{ "grib2_table_4_2_0_3.csv", grib2_table_4_2_0_3_csv }, +{ "grib2_table_4_2_0_4.csv", grib2_table_4_2_0_4_csv }, +{ "grib2_table_4_2_0_5.csv", grib2_table_4_2_0_5_csv }, +{ "grib2_table_4_2_0_6.csv", grib2_table_4_2_0_6_csv }, +{ "grib2_table_4_2_0_7.csv", grib2_table_4_2_0_7_csv }, +{ "grib2_table_4_2_10_0.csv", grib2_table_4_2_10_0_csv }, +{ "grib2_table_4_2_10_1.csv", grib2_table_4_2_10_1_csv }, +{ "grib2_table_4_2_10_191.csv", grib2_table_4_2_10_191_csv }, +{ "grib2_table_4_2_10_2.csv", grib2_table_4_2_10_2_csv }, +{ "grib2_table_4_2_10_3.csv", grib2_table_4_2_10_3_csv }, +{ "grib2_table_4_2_10_4.csv", grib2_table_4_2_10_4_csv }, +{ "grib2_table_4_2_1_0.csv", grib2_table_4_2_1_0_csv }, +{ "grib2_table_4_2_1_1.csv", grib2_table_4_2_1_1_csv }, +{ "grib2_table_4_2_1_2.csv", grib2_table_4_2_1_2_csv }, +{ "grib2_table_4_2_20_0.csv", grib2_table_4_2_20_0_csv }, +{ "grib2_table_4_2_20_1.csv", grib2_table_4_2_20_1_csv }, +{ "grib2_table_4_2_20_2.csv", grib2_table_4_2_20_2_csv }, +{ "grib2_table_4_2_2_0.csv", grib2_table_4_2_2_0_csv }, +{ "grib2_table_4_2_2_3.csv", grib2_table_4_2_2_3_csv }, +{ "grib2_table_4_2_2_4.csv", grib2_table_4_2_2_4_csv }, +{ "grib2_table_4_2_2_5.csv", grib2_table_4_2_2_5_csv }, +{ "grib2_table_4_2_2_6.csv", grib2_table_4_2_2_6_csv }, +{ "grib2_table_4_2_3_0.csv", grib2_table_4_2_3_0_csv }, +{ "grib2_table_4_2_3_1.csv", grib2_table_4_2_3_1_csv }, +{ "grib2_table_4_2_3_2.csv", grib2_table_4_2_3_2_csv }, +{ "grib2_table_4_2_3_3.csv", grib2_table_4_2_3_3_csv }, +{ "grib2_table_4_2_3_4.csv", grib2_table_4_2_3_4_csv }, +{ "grib2_table_4_2_3_5.csv", grib2_table_4_2_3_5_csv }, +{ "grib2_table_4_2_3_6.csv", grib2_table_4_2_3_6_csv }, +{ "grib2_table_4_2_4_0.csv", grib2_table_4_2_4_0_csv }, +{ "grib2_table_4_2_4_1.csv", grib2_table_4_2_4_1_csv }, +{ "grib2_table_4_2_4_10.csv", grib2_table_4_2_4_10_csv }, +{ "grib2_table_4_2_4_2.csv", grib2_table_4_2_4_2_csv }, +{ "grib2_table_4_2_4_3.csv", grib2_table_4_2_4_3_csv }, +{ "grib2_table_4_2_4_4.csv", grib2_table_4_2_4_4_csv }, +{ "grib2_table_4_2_4_5.csv", grib2_table_4_2_4_5_csv }, +{ "grib2_table_4_2_4_6.csv", grib2_table_4_2_4_6_csv }, +{ "grib2_table_4_2_4_7.csv", grib2_table_4_2_4_7_csv }, +{ "grib2_table_4_2_4_8.csv", grib2_table_4_2_4_8_csv }, +{ "grib2_table_4_2_4_9.csv", grib2_table_4_2_4_9_csv }, +{ "grib2_table_4_2_local_Canada.csv", grib2_table_4_2_local_Canada_csv }, +{ "grib2_table_4_2_local_HPC.csv", grib2_table_4_2_local_HPC_csv }, +{ "grib2_table_4_2_local_MRMS.csv", grib2_table_4_2_local_MRMS_csv }, +{ "grib2_table_4_2_local_NCEP.csv", grib2_table_4_2_local_NCEP_csv }, +{ "grib2_table_4_2_local_NDFD.csv", grib2_table_4_2_local_NDFD_csv }, +{ "grib2_table_4_2_local_index.csv", grib2_table_4_2_local_index_csv }, +{ "grib2_table_4_5.csv", grib2_table_4_5_csv }, +{ "grib2_table_versions.csv", grib2_table_versions_csv }, diff --git a/frmts/grib/degrib/degrib/metaname.cpp b/frmts/grib/degrib/degrib/metaname.cpp index 4ab5576c7833..d0192d7fd93c 100644 --- a/frmts/grib/degrib/degrib/metaname.cpp +++ b/frmts/grib/degrib/degrib/metaname.cpp @@ -27,8 +27,40 @@ #include +#ifdef EMBED_RESOURCE_FILES +#include "embedded_resources.h" +#include +#include +static std::mutex goMutex; +static std::map* pgoMapResourceFiles = nullptr; +#endif + +void MetanameCleanup(void) +{ +#ifdef EMBED_RESOURCE_FILES + std::lock_guard oGuard(goMutex); + if( pgoMapResourceFiles ) + { + for( const auto& oIter: *pgoMapResourceFiles ) + { + VSIUnlink(oIter.second.c_str()); + } + delete pgoMapResourceFiles; + } + pgoMapResourceFiles = nullptr; +#endif +} + static const char* GetGRIB2_CSVFilename(const char* pszFilename) { +#ifdef EMBED_RESOURCE_FILES + std::lock_guard oGuard(goMutex); + if( !pgoMapResourceFiles ) + pgoMapResourceFiles = new std::map(); + const auto oIter = pgoMapResourceFiles->find(pszFilename); + if( oIter != pgoMapResourceFiles->end() ) + return oIter->second.c_str(); +#endif const char* pszGribTableDirectory = CPLGetConfigOption("GRIB_RESOURCE_DIR", nullptr); if( pszGribTableDirectory ) { @@ -38,11 +70,35 @@ static const char* GetGRIB2_CSVFilename(const char* pszFilename) return pszFullFilename; return nullptr; } - const char* pszRet = CSVFilename(pszFilename); + const char* pszRet = nullptr; + CPL_IGNORE_RET_VAL(pszRet); +#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES + pszRet = CSVFilename(pszFilename); // CSVFilename() returns the same content as pszFilename if it does not // find the file. if( pszRet && strcmp(pszRet, pszFilename) == 0 ) - return nullptr; +#endif + { +#ifdef EMBED_RESOURCE_FILES + const char* pszFileContent = GRIBGetCSVFileContent(pszFilename); + if( pszFileContent ) + { + const std::string osTmpFilename = VSIMemGenerateHiddenFilename(pszFilename); + VSIFCloseL(VSIFileFromMemBuffer( + osTmpFilename.c_str(), + const_cast( + reinterpret_cast(pszFileContent)), + static_cast(strlen(pszFileContent)), + /* bTakeOwnership = */ false)); + (*pgoMapResourceFiles)[pszFilename] = osTmpFilename; + pszRet = (*pgoMapResourceFiles)[pszFilename].c_str(); + } + else +#endif + { + return nullptr; + } + } return pszRet; } diff --git a/frmts/grib/degrib/degrib/metaname.h b/frmts/grib/degrib/degrib/metaname.h index 3408adb5980b..4df04802d4c6 100644 --- a/frmts/grib/degrib/degrib/metaname.h +++ b/frmts/grib/degrib/degrib/metaname.h @@ -46,6 +46,8 @@ void ParseLevelName (unsigned short int center, unsigned short int subcenter, double sndValue, char **shortLevelName, char **longLevelName); +void MetanameCleanup(void); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/frmts/grib/degrib/generate_embedded_resources.py b/frmts/grib/degrib/generate_embedded_resources.py new file mode 100755 index 000000000000..4a45fc0b1cfe --- /dev/null +++ b/frmts/grib/degrib/generate_embedded_resources.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: MIT +# Copyright (C) 2024 Even Rouault + +import glob +import os + +embedded_resources_gen1 = open( + os.path.join(os.path.dirname(__file__), "degrib/embedded_resources_gen1.c"), "wt" +) +embedded_resources_gen2 = open( + os.path.join(os.path.dirname(__file__), "degrib/embedded_resources_gen2.c"), "wt" +) + +embedded_resources_gen1.write( + "// File generated by generate_embedded_resources.py. DO NOT EDIT\n" +) +embedded_resources_gen2.write( + "// File generated by generate_embedded_resources.py. DO NOT EDIT\n" +) + +for f in sorted(glob.glob(os.path.join(os.path.dirname(__file__), "../data/*.csv"))): + f = os.path.basename(f) + c = f.replace(".", "_") + embedded_resources_gen1.write( + """ +static const char %s [] = { + #embed "../../data/%s" + , 0 +}; +""" + % (c, f) + ) + embedded_resources_gen2.write("""{ "%s", %s },\n""" % (f, c)) + +embedded_resources_gen1.close() +embedded_resources_gen2.close() diff --git a/frmts/grib/gribdataset.cpp b/frmts/grib/gribdataset.cpp index 9e506e379760..f118e257cf39 100644 --- a/frmts/grib/gribdataset.cpp +++ b/frmts/grib/gribdataset.cpp @@ -27,12 +27,6 @@ #include #endif -#ifndef _WIN32 -#include // isatty() -#else -#include // _isatty() -#endif - #include #include #include @@ -901,11 +895,7 @@ static bool IsGdalinfoInteractive() { static const bool bIsGdalinfoInteractive = []() { -#ifndef _WIN32 - if (isatty(static_cast(fileno(stdout)))) -#else - if (_isatty(_fileno(stdout))) -#endif + if (CPLIsInteractive(stdout)) { std::string osPath; osPath.resize(1024); @@ -2815,6 +2805,7 @@ static void GDALDeregister_GRIB(GDALDriver *) { if (hGRIBMutex != nullptr) { + MetanameCleanup(); CPLDestroyMutex(hGRIBMutex); hGRIBMutex = nullptr; } diff --git a/frmts/gtiff/cogdriver.cpp b/frmts/gtiff/cogdriver.cpp index 69a72ec7488d..ac56ca0d2845 100644 --- a/frmts/gtiff/cogdriver.cpp +++ b/frmts/gtiff/cogdriver.cpp @@ -1505,13 +1505,13 @@ void GDALCOGDriver::InitializeCreationOptionList() "1(fast)-9(slow)' default='5'/>" "