From 78c7886935b017b70318e91a8f12bb6ea5727b3e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 20 May 2024 01:46:37 +0200 Subject: [PATCH 01/31] OpenFileGDB: add partial read-only support for tables with 64-bit ObjectIDs Tables with non-sparse pages in .gdbtablx are fully supported. A subset of tables with sparse pages are properly supported (cf https://github.com/rouault/dump_gdbtable/wiki/FGDB-Spec#trailing-section-16-bytes--variable-number-) When we detect that we can't properly decode ObjectID in the unsupported subset of sparse tables, we emit a warning, and turn off use of attribute/spatial indices. Read-only support for now. --- .../3features.gdb/a00000001.TablesByName.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000001.gdbindexes | Bin 0 -> 110 bytes .../3features.gdb/a00000001.gdbtable | Bin 0 -> 322 bytes .../3features.gdb/a00000001.gdbtablx | Bin 0 -> 5152 bytes .../3features.gdb/a00000002.gdbtable | Bin 0 -> 2055 bytes .../3features.gdb/a00000002.gdbtablx | Bin 0 -> 5152 bytes .../3features.gdb/a00000003.gdbindexes | Bin 0 -> 42 bytes .../3features.gdb/a00000003.gdbtable | Bin 0 -> 1016 bytes .../3features.gdb/a00000003.gdbtablx | Bin 0 -> 5152 bytes .../a00000004.CatItemsByPhysicalName.atx | Bin 0 -> 4118 bytes .../a00000004.CatItemsByType.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000004.FDO_UUID.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000004.gdbindexes | Bin 0 -> 310 bytes .../3features.gdb/a00000004.gdbtable | Bin 0 -> 10304 bytes .../3features.gdb/a00000004.gdbtablx | Bin 0 -> 5152 bytes .../3features.gdb/a00000004.horizon | Bin 0 -> 32 bytes .../objectid64/3features.gdb/a00000004.spx | Bin 0 -> 4118 bytes .../a00000005.CatItemTypesByName.atx | Bin 0 -> 12310 bytes .../a00000005.CatItemTypesByParentTypeID.atx | Bin 0 -> 4118 bytes .../a00000005.CatItemTypesByUUID.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000005.gdbindexes | Bin 0 -> 296 bytes .../3features.gdb/a00000005.gdbtable | Bin 0 -> 2071 bytes .../3features.gdb/a00000005.gdbtablx | Bin 0 -> 5152 bytes .../a00000006.CatRelsByDestinationID.atx | Bin 0 -> 4118 bytes .../a00000006.CatRelsByOriginID.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000006.CatRelsByType.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000006.FDO_UUID.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000006.gdbindexes | Bin 0 -> 318 bytes .../3features.gdb/a00000006.gdbtable | Bin 0 -> 263 bytes .../3features.gdb/a00000006.gdbtablx | Bin 0 -> 5152 bytes .../a00000007.CatRelTypesByBackwardLabel.atx | Bin 0 -> 12310 bytes .../a00000007.CatRelTypesByDestItemTypeID.atx | Bin 0 -> 4118 bytes .../a00000007.CatRelTypesByForwardLabel.atx | Bin 0 -> 12310 bytes .../a00000007.CatRelTypesByName.atx | Bin 0 -> 12310 bytes ...00000007.CatRelTypesByOriginItemTypeID.atx | Bin 0 -> 4118 bytes .../a00000007.CatRelTypesByUUID.atx | Bin 0 -> 4118 bytes .../3features.gdb/a00000007.gdbindexes | Bin 0 -> 602 bytes .../3features.gdb/a00000007.gdbtable | Bin 0 -> 3626 bytes .../3features.gdb/a00000007.gdbtablx | Bin 0 -> 5152 bytes .../3features.gdb/a00000009.gdbindexes | Bin 0 -> 116 bytes .../3features.gdb/a00000009.gdbtable | Bin 0 -> 1384 bytes .../3features.gdb/a00000009.gdbtablx | Bin 0 -> 5156 bytes .../3features.gdb/a00000009.horizon | 1 + .../objectid64/3features.gdb/a00000009.spx | Bin 0 -> 65566 bytes .../data/filegdb/objectid64/3features.gdb/gdb | Bin 0 -> 4 bytes .../objectid64/3features.gdb/timestamps | Bin 0 -> 400 bytes .../with_holes_8.gdb/a00000001.gdbtable | Bin 0 -> 505 bytes .../with_holes_8.gdb/a00000001.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000002.gdbtable | Bin 0 -> 2094 bytes .../with_holes_8.gdb/a00000002.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000003.gdbtable | Bin 0 -> 797 bytes .../with_holes_8.gdb/a00000003.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000004.freelist | Bin 0 -> 4440 bytes .../with_holes_8.gdb/a00000004.gdbtable | Bin 0 -> 47714 bytes .../with_holes_8.gdb/a00000004.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000005.gdbtable | Bin 0 -> 2060 bytes .../with_holes_8.gdb/a00000005.gdbtablx | Bin 0 -> 4128 bytes .../a00000006.FDO_OriginID.atx | Bin 0 -> 4118 bytes .../with_holes_8.gdb/a00000006.gdbindexes | Bin 0 -> 66 bytes .../with_holes_8.gdb/a00000006.gdbtable | Bin 0 -> 740 bytes .../with_holes_8.gdb/a00000006.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000007.gdbtable | Bin 0 -> 3665 bytes .../with_holes_8.gdb/a00000007.gdbtablx | Bin 0 -> 4128 bytes .../with_holes_8.gdb/a00000009.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a00000009.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a00000009.gdbtablx | Bin 0 -> 37990 bytes .../with_holes_8.gdb/a00000009.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a00000009.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000a.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000a.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a0000000a.gdbtablx | Bin 0 -> 37990 bytes .../with_holes_8.gdb/a0000000a.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000a.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000b.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000b.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a0000000b.gdbtablx | Bin 0 -> 37990 bytes .../with_holes_8.gdb/a0000000b.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000b.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000c.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000c.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a0000000c.gdbtablx | Bin 0 -> 37990 bytes .../with_holes_8.gdb/a0000000c.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000c.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000d.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000d.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a0000000d.gdbtablx | Bin 0 -> 37990 bytes .../with_holes_8.gdb/a0000000d.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000d.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000e.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000e.gdbtable | Bin 0 -> 566 bytes .../with_holes_8.gdb/a0000000e.gdbtablx | Bin 0 -> 155936 bytes .../with_holes_8.gdb/a0000000e.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000e.spx | Bin 0 -> 65566 bytes .../with_holes_8.gdb/a0000000f.gdbindexes | Bin 0 -> 116 bytes .../with_holes_8.gdb/a0000000f.gdbtable | Bin 0 -> 642 bytes .../with_holes_8.gdb/a0000000f.gdbtablx | Bin 0 -> 176416 bytes .../with_holes_8.gdb/a0000000f.horizon | Bin 0 -> 32 bytes .../objectid64/with_holes_8.gdb/a0000000f.spx | Bin 0 -> 65566 bytes .../filegdb/objectid64/with_holes_8.gdb/gdb | Bin 0 -> 8 bytes .../objectid64/with_holes_8.gdb/timestamps | Bin 0 -> 400 bytes autotest/ogr/ogr_openfilegdb.py | 157 ++++- ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp | 615 +++++++++++------- .../openfilegdb/filegdbindex_write.cpp | 42 +- ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp | 285 ++++++-- ogr/ogrsf_frmts/openfilegdb/filegdbtable.h | 60 +- .../openfilegdb/filegdbtable_write.cpp | 67 +- .../openfilegdb/filegdbtable_write_fields.cpp | 2 +- .../openfilegdb/gdalopenfilegdbrasterband.cpp | 8 +- ogr/ogrsf_frmts/openfilegdb/ogr_openfilegdb.h | 4 +- .../openfilegdb/ogropenfilegdbdatasource.cpp | 2 +- .../ogropenfilegdbdatasource_write.cpp | 51 +- .../openfilegdb/ogropenfilegdblayer.cpp | 104 +-- .../openfilegdb/ogropenfilegdblayer_write.cpp | 13 +- 113 files changed, 959 insertions(+), 452 deletions(-) create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.TablesByName.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.CatItemsByPhysicalName.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.CatItemsByType.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.FDO_UUID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByName.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByParentTypeID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByUUID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByDestinationID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByOriginID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByType.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.FDO_UUID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByBackwardLabel.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByDestItemTypeID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByForwardLabel.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByName.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByUUID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/gdb create mode 100644 autotest/ogr/data/filegdb/objectid64/3features.gdb/timestamps create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000001.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000001.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000002.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000002.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.freelist create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000005.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000005.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.FDO_OriginID.atx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.gdbindexes create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.gdbtable create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.gdbtablx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.horizon create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.spx create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/gdb create mode 100644 autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/timestamps diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.TablesByName.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.TablesByName.atx new file mode 100644 index 0000000000000000000000000000000000000000..f4eaf02531704846ef18b407c3976fa7ac1ec45f GIT binary patch literal 4118 zcmeH~?Fxc06oy~>?k>Eu=)=}v)yVb3yXjJUk1|3j3V%3*GkNC}g~w-S0|8*oNSiPV zX35N$IkRG>Ovc3LIPBF74%lH+_W^fYP#_=C5&0|v$~#53dHTjDb>~@m7JF8oW9^wl71t(>{z}p{t3v36_BeR0waJ>R z7hgr5W(|wMFcOPGBwU^|>bOQty{o+Yk~H^vZR+}x8c+jjKn6orp9BUJH!M$m<8pTb2giXbkk`ymXm7Ge`@X3@8qN1c;sT^TrV^L^*u zHh`E}GapRzlGbBdJ%RVa&*W`Hd!#@|MB)gE8x-YVXmqzIpb54xxKl|uE(w7d{dkuq z->15A>b8+iYLgRU$aK9)lk;!MTfnEsp=_lqq|Vf(@_fL=p6_`gkO@h@A?MnvKo_K6 u4b7D15mQ#z!UnsfoxiiM{V5v*-^i5J#g9CgJR%w~J*O&XjjnT3@{51d`8)go literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.gdbtablx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000001.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..3d6f920bbf82725a7a359819071aa1ea475639f6 GIT binary patch literal 5152 zcmeI$F%5uF5JbTvgaQS)Ko~mkfDAOC0wwg^7Kk93pX{rCHRX*}HT34*)Z2QsCr5H7 pS8^v$vgH4ocmxO#AV7cs0RjXF5FkK+009C72>ebUr@imI_W~9`1swnY literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtable b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..a0af90eaae1fb1738ddead12a36441ef8292c88d GIT binary patch literal 2055 zcma)7-*3|}5H{*c$JBM>=L-l0#>7CQsAyD!Hz!THh%}iJM;g3DrcFkqCP7ImHvV{o z?KWwh#Dz$f9Q*G3zB`}KEdam~89o_pGK#OWyPTb@7U=#Co!TX$U4{6>AOIn_1`aVw$t)&wf=(#gU`UYu?2c3ox(0GO zuI?fA3LT*zzBKa#yqM?G4Cgoq=e{&e;vl?Uq&N?gxKA~#Jr9)&XhQ|5wx?P41fm@m zY3IkB&B0i;CMvSOu1xlnB>I{9S?14Pa)yVcp@S?Nj@7$*Y=p7@oU`i6h9!Hv#WT)g ze*&*OO*d4}(Z8v!0g@$dzh^^-D&{dhQJX!Q29Lxr7#2x+MU%o-q};#Br65TooL90! z4|mhKmX|g|xS(eDIeFgMST3keDHZvbB774@i)eiYS@^qp5G=`ctV-|%&7hXcT4c>w62nN@a;TA-8V{K>>}pstJ{f{uzx zwLPMCxx@M@yVvWi+c+c2B|**yRJ90~?_BEwV8tnL66iN6&m&&HZqj>EQP1vYlw>%Z zvWArTxfI6is_^+f6(ZK!A$ulRq>T!y*LmZ~=!;3p`sW+_TcHP+nbl)MpW^1U@a>I0 KE8>8gpP0Wjd{iF* literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtablx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000002.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..7c12c568195050346285f43f1164067d9ce86e7c GIT binary patch literal 5152 zcmeI$uMNUL97pkkmOm2^Jhm2##9$eqEP*Rw1rkOeL9hS=;0Ue>8dF=-gapO!jZptC z`F_laOP;(er8I~wVx!n8mY&eUoVyj2aD)qdU^)!8P{9j+u+D-5Jm3qX*gyq0c!S3R7O;Q?EMNf(Sik}nuz&?DU;ztQzycQNS%9xS#`yy^ CN)wa- literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbindexes b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..58df68d525b47a895d5876a8bf5f437a90afc3f0 GIT binary patch literal 42 mcmZQ%U|?VaVmAgC27iWl22UW(z#ss`|Nk=rNhTnMhyegseFSR& literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbtable b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000003.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..f8006ccdaad9fa9e214675facdf868ce17284898 GIT binary patch literal 1016 zcmbtSy^hmB5S~l!!U<@2g{>&8$UkwCZGb3oj88a@YzOWVMb_fm%Na$EodA+*5 z0hSnd#Tt1xp=3b^pt4)4c;JE$1{~sygncnpy1-xxbKnR=d4DgD9IcUShj0V~h#`U$ znMKxcgAD({u{RvM$hjeE`!K^ha7gejF@(6vdR!k~nX#|Rd=>9rz1lGi$838ef>ZFw z^a6!8Yeed_{DD0pogC!#?;EaTbw=cZR@WL$MQc=LNu|xIsy9?s)9c!JEcIQ>Hn4)3 za2`L8!c!v2Re3B8dX|rM`2_I{QW`Z~Q>m&el-4vnD((e~lXDSZ@i2>v7u8m_f3nQ> zaQ5L|asTrByEi{Sp5Ix${qf0u+`qex;wOOB>vG?94#YzZ-_2f0gMn`c!&%G&P3;jW zFT`(7PKg|LA%(fFB@^iqDO5#P{!0o}(K#^MzUA~r#7|f}f5wwJ4{V-IfKp9D4q~Wm&Fj8|HbLelJqnq74vN#rwN;i4$*PMVj|H(1utY%9&}kK z`lF|m()u4Q9#6(EadQ7LeooleDqR)!OB-><$dp$1}hPJ zS7r*8%)!RZV&@(x^JRHXR?CSe z;LLFzz8#57L@76BYEg~+4BVwoj>TW6c0VHbLY)1Y8a0tlHVQ}5p{bdqH8r&jGjAkO zGUcvJwUrnmoN4hP&=SoH)4+8~(OVoEK6eWDtb^RSWo63ES7toeSo8PJF)92>_q`Xu-zKjB literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtable b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..0cbf02d63282a4455bcb2f2593fd2becef0b3594 GIT binary patch literal 10304 zcmeHNYiuJ|6~3EI+U@R^R*3#U0?70G?AIuH_q&i zXWW@_;|PHk(Ul4mf#@$vRSHNH_<>raN(d0DL?03oNKhn7fAIQ&5CRE=)K*oYeD}_s zhacN*QH3hBj-0vYobTNCIrrQ%K1c{Ti0^|RB4i3r@3VN9TM`el2hb!~lX$+D94eD- za*0sVBn~N(JRvttPE4+ob(c7DY+|xb_DL5Aa`@Qf)hY^#Hp^hnXv_f_vw~2-t zRHBn%z+?q%2Ou&zd4Or$mgfu=IuwE$>7fUXXd^+$fywJ&k}l@xU_C`*kR4(Y)G+ym z+qLlks0+Zsby6j(K6yBg4((td2EGoyCQ9TenYuNPViWyj12-UI0}8~4iXbV3UKg0o z;R>;!#pcc8Xo~Uju^n=oQBj``3Al&I)a^Om8mrbIZPCm*KA}BOE@-NTX;j^{xS8B6 z9ats>QYOnJhgrObNTSpr=TWMW2IgX(WXKF8QLO9|sbg779GQo79I8etWCgWWQ7_vK ztk!Y3S&|}is5`-*F%m}$N#0rv&opF7{H0K0$SwRuQF9e_mQcIfhh@wRt~#>+8BNvd#Q*%vp%pnYYTjUN}!z?w2|U%vv%2t=fO*fR&T~mAmqx<)9gX6oa4_c z_s5nved$$BlLvmXI81;3Z%;k;%5zU=U;p+$ep%YC-0eM|`1SNelis=btItf9&AcPoqB1Yi5)r5`yDJ#hEKUuK`*+|4T|cP)Mth5jptzw_f) z9qa1qBRkK3>;2#O=BLbT|DQjl?<;)zj-St8ed&cKu}wA!OohBO0r<-=9QoI&{^RRs zzxUbO?s)#I)sNlwZ~M6`R(bJiGSb_6@%5W-Id58*?XKFSx129m&(TR$ETS;|cjfX|>ke zr5$xzGi*mSnl#Y5zLJM^+aADE+tOl9vm@KQdRtzyn#;wSoQTIV$FoXKbyVGKS5#*= zyQ#=wQdv_wG^@z&Be$#REsGk3epfYGw3Xdab(=zCn6LPgQpZeWS^KMIQ`PfmXP3;Nyk_Inyv`)6q<$=2)8126U7aaEiu`$rsAdbgYxr zqsQf}k~cdl=6rMomn=)&FYmZ=OqDE+f}=i{O~yT{%NW@J&Wda)wcbaWg9SL0>c<@pT6rYAvE@JMlm3MyY|5h z!KRLH9r=2Any`)`xstD8#j(ZsQxV%Hu)cr+OE$b+<(X!{QD6l-TDxbd%v#RU{XNra zx#lv4U8VQ-G>f*Dso4@%o6|LFI9>yZR4mE1=yhPqifWh9e`acmEC>ah(;C%VeBR2Z z&J=R>VqTF4*vfK+W7x(j^sjB412EX=06l8b17g63y5O$llCG)tkR;$KykGv}`4qaZ z$Uc=-iuPJh*VS#EdY&@Gft9;cC#J?Hl|oB{n+%kET9F4j|KAOjo3H25gE7@smMR5B z4j9)OBsN5cI@bQRM(Qr~TN>y)Hk`&PHQLUuH>zL>*vaEld9#OEeFysOFNiQQpMK{S zsIg>G_1!m~!4&$j7Ho6>_uF@pvYlK0TM4NzP_I z0WZqYO5s9Xl;ogTr8;*^_O90D2MRO4F;>sn>?j_ukX1rkIJ})pW{c{sb<3eLJkqjm zZmT+mA(k`DQamBwMM8ZHud3P(wZfngPNDB0D1_@pOw*;7qm3J#knPu~gE4v(4?{*> znTSq26m+~In;);tQJ}}PkM%PR$1-)DS|!@qMmU4nA2aEqrEzH1)Hpsi2eS*)s#|LF zG9t5r!JL{OG>#W4IqZ8Vg=cb<#W}RK12rGR{VD z*lut}tqaS-4oA%14`c&eh54JHV&0DML4y*-b+yA#&jAlh6~617(Y;$R?xQ;FU=91{ z0>a~LIy0L}WMT=pjCuZd@Lgyg>&eJAbyGtyIfraT0mv`a*GrHA~w-1j5AJOgfdEPo@*|^E0Uw zuddAFx$>G&fdeq?``}w>qeQJHGjCTPO)piKN`-o%Dxjy?g|bnAGljztjkwFrx-lNT zW?If}0OuVZM&=N>TB^?4*IJrth^ptu)mps@c&FtirCGUW+!$JB_lX zwKcEe;aMwDoDQNg?9#31(t3TRTrJiwih&m@wPhhsrZcJJ))ub3*gVPwx8Ce@I+2WH z@4_eqN4e;U(76ok23y(_UAV|ckp-8l2vXVbRJZ^d9z)p}P!N<-8peAJr8IySlyc!> z-NaeKau-?5K|F!(hFHqRh&aSiN+Z$`M_JL@yA_KzHGbt5_lW@>%nheFpFcKxC@WYY z+-<}B5C;y|^^CrCd&i`6!|;cLAm-5}(#| zIP=kfmWM0+IFOFntILbSxtIUYM$8aA~S5Moyd-+Q+rdjnr+Qy%*`sbIm+Hd%S>t8>~D%E zkgT0ZNi|S+2T=j9){Z&b{Z02)AttNIa+*?vO~OgZn*}Mt26<9)2QFntH6pnao9i}W zolRuZk>T3RQ~R=Gc6lbX-P+zPv6ME87ZWj%kx&*fBCY|E4@cIhW<<`Y5lO_3k{<{T zgpHCHF-il$q7-%GL@5*^N>M*Ll)S)Die5J+l>Bf|iUtRQt9g6Fua7f4Whod=Z)SnNGmxpgsltpe^3tl7^nHF(Z zg!{sUX*Z@COYvA}A^g8+55zMPR^9uPiY;tR4KHgV9nFcHGc6sPRpK{#W(og8lUVPz z*mZ`#HhNQINJ-f+_HCL$_&r19nC>jF1PRXu)sQ4**REr2XMqWjv%b8en|mI;#GEU4 ztsZBG<&_%-l1Gf=<(8SQJSbu;m!q<5?={fAjMp&QoIKp4VR}?$up2z_`qsKP41L>u zlgG|$Xk*E~*X6Gu;HpHXiUr2}WRSgsBYW%NH*Qr5wQ8|Z#ATPAE6nF3zZGfrboN^4 zH`hAEGARiFf5rr7|GsSilbnfrz0b9$vB?rbm)x;@C+yL;+2Z+(U-+5 z7IuahF4SWQ4UarWaYk6o)ZQF7(4jZ2(N8I{4tmWDwK{$@*8;m~tn1CiwX~P>r$+kA zC-1B~N&cj{5zr8s#S|hN&c@=(y(t#dIh?1a#SZHcR7ilGy9L(z>KRn=N;sO zHCScK4PoT5Yysy-KDJy4I=}*qvP7@L>hE`uTryNFDldVv?{j#GV~}PN7?e21#$jUE63CMu&mBF`%Z@A+_iG>KiHZMeOWUFputA?{x$VH88BWx|T$)b^v z!E)fJo^0;FTU1PtAh)zu% zxcbKzH~#RsU!3~W7Y{s>x%$kLPaXK$!(YF8;JXv|8{dC);*E*FPdsb9{OH7$gZ}}( CpD$kk literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtablx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..93fec31b61e3d8dce2c753b4624c1f43c4d16239 GIT binary patch literal 5152 zcmeI$!3_W*3vQKmY&$ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.horizon b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.spx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000004.spx new file mode 100644 index 0000000000000000000000000000000000000000..15bba5bf1c63e26f4de207a8e38a22b89fc95ae0 GIT binary patch literal 4118 zcmeI$u?@f=5JOQT8cIfB1vbd2lw`}CK`ek$ek5ar#1*eZl5)h$eX=gt5I_I{1g->n m)tE^&y1l!H`)U}E5I_I{1Q0*~0R#~EL7+S-ra0#pz~>L^aRjRX literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByName.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByName.atx new file mode 100644 index 0000000000000000000000000000000000000000..5f5004620ba9c73658f069eeace1557d8df8cebe GIT binary patch literal 12310 zcmeI2S#Q%o6oqft_kCZ@{yR`Y0Er^h26#@=6e`lBvRx?uhM&rJ#tBh|JXAcy zvI1FwtUy*EE09tFt@&@w2;AX*44egzz!mTmJOBl_4W5IW;2d}XZUDW1`8?mDoOTB4KiNe)23tWnWt}0< zCkbcfHTYHVlWqM~2Y<(cxBL>vYCIpO_R0?-Qiu4P;?|pbg)Q^z!gesrG^v$qK7SoX zicihG9%~u#Q+{qTaa`@E*KnDiKmSuURz)?>&y{^87w%$hAC*?#UuKd04~N$4Ixc`S z;0d?}?tzEkG&lk71N927gJ(d$8Mp+lf`c5feY|;i!>`24wc?-P=L=u=;N#*~QG;lG zqgg6_##&~u5tx04*-w}A=LWn~MS7gurR0KN94X5>ry6r+^kb;p^!vLysOOF2 zXY8s@NrjcQL*Q+OTg7ixPc)US4^o5W|H%qu1+oHJfvkWO_*JyHY9+7Td>tD925`t0 An*aa+ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByParentTypeID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.CatItemTypesByParentTypeID.atx new file mode 100644 index 0000000000000000000000000000000000000000..269f1f31a465213f69d5cad44ad6d2744b796761 GIT binary patch literal 4118 zcmeI$$%@1v5C-6$ec$(e?>QA1HCi5xS}zsy?(=7uzRamFGtjgJUks!O$&VP!WHOn2 z!dTXLyAdSeEM5PtiF3I?S#Oz2+Q4;Xop+$Yz&`gLY&~(Ou&j=Y6hSHHpQU6W(h) ztTGgg60*L{6pIz=ICv#iiwSnn<^au1d^ab=O^a}a_`WyNU`VRU5dLQ}*`W1Qsy$fBr{(?S+K7u}neuCbD zUP9kOA3{GuKSQ5E-$S23UqN3(??JDjcc5ROKcP3}xiw!gwhubT3(3+&KAMv3-DcIQ zX6t@ytIV_eBGp7xXC?0}G)MMgRwK6DZ``MrdJF-%N#Y!eHI2+lTkJe$ZHO|pHdn)k z50iLSoH#pq3Qs(`y4`Owq(`Zpmexihz9_agYLa{U=*43rYA=cIo3zzKQiN&rjTU-2 zGo+uKdB62JRJ$6T=_qckO@n>Xl*uNII}h}TMwtib|T&0Ij$Fn7%N5V%`8(hd(y+c5W&hW9R(a*)*<5-Mm` z+aoGhM7hl!jKafl;<(aGaq6XDDpmUI;N?2PQAvP z=wJLIhdEKr>CJs|DkCac9ZsXD4K7T>w3VH5q{`N@hoAc`xZr`S3I>AeHE67ZR~pCL z2wn$XMQa5>2f;~jA@b0SU8wRtC(BsNa3Wlaczg&B1 zXV9>Y3H4M&H6~rFVo!@rfpgYyX?0Ip9lI)qghL!u~O;Sw1ds&t%tN< z?0Xe*#zmEz2kMH&6S5LL}D5F!FlknsCrsfDqUmDbI7PK|7OI ztyu7j>N%^xNls@QJh2fviY2ZM-yVG@Qr_Q`zgUkx9|Rr*9t0i)9t0i){%-_cKYug7 MzMcQ~Ux4fNzfc=rO#lD@ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbindexes b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..bc887093f340449a4e4f40224ff3a487814c09a8 GIT binary patch literal 296 zcmZQ!U|`?@VmAgC27iWlAWmY)Vn}63W+-9s1d1^*2mtZ_|BOJA1Bfx@g@B^Y42fX% zB|x3I3?U4a3dhbTnfm;XF>qb^dg{nc|bd0W}}-3 X@&j&lgxu%{v>_L00|Nu98xi6F*S08% literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbtable b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000005.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..bf93ec49a465f6bf1205fb164e92bee5b3303828 GIT binary patch literal 2071 zcmZXV4@?tR9LGja+h|ws-Oi@JHbdkjWI+UoZW*Rpq%G@ARR|)Jr+GE+X$+g#e-_P&;-tXVJ z5(F_1UZ22gGQ5P(40p#}kn;gr7oqz}=+`}!&=KniKYXQqTPhL8bUsT)Dms%dEx6XL7ZwaoII8Q9t zM=rG(`qvz6+OH@JzLPZ+DeBVAuD|IQPA6Df=Rfa0OGSBWNR9`rfXE=S?6lhzt7TR4 zA$zN#enB zzO_s17Vq1b-0=Q|UN!Y(JDzl3IgjEKG3*ax8aQ6 zgjZ6fg$FbTQ;JFq$U=V0i?u6y!qootdHW{eoOgNwTL_AUt%4!ar^zqO9SoFj=HtBL z4QertxN8|R1DmTj8j7`-(=T64J|S=1lVkh%!nbGe_|8gZ^p5({fLy==T2IjvZHdYz zwe#ZfZPhjVlm{{?zMD@1U`&9A$q{A-Dx1%#w}0+jwm$Bg{q*D3-u6D65^EX8 zsADWB+^T$zKW@v6s}^;!>d>|jTq}tOBwxV7T6PQ=R5o~9;fzY4`! z+&Z)DP4bE+x+&nx^b(x#)d1soAh>mu+0krh!v5m|Zm-_mtF+EF-DsXxl7M5L>BaQK z=nH^Af9bu~fk`J~8fQNH@y3jRHvm^HMWeXliNFo}Oq7XmZsd*uG-PdleLrPK?XmCp zx#?D%a=7E{GZZKf3uj(f>6)~6&cbH{S$phw;Jk$3j*l0VI^1MX)Acd)XrTJ54ISHR z)|?*}JJW!w_vNI{h^xx6q$4MaT>3Wc@x`qR1JXI}QT5{HZXKCi@oMY5xb25Oym((6 zR^_e%98&-%J$l39qYd}X&GBYK2fY?E@R>pOP@F B2><{9 literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByOriginID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByOriginID.atx new file mode 100644 index 0000000000000000000000000000000000000000..1664d21885e8448309fe4bc31b7f2754ba2770b8 GIT binary patch literal 4118 zcmeI#OA3H63`Nnn8D{~jwVz=ux+!t*TBU0!!Y!mBql7biA|k_6^|k&@{5=HDBW=6c zWE$t_)t9ttE+J@%mRz>QLULD+%h-tk0tg_000IagfB*srAn<2_@lE|T`M306`wgwf B2_gUh literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByType.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.CatRelsByType.atx new file mode 100644 index 0000000000000000000000000000000000000000..0413692ccd84d77f05a97571224962a84f049876 GIT binary patch literal 4118 zcmeI#Q3`-C2nJxgOm9MCg1tItmx}IQ7CXZT{(%#pl<(~)B3hoxH~Kf>pCNE|*ZnBg zxJn31HRoocTwKCh-$Grl8JA%v1Rwwb2tWV=5P$##AOHaf{8?aqQ-3D^k^Zya@mvX9 literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.FDO_UUID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.FDO_UUID.atx new file mode 100644 index 0000000000000000000000000000000000000000..282a6a2783509cad39c2bb11f6f1e7b6c72073d0 GIT binary patch literal 4118 zcmeI#O$vY@5QgFDGVKNw%%7#1UMjkKS$GBsUSS+&HN&&}L`0US`WyY5_-6>5-q@PA z;`K;%R~JguSWVVaQVKRUo1434T*giW5I_I{1Q0*~0R#|00D(UXtZ(Y?$$zB(-fy!1 B2}S?_ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbindexes b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..c608a88be082bd48fb5821f6bacf48df11c2495d GIT binary patch literal 318 zcmZvXu?oUK6a+_6v526JAEZG@C5Wj-MI<6B#>TIYbHordSM0&Zv3dAtL4iW?c4G0C6X92N+;_M6@{tQVBSq!NR$qXe7o(wJw3~Ve+ zETIgc5DvE*6HrAFLncEy5a&Szxzt$LT!31NVOoF!EFlb)3bd^TFzhfWny&2}TxH23Mei{vKN@m+<=L3HLs(GcybOrM_^j+V`WR zsMTjq+VK^ViW>w7o|vq0-sLXixB9~K;~<{`0C68T Axc~qF literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbtablx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000006.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..431307d13c60381f56be9a7c0c765046b36cb27f GIT binary patch literal 5152 zcmeI$!3h8$3-MA9039Z2oNAZfB*pk1PBlyK!5-N L0{;rUe_$^>!R-Lc literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByBackwardLabel.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByBackwardLabel.atx new file mode 100644 index 0000000000000000000000000000000000000000..8797338e7a9c750c8dff7448fb1f9ba3da067822 GIT binary patch literal 12310 zcmeI1=}y8x6h<%Z`@a9`yFf4|Mvcn@Ac7K+62KVW#z*zs)_I{>NWyKzz7%tBVYuKfDtePM!*Od0V9wn0bTQ-{20*hJ#>ur(E(aT+vpUnqXIfY zJLnK?pk1_vHqkO#L}zFVC3%Gfj*Ifb+P*1ItmMnA>h-9`+CGqLse51DyR!#HJhjL; zRB+kCnfi<~6J<)8t$4?6O-j*mh0KNKC*1X>b;T%ue`%)mf0NcT%%74BawFD%2h;9v zuwTVdaYrt!MaC?Dii7^bC~9wsf;O?NS-m);r*SiQZGTJnDV1>2B_g%Ysgp(&&(kux z1mennp5s51w`f%9L_D+2fx7%%;x2CEgsirG9qFj3eN@Rmdu#n!_cxf&&N@b1{HEPs z?7vE1YkKQrqedndzuEFvW{i9!a&_Dub=T>gn~dM|`RjkurN?F5nq2&TXA1c^b@{uG zo)jJWT|6rs_rBA*&h0N`CZjXykf)fwfy{Py`%`v0{$)EaNGQy@KYPE-B7DfEYjoX~ z&`NaV=fl;1@@}7a)YQ4GuIx1v5a$5;SN{Qdj3#l|Wltg_n{Pj@Vzn9pBVYuKfDteP dM!*Od0V7}pjKF^&@D&ueP0ax48++Zf{0+VJ$xQ$N literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByDestItemTypeID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByDestItemTypeID.atx new file mode 100644 index 0000000000000000000000000000000000000000..47d2132ee8b7ee9a518ea5306182b61d0b93f04a GIT binary patch literal 4118 zcmeI$xsKg1429vFWU@{6WhTo5q$8k(qNtJ*rI#}N?m0I=$|=$W2_OLjY?Bb*?3Gtk%M~PCapa1Ms4K-H{jkKgeY#~#dDN!18Oe?+hW#_S{{;Y&@ zDCS;BeJOEDm2}ylyoUKi@~mDd0c7a`B7uW^*0!N_*mv1pEVm2FJ@jCOJ#lYKm8GZL$riEF0g>s(Kgyg2k032 zeT!%rZK4c$dvj<7t)exwfwoY?N__nx{(OhrJ^Iti*LVNE;CyWVep36t(Z13i$2=#! zUHtlw{Lbms0t@3mdzvSF7F)W)&VChh^07D?r5>sB5j#!W-*L_RSlBf^dqe!o<6tRl zvhQoXAbx+FUz7QxLnPb3-EwQ%eG;GA!2FW=#G~pyyLqI&WNx|Rc#w#-qdB+>zuRhE zP+(n3YXsRB#!{V9>c9WUzB;`c?h(JhqE1Qt>QD>!q|Of)ILYKbzNZs3UzOtQQtzJ@ z|48Vu_DE^X5k_|&r&H9b2#A0Ph=2%)fCz|y2#A0Ph`_%f@S115r$+$X`}gF>Z!;8~ Ax&QzG literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByName.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByName.atx new file mode 100644 index 0000000000000000000000000000000000000000..70ed36c3fea8bbfec79d541e8f5931e81151e214 GIT binary patch literal 12310 zcmeI1`%c0z6vhwU@Avy(dlw0Y#Hi8v0E&P_WM<45-o{7u`#KCOfd&({F7b?fvTo9S z`?cL^&qPEj&;qmw<++9OB+t^7rqrb;cPPcuK&dM=b0(G!%7M2}p7*YGD*+{-1eAah zPy$Lo2`B+2pahhF67WfYYyPvLfdhOWLc7p9bPgRsn-FsZd(alN0bM{l5P!b}v5r${ z6@6!X;$Zu` zm6qJY*#{glZ}L@<&eWA?vrG?uqQc)I*u!w1Jm0=f~{ za^$oiK6ltd$-JF2ahE@vmYeV`N}4{{Dr2k#lz7(ZG`1} z-0d$yBw@6RFb)QVjMwqIb^9YbzAdmv%6J`lmRNt4;6D)f4l1}!uKf8%pBu|R D1zNXT literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx new file mode 100644 index 0000000000000000000000000000000000000000..139b478cc0a4b689389507d63d6ff6ec57496bfa GIT binary patch literal 4118 zcmeI0$*LPM5Jg|+d7kqD*%e5tNm^-NxLZfXasWanRL#a;Tfn`hmdbuqnv7~Da zvXVDR5~iksa+>z&qI-^cz36CWYEFApNBb4jxDFaJ*rl*?LwifnCCyF65ETtHTzivm z!(Nr^j(Tg#le5>U-HiGkjbzkHT{Y9(yQdzxSV>69 zk&{Mi!jcqq_(}qOUL?45a zX$iJ1k>K9l^Paw_AC_bzUkjoJPuZjO%=y+7E|E~F5oS?HnlX(Yi89CDGqtsxub10W zb1DPWowK9;aB95`GFemGY^@E<@wU{erm5^k&9!3M)2OY&WNYP~a`GWCv09aqW6;uy z_MS0EsVKmlWTl$Zw(BKE;69kRJG$dU4;;XLuP?%vVHC4sLbs(h&JBlcGClUFS*UVW zn!&B=>Wc^5|HIVZ@A~tb)4*xqG;kU?4V(r}1E+!0!2dDu>;8%TcxwOi7U1~&8=s}> AV*mgE literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByUUID.atx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.CatRelTypesByUUID.atx new file mode 100644 index 0000000000000000000000000000000000000000..dea48d3ebdbbb51845e4bdaaec08af989846aba8 GIT binary patch literal 4118 zcmeH~Nv`EK3`Kt>skD{8sb&G05fCLxqK0gwO+CE5*YyW7tZ7CzZ~@uACaFVe;Fp(| zmru}jeSrRg-ayy0AEEcq&(J^6N9Y~&1N1xe9rQQ!HS`Vi2lN~C7Wx+Y75WAG3HlTI z9{LKpuP+mLl;q;gCe<^9r0+pKEpQuk`l;MnJRdxFQA?-ljY%?+^kh<=Ue{JVD((5G z2zEvv#NtLG;k0ZN>Y6o0&NPLq_pApYD4ZwBZk0>}tJxQCtT;DZ+N`@7wce4DZIhZQ zF>9Z^PE%cQWtLz&h}DwhY$FZ$aqFfzXQab*R?)r-O;k$Kx{&P_)FhB)ZksC4$JjeF zW7CkcKtdKFf6^)4+K5{E&gWxRRF&rVxL1@Zp~0~s%>#)Vphh?f|Pu}5$)8M6Q*gQ~_Ltq9Fc(9U-D;0c#R%E7x<<_52VJ*qb9 zc&~AI*;crE)M_V@rwzC`$r&Vg&SkZa7+#+b4zpKv5~6CFAf&P=!kmzfR7^NozP{ZZe5OHdS6!@ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbindexes b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..2a98c93adab6374c865fccc1045125683eb4a05b GIT binary patch literal 602 zcmaixVGF@f7=<5Fd_a6e{>-eEl9l)@8wnelmHhfSvslCKT4%fW-MP=X_q|P^Ejie@ ztR;^lJA7iMszFamlA7Kv>GywM)d?H<7g`sv6;HpM-C2SERT9zjtKJvR}JS&t08k_L_S#SGPiCL9r)!ePG?#u zD|Q4ysahx?6s@(Qww0my02RfzfQlWpSlXhFMbxo-$p(^zHa0Uicf0%l`~8pa|98_L z05A$Z58z`5AJcD=lkv+NO)%@W%(o2y_P|~YV!;YP02!cx0Pp}9VB60&8blihgD2QJ ziGdc7KnW*kC>IYTAQ>dUFM~8Az=H#wLA`GQTM>{#Jp$tHGk`nrff{bQ8cu709E8Jn zEL>JWfwxe=4mbwGDH&M#Um^mC$%Xq+5LpThqzn+u2!MWe;2ZerT=e2Hr;1g_%U!3&$sD1Sk}3%u2dQ!lx|2VWYH3nNLMgZ; zpg5+jIA(is&}IO70*%$LaC^1qXV0L$MRmGBY9jAaVB2!#gw~K#Z$FLO5w-pCoRb4C zv{D%%BIwm>?Mg?JHHNj(xD^Ib4w3~x&%oA-FYH`5 zbqKe3&97R7Uu^mA2<;V>|FR!(bIF&rA^BeMm-o4>4IL4cOvzAo1r#%gBob7lOzOaV zoAT~~gBuo+qz&XgL_T-_di|^&R|C`L|8o1yT%Q&^8aOG2BT`s~!IBp5=yb=ZBU@AM z`tnO$0%{gIkgRP7uZqUTaszYP)19kAUY~s8nRaW#HP_IcY22wodzTk%WO=vWXgmzP zX+m!DoQ-?l?eu2G=?A>LM}D2d7G@XD6xB4>%)?W@6@dM14lBg@UHpm-xpSX-Ky@vl zBPiNXu408bB~1J<7ACAebJ3CALui~u1*4)otEo@Ebu1_-oAtf_Jg*fe@JweG-1c7o zHeQ%HLDX6)Ri?6#_hN+^1^S8rlH-UK9CJ>fqAZD-y(D5rc~DmI&~(k-S(&8Y`u)|* zWP+-Du7AEB)U3lJf)ynUuZ&RfrLkI4W~~Hz6{S&`W7Gy`XcG8P_;xYRZH{sazg&lvLz2&hm(9Z%5En0G8oPn;j!8}tIAh)txUESgAsTPOo{-cFlP9*jB7X*5+0oKQ;hq{ zoVAU3kHE`?-CZuI1E6sq6*C$wanuT?h?#C`)=>a393HZCpCh}5_zHK_*vq}jcgLo_ z)~w#LzfrtlyeyynTj4Z345sFQ4tzvUgk&ukz4|CpH+4Ax5}BBgYbK@MEG=44W?vw@ zv&p*v=P1qy7HZ)}Mj%@(DPfh0NeuAP*|Neaei(1_fSaYQhrj(mpwZDsn-&LW+8)=6 zd=F0jw`2$?L=J@Q!d1i0N>VzLWA11I_Dp!Su@ts}S22iP+L6@s>+~{iRj7xkwBgYi z^}M8K=LuHG#^Dz~OuyY&iDNq>EQqaE$q8*QOjZvjNtq0>0dWl2oveH8X+9T(+neu3 z7G!jC@y**i{5RR^6=4S!c>LhKB|zrPQ^!Mp1QY^w?s3qt&?go!9>@v&tn8sfYr{5> zuHxqT57{`w@NAk6WJ_vUrqeo+4CKDp+)lle3qq<6{q^z8YP)v)Hj6`hA|jdYo@Q6i zK3VlYFRigm=qpY)d@{GLO{|Zon^Sxvc8=T_mAmnvo<tL4H9cf zB_yd>;lv|!G)Wn5C2$D9XN)&Z&#y$Y>wHDUXF`{b(Q8`#6KAZF`&EP;_%SIU#rtR- Z-lT?Oy_(Q!A$zTLr(Sm~iBS)Le*nHMfzbc} literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbtablx b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000007.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..bf096e13d28fc61c7dc794d7b965ba6083f1f807 GIT binary patch literal 5152 zcmeI$tqlS}00rTN;~y?~+Q3u685C$bpaX)#(*j-KNC*~%q6ZYoIo=e3n@!%AY_e+P zR;83qw#fR~oXZuK-Ef0V6`s)a!X4H;e!zARK5!g{77n9O#@QL_NjSk5cGK{N{Vcp- nRfh|BEMNf(Sik}nuz&?DU;ztQzycPqfCVgIfqxd@_m6RY0`v~Q literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbindexes b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..c9d0caa2233c4a1fcbaf3847dab2f14df879a2b9 GIT binary patch literal 116 zcmZQ#U|`?@VmAgC27iWlAa-K#VsK?}W(Z;M1d1^*2mtZ_|BOJA1Bfx@If0@u-N6hQ X42cW{45>gd79eI|0O?}|Vq_Tr4EhZd literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbtable b/autotest/ogr/data/filegdb/objectid64/3features.gdb/a00000009.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..451b4e263cc6b611ab030b0bcb1985aa6e4e0b3a GIT binary patch literal 1384 zcmaJ=Ye-XJ7=DlEoH@;Yg#8Rl1BnoEURL`N?s!d8C(aRNvdl~^saUJ1KM4a{&Lk^; zX!WNHxuSw@ts>3raWT4IXS;>OET*RGphz_FCN(mHCdB~2TOjMZg_SLI1>oPbjJB<`z zS%AiRr2*;Rek{*S`Lq>hj2olQvNQkU700{l!DJIy>UfN$IMl{--m!>DIE&{)hZ7wY zW5fWt0GPQzS6O6|4VucJ>^N0YDg0p!;)tQKd<&FjfwjuGjLH}D16exsNklj1ih=iK z-Wjl$fD!1*eD+<338)OQ5NIXneT0sIgAI}So6F~=II+WX8}d}<3b-qrdm(p+A!Hz{ zYOyXI^CPIK1Y$YYkpvaK;Zu|kjQ`{0@ve4w5kLw!vV46i_t4JuRQWYj(q3RNh0HI5 z-__+Y%hV>jED|}pjXdt3ea7Q5>pA!IK<=LW za&47MuqfjvSKk%oD5ZL2iyWb`C;|tZJ>`p}PO$r-(HIu^h^>?Zy6m#qWs#^ADfG`% z1yagj2|w9|^4aWgS!j6L@OnsSum3#ekv@0a)I4bz6YqB2YH^DXx@ulHLY$8?f86g8 zzP|dkqGxJG_TDj_`Ep*k(tSGYUYlD}-7?(D4cR_ho*vUTKI(VxI^7<> zZN_4)b5r3OJEApm#^e+2xsLlpjF!Y9Zmpv}Bta5jNQ%~w7-@uDmpqhat>1KsKqEmydIb~Wf8Sr^#cmuO<_!t$I9D^JbWSXbk(o4{@ zf(BfoM6I$!q*lFxdQqkR literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000001.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000001.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..52fde329b77d5952c4cc4ba8ed62037af103eb28 GIT binary patch literal 4128 zcmeI#F%5t~5Cze_a6Jalf(u-rgB}4WpryvZ^$4Is#pQ+q5hSz8>$aNicM*}$BV)ym u6Biy-3_OjPFk``n0|hrc3s}Gc7O;Q?EMNf(Sik}nuz&^rQ{bKQ-<)sz6$y9% literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000002.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000002.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..0b29eb53e07d77c606e25ebcbd77f7e494d1f585 GIT binary patch literal 2094 zcma)7?QYXB6gAq)#`H7hfha^j_g-ky`G&;7oO4oeiv!*s8&Cy z9k}G{{_&`F*sLEOk;cC3$LSS=SZ3z809XPmr+|Pf@C`&D1}@!!e0$>zICQ}Q0o_jN zh=D%+%1qWf)F%WU)pP+YY82C1Naqxts;YorKYsphQnWv-U8RSH8n#x3s*RO%yp2B- z=n~JFj8o`3spI)$0!LBk`4^MeN&P6SF{tuE7{$qo=95k_p&h0*px-d1G2_GWNF;l!dCjL!k_h74CcHQswo!K>A!HZI7c* zg>YY$cnifpBf`w|aULPkyLLG%KkAZDozR6P5?cSly&-shS~qEu#L{D&Tl9XKNY(AE zMQ!pHb-k9is+7dBPi7>3~w5w)?kAu%Bgzy!o0Xl*TRY#9KTpo6f2i6fX;xPSxL5jqp%i#yPL zmpsM&#a;4tccG*JDVDgQ$By9vCsgS0z-A$3DA8a*5=v*Z_+ZaLM^tztj-&!Lp7>7t wm2|}&+qIPAf(9>)@NxkcZ~+%^0T*xq7jOX=Z~+%^0T-CB0MGpYpTKm(563+c?f?J) literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..5300ca64943170bf47c6e0c42c14ffa4c47a1995 GIT binary patch literal 797 zcmZQ(U|?VZ;y*yl3dArV%M4+8Lj~l4w7ZL=kAktDp@E)3N@|&|WwM2lnW2$!s)=q& zW>HBc$OsmYaz+LQ9tI{)1{VefHWntfV1^)u5C&I<2p|bmzyTuI+!zuWau|vkQh~w@ z%p9!9B9Rag_6UYZhERq)hD?SMph|RAQBaX6JZgNQYJAbvaD&VaVaNx%A(f#B=$1T& zWFQM{2UiqM8DCs7_tx%v{}kl+KNA_SZO-%=igM|#uO-zlAER9SJEiFxqfm%WCVrWP^0HPQg!oYMx zi=HExpY;RGXXrTWfX?rIyLG|$V+ZX1to(S%Hz~j#Mh|X}GE_o51f^5Yo+aqf&wtob Ky1bk*JPH7ebh7#Y literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000003.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..730418420e468d11c7758cf599bd0f363c50b722 GIT binary patch literal 4128 zcmeI$xeWjy3*ltdTw-w)?D(44e^(+Rz*YNVNVx;aiCMt}eT0t5&UAV7cs0RjXF Ld@a!bfUfrd*zEw1 literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.freelist b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.freelist new file mode 100644 index 0000000000000000000000000000000000000000..040b0de5c092ac282111f17f566bc3fa2dde68f5 GIT binary patch literal 4440 zcmZQ)U|{$U1eZA(80@5g91!D)E|k&b2xV+HhcX0Jpo~s7Amb5Gaf|?vF$zXQU^E0q zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1O{;kFaq5M>i>fP$eGlpL1wbkz+7s( IliKD10Ap{p$p8QV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..56afc1dd9910f94ae71d40779de43326c9e39027 GIT binary patch literal 47714 zcmeHQYiu0V6}}-hN!&mLO{G>U)Y=~;Ew=aN$2v?lc6Qh6P3(B(U6U87JKnwCov@xU zvtw+lYL%8pp_CsyRP`aX^z{QlOHqqj^%0Ozk!V$Z07A4skSeu*5Tz7KK$Mc6d+*$N z?f4N|(}K@N&dfdc+?hLP?w$G0J@vVDTgkkf>c3b9Cmz+o?L zyQpo7Oo_yYt|Dz^vP|YdgLGWg)^>!bWR4};?t^2rlbuCUhx{hd;0P*dkf1_a5yTcK zk+!xiti-M?7pPE%LPm)!Kn*PW9C1Rnw5_3&5IQ^k$0-n<%o2mZ5!%j+s)Z{+aa&>Q z6qzJrPO>8l6`F+$(c#a6KLhs2PSSo!7IrI8Pj=xtWU&i!v$q!@Ng-2KftBef5)(=> zyO~^@!u9fN+b&L{!tolA1JdoJy}bk$QiYkfk!_JJd6FX&Bu_G6ppztG?@g1Vuva3} zaG8T7MS3Bp3vSvlDZ?Eb=bBl_4bv)-A{m9FPQr27jb=v;)9WWmG62Vp^KB1_!5I?# zY(22;gS-U%OTr%0JO%%{;K-A3%yBrns6!qunsQ?;f}tvKZLFR#!j8uDr=Xb>pw6rg z%p5ga-_?r4d3xa-Nq%m2UUtQUAWPb3-3B+AkTbK7u?;zME#IEGxo4QCPru32li?fd`!^!QBifZT%K?T@_qkF$Yo+D@klbQtp1=THC2y63h7-}pNF z{ow$J;J0_^g&&^%+tskQ^u|56y*#j^<6qhP=P$YW$W43B8tJ9ieoJ@fZu-bmgSY(g znTMejTt}cMkcTe<^>eR$`s&|Yvh($cyPx{`XI{vD^@?5ZG@m$QP7K|W=v+AU{M#4r zIBJ;JH|Ny~-Enkc@^Gm*oXHXLTAOlUX|Bitkmh8+KdjHm3*NT zg=a`IVi#L#YQ8dy$sWV3MH8`DJbG+=ti(%pYkJdC^$K-Mtz8Ion;Ngy{)njDK2NP9 z>5k4$C8Jtu!>AQitDe42iF%VP`-nP6qe8K+HL51nb4&B8UZvIatlDT&C=ryiVvcFb zV3Rbn!kAG}8(BEnjM}75wi0!Axu&n|h+44>bGc|5wL;degrV!SV!@p$Sti_&ptPXR z8vLSltpV3;F3^Zj$r^Jin6T9#!=|Y&Pt1yZTX|ePZJ5`J7LZOVQD2%>aa_|~g%~J! z(@f#uLTy%Cl7|2dzo0{jdTs>k2qb>){Q04CcUyn@@)dV|Y~sw$hrX82m(T22YR-4v zvh~onB3&=Nc=DTXUAm}Q_366Npv~#jw0h~$LV0w0bYd)5g5>b7>-^cQGqAt*(&%t8 zM_#;qy=~iREF)C47EGFHsLiIZ?fa;u2P+@a;QjUrZ~p0^66+He-X&PuZ&J(Y5ZSV~ ze$XI1`K$?LONA_Je0<9Uc58Bbo-1J48C9325!Ykrf z(n;m1F-I+PS&2F#cAt2WLi5N%qoK|;sBHm{Kp(aIi1f<N9R6IG*6YEM0rg{>60|{t8{mE0Ox{8y9@f;i>Z&Io+YL$AV3%tHlT~kL2WjHRa zft-ut1HFC8#9*Q?J~-H$grb0Gpju!Bu*0zlQ>$ruy1O?94+ohNdB=TuGG&)LE*>QD znEXL$TP8G4$kIYZiAsT;?I{m=p2w0h9xN#ZqU5-*q@9wrDg*8?83;|MdInRhpAE`i zqL+5Z23a*-CA%)J08AKjugSi)1XR#fv1{Q9Rw^^@s)#b9TCMIrn(eooPbTz^9)<@+ zZ!*!(?iB0_=@_)Grsc`=2tWh+spvyag9Q&DYH`31nA-a$XYa>!2ZLP^4(Yy>@Bj7g zpFNuY!B1}QQc~0Bf{^YqKfIXHnwR-e-N8cX@&K>{r2w{j1+g96C9n6=YO-~~J20&a zHk20UMnbDFr^k!nn9G9*k2-#*5A4_vaJXUKx7>PQi+p*mfn7xJUm3a}!eclX5gsBu zM0kks;87qzkQclm!dnNz6N7eN_~f_m>b&FL;fMBbE4}m4kG~p3c>Y1V8ULW&!N8#1 zOaR#7pdEwXD+9Zk^`JUXlIQ{<9rIcoTxJh{hrzS|WH1T`?OYRw*;Ybd9JIqhJGF^} zb~tF)so2AGe1HrG?PS13AXCr4+y)QY2}pNy`{Unvn{L1Q(uHuJu znE<4#Af)pJk=KgDR@MXQKnWoo#$kC-mOmo^Qi<3F#$nws2-w0+{Flvyaad?_7>9+) zUKoe%#yG4TMEw7Y!wN`u#hqt=_s^jdLyu0~`pVtMrt+&px~d=2Wdo3|ijeMtAYFAm zkPeg((jlZnNSDT`b?oWlOqmjv}PHAV|lS{&?07LAMmMxqQE&UTz}xF#)+Xjet+ul_jX4Ptq$pC z{g7@X0O@8C(p?awn_Um210{rX2PKiHBW|F<;o17vIVcyhZq-jqvKlAJ>Vzap#?WWA+JXtp1crW?E-%6we|VaPizRgi zEdQ+96hfJX1{?#D@g$z?fpz3-s)cf(tcw7}zZk;9ej-~~*g^`pmA=XUt?km64eI>X zc^6My2-OLYR;G@xZYir1M3)+;fAI*`Ng<7u56`?3Xqa`ucvjb>dF_C zt7(dl)De}(ee#r2FO0La(%B?q{vKn2nvj17x2g?RUq0hJ%VV9gb!OW zF97(t=Ba6EVOMA8EtjYT*CijnAB)3^FvyISmt8G8v0<<=*b19j`zcz+Re6%$Vo0U(x z29wP?Xl%Trvg`%EUq0t^`i-;!4gLr;lw-jA#%Q1TRyl5`_!jciC6NbP$P+2PRf1e1 ztz7|LbSseGz-c}6Ew8#^!LY9JlHUl4&442kE)9;I>2TqQv?Ro_bJt|W5qb_aj$I6f zD{X!pArK@vZf1}vr)zksJUTI1C?7v5lyb#V-d1PyERsrR1r{AeQs#&m05YXWo*YR` zGuR{cQYn{1N0DqfEz>r`7vZc-nxlmBuZ67Dw?A;0U%Hk068&$6h6qx{|PzVrP;foeyz1|X+ae8As+Etyb;sne?SatuK$4n%BKP6N)IZ)g=Y zP_uy4_Ivt!ZSW6oQNY@eWIUun?>Wt)CRet%Z+Yqd7pGpj_w`Th-TUA(&pxo_K}eFl zdkLx6IbHV9bMFnc)$^Oo#peuE-l;WVUcO6HwV{qaOho1Uqv+^60MPawf}XoC9RldQ z8Ek4mC5;#Ex=BVHVl?|Stf}XuEgfBe%4JEqWWS)J1w>HGH zm!!2D>e<7OVwe=OJe z&}Yb8E}Wc?lk?@5aM80z&t81x8JC`z3&8j1UykygEImQbKE$)%P`>j`=Gh16%R@YS z$z8sop8WuN_87CDHF8UG0RejUC0MlxlcSH7#mas3?9sD_^?A7L9+s@9aoIgCyN}_r zdz%R$fy?f3+zQ97-e2QZ=-G#O_8a=yi%sU)2fktv;@QhjSZt_g59Bl$v&Wb{diLC( zcinmP>@|O{z?gl@)O(zbk1_jsjM*dQn1^%^3H4}c87N%mkS@{=&P#$wq>I#Q^`c2D z8vlZa&7bO%Va)!$C)5+O@i+AAN1M#E4}8oh#Iu)QGTKnjeh@u-jM-z%-VV>BXOC3* sHkDnFM;~MM7_&#J{A;>^;T>c47_+xOT!kzW$RhDRW0Am^y)S0}Z@7a?AOHXW literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000004.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..0a43ae9a20008fb30e3e2b4c91943ddc39eb1dba GIT binary patch literal 4128 zcmeI$F%5t~5Czd8M}rGoAOZ;l1vo$fK>|Gu1!_=3#XlJ&v&l4@tzNkWXi}HjbQXBh sJi;(Rzred@x5ItFcIC)XKmi35P(T3%6i`3`1r$&~f!_<%e?YCy2M=oqqyPW_ literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000005.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000005.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..9a0fe221dc4aa99430abf335eefce0eb0d7f5564 GIT binary patch literal 2060 zcmZ{k4@?tR9LEnRe{?|Qk8yEkR)GpCtWXh|qChDHgd%O3E==6@`ltu(UAa4;3~{KS z^2a|6ryEeBf-x>CF;f%~7F|XH{vS%zRWivo2FlnZQ&)-X9kjZ=WtY6%yLaEu-TVE1 z-}iIDFiZ?bHXN>SI4)0*>DgNQ!hCpFB*dkPq+#KbFo_NnhDH-BB9@0oNWrpD9m#OT z{3isw+=;(?q+?nv7Xui9aab~zfMIR|XF(>GIsGtGEbv^1;TQuI6jW%iV$1~RQ%d+! zpoA}tZ~+q>p8D>q}ACJpt!%7I1-3+h6+dWl+lb*!sCozP&vsP=T^88j=cwI_-kmJATV3 zg{5WjSjS~D8r6BlX<3fJ2~LA+4L}NsN5|dTvTTjA@`%ubVL(>K=ZC(tu!dGl}8IvAc5g{trcyGcD)y`NCLuE}k0hU2^ z(X%FN^W5X|s_T;XRnE7psM_zfS|m4ffMQ{7`0TofLyClT8x^P1FB}raj~_repVw+f z9K5+l|A^WtsL4rcNtt&cxOx?VQf>*f5hp1=F)6)+h{ z0_W}VctQF7?RWhY2g+rVG>=)&P_59eb!>$X;HNg7RcP&$o&ED^K!x@9OqB6SB3X`x z?!XPAG@Qu;4Bu~7UUIu`F=wQHWoyybE~y0Q32>cqI)651m|4@(3YV0og-f4|WHs1O zrI*99Qzqx#)U5B%KGAqIp)BNn*3&3KJ4$$~aFzp%BM2UO?$j1}Lq3^u>U@!N-;M^~ z|3hT(Y(vxXxhkE=FRE_tOWz(uSmq2L2}cVlv>yh?Z}?h76+{;5q^(IUUFKC=g3nz= z8Fvj}7+5zxZc7dAWFowFu9SC*aJf!@yt!P*b-Z} z*T+0X34d5U2C#tQraHr$t`Ip=V&0jOp4^eN?yZCVSsPGLXb;;F3ZJp7F>x=F7Dgt2 zGaSAtdTYikl<`iZS^Ul63x&w;6P7EhYo5!iKYIJ;cZa&pq9*oK;glY>o+<0{P0qJ^ zc}>`nt*HlB^mHr9;e9CJE2DKlC;9+46gV*cKYDO5YuC$xw7TlEUvmL!3+hNubumSV z>{jN1znsCn4TrkrmH^}J!?Q~M)262ZdrRbN|NRnu)Oh>lq{?$OvW^bpdNg=pCPx}b zuGrBK`Fh@r9r;)<9+Y=Ze!BTH^}b(G#!CYmJa7Dk#NI>E`0)_zysjjW*5CjYxR8e+lf~l%xfiA&MXF#5p zeCCQPes^D#6tG5u1_J`a4Nj=hV#I1D9Z}oYyVL-7s-56uchUyv6h$b|n1+A#v zW8~>9aJx{7YII_gjpF^4CBKfLM;&F?8d0SZun0u-PC1t>rP3j9o9 N|F-_R`L`{<&M#+pK3@O; literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbindexes b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..9e15d35d613629194e6fd897de84c6d110e2bfa8 GIT binary patch literal 66 vcmZQ%U|`?@VmAgC27iWlATDCaWJm|%JO)po7y|n=;#MK literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000006.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..1da41b719ca6c0ab6209542a3d8617aeaf4cbc11 GIT binary patch literal 740 zcmZQ(U|?Ve;%Pw43dAt*gbBj(h6>07X?GV#9|dDQLjyg7l+-d^%VY~9GeaZeR1@8l z%%YM?kP$2(<%}RBIs6%t7_t~r8Il=F7(5wV7#P@Cm{>v?LLnS(H5Q z$Gz|ON{ze@M=5lJWPA6W{_POuus^iz!=#8%cbo=Iy~HoX`Ip6QwwCDKO>XO-q8qek z5tqoiPjU`z_uBp+JNZrxr$K>-85KGf*7!Ate7JY9yki!+K~FYhtlp9z<7>3Mim}0tzUgfC38qN1*-zRqX?%$^%;f literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..231ae0568629460f3704635485ad82e2e607d757 GIT binary patch literal 3665 zcmb7G3s4hR6n!WtDhLXK3QEw?qWIyXh>8j%1R^9UKR;zUx+I&h7&fq3lqg!uAB;k+ z*bxM^YN3i!q}GaJD?{-EDvEyr(RRdQso$}RIQDI_0rFrPo0-h+eS7ac=bm@p-5m@7 z7!UvL@NWkH`d{cs-CZ!d-~_LMJiic*=PY-(S#DB%?acWSZx1hb4^Mo~OesOpY9@s# z<8A_gIWQN4Xb=lqK2XA>>)I857jfHw)G?Rm zr{8U=K<$xyr9z5R0#bLi{uHBdlIG|vj#y^JjUg1$UfTq6hcM-58$VxE-yzn7)-No* zm3$9Pd*g^%+MBJP_~(`_x!5|?Gs8ka<8gWrORkcnF@jWam{l9r!lfJm$pV>xBnD;W z0?EEAeh@En$nC=qkNq%9kf5TEH!t_kHa$s+JPx{L7vFW+{2@-5JyrCu0+VmF=V3Ia z!f6PEV{{^gv$k~)AiLmTh-u(#0@a1sho2`k|1qbGTNUUiI^5WP4y_Qyw(7;!D%uaz zwilBiIjpw{h+)9)<+RzmyIvCJw%iXZ&U(W|lXpI!mtm@r1sz!G__NCvABh!D83zRv zkZc;O-U77`2E+oe$C$#7L{VXL90Vo#kuPV;C+$YDP}`2hP=(S}t7wA6XhO-D0HiW7))iTeBgRehul^_deT(hYj;64LwX)7B+)Ucvk}nqx z9hRhupOHRzY4M~Pe|1C^cGV3N=TFk(eA;aac!= z)^jXuSelR!&$dD~5~oa&6xsi|&mHCEXvm9A#%_%!il&fDr$zU!ok=C}oTu4BtMy*vgZ=T^gKt$->e>e*0Ftf6*q#cSdEtYZZRI<;O~Ta(L0(LHO+z z-_!*ZO^&;|*Lp+X*l;x|=_#8r*whxxy}oEiwC%JHdn(s<-T8zD^_>%betEV(&09P< z_7sYQR&L9pie-H$3{bXSA^3{m^yS13#P`!{^1TjC;8x_@tG``VckG(0eBR5GXM?hj zYq+OQ#Vtp>P~TYW`y!Gi6a=kiN0G7P*KaDZ;k`pzI}k5CxY^*n{km`3lHcyWU*y_~ zazYQFejpl>F+76Cp|!K9PJC&{CbtpX($rOtG|vkjU-9i&EuZ=@p!&lXG26p)AER<0 zvnYbXB=|>)(QhbdgL|dWc7-O^?%nQRo&ENd)T?~SX*`qOz~3{#%U)mQRN2<^3RbVU zc3r$aHKHNB;bO=MZb6~6dF6#&)udDoslJ@iFa!V)qRw@XnI`Zr@uNt#Xlq{-}Gpmw5b)U=wO7=fF{R_fzjWYlM literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000007.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..5ed1ca9acb224f26cbb87a4b3dbafe1271471552 GIT binary patch literal 4128 zcmeI$s|^BC5Jus{-d)}#=q-d2sDV0GfdPTXq9}#X2^B!-fN*XVn17N_Ofu@oDZog* zT5VizDh-_U(yqWmTMwQZI(TasLo}f%*@q>iht#xDERN literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbindexes b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbindexes new file mode 100644 index 0000000000000000000000000000000000000000..cc24e2a06b9b9cd5f6fe51035fb99fb73263853b GIT binary patch literal 116 zcmZY0I|_h600Yrc6cjAHl3&(sR?y!2pJE*gL9lo!5>f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..a3af82aa62a81940b5bef5675e1bb499990740a4 GIT binary patch literal 37990 zcmeI$%L#-q5CG6oRy$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a00000009.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..d4842ffc23d1d41f9374b4e50a0e78b172693d57 GIT binary patch literal 37990 zcmeI$yA6Xd5CG5vg^n$<2z91l6Bvbp$ub3UlW+ngBnsLndMDXFe+v6m);%JkT3bf5 z4SE6u2oNAZfB*pk1nvc%_5B$q2oNAZfB*pkKPS+=w`$;P%(`Ia)7+PK{n}){43`TZ z#ly>n1p)*J5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAn?xu)#-85+?TeF MIIrdQG5_rG1&m)1>i_@% literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.horizon b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000a.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..eee05bd71076531d442af3f15f43fb65db34b54f GIT binary patch literal 37990 zcmeI$!3l&g5CFhc&Yv$U2wXE_RVN5`Xgivy2bK_I7Lt(7XLmk<{X|5QSBdUA>;wo9 zAV7cs0RjXF5U4=l+g4D75(p3=K!CvO1lDahdzyY_E?6dmX`Gin|GW2;Fz0KGRAW(U zB0zuu0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5;&7Yj6x-qJWPeH-!2mFNF` GzvBQTPYwM5 literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.horizon b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000b.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..b83ce08a274d3517fcfd1c066e081bf9228a80b6 GIT binary patch literal 37990 zcmeI*u?>VE6aY}pc%`+iiQ8DXgBv)4gEi%Qd=THo OxoxU*UQRCiza#z|+YS)` literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.horizon b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000c.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..f343e6d9eb39e33e7edb1a3c4dae26d688aa3d6f GIT binary patch literal 37990 zcmeI$I|{-;5CG6gh-j0LMsk3?ND4vl3SPzwdM4e$2$nY6gg3BXmVtTQ&JYn@9T|tu zVkSU<0D(pVrEip2S^@+J5FkK+009C72oNCfr@;AnKjzchBm08c8JyR$jP=Kw`)ge4 zEC}pX;I!Am)J1>*0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!894fv)D9w!HIN Pma&h>yWjYpUt7EZx&#al literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.horizon b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000d.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..c99336adb5db16106db7d62144923ba71a1b38b3 GIT binary patch literal 566 zcmaJ;yGjE=6g?Zw14R4;C3UjwBd$rKu8C}n#)NEwg-F2cqOq_DqNKDF3;)ByUyy)x zK!UZOpoon{P|wWlHWHi*bMDMNbI#nEd8qNC0ZFC=Ih=&T-Dtf`^eXP&imjW5ZUo_F zzSJvPj%nFpAs_Sy7sF?sO_8R}0SP31RN za!sqZ9*$!C;RwcKSFX>#)c(!W=HK<_G!@*|%Inknk4gQwxe=q}{WrzGEjXD5woqfk cWws%n6RAw%_2Kb-iN)_l?RxZfH(5~gFLv8qJ^%m! literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtablx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.gdbtablx new file mode 100644 index 0000000000000000000000000000000000000000..3a77f69dfcb1325bbc278c475b3bd6f26a01ef60 GIT binary patch literal 155936 zcmeI$%L&3j5CG8GMZrr>DZxg(Y5}Ro5(0u%q#7I2U6Mcw@L=3Gu>Z@me^^<9prciULc?$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000e.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3f)4R%Q~HMj{InCz~Dg%DFHb V{rA@PeIEn{cK=Wbo?or+as&A53=;qV literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.gdbtable b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.gdbtable new file mode 100644 index 0000000000000000000000000000000000000000..e035332fb4d83d3d25f7581e1cf0deff6deab17e GIT binary patch literal 642 zcmb_ZyG{Z@6g`XbNR06lG}N%!2O=~oE@lx$2n(?=hD3IwSXdCEq14J)_#YPjf+4Xc zF~r(W&=?yFqMn&qEDW*o-t3$^GiT13J39*%PZ|(qDLxUG5rx|dut;=^g`Fi^Hx1qJ zgR^wLowKs0We1tG-|6=HkCcd!rp*8mL_L&HKng12`N@Y`zj|2zJ%1m^{Ohu_I=+1yRSs)w6O_Ea2L7?1NK~vVJN326Uas&tvAV7cs0RjXF5U5FD=@zf1k4vot2oNAZfB*pk1PBly zK!5-N0_OrNkGRfJ93en}009C72oNAZfB*pk1PBlyK!5-N0t5&UAVAZcuM>4@s_r|=!T3So<~rF5(p3=P`$uNzSFCB z^acnJs7m0ms%jN`S-{2YvXK!42oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0@o9mo}8-P?ho@O zA}zfr|GC-jj``9r?IMqEzxw%dp3jp1>E|Ev@mX&Fa^$Xt009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly mK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNA}2Z70z=llo3t{Lh8 literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.horizon b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.horizon new file mode 100644 index 0000000000000000000000000000000000000000..b64b92356a70378da21d5ffdecaaca5124025076 GIT binary patch literal 32 YcmZQ#0D;N_OdzTu3_>$hIzagj07ZNRrvLx| literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.spx b/autotest/ogr/data/filegdb/objectid64/with_holes_8.gdb/a0000000f.spx new file mode 100644 index 0000000000000000000000000000000000000000..5fa796d466b1859b760d8a500afd3d9fc8111303 GIT binary patch literal 65566 zcmeIup%DNu3Reset(); } - virtual int GetNextRowSortedByFID() override; + virtual int64_t GetNextRowSortedByFID() override; - virtual int GetRowCount() override + virtual int64_t GetRowCount() override { return poTable->GetTotalRecordCount(); } - virtual int GetNextRowSortedByValue() override + virtual int64_t GetNextRowSortedByValue() override { return poParentIter->GetNextRowSortedByValue(); } @@ -129,8 +129,8 @@ class FileGDBTrivialIterator final : public FileGDBIterator return poParentIter->GetMaxValue(eOutType); } - virtual int GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, - int &nCount) override + virtual bool GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, + int &nCount) override { return poParentIter->GetMinMaxSumCount(dfMin, dfMax, dfSum, nCount); } @@ -144,8 +144,8 @@ class FileGDBNotIterator final : public FileGDBIterator { FileGDBIterator *poIterBase = nullptr; FileGDBTable *poTable = nullptr; - int iRow = 0; - int iNextRowBase = -1; + int64_t iRow = 0; + int64_t iNextRowBase = -1; int bNoHoles = 0; FileGDBNotIterator(const FileGDBNotIterator &) = delete; @@ -161,8 +161,8 @@ class FileGDBNotIterator final : public FileGDBIterator } virtual void Reset() override; - virtual int GetNextRowSortedByFID() override; - virtual int GetRowCount() override; + virtual int64_t GetNextRowSortedByFID() override; + virtual int64_t GetRowCount() override; }; /************************************************************************/ @@ -173,8 +173,8 @@ class FileGDBAndIterator final : public FileGDBIterator { FileGDBIterator *poIter1 = nullptr; FileGDBIterator *poIter2 = nullptr; - int iNextRow1 = -1; - int iNextRow2 = -1; + int64_t iNextRow1 = -1; + int64_t iNextRow2 = -1; bool m_bTakeOwnershipOfIterators = false; FileGDBAndIterator(const FileGDBAndIterator &) = delete; @@ -191,7 +191,7 @@ class FileGDBAndIterator final : public FileGDBIterator } virtual void Reset() override; - virtual int GetNextRowSortedByFID() override; + virtual int64_t GetNextRowSortedByFID() override; }; /************************************************************************/ @@ -203,8 +203,8 @@ class FileGDBOrIterator final : public FileGDBIterator FileGDBIterator *poIter1 = nullptr; FileGDBIterator *poIter2 = nullptr; int bIteratorAreExclusive = false; - int iNextRow1 = -1; - int iNextRow2 = -1; + int64_t iNextRow1 = -1; + int64_t iNextRow2 = -1; bool bHasJustReset = true; FileGDBOrIterator(const FileGDBOrIterator &) = delete; @@ -221,8 +221,8 @@ class FileGDBOrIterator final : public FileGDBIterator } virtual void Reset() override; - virtual int GetNextRowSortedByFID() override; - virtual int GetRowCount() override; + virtual int64_t GetNextRowSortedByFID() override; + virtual int64_t GetRowCount() override; }; /************************************************************************/ @@ -230,7 +230,9 @@ class FileGDBOrIterator final : public FileGDBIterator /************************************************************************/ constexpr int MAX_DEPTH = 3; -constexpr int FGDB_PAGE_SIZE = 4096; +constexpr int FGDB_PAGE_SIZE_V1 = 4096; +constexpr int FGDB_PAGE_SIZE_V2 = 65536; +constexpr int MAX_FGDB_PAGE_SIZE = FGDB_PAGE_SIZE_V2; class FileGDBIndexIteratorBase : virtual public FileGDBIterator { @@ -238,40 +240,67 @@ class FileGDBIndexIteratorBase : virtual public FileGDBIterator FileGDBTable *poParent = nullptr; bool bAscending = false; VSILFILE *fpCurIdx = nullptr; + + //! Version of .atx/.spx: 1 or 2 + GUInt32 m_nVersion = 0; + + // Number of pages of size m_nPageSize GUInt32 m_nPageCount = 0; + + //! Page size in bytes: 4096 for v1 format, 65536 for v2 + int m_nPageSize = 0; + + //! Maximum number of features or sub-pages referenced by a page. GUInt32 nMaxPerPages = 0; + + //! Size of ObjectID referenced in pages, in bytes. + // sizeof(uint32_t) for V1, sizeof(uint64_t) for V2 + GUInt32 m_nObjectIDSize = 0; + + //! Size of the indexed value, in bytes. GUInt32 m_nValueSize = 0; - GUInt32 nOffsetFirstValInPage = 0; - GUInt32 nValueCountInIdx = 0; + + //! Non-leaf page header size in bytes. 8 for V1, 12 for V2 + GUInt32 m_nNonLeafPageHeaderSize = 0; + + //! Leaf page header size in bytes. 12 for V1, 20 for V2 + GUInt32 m_nLeafPageHeaderSize = 0; + + //! Offset within a page at which the first indexed value is found. + GUInt32 m_nOffsetFirstValInPage = 0; + + //! Number of values referenced in the index. + GUInt64 m_nValueCountInIdx = 0; + GUInt32 nIndexDepth = 0; #ifdef DEBUG - int iLoadedPage[MAX_DEPTH]; + uint64_t iLoadedPage[MAX_DEPTH]; #endif int iFirstPageIdx[MAX_DEPTH]; int iLastPageIdx[MAX_DEPTH]; int iCurPageIdx[MAX_DEPTH]; GUInt32 nSubPagesCount[MAX_DEPTH]; - GUInt32 nLastPageAccessed[MAX_DEPTH]; + uint64_t nLastPageAccessed[MAX_DEPTH]; int iCurFeatureInPage = -1; int nFeaturesInPage = 0; bool bEOF = false; - GByte abyPage[MAX_DEPTH][FGDB_PAGE_SIZE]; - GByte abyPageFeature[FGDB_PAGE_SIZE]; + GByte abyPage[MAX_DEPTH][MAX_FGDB_PAGE_SIZE]; + GByte abyPageFeature[MAX_FGDB_PAGE_SIZE]; - typedef lru11::Cache> CacheType; + typedef lru11::Cache> CacheType; std::array m_oCachePage{ {CacheType{2, 0}, CacheType{2, 0}, CacheType{2, 0}}}; CacheType m_oCacheFeaturePage{2, 0}; bool ReadTrailer(const std::string &osFilename); - int ReadPageNumber(int iLevel); - int LoadNextPage(int iLevel); - virtual bool FindPages(int iLevel, int nPage) = 0; - int LoadNextFeaturePage(); + uint64_t ReadPageNumber(int iLevel); + bool LoadNextPage(int iLevel); + virtual bool FindPages(int iLevel, uint64_t nPage) = 0; + bool LoadNextFeaturePage(); FileGDBIndexIteratorBase(FileGDBTable *poParent, int bAscending); @@ -307,7 +336,7 @@ class FileGDBIndexIterator final : public FileGDBIndexIteratorBase int iSorted = 0; int nSortedCount = -1; - int *panSortedRows = nullptr; + int64_t *panSortedRows = nullptr; int SortRows(); GUInt16 asUTF16Str[MAX_CAR_COUNT_INDEXED_STR]; @@ -321,8 +350,8 @@ class FileGDBIndexIterator final : public FileGDBIndexIteratorBase const OGRField *GetMinMaxValue(OGRField *psField, int &eOutType, int bIsMin); - virtual bool FindPages(int iLevel, int nPage) override; - int GetNextRow(); + virtual bool FindPages(int iLevel, uint64_t nPage) override; + int64_t GetNextRow(); FileGDBIndexIterator(FileGDBTable *poParent, int bAscending); int SetConstraint(int nFieldIdx, FileGDBSQLOp op, @@ -343,19 +372,19 @@ class FileGDBIndexIterator final : public FileGDBIndexIteratorBase OGRFieldType eOGRFieldType, const OGRField *psValue); - virtual int GetNextRowSortedByFID() override; - virtual int GetRowCount() override; + virtual int64_t GetNextRowSortedByFID() override; + virtual int64_t GetRowCount() override; virtual void Reset() override; - virtual int GetNextRowSortedByValue() override + virtual int64_t GetNextRowSortedByValue() override { return GetNextRow(); } virtual const OGRField *GetMinValue(int &eOutType) override; virtual const OGRField *GetMaxValue(int &eOutType) override; - virtual int GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, - int &nCount) override; + virtual bool GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, + int &nCount) override; }; /************************************************************************/ @@ -384,7 +413,7 @@ const OGRField *FileGDBIterator::GetMaxValue(int &eOutType) /* GetNextRowSortedByValue() */ /************************************************************************/ -int FileGDBIterator::GetNextRowSortedByValue() +int64_t FileGDBIterator::GetNextRowSortedByValue() { PrintError(); return -1; @@ -394,15 +423,15 @@ int FileGDBIterator::GetNextRowSortedByValue() /* GetMinMaxSumCount() */ /************************************************************************/ -int FileGDBIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, - double &dfSum, int &nCount) +bool FileGDBIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, + double &dfSum, int &nCount) { PrintError(); dfMin = 0.0; dfMax = 0.0; dfSum = 0.0; nCount = 0; - return FALSE; + return false; } /************************************************************************/ @@ -475,10 +504,10 @@ FileGDBIterator *FileGDBIterator::BuildOr(FileGDBIterator *poIter1, /* GetRowCount() */ /************************************************************************/ -int FileGDBIterator::GetRowCount() +int64_t FileGDBIterator::GetRowCount() { Reset(); - int nCount = 0; + int64_t nCount = 0; while (GetNextRowSortedByFID() >= 0) nCount++; Reset(); @@ -498,7 +527,7 @@ FileGDBTrivialIterator::FileGDBTrivialIterator(FileGDBIterator *poParentIterIn) /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBTrivialIterator::GetNextRowSortedByFID() +int64_t FileGDBTrivialIterator::GetNextRowSortedByFID() { if (iRow < poTable->GetTotalRecordCount()) return iRow++; @@ -541,7 +570,7 @@ void FileGDBNotIterator::Reset() /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBNotIterator::GetNextRowSortedByFID() +int64_t FileGDBNotIterator::GetNextRowSortedByFID() { if (iNextRowBase < 0) { @@ -579,7 +608,7 @@ int FileGDBNotIterator::GetNextRowSortedByFID() /* GetRowCount() */ /************************************************************************/ -int FileGDBNotIterator::GetRowCount() +int64_t FileGDBNotIterator::GetRowCount() { return poTable->GetValidRecordCount() - poIterBase->GetRowCount(); } @@ -626,7 +655,7 @@ void FileGDBAndIterator::Reset() /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBAndIterator::GetNextRowSortedByFID() +int64_t FileGDBAndIterator::GetNextRowSortedByFID() { if (iNextRow1 == iNextRow2) { @@ -697,7 +726,7 @@ void FileGDBOrIterator::Reset() /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBOrIterator::GetNextRowSortedByFID() +int64_t FileGDBOrIterator::GetNextRowSortedByFID() { if (bHasJustReset) { @@ -708,19 +737,19 @@ int FileGDBOrIterator::GetNextRowSortedByFID() if (iNextRow1 < 0) { - int iVal = iNextRow2; + auto iVal = iNextRow2; iNextRow2 = poIter2->GetNextRowSortedByFID(); return iVal; } if (iNextRow2 < 0 || iNextRow1 < iNextRow2) { - int iVal = iNextRow1; + auto iVal = iNextRow1; iNextRow1 = poIter1->GetNextRowSortedByFID(); return iVal; } if (iNextRow2 < iNextRow1) { - int iVal = iNextRow2; + auto iVal = iNextRow2; iNextRow2 = poIter2->GetNextRowSortedByFID(); return iVal; } @@ -728,7 +757,7 @@ int FileGDBOrIterator::GetNextRowSortedByFID() if (bIteratorAreExclusive) PrintError(); - int iVal = iNextRow1; + auto iVal = iNextRow1; iNextRow1 = poIter1->GetNextRowSortedByFID(); iNextRow2 = poIter2->GetNextRowSortedByFID(); return iVal; @@ -738,7 +767,7 @@ int FileGDBOrIterator::GetNextRowSortedByFID() /* GetRowCount() */ /************************************************************************/ -int FileGDBOrIterator::GetRowCount() +int64_t FileGDBOrIterator::GetRowCount() { if (bIteratorAreExclusive) return poIter1->GetRowCount() + poIter2->GetRowCount(); @@ -790,61 +819,109 @@ bool FileGDBIndexIteratorBase::ReadTrailer(const std::string &osFilename) VSIFSeekL(fpCurIdx, 0, SEEK_END); vsi_l_offset nFileSize = VSIFTellL(fpCurIdx); - returnErrorIf(nFileSize < FGDB_PAGE_SIZE + 22); - - VSIFSeekL(fpCurIdx, nFileSize - 22, SEEK_SET); - GByte abyTrailer[22]; - returnErrorIf(VSIFReadL(abyTrailer, 22, 1, fpCurIdx) != 1); + constexpr int V1_TRAILER_SIZE = 22; + constexpr int V2_TRAILER_SIZE = 30; + returnErrorIf(nFileSize < V1_TRAILER_SIZE); + + GByte abyTrailer[V2_TRAILER_SIZE]; + VSIFSeekL(fpCurIdx, nFileSize - sizeof(uint32_t), SEEK_SET); + returnErrorIf(VSIFReadL(abyTrailer, sizeof(uint32_t), 1, fpCurIdx) != 1); + m_nVersion = GetUInt32(abyTrailer, 0); + returnErrorIf(m_nVersion != 1 && m_nVersion != 2); + + if (m_nVersion == 1) + { + m_nPageSize = FGDB_PAGE_SIZE_V1; + VSIFSeekL(fpCurIdx, nFileSize - V1_TRAILER_SIZE, SEEK_SET); + returnErrorIf(VSIFReadL(abyTrailer, V1_TRAILER_SIZE, 1, fpCurIdx) != 1); + + m_nPageCount = + static_cast((nFileSize - V1_TRAILER_SIZE) / m_nPageSize); + + m_nValueSize = abyTrailer[0]; + m_nObjectIDSize = static_cast(sizeof(uint32_t)); + m_nNonLeafPageHeaderSize = 8; + m_nLeafPageHeaderSize = 12; + + nMaxPerPages = (m_nPageSize - m_nLeafPageHeaderSize) / + (m_nObjectIDSize + m_nValueSize); + m_nOffsetFirstValInPage = + m_nLeafPageHeaderSize + nMaxPerPages * m_nObjectIDSize; + + GUInt32 nMagic1 = GetUInt32(abyTrailer + 2, 0); + returnErrorIf(nMagic1 != 1); + + nIndexDepth = GetUInt32(abyTrailer + 6, 0); + /* CPLDebug("OpenFileGDB", "nIndexDepth = %u", nIndexDepth); */ + returnErrorIf(!(nIndexDepth >= 1 && nIndexDepth <= MAX_DEPTH + 1)); + + m_nValueCountInIdx = GetUInt32(abyTrailer + 10, 0); + /* CPLDebug("OpenFileGDB", "m_nValueCountInIdx = %u", m_nValueCountInIdx); */ + /* negative like in sample_clcV15_esri_v10.gdb/a00000005.FDO_UUID.atx */ + if ((m_nValueCountInIdx >> (8 * sizeof(m_nValueCountInIdx) - 1)) != 0) + { + CPLDebugOnly("OpenFileGDB", "m_nValueCountInIdx=%u", + static_cast(m_nValueCountInIdx)); + return false; + } - m_nPageCount = static_cast((nFileSize - 22) / FGDB_PAGE_SIZE); + /* QGIS_TEST_101.gdb/a00000006.FDO_UUID.atx */ + /* or .spx file from test dataset https://github.com/OSGeo/gdal/issues/5888 + */ + if (m_nValueCountInIdx == 0 && nIndexDepth == 1) + { + VSIFSeekL(fpCurIdx, 4, SEEK_SET); + GByte abyBuffer[4]; + returnErrorIf(VSIFReadL(abyBuffer, 4, 1, fpCurIdx) != 1); + m_nValueCountInIdx = GetUInt32(abyBuffer, 0); + } + /* PreNIS.gdb/a00000006.FDO_UUID.atx has depth 2 and the value of */ + /* m_nValueCountInIdx is 11 which is not the number of non-null values */ + else if (m_nValueCountInIdx < nMaxPerPages && nIndexDepth > 1) + { + if (m_nValueCountInIdx > 0 && poParent->IsFileGDBV9() && + strstr(osFilename.c_str(), "blk_key_index.atx")) + { + // m_nValueCountInIdx not reliable in FileGDB v9 .blk_key_index.atx + // but index seems to be OK + return true; + } - m_nValueSize = abyTrailer[0]; + CPLDebugOnly( + "OpenFileGDB", + "m_nValueCountInIdx=%u < nMaxPerPages=%u, nIndexDepth=%u", + static_cast(m_nValueCountInIdx), nMaxPerPages, + nIndexDepth); + return false; + } + } + else + { + m_nPageSize = FGDB_PAGE_SIZE_V2; + VSIFSeekL(fpCurIdx, nFileSize - V2_TRAILER_SIZE, SEEK_SET); + returnErrorIf(VSIFReadL(abyTrailer, V2_TRAILER_SIZE, 1, fpCurIdx) != 1); - nMaxPerPages = (FGDB_PAGE_SIZE - 12) / (4 + m_nValueSize); - nOffsetFirstValInPage = 12 + nMaxPerPages * 4; + m_nPageCount = + static_cast((nFileSize - V2_TRAILER_SIZE) / m_nPageSize); - GUInt32 nMagic1 = GetUInt32(abyTrailer + 2, 0); - returnErrorIf(nMagic1 != 1); + m_nValueSize = abyTrailer[0]; + m_nObjectIDSize = static_cast(sizeof(uint64_t)); + m_nNonLeafPageHeaderSize = 12; + m_nLeafPageHeaderSize = 20; - nIndexDepth = GetUInt32(abyTrailer + 6, 0); - /* CPLDebug("OpenFileGDB", "nIndexDepth = %u", nIndexDepth); */ - returnErrorIf(!(nIndexDepth >= 1 && nIndexDepth <= MAX_DEPTH + 1)); + nMaxPerPages = (m_nPageSize - m_nLeafPageHeaderSize) / + (m_nObjectIDSize + m_nValueSize); + m_nOffsetFirstValInPage = + m_nLeafPageHeaderSize + nMaxPerPages * m_nObjectIDSize; - nValueCountInIdx = GetUInt32(abyTrailer + 10, 0); - /* CPLDebug("OpenFileGDB", "nValueCountInIdx = %u", nValueCountInIdx); */ - /* negative like in sample_clcV15_esri_v10.gdb/a00000005.FDO_UUID.atx */ - if ((nValueCountInIdx >> (8 * sizeof(nValueCountInIdx) - 1)) != 0) - { - CPLDebugOnly("OpenFileGDB", "nValueCountInIdx=%u", nValueCountInIdx); - return false; - } + GUInt32 nMagic1 = GetUInt32(abyTrailer + 2, 0); + returnErrorIf(nMagic1 != 1); - /* QGIS_TEST_101.gdb/a00000006.FDO_UUID.atx */ - /* or .spx file from test dataset https://github.com/OSGeo/gdal/issues/5888 - */ - if (nValueCountInIdx == 0 && nIndexDepth == 1) - { - VSIFSeekL(fpCurIdx, 4, SEEK_SET); - GByte abyBuffer[4]; - returnErrorIf(VSIFReadL(abyBuffer, 4, 1, fpCurIdx) != 1); - nValueCountInIdx = GetUInt32(abyBuffer, 0); - } - /* PreNIS.gdb/a00000006.FDO_UUID.atx has depth 2 and the value of */ - /* nValueCountInIdx is 11 which is not the number of non-null values */ - else if (nValueCountInIdx < nMaxPerPages && nIndexDepth > 1) - { - if (nValueCountInIdx > 0 && poParent->IsFileGDBV9() && - strstr(osFilename.c_str(), "blk_key_index.atx")) - { - // nValueCountInIdx not reliable in FileGDB v9 .blk_key_index.atx - // but index seems to be OK - return true; - } + nIndexDepth = GetUInt32(abyTrailer + 6, 0); + /* CPLDebug("OpenFileGDB", "nIndexDepth = %u", nIndexDepth); */ + returnErrorIf(!(nIndexDepth >= 1 && nIndexDepth <= MAX_DEPTH + 1)); - CPLDebugOnly("OpenFileGDB", - "nValueCountInIdx=%u < nMaxPerPages=%u, nIndexDepth=%u", - nValueCountInIdx, nMaxPerPages, nIndexDepth); - return false; + m_nValueCountInIdx = GetUInt64(abyTrailer + 10, 0); } return true; @@ -970,15 +1047,40 @@ int FileGDBIndex::GetMaxWidthInBytes(const FileGDBTable *poTable) const VSIFSeekL(fpCurIdx, 0, SEEK_END); vsi_l_offset nFileSize = VSIFTellL(fpCurIdx); - if (nFileSize < FGDB_PAGE_SIZE + 22) + + constexpr int V1_TRAILER_SIZE = 22; + constexpr int V2_TRAILER_SIZE = 30; + + if (nFileSize < FGDB_PAGE_SIZE_V1 + V1_TRAILER_SIZE) + { + VSIFCloseL(fpCurIdx); + return 0; + } + + GByte abyTrailer[V2_TRAILER_SIZE]; + VSIFSeekL(fpCurIdx, nFileSize - sizeof(uint32_t), SEEK_SET); + if (VSIFReadL(abyTrailer, sizeof(uint32_t), 1, fpCurIdx) != 1) + { + VSIFCloseL(fpCurIdx); + return 0; + } + const auto nVersion = GetUInt32(abyTrailer, 0); + if (nVersion != 1 && nVersion != 2) { VSIFCloseL(fpCurIdx); return 0; } - VSIFSeekL(fpCurIdx, nFileSize - 22, SEEK_SET); - GByte abyTrailer[22]; - if (VSIFReadL(abyTrailer, 22, 1, fpCurIdx) != 1) + const int nTrailerSize = nVersion == 1 ? V1_TRAILER_SIZE : V2_TRAILER_SIZE; + + if (nVersion == 2 && nFileSize < FGDB_PAGE_SIZE_V2 + V2_TRAILER_SIZE) + { + VSIFCloseL(fpCurIdx); + return 0; + } + + VSIFSeekL(fpCurIdx, nFileSize - nTrailerSize, SEEK_SET); + if (VSIFReadL(abyTrailer, nTrailerSize, 1, fpCurIdx) != 1) { VSIFCloseL(fpCurIdx); return 0; @@ -1022,8 +1124,8 @@ int FileGDBIndexIterator::SetConstraint(int nFieldIdx, FileGDBSQLOp op, if (!ReadTrailer(pszAtxName)) return FALSE; - returnErrorIf(nValueCountInIdx > - static_cast(poParent->GetValidRecordCount())); + returnErrorIf(m_nValueCountInIdx > + static_cast(poParent->GetValidRecordCount())); switch (eFieldType) { @@ -1156,7 +1258,7 @@ int FileGDBIndexIterator::SetConstraint(int nFieldIdx, FileGDBSQLOp op, break; } - if (nValueCountInIdx > 0) + if (m_nValueCountInIdx > 0) { if (nIndexDepth == 1) { @@ -1208,21 +1310,21 @@ static int FileGDBUTF16StrCompare(const GUInt16 *pasFirst, /* FindPages() */ /************************************************************************/ -bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) +bool FileGDBIndexIterator::FindPages(int iLevel, uint64_t nPage) { const bool errorRetValue = false; - VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * FGDB_PAGE_SIZE, + VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * m_nPageSize, SEEK_SET); #ifdef DEBUG iLoadedPage[iLevel] = nPage; #endif - returnErrorIf(VSIFReadL(abyPage[iLevel], FGDB_PAGE_SIZE, 1, fpCurIdx) != 1); + returnErrorIf(VSIFReadL(abyPage[iLevel], m_nPageSize, 1, fpCurIdx) != 1); - nSubPagesCount[iLevel] = GetUInt32(abyPage[iLevel] + 4, 0); + nSubPagesCount[iLevel] = GetUInt32(abyPage[iLevel] + m_nObjectIDSize, 0); returnErrorIf(nSubPagesCount[iLevel] == 0 || nSubPagesCount[iLevel] > nMaxPerPages); if (nIndexDepth == 2) - returnErrorIf(nValueCountInIdx > + returnErrorIf(m_nValueCountInIdx > nMaxPerPages * (nSubPagesCount[0] + 1)); if (eOp == FGSO_ISNOTNULL) @@ -1250,7 +1352,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_INT16: { GInt16 nVal = - GetInt16(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetInt16(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && nVal < nLastMax); nLastMax = nVal; @@ -1262,7 +1364,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_INT32: { GInt32 nVal = - GetInt32(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetInt32(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && nVal < nLastMax); nLastMax = nVal; @@ -1274,7 +1376,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_INT64: { int64_t nVal = - GetInt64(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetInt64(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && nVal < nLastMax); nLastMax = nVal; @@ -1286,7 +1388,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_FLOAT32: { float fVal = - GetFloat32(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetFloat32(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && fVal < dfLastMax); dfLastMax = fVal; @@ -1298,7 +1400,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_FLOAT64: { const double dfVal = - GetFloat64(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetFloat64(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && dfVal < dfLastMax); dfLastMax = dfVal; @@ -1312,7 +1414,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_TIME: { const double dfVal = - GetFloat64(abyPage[iLevel] + nOffsetFirstValInPage, i); + GetFloat64(abyPage[iLevel] + m_nOffsetFirstValInPage, i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && dfVal < dfLastMax); dfLastMax = dfVal; @@ -1332,7 +1434,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) GUInt16 asMax[MAX_CAR_COUNT_INDEXED_STR]; pasMax = asMax; memcpy(asMax, - abyPage[iLevel] + nOffsetFirstValInPage + + abyPage[iLevel] + m_nOffsetFirstValInPage + nStrLen * sizeof(GUInt16) * i, nStrLen * sizeof(GUInt16)); for (int j = 0; j < nStrLen; j++) @@ -1350,7 +1452,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) case FGFT_GLOBALID: { const char *psNonzMaxUUID = reinterpret_cast( - abyPage[iLevel] + nOffsetFirstValInPage + + abyPage[iLevel] + m_nOffsetFirstValInPage + UUID_LEN_AS_STRING * i); #ifdef DEBUG_INDEX_CONSISTENCY returnErrorIf(i > 0 && memcmp(psNonzMaxUUID, szLastMaxUUID, @@ -1458,14 +1560,14 @@ bool FileGDBIndexIterator::FindPages(int iLevel, int nPage) void FileGDBIndexIteratorBase::Reset() { iCurPageIdx[0] = (bAscending) ? iFirstPageIdx[0] - 1 : iLastPageIdx[0] + 1; - memset(iFirstPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(int)); - memset(iLastPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(int)); - memset(iCurPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(int)); - memset(nLastPageAccessed, 0, MAX_DEPTH * sizeof(int)); + memset(iFirstPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(iFirstPageIdx[0])); + memset(iLastPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(iLastPageIdx[0])); + memset(iCurPageIdx + 1, 0xFF, (MAX_DEPTH - 1) * sizeof(iCurPageIdx[0])); + memset(nLastPageAccessed, 0, MAX_DEPTH * sizeof(nLastPageAccessed[0])); iCurFeatureInPage = 0; nFeaturesInPage = 0; - bEOF = (nValueCountInIdx == 0); + bEOF = (m_nValueCountInIdx == 0); } /************************************************************************/ @@ -1483,15 +1585,33 @@ void FileGDBIndexIterator::Reset() /* ReadPageNumber() */ /************************************************************************/ -int FileGDBIndexIteratorBase::ReadPageNumber(int iLevel) +uint64_t FileGDBIndexIteratorBase::ReadPageNumber(int iLevel) { const int errorRetValue = 0; - GUInt32 nPage = GetUInt32(abyPage[iLevel] + 8, iCurPageIdx[iLevel]); - if (nPage == nLastPageAccessed[iLevel]) + uint64_t nPage; + if (m_nVersion == 1) { - if (!LoadNextPage(iLevel)) - return 0; - nPage = GetUInt32(abyPage[iLevel] + 8, iCurPageIdx[iLevel]); + nPage = GetUInt32(abyPage[iLevel] + m_nNonLeafPageHeaderSize, + iCurPageIdx[iLevel]); + if (nPage == nLastPageAccessed[iLevel]) + { + if (!LoadNextPage(iLevel)) + return 0; + nPage = GetUInt32(abyPage[iLevel] + m_nNonLeafPageHeaderSize, + iCurPageIdx[iLevel]); + } + } + else + { + nPage = GetUInt64(abyPage[iLevel] + m_nNonLeafPageHeaderSize, + iCurPageIdx[iLevel]); + if (nPage == nLastPageAccessed[iLevel]) + { + if (!LoadNextPage(iLevel)) + return 0; + nPage = GetUInt64(abyPage[iLevel] + m_nNonLeafPageHeaderSize, + iCurPageIdx[iLevel]); + } } nLastPageAccessed[iLevel] = nPage; returnErrorIf(nPage < 2); @@ -1502,16 +1622,16 @@ int FileGDBIndexIteratorBase::ReadPageNumber(int iLevel) /* LoadNextPage() */ /************************************************************************/ -int FileGDBIndexIteratorBase::LoadNextPage(int iLevel) +bool FileGDBIndexIteratorBase::LoadNextPage(int iLevel) { - const int errorRetValue = FALSE; + const bool errorRetValue = false; if ((bAscending && iCurPageIdx[iLevel] == iLastPageIdx[iLevel]) || (!bAscending && iCurPageIdx[iLevel] == iFirstPageIdx[iLevel])) { if (iLevel == 0 || !LoadNextPage(iLevel - 1)) - return FALSE; + return false; - GUInt32 nPage = ReadPageNumber(iLevel - 1); + const auto nPage = ReadPageNumber(iLevel - 1); returnErrorIf(!FindPages(iLevel, nPage)); iCurPageIdx[iLevel] = @@ -1525,23 +1645,23 @@ int FileGDBIndexIteratorBase::LoadNextPage(int iLevel) iCurPageIdx[iLevel]--; } - return TRUE; + return true; } /************************************************************************/ /* LoadNextFeaturePage() */ /************************************************************************/ -int FileGDBIndexIteratorBase::LoadNextFeaturePage() +bool FileGDBIndexIteratorBase::LoadNextFeaturePage() { - const int errorRetValue = FALSE; - GUInt32 nPage; + const bool errorRetValue = false; + GUInt64 nPage; if (nIndexDepth == 1) { if (iCurPageIdx[0] == iLastPageIdx[0]) { - return FALSE; + return false; } if (bAscending) iCurPageIdx[0]++; @@ -1553,7 +1673,7 @@ int FileGDBIndexIteratorBase::LoadNextFeaturePage() { if (!LoadNextPage(nIndexDepth - 2)) { - return FALSE; + return false; } nPage = ReadPageNumber(nIndexDepth - 2); returnErrorIf(nPage < 2); @@ -1563,7 +1683,7 @@ int FileGDBIndexIteratorBase::LoadNextFeaturePage() m_oCacheFeaturePage.getPtr(nPage); if (cachedPagePtr) { - memcpy(abyPageFeature, cachedPagePtr->data(), FGDB_PAGE_SIZE); + memcpy(abyPageFeature, cachedPagePtr->data(), m_nPageSize); } else { @@ -1574,37 +1694,32 @@ int FileGDBIndexIteratorBase::LoadNextFeaturePage() cachedPage.clear(); } - VSIFSeekL(fpCurIdx, - static_cast(nPage - 1) * FGDB_PAGE_SIZE, + VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * m_nPageSize, SEEK_SET); #ifdef DEBUG iLoadedPage[nIndexDepth - 1] = nPage; #endif - returnErrorIf(VSIFReadL(abyPageFeature, FGDB_PAGE_SIZE, 1, fpCurIdx) != - 1); + returnErrorIf(VSIFReadL(abyPageFeature, m_nPageSize, 1, fpCurIdx) != 1); cachedPage.insert(cachedPage.end(), abyPageFeature, - abyPageFeature + FGDB_PAGE_SIZE); + abyPageFeature + m_nPageSize); m_oCacheFeaturePage.insert(nPage, std::move(cachedPage)); } - GUInt32 nFeatures = GetUInt32(abyPageFeature + 4, 0); + const GUInt32 nFeatures = GetUInt32(abyPageFeature + m_nObjectIDSize, 0); returnErrorIf(nFeatures > nMaxPerPages); nFeaturesInPage = static_cast(nFeatures); iCurFeatureInPage = (bAscending) ? 0 : nFeaturesInPage - 1; - if (nFeatures == 0) - return FALSE; - - return TRUE; + return nFeatures != 0; } /************************************************************************/ /* GetNextRow() */ /************************************************************************/ -int FileGDBIndexIterator::GetNextRow() +int64_t FileGDBIndexIterator::GetNextRow() { - const int errorRetValue = -1; + const int64_t errorRetValue = -1; if (bEOF) return -1; @@ -1632,7 +1747,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_INT16: { const GInt16 nVal = - GetInt16(abyPageFeature + nOffsetFirstValInPage, + GetInt16(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); nComp = COMPARE(sValue.Integer, nVal); break; @@ -1641,7 +1756,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_INT32: { const GInt32 nVal = - GetInt32(abyPageFeature + nOffsetFirstValInPage, + GetInt32(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); nComp = COMPARE(sValue.Integer, nVal); break; @@ -1650,7 +1765,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_FLOAT32: { const float fVal = - GetFloat32(abyPageFeature + nOffsetFirstValInPage, + GetFloat32(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); nComp = COMPARE(sValue.Real, fVal); break; @@ -1659,7 +1774,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_FLOAT64: { const double dfVal = - GetFloat64(abyPageFeature + nOffsetFirstValInPage, + GetFloat64(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); nComp = COMPARE(sValue.Real, dfVal); break; @@ -1671,7 +1786,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_DATETIME_WITH_OFFSET: { const double dfVal = - GetFloat64(abyPageFeature + nOffsetFirstValInPage, + GetFloat64(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); if (sValue.Real + 1e-10 < dfVal) nComp = -1; @@ -1686,7 +1801,7 @@ int FileGDBIndexIterator::GetNextRow() { GUInt16 asVal[MAX_CAR_COUNT_INDEXED_STR]; memcpy(asVal, - abyPageFeature + nOffsetFirstValInPage + + abyPageFeature + m_nOffsetFirstValInPage + nStrLen * 2 * iCurFeatureInPage, nStrLen * 2); for (int j = 0; j < nStrLen; j++) @@ -1699,7 +1814,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_GLOBALID: { nComp = memcmp(szUUID, - abyPageFeature + nOffsetFirstValInPage + + abyPageFeature + m_nOffsetFirstValInPage + UUID_LEN_AS_STRING * iCurFeatureInPage, UUID_LEN_AS_STRING); break; @@ -1708,7 +1823,7 @@ int FileGDBIndexIterator::GetNextRow() case FGFT_INT64: { const int64_t nVal = - GetInt64(abyPageFeature + nOffsetFirstValInPage, + GetInt64(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); nComp = COMPARE(sValue.Integer64, nVal); break; @@ -1767,17 +1882,21 @@ int FileGDBIndexIterator::GetNextRow() if (bMatch) { - const GUInt32 nFID = - GetUInt32(abyPageFeature + 12, iCurFeatureInPage); + const GUInt64 nFID = + m_nVersion == 1 + ? GetUInt32(abyPageFeature + m_nLeafPageHeaderSize, + iCurFeatureInPage) + : GetUInt64(abyPageFeature + m_nLeafPageHeaderSize, + iCurFeatureInPage); if (bAscending) iCurFeatureInPage++; else iCurFeatureInPage--; returnErrorAndCleanupIf( - nFID < 1 || nFID > static_cast( + nFID < 1 || nFID > static_cast( poParent->GetTotalRecordCount()), bEOF = true); - return static_cast(nFID - 1); + return static_cast(nFID - 1); } else { @@ -1801,14 +1920,15 @@ int FileGDBIndexIterator::SortRows() Reset(); while (true) { - int nRow = GetNextRow(); + int64_t nRow = GetNextRow(); if (nRow < 0) break; if (nSortedCount == nSortedAlloc) { int nNewSortedAlloc = 4 * nSortedAlloc / 3 + 16; - int *panNewSortedRows = static_cast(VSI_REALLOC_VERBOSE( - panSortedRows, sizeof(int) * nNewSortedAlloc)); + int64_t *panNewSortedRows = + static_cast(VSI_REALLOC_VERBOSE( + panSortedRows, sizeof(int64_t) * nNewSortedAlloc)); if (panNewSortedRows == nullptr) { nSortedCount = 0; @@ -1822,8 +1942,8 @@ int FileGDBIndexIterator::SortRows() if (nSortedCount == 0) return FALSE; std::sort(panSortedRows, panSortedRows + nSortedCount); -#ifdef nValueCountInIdx_reliable - if (eOp == FGSO_ISNOTNULL && (int)nValueCountInIdx != nSortedCount) +#ifdef m_nValueCountInIdx_reliable + if (eOp == FGSO_ISNOTNULL && (int64_t)m_nValueCountInIdx != nSortedCount) PrintError(); #endif return TRUE; @@ -1833,7 +1953,7 @@ int FileGDBIndexIterator::SortRows() /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBIndexIterator::GetNextRowSortedByFID() +int64_t FileGDBIndexIterator::GetNextRowSortedByFID() { if (eOp == FGSO_EQ) return GetNextRow(); @@ -1857,21 +1977,21 @@ int FileGDBIndexIterator::GetNextRowSortedByFID() /* GetRowCount() */ /************************************************************************/ -int FileGDBIndexIterator::GetRowCount() +int64_t FileGDBIndexIterator::GetRowCount() { - // The nValueCountInIdx value has been found to be unreliable when the index + // The m_nValueCountInIdx value has been found to be unreliable when the index // is built as features are inserted (and when they are not in increasing // order) (with FileGDB SDK 1.3) So disable this optimization as there's no // fast way to know if the value is reliable or not. -#ifdef nValueCountInIdx_reliable +#ifdef m_nValueCountInIdx_reliable if (eOp == FGSO_ISNOTNULL) - return (int)nValueCountInIdx; + return (int64_t)m_nValueCountInIdx; #endif if (nSortedCount >= 0) return nSortedCount; - int nRowCount = 0; + int64_t nRowCount = 0; bool bSaveAscending = bAscending; bAscending = true; /* for a tiny bit of more efficiency */ Reset(); @@ -1891,38 +2011,52 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, { const OGRField *errorRetValue = nullptr; eOutType = -1; - if (nValueCountInIdx == 0) + if (m_nValueCountInIdx == 0) return nullptr; - GByte l_abyPage[FGDB_PAGE_SIZE]; - GUInt32 nPage = 1; + std::vector l_abyPageV; + try + { + l_abyPageV.resize(m_nPageSize); + } + catch (const std::exception &) + { + return nullptr; + } + GByte *l_abyPage = l_abyPageV.data(); + uint64_t nPage = 1; for (GUInt32 iLevel = 0; iLevel < nIndexDepth - 1; iLevel++) { - VSIFSeekL(fpCurIdx, - static_cast(nPage - 1) * FGDB_PAGE_SIZE, + VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * m_nPageSize, SEEK_SET); #ifdef DEBUG iLoadedPage[iLevel] = nPage; #endif - returnErrorIf(VSIFReadL(l_abyPage, FGDB_PAGE_SIZE, 1, fpCurIdx) != 1); - GUInt32 l_nSubPagesCount = GetUInt32(l_abyPage + 4, 0); + returnErrorIf(VSIFReadL(l_abyPage, m_nPageSize, 1, fpCurIdx) != 1); + GUInt32 l_nSubPagesCount = GetUInt32(l_abyPage + m_nObjectIDSize, 0); returnErrorIf(l_nSubPagesCount == 0 || l_nSubPagesCount > nMaxPerPages); - if (bIsMin) - nPage = GetUInt32(l_abyPage + 8, 0); + if (m_nVersion == 1) + { + nPage = GetUInt32(l_abyPage + m_nNonLeafPageHeaderSize, + bIsMin ? 0 : l_nSubPagesCount); + } else - nPage = GetUInt32(l_abyPage + 8, l_nSubPagesCount); + { + nPage = GetUInt64(l_abyPage + m_nNonLeafPageHeaderSize, + bIsMin ? 0 : l_nSubPagesCount); + } returnErrorIf(nPage < 2); } - VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * FGDB_PAGE_SIZE, + VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * m_nPageSize, SEEK_SET); #ifdef DEBUG iLoadedPage[nIndexDepth - 1] = nPage; #endif - returnErrorIf(VSIFReadL(l_abyPage, FGDB_PAGE_SIZE, 1, fpCurIdx) != 1); + returnErrorIf(VSIFReadL(l_abyPage, m_nPageSize, 1, fpCurIdx) != 1); - GUInt32 nFeatures = GetUInt32(l_abyPage + 4, 0); + GUInt32 nFeatures = GetUInt32(l_abyPage + m_nObjectIDSize, 0); returnErrorIf(nFeatures < 1 || nFeatures > nMaxPerPages); int iFeature = (bIsMin) ? 0 : nFeatures - 1; @@ -1932,7 +2066,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_INT16: { const GInt16 nVal = - GetInt16(l_abyPage + nOffsetFirstValInPage, iFeature); + GetInt16(l_abyPage + m_nOffsetFirstValInPage, iFeature); psField->Integer = nVal; eOutType = OFTInteger; return psField; @@ -1941,7 +2075,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_INT32: { const GInt32 nVal = - GetInt32(l_abyPage + nOffsetFirstValInPage, iFeature); + GetInt32(l_abyPage + m_nOffsetFirstValInPage, iFeature); psField->Integer = nVal; eOutType = OFTInteger; return psField; @@ -1950,7 +2084,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_FLOAT32: { const float fVal = - GetFloat32(l_abyPage + nOffsetFirstValInPage, iFeature); + GetFloat32(l_abyPage + m_nOffsetFirstValInPage, iFeature); psField->Real = fVal; eOutType = OFTReal; return psField; @@ -1959,7 +2093,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_FLOAT64: { const double dfVal = - GetFloat64(l_abyPage + nOffsetFirstValInPage, iFeature); + GetFloat64(l_abyPage + m_nOffsetFirstValInPage, iFeature); psField->Real = dfVal; eOutType = OFTReal; return psField; @@ -1969,7 +2103,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_DATETIME_WITH_OFFSET: { const double dfVal = - GetFloat64(l_abyPage + nOffsetFirstValInPage, iFeature); + GetFloat64(l_abyPage + m_nOffsetFirstValInPage, iFeature); FileGDBDoubleDateToOGRDate(dfVal, false, psField); eOutType = OFTDateTime; return psField; @@ -1978,7 +2112,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_DATE: { const double dfVal = - GetFloat64(l_abyPage + nOffsetFirstValInPage, iFeature); + GetFloat64(l_abyPage + m_nOffsetFirstValInPage, iFeature); FileGDBDoubleDateToOGRDate(dfVal, false, psField); eOutType = OFTDate; return psField; @@ -1987,7 +2121,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_TIME: { const double dfVal = - GetFloat64(l_abyPage + nOffsetFirstValInPage, iFeature); + GetFloat64(l_abyPage + m_nOffsetFirstValInPage, iFeature); FileGDBDoubleTimeToOGRTime(dfVal, psField); eOutType = OFTTime; return psField; @@ -1999,7 +2133,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, for (int j = 0; j < nStrLen; j++) { GUInt16 nCh = - GetUInt16(l_abyPage + nOffsetFirstValInPage + + GetUInt16(l_abyPage + m_nOffsetFirstValInPage + nStrLen * sizeof(GUInt16) * iFeature, j); awsVal[j] = nCh; @@ -2021,7 +2155,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_GLOBALID: { memcpy(psField->String, - l_abyPage + nOffsetFirstValInPage + + l_abyPage + m_nOffsetFirstValInPage + UUID_LEN_AS_STRING * iFeature, UUID_LEN_AS_STRING); psField->String[UUID_LEN_AS_STRING] = 0; @@ -2032,7 +2166,7 @@ const OGRField *FileGDBIndexIterator::GetMinMaxValue(OGRField *psField, case FGFT_INT64: { const int64_t nVal = - GetInt64(l_abyPage + nOffsetFirstValInPage, iFeature); + GetInt64(l_abyPage + m_nOffsetFirstValInPage, iFeature); psField->Integer64 = nVal; eOutType = OFTInteger64; return psField; @@ -2140,7 +2274,7 @@ void FileGDBIndexIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, } } - dfVal = Getter::GetAsDouble(abyPageFeature + nOffsetFirstValInPage, + dfVal = Getter::GetAsDouble(abyPageFeature + m_nOffsetFirstValInPage, iCurFeatureInPage); dfLocalSum += dfVal; @@ -2155,10 +2289,10 @@ void FileGDBIndexIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, dfMax = dfVal; } -int FileGDBIndexIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, - double &dfSum, int &nCount) +bool FileGDBIndexIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, + double &dfSum, int &nCount) { - const int errorRetValue = FALSE; + const bool errorRetValue = false; dfMin = 0.0; dfMax = 0.0; dfSum = 0.0; @@ -2213,7 +2347,7 @@ int FileGDBIndexIterator::GetMinMaxSumCount(double &dfMin, double &dfMax, bAscending = bSaveAscending; Reset(); - return TRUE; + return true; } /************************************************************************/ @@ -2225,7 +2359,7 @@ class FileGDBSpatialIndexIteratorImpl final : public FileGDBIndexIteratorBase, { OGREnvelope m_sFilterEnvelope; bool m_bHasBuiltSetFID = false; - std::vector m_oFIDVector{}; + std::vector m_oFIDVector{}; size_t m_nVectorIdx = 0; int m_nGridNo = 0; GInt64 m_nMinVal = 0; @@ -2233,7 +2367,7 @@ class FileGDBSpatialIndexIteratorImpl final : public FileGDBIndexIteratorBase, GInt32 m_nCurX = 0; GInt32 m_nMaxX = 0; - virtual bool FindPages(int iLevel, int nPage) override; + virtual bool FindPages(int iLevel, uint64_t nPage) override; int GetNextRow(); bool ReadNewXRange(); bool ResetInternal(); @@ -2252,7 +2386,7 @@ class FileGDBSpatialIndexIteratorImpl final : public FileGDBIndexIteratorBase, return poParent; } // avoid MSVC C4250 inherits via dominance warning - virtual int GetNextRowSortedByFID() override; + virtual int64_t GetNextRowSortedByFID() override; virtual void Reset() override; virtual bool SetEnvelope(const OGREnvelope &sFilterEnvelope) override; @@ -2426,7 +2560,7 @@ bool FileGDBSpatialIndexIteratorImpl::ReadNewXRange() } const bool errorRetValue = false; - if (nValueCountInIdx > 0) + if (m_nValueCountInIdx > 0) { if (nIndexDepth == 1) { @@ -2500,7 +2634,7 @@ static bool FindMinMaxIdx(const GByte *pBaseAddr, const int nVals, /* FindPages() */ /************************************************************************/ -bool FileGDBSpatialIndexIteratorImpl::FindPages(int iLevel, int nPage) +bool FileGDBSpatialIndexIteratorImpl::FindPages(int iLevel, uint64_t nPage) { const bool errorRetValue = false; @@ -2510,7 +2644,7 @@ bool FileGDBSpatialIndexIteratorImpl::FindPages(int iLevel, int nPage) m_oCachePage[iLevel].getPtr(nPage); if (cachedPagePtr) { - memcpy(abyPage[iLevel], cachedPagePtr->data(), FGDB_PAGE_SIZE); + memcpy(abyPage[iLevel], cachedPagePtr->data(), m_nPageSize); } else { @@ -2521,35 +2655,46 @@ bool FileGDBSpatialIndexIteratorImpl::FindPages(int iLevel, int nPage) cachedPage.clear(); } - VSIFSeekL(fpCurIdx, - static_cast(nPage - 1) * FGDB_PAGE_SIZE, + VSIFSeekL(fpCurIdx, static_cast(nPage - 1) * m_nPageSize, SEEK_SET); #ifdef DEBUG iLoadedPage[iLevel] = nPage; #endif - returnErrorIf(VSIFReadL(abyPage[iLevel], FGDB_PAGE_SIZE, 1, fpCurIdx) != + returnErrorIf(VSIFReadL(abyPage[iLevel], m_nPageSize, 1, fpCurIdx) != 1); cachedPage.insert(cachedPage.end(), abyPage[iLevel], - abyPage[iLevel] + FGDB_PAGE_SIZE); + abyPage[iLevel] + m_nPageSize); m_oCachePage[iLevel].insert(nPage, std::move(cachedPage)); } - nSubPagesCount[iLevel] = GetUInt32(abyPage[iLevel] + 4, 0); + nSubPagesCount[iLevel] = GetUInt32(abyPage[iLevel] + m_nObjectIDSize, 0); returnErrorIf(nSubPagesCount[iLevel] == 0 || nSubPagesCount[iLevel] > nMaxPerPages); - if (GetInt64(abyPage[iLevel] + nOffsetFirstValInPage, 0) > m_nMaxVal) + if (GetInt64(abyPage[iLevel] + m_nOffsetFirstValInPage, 0) > m_nMaxVal) { iFirstPageIdx[iLevel] = 0; - // nSubPagesCount[iLevel] == 1 && GetUInt32(abyPage[iLevel] + 12, 0) == + // nSubPagesCount[iLevel] == 1 && GetUInt32(abyPage[iLevel] + m_nLeafPageHeaderSize, 0) == // 0 should only happen on non-nominal cases where one forces the depth // of the index to be greater than needed. - iLastPageIdx[iLevel] = (nSubPagesCount[iLevel] == 1 && - GetUInt32(abyPage[iLevel] + 12, 0) == 0) - ? 0 - : 1; + if (m_nVersion == 1) + { + iLastPageIdx[iLevel] = + (nSubPagesCount[iLevel] == 1 && + GetUInt32(abyPage[iLevel] + m_nLeafPageHeaderSize, 0) == 0) + ? 0 + : 1; + } + else + { + iLastPageIdx[iLevel] = + (nSubPagesCount[iLevel] == 1 && + GetUInt64(abyPage[iLevel] + m_nLeafPageHeaderSize, 0) == 0) + ? 0 + : 1; + } } - else if (!FindMinMaxIdx(abyPage[iLevel] + nOffsetFirstValInPage, + else if (!FindMinMaxIdx(abyPage[iLevel] + m_nOffsetFirstValInPage, static_cast(nSubPagesCount[iLevel]), m_nMinVal, m_nMaxVal, iFirstPageIdx[iLevel], iLastPageIdx[iLevel])) @@ -2582,7 +2727,7 @@ int FileGDBSpatialIndexIteratorImpl::GetNextRow() int nMinIdx = 0; int nMaxIdx = 0; if (!LoadNextFeaturePage() || - !FindMinMaxIdx(abyPageFeature + nOffsetFirstValInPage, + !FindMinMaxIdx(abyPageFeature + m_nOffsetFirstValInPage, nFeaturesInPage, m_nMinVal, m_nMaxVal, nMinIdx, nMaxIdx) || nMinIdx > nMaxIdx) @@ -2623,17 +2768,21 @@ int FileGDBSpatialIndexIteratorImpl::GetNextRow() } #ifdef DEBUG - const GInt64 nVal = - GetInt64(abyPageFeature + nOffsetFirstValInPage, iCurFeatureInPage); + const GInt64 nVal = GetInt64(abyPageFeature + m_nOffsetFirstValInPage, + iCurFeatureInPage); CPL_IGNORE_RET_VAL(nVal); CPLAssert(nVal >= m_nMinVal && nVal <= m_nMaxVal); #endif - const GUInt32 nFID = GetUInt32(abyPageFeature + 12, iCurFeatureInPage); + const GUInt64 nFID = + m_nVersion == 1 ? GetUInt32(abyPageFeature + m_nLeafPageHeaderSize, + iCurFeatureInPage) + : GetUInt64(abyPageFeature + m_nLeafPageHeaderSize, + iCurFeatureInPage); iCurFeatureInPage++; returnErrorAndCleanupIf( nFID < 1 || - nFID > static_cast(poParent->GetTotalRecordCount()), + nFID > static_cast(poParent->GetTotalRecordCount()), bEOF = true); return static_cast(nFID - 1); } @@ -2673,7 +2822,7 @@ void FileGDBSpatialIndexIteratorImpl::Reset() /* GetNextRowSortedByFID() */ /************************************************************************/ -int FileGDBSpatialIndexIteratorImpl::GetNextRowSortedByFID() +int64_t FileGDBSpatialIndexIteratorImpl::GetNextRowSortedByFID() { if (m_nVectorIdx == 0) { @@ -2684,7 +2833,7 @@ int FileGDBSpatialIndexIteratorImpl::GetNextRowSortedByFID() // than using a unordered_set (or set) while (true) { - const int nFID = GetNextRow(); + const auto nFID = GetNextRow(); if (nFID < 0) break; m_oFIDVector.push_back(nFID); @@ -2694,16 +2843,16 @@ int FileGDBSpatialIndexIteratorImpl::GetNextRowSortedByFID() if (m_oFIDVector.empty()) return -1; - const int nFID = m_oFIDVector[m_nVectorIdx]; + const auto nFID = m_oFIDVector[m_nVectorIdx]; ++m_nVectorIdx; return nFID; } - const int nLastFID = m_oFIDVector[m_nVectorIdx - 1]; + const auto nLastFID = m_oFIDVector[m_nVectorIdx - 1]; while (m_nVectorIdx < m_oFIDVector.size()) { // Do not return consecutive identical FID - const int nFID = m_oFIDVector[m_nVectorIdx]; + const auto nFID = m_oFIDVector[m_nVectorIdx]; ++m_nVectorIdx; if (nFID == nLastFID) { diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbindex_write.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbindex_write.cpp index fb477682c525..b2561ed88b1d 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbindex_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbindex_write.cpp @@ -305,7 +305,7 @@ void FileGDBTable::ComputeOptimalSpatialIndexGridResolution() { // For point, use the density as the grid resolution int nValid = 0; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -349,7 +349,7 @@ void FileGDBTable::ComputeOptimalSpatialIndexGridResolution() int64_t nValid = 0; auto poGeomConverter = std::unique_ptr( FileGDBOGRGeometryConverter::BuildConverter(poGeomField)); - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -403,7 +403,7 @@ void FileGDBTable::ComputeOptimalSpatialIndexGridResolution() // of all geometries double dfMaxSize = 0; OGREnvelope sEnvelope; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -847,7 +847,7 @@ bool FileGDBTable::CreateSpatialIndex() } auto poGeomConverter = std::unique_ptr( FileGDBOGRGeometryConverter::BuildConverter(poGeomField)); - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; const double dfGridStep = m_adfSpatialIndexGridResolution.back(); @@ -1159,11 +1159,11 @@ bool FileGDBTable::CreateSpatialIndex() }; std::vector aSetValues; - int iLastReported = 0; + int64_t iLastReported = 0; const auto nReportIncrement = m_nTotalRecordCount / 20; try { - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { if (m_nTotalRecordCount > 10000 && (iCurFeat + 1 == m_nTotalRecordCount || @@ -1343,9 +1343,10 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) const auto eFieldType = m_apoFields[iField]->GetType(); if (eFieldType == FGFT_INT16) { - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1368,9 +1369,10 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) } else if (eFieldType == FGFT_INT32) { - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1393,9 +1395,10 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) } else if (eFieldType == FGFT_INT64) { - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1418,9 +1421,10 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) } else if (eFieldType == FGFT_FLOAT32) { - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1444,11 +1448,12 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) eFieldType == FGFT_DATE || eFieldType == FGFT_TIME || eFieldType == FGFT_DATETIME_WITH_OFFSET) { - typedef std::pair ValueOIDPair; + typedef std::pair ValueOIDPair; std::vector asValues; // Hack to force reading DateTime as double m_apoFields[iField]->m_bReadAsDouble = true; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1471,13 +1476,14 @@ bool FileGDBTable::CreateAttributeIndex(const FileGDBIndex *poIndex) } else if (eFieldType == FGFT_STRING) { - typedef std::pair, int> ValueOIDPair; + typedef std::pair, int64_t> ValueOIDPair; std::vector asValues; bRet = true; const bool bIsLower = STARTS_WITH_CI(poIndex->GetExpression().c_str(), "LOWER("); int maxStrSize = 0; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; + ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp index a4a7a235891c..89f07f9cb451 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp @@ -30,6 +30,7 @@ #include "filegdbtable.h" #include +#include #include #include #include @@ -569,45 +570,55 @@ bool FileGDBTable::GuessFeatureLocations() } } - int nInvalidRecords = 0; - while (nOffset < m_nFileSize) + int64_t nInvalidRecords = 0; + try { - GUInt32 nSize; - int bDeletedRecord; - if (!IsLikelyFeatureAtOffset(nOffset, &nSize, &bDeletedRecord)) - { - nOffset++; - } - else + while (nOffset < m_nFileSize) { - /*CPLDebug("OpenFileGDB", "Feature found at offset %d (size = %d)", - nOffset, nSize);*/ - if (bDeletedRecord) + GUInt32 nSize; + int bDeletedRecord; + if (!IsLikelyFeatureAtOffset(nOffset, &nSize, &bDeletedRecord)) + { + nOffset++; + } + else { - if (bReportDeletedFeatures) + /*CPLDebug("OpenFileGDB", "Feature found at offset %d (size = %d)", + nOffset, nSize);*/ + if (bDeletedRecord) { - m_bHasDeletedFeaturesListed = TRUE; - m_anFeatureOffsets.push_back(MARK_DELETED(nOffset)); + if (bReportDeletedFeatures) + { + m_bHasDeletedFeaturesListed = TRUE; + m_anFeatureOffsets.push_back(MARK_DELETED(nOffset)); + } + else + { + nInvalidRecords++; + m_anFeatureOffsets.push_back(0); + } } else - { - nInvalidRecords++; - m_anFeatureOffsets.push_back(0); - } + m_anFeatureOffsets.push_back(nOffset); + nOffset += nSize; } - else - m_anFeatureOffsets.push_back(nOffset); - nOffset += nSize; } } - m_nTotalRecordCount = static_cast(m_anFeatureOffsets.size()); + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory in FileGDBTable::GuessFeatureLocations()"); + return false; + } + m_nTotalRecordCount = static_cast(m_anFeatureOffsets.size()); if (m_nTotalRecordCount - nInvalidRecords > m_nValidRecordCount) { if (!m_bHasDeletedFeaturesListed) { CPLError(CE_Warning, CPLE_AppDefined, - "More features found (%d) than declared number of valid " - "features (%d). " + "More features found (%" PRId64 + ") than declared number of valid " + "features ((%" PRId64 "). " "So deleted features will likely be reported.", m_nTotalRecordCount - nInvalidRecords, m_nValidRecordCount); @@ -619,16 +630,26 @@ bool FileGDBTable::GuessFeatureLocations() } /************************************************************************/ -/* ReadTableXHeader() */ +/* ReadTableXHeaderV3() */ /************************************************************************/ -int FileGDBTable::ReadTableXHeader() +bool FileGDBTable::ReadTableXHeaderV3() { - const int errorRetValue = FALSE; + const bool errorRetValue = false; GByte abyHeader[16]; // Read .gdbtablx file header returnErrorIf(VSIFReadL(abyHeader, 16, 1, m_fpTableX) != 1); + + const int nGDBTablxVersion = GetUInt32(abyHeader, 0); + if (nGDBTablxVersion != static_cast(m_eGDBTableVersion)) + { + CPLError(CE_Failure, CPLE_AppDefined, + ".gdbtablx version is %d whereas it should be %d", + nGDBTablxVersion, static_cast(m_eGDBTableVersion)); + return false; + } + m_n1024BlocksPresent = GetUInt32(abyHeader + 4, 0); m_nTotalRecordCount = GetInt32(abyHeader + 8, 0); @@ -697,7 +718,100 @@ int FileGDBTable::ReadTableXHeader() returnErrorIf(nCountBlocks != m_n1024BlocksPresent); } } - return TRUE; + return true; +} + +/************************************************************************/ +/* ReadTableXHeaderV4() */ +/************************************************************************/ + +bool FileGDBTable::ReadTableXHeaderV4() +{ + const bool errorRetValue = false; + GByte abyHeader[16]; + + // Read .gdbtablx file header + returnErrorIf(VSIFReadL(abyHeader, 16, 1, m_fpTableX) != 1); + + const int nGDBTablxVersion = GetUInt32(abyHeader, 0); + if (nGDBTablxVersion != static_cast(m_eGDBTableVersion)) + { + CPLError(CE_Failure, CPLE_AppDefined, + ".gdbtablx version is %d whereas it should be %d", + nGDBTablxVersion, static_cast(m_eGDBTableVersion)); + return false; + } + + m_n1024BlocksPresent = GetUInt64(abyHeader + 4, 0); + + m_nTablxOffsetSize = GetUInt32(abyHeader + 12, 0); + returnErrorIf(m_nTablxOffsetSize < 4 || m_nTablxOffsetSize > 6); + + m_nOffsetTableXTrailer = + 16 + m_nTablxOffsetSize * 1024 * + static_cast(m_n1024BlocksPresent); + if (m_n1024BlocksPresent != 0) + { + GByte abyTrailer[12]; + + VSIFSeekL(m_fpTableX, m_nOffsetTableXTrailer, SEEK_SET); + returnErrorIf(VSIFReadL(abyTrailer, 12, 1, m_fpTableX) != 1); + + m_nTotalRecordCount = GetUInt64(abyTrailer, 0); + + // Cf https://github.com/rouault/dump_gdbtable/wiki/FGDB-Spec#trailing-section-16-bytes--variable-number- + // for all below magic numbers and byte sequences + GUInt32 nSizeBitmapSection = GetUInt32(abyTrailer + 8, 0); + if (nSizeBitmapSection == 0) + { + // no bitmap. Fine + } + else if (nSizeBitmapSection == 22 + 32768 + 52 && + m_nTotalRecordCount <= 32768 * 1024 * 8) + { + try + { + std::vector abyBitmapSection(nSizeBitmapSection); + returnErrorIf(VSIFReadL(abyBitmapSection.data(), + abyBitmapSection.size(), 1, + m_fpTableX) != 1); + if (memcmp(abyBitmapSection.data(), "\x01\x00\x01\x00\x00\x00", + 6) == 0 && + memcmp(abyBitmapSection.data() + 22 + 32768, + "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 12) == 0) + { + m_abyTablXBlockMap.insert( + m_abyTablXBlockMap.end(), abyBitmapSection.data() + 22, + abyBitmapSection.data() + 22 + 32768); + } + else + { + m_bReliableObjectID = false; + } + } + catch (const std::exception &e) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Cannot allocate m_abyTablXBlockMap: %s", e.what()); + return false; + } + } + else + { + m_bReliableObjectID = false; + } + if (!m_bReliableObjectID) + { + m_nTotalRecordCount = 1024 * m_n1024BlocksPresent; + CPLError(CE_Warning, CPLE_AppDefined, + "Due to partial reverse engineering of the format, " + "ObjectIDs will not be accurate and attribute and spatial " + "indices cannot be used on %s", + m_osFilenameWithLayerName.c_str()); + } + } + return true; } /************************************************************************/ @@ -713,7 +827,7 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, m_bUpdate = bUpdate; m_osFilename = pszFilename; - CPLString m_osFilenameWithLayerName(m_osFilename); + m_osFilenameWithLayerName = m_osFilename; if (pszLayerName) m_osFilenameWithLayerName += CPLSPrintf(" (layer %s)", pszLayerName); @@ -725,11 +839,44 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, return false; } - // Read .gdtable file header + // Read .gdbtable file header GByte abyHeader[40]; returnErrorIf(VSIFReadL(abyHeader, 40, 1, m_fpTable) != 1); - m_nValidRecordCount = GetInt32(abyHeader + 4, 0); - returnErrorIf(m_nValidRecordCount < 0); + + int nGDBTableVersion = GetInt32(abyHeader, 0); + if (nGDBTableVersion == 3) + { + m_eGDBTableVersion = GDBTableVersion::V3; + } + else if (nGDBTableVersion == 4) + { + m_eGDBTableVersion = GDBTableVersion::V4; + if (m_bUpdate) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Version 4 of the FileGeodatabase format is not supported " + "for update."); + return false; + } + } + else + { + CPLError(CE_Failure, CPLE_NotSupported, + "Version %u of the FileGeodatabase format is not supported.", + nGDBTableVersion); + return false; + } + + if (m_eGDBTableVersion == GDBTableVersion::V3) + { + m_nValidRecordCount = GetInt32(abyHeader + 4, 0); + returnErrorIf(m_nValidRecordCount < 0); + } + else + { + m_nValidRecordCount = GetInt64(abyHeader + 16, 0); + returnErrorIf(m_nValidRecordCount < 0); + } m_nHeaderBufferMaxSize = GetInt32(abyHeader + 8, 0); @@ -765,7 +912,11 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, returnErrorIf(m_fpTableX == nullptr); } } - else if (!ReadTableXHeader()) + else if (m_eGDBTableVersion == GDBTableVersion::V3 && + !ReadTableXHeaderV3()) + return false; + else if (m_eGDBTableVersion == GDBTableVersion::V4 && + !ReadTableXHeaderV4()) return false; } @@ -778,8 +929,8 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, { /* Potentially unsafe. See #5842 */ CPLDebug("OpenFileGDB", - "%s: nTotalRecordCount (was %d) forced to " - "nValidRecordCount=%d", + "%s: nTotalRecordCount (was %" PRId64 ") forced to " + "nValidRecordCount=%" PRId64, m_osFilenameWithLayerName.c_str(), m_nTotalRecordCount, m_nValidRecordCount); m_nTotalRecordCount = m_nValidRecordCount; @@ -789,8 +940,10 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, /* By default err on the safe side */ CPLError( CE_Warning, CPLE_AppDefined, - "File %s declares %d valid records, but %s declares " - "only %d total records. Using that later value for safety " + "File %s declares %" PRId64 + " valid records, but %s declares " + "only %" PRId64 + " total records. Using that later value for safety " "(this possibly ignoring features). " "You can also try setting OPENFILEGDB_IGNORE_GDBTABLX=YES " "to " @@ -809,7 +962,8 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, else if (m_nTotalRecordCount != m_nValidRecordCount) { CPLDebug("OpenFileGDB", - "%s: nTotalRecordCount=%d nValidRecordCount=%d", + "%s: nTotalRecordCount=%" PRId64 + " nValidRecordCount=%" PRId64, pszFilename, m_nTotalRecordCount, m_nValidRecordCount); } #endif @@ -836,18 +990,19 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, returnErrorIf(VSIFReadL(abyHeader, 14, 1, m_fpTable) != 1); m_nFieldDescLength = GetUInt32(abyHeader, 0); - const auto nVersion = GetUInt32(abyHeader + 4, 0); - // nVersion == 6 is used in table arcgis_pro_32_types.gdb/a0000000b.gdbtable (big_int) + const auto nSecondaryHeaderVersion = GetUInt32(abyHeader + 4, 0); + // nSecondaryHeaderVersion == 6 is used in table arcgis_pro_32_types.gdb/a0000000b.gdbtable (big_int) // Not sure why... - if (m_bUpdate && nVersion != 4 && nVersion != 6) // FileGDB v10 + if (m_bUpdate && nSecondaryHeaderVersion != 4 && + nSecondaryHeaderVersion != 6) // FileGDB v10 { CPLError(CE_Failure, CPLE_NotSupported, - "Version %u of the FileGeodatabase format is not supported " - "for update.", - nVersion); + "Version %u of the secondary header of the FileGeodatabase " + "format is not supported for update.", + nSecondaryHeaderVersion); return false; } - m_bIsV9 = (nVersion == 3); + m_bIsV9 = (nSecondaryHeaderVersion == 3); returnErrorIf(m_nOffsetFieldDesc > std::numeric_limits::max() - m_nFieldDescLength); @@ -1364,25 +1519,27 @@ static void ReadVarIntAndAddNoCheck(GByte *&pabyIter, GIntBig &nOutVal) /************************************************************************/ vsi_l_offset -FileGDBTable::GetOffsetInTableForRow(int iRow, vsi_l_offset *pnOffsetInTableX) +FileGDBTable::GetOffsetInTableForRow(int64_t iRow, + vsi_l_offset *pnOffsetInTableX) { const int errorRetValue = 0; if (pnOffsetInTableX) *pnOffsetInTableX = 0; returnErrorIf(iRow < 0 || iRow >= m_nTotalRecordCount); - m_bIsDeleted = FALSE; + m_bIsDeleted = false; if (m_fpTableX == nullptr) { - m_bIsDeleted = IS_DELETED(m_anFeatureOffsets[iRow]); - return GET_OFFSET(m_anFeatureOffsets[iRow]); + m_bIsDeleted = + IS_DELETED(m_anFeatureOffsets[static_cast(iRow)]); + return GET_OFFSET(m_anFeatureOffsets[static_cast(iRow)]); } vsi_l_offset nOffsetInTableX; if (!m_abyTablXBlockMap.empty()) { GUInt32 nCountBlocksBefore = 0; - int iBlock = iRow / 1024; + const int iBlock = static_cast(iRow / 1024); // Check if the block is not empty if (TEST_BIT(m_abyTablXBlockMap.data(), iBlock) == 0) @@ -1406,7 +1563,8 @@ FileGDBTable::GetOffsetInTableForRow(int iRow, vsi_l_offset *pnOffsetInTableX) } m_nCountBlocksBeforeIBlockIdx = iBlock; m_nCountBlocksBeforeIBlockValue = nCountBlocksBefore; - const int iCorrectedRow = nCountBlocksBefore * 1024 + (iRow % 1024); + const int64_t iCorrectedRow = + static_cast(nCountBlocksBefore) * 1024 + (iRow % 1024); nOffsetInTableX = 16 + static_cast(m_nTablxOffsetSize) * iCorrectedRow; } @@ -1455,9 +1613,9 @@ uint64_t FileGDBTable::ReadFeatureOffset(const GByte *pabyBuffer) /* GetAndSelectNextNonEmptyRow() */ /************************************************************************/ -int FileGDBTable::GetAndSelectNextNonEmptyRow(int iRow) +int64_t FileGDBTable::GetAndSelectNextNonEmptyRow(int64_t iRow) { - const int errorRetValue = -1; + const int64_t errorRetValue = -1; returnErrorAndCleanupIf(iRow < 0 || iRow >= m_nTotalRecordCount, m_nCurRow = -1); @@ -1465,17 +1623,18 @@ int FileGDBTable::GetAndSelectNextNonEmptyRow(int iRow) { if (!m_abyTablXBlockMap.empty() && (iRow % 1024) == 0) { - int iBlock = iRow / 1024; + int iBlock = static_cast(iRow / 1024); if (TEST_BIT(m_abyTablXBlockMap.data(), iBlock) == 0) { - int nBlocks = DIV_ROUND_UP(m_nTotalRecordCount, 1024); + int nBlocks = + static_cast(DIV_ROUND_UP(m_nTotalRecordCount, 1024)); do { iBlock++; } while (iBlock < nBlocks && TEST_BIT(m_abyTablXBlockMap.data(), iBlock) == 0); - iRow = iBlock * 1024; + iRow = static_cast(iBlock) * 1024; if (iRow >= m_nTotalRecordCount) return -1; } @@ -1495,7 +1654,7 @@ int FileGDBTable::GetAndSelectNextNonEmptyRow(int iRow) /* SelectRow() */ /************************************************************************/ -int FileGDBTable::SelectRow(int iRow) +bool FileGDBTable::SelectRow(int64_t iRow) { const int errorRetValue = FALSE; returnErrorAndCleanupIf(iRow < 0 || iRow >= m_nTotalRecordCount, @@ -1539,7 +1698,8 @@ int FileGDBTable::SelectRow(int iRow) "NO"))) { CPLError(CE_Failure, CPLE_AppDefined, - "Invalid row length (%u) on feature %u compared " + "Invalid row length (%u) on feature %" PRId64 + " compared " "to the maximum size in the header (%u)", m_nRowBlobLength, iRow + 1, m_nHeaderBufferMaxSize); @@ -1549,7 +1709,8 @@ int FileGDBTable::SelectRow(int iRow) else { CPLDebug("OpenFileGDB", - "Invalid row length (%u) on feature %u compared " + "Invalid row length (%u) on feature %" PRId64 + " compared " "to the maximum size in the header (%u)", m_nRowBlobLength, iRow + 1, m_nHeaderBufferMaxSize); @@ -1571,7 +1732,7 @@ int FileGDBTable::SelectRow(int iRow) if (nOffsetTable + 4 + m_nRowBlobLength > m_nFileSize) { CPLError(CE_Failure, CPLE_AppDefined, - "Invalid row length (%u) on feature %u", + "Invalid row length (%u) on feature %" PRId64, m_nRowBlobLength, iRow + 1); m_nCurRow = -1; return errorRetValue; @@ -2253,7 +2414,7 @@ const OGRField *FileGDBTable::GetFieldValue(int iCol) if (iCol == static_cast(m_apoFields.size()) - 1 && m_pabyIterVals < pabyEnd) { - CPLDebug("OpenFileGDB", "%d bytes remaining at end of record %d", + CPLDebug("OpenFileGDB", "%d bytes remaining at end of record %" PRId64, static_cast(pabyEnd - m_pabyIterVals), m_nCurRow); } diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h index a5a4fc67f126..d057f3f5935e 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h @@ -428,10 +428,19 @@ class FileGDBTable { VSILFILE *m_fpTable = nullptr; VSILFILE *m_fpTableX = nullptr; + + enum class GDBTableVersion + { + V3 = 3, // 32-bit object id + V4 = 4, // 64-bit object id (ince ArcGIS Pro 3.2) + }; + GDBTableVersion m_eGDBTableVersion = GDBTableVersion::V3; vsi_l_offset m_nFileSize = 0; /* only read when needed */ bool m_bUpdate = false; + bool m_bReliableObjectID = true; // can be set to false on some V4 files std::string m_osFilename{}; + std::string m_osFilenameWithLayerName{}; bool m_bIsV9 = false; std::vector> m_apoFields{}; int m_iObjectIdField = -1; @@ -465,7 +474,7 @@ class FileGDBTable no .gdbtablx file */ uint64_t m_nOffsetTableXTrailer = 0; - uint32_t m_n1024BlocksPresent = 0; + uint64_t m_n1024BlocksPresent = 0; std::vector m_abyTablXBlockMap{}; int m_nCountBlocksBeforeIBlockIdx = 0; /* optimization */ int m_nCountBlocksBeforeIBlockValue = 0; /* optimization */ @@ -479,9 +488,9 @@ class FileGDBTable int m_nChSaved = -1; int m_bError = FALSE; - int m_nCurRow = -1; + int64_t m_nCurRow = -1; int m_bHasDeletedFeaturesListed = FALSE; - int m_bIsDeleted = FALSE; + bool m_bIsDeleted = false; int m_nLastCol = -1; GByte *m_pabyIterVals = nullptr; int m_iAccNullable = 0; @@ -494,8 +503,8 @@ class FileGDBTable bool m_bStringsAreUTF8 = true; // if false, UTF16 std::string m_osTempString{}; // used as a temporary to store strings // recoded from UTF16 to UTF8 - int m_nValidRecordCount = 0; - int m_nTotalRecordCount = 0; + int64_t m_nValidRecordCount = 0; + int64_t m_nTotalRecordCount = 0; int m_iGeomField = -1; int m_nCountNullableFields = 0; int m_nNullableFieldsSizeInBytes = 0; @@ -556,7 +565,8 @@ class FileGDBTable bool WriteHeader(VSILFILE *fpTable); bool WriteHeaderX(VSILFILE *fpTableX); - int ReadTableXHeader(); + bool ReadTableXHeaderV3(); + bool ReadTableXHeaderV4(); int IsLikelyFeatureAtOffset(vsi_l_offset nOffset, GUInt32 *pnSize, int *pbDeletedRecord); bool GuessFeatureLocations(); @@ -624,12 +634,12 @@ class FileGDBTable return m_bGeomTypeHasM; } - int GetValidRecordCount() const + int64_t GetValidRecordCount() const { return m_nValidRecordCount; } - int GetTotalRecordCount() const + int64_t GetTotalRecordCount() const { return m_nTotalRecordCount; } @@ -670,6 +680,15 @@ class FileGDBTable return m_apoIndexes[i].get(); } + /** Return if we can use attribute or spatial indices. + * This can be false for some sparse tables with 64-bit ObjectID since + * the format of the sparse bitmap isn't fully understood yet. + */ + bool CanUseIndices() const + { + return m_bReliableObjectID; + } + bool HasSpatialIndex(); bool CreateIndex(const std::string &osIndexName, const std::string &osExpression); @@ -677,7 +696,8 @@ class FileGDBTable bool CreateSpatialIndex(); vsi_l_offset - GetOffsetInTableForRow(int iRow, vsi_l_offset *pnOffsetInTableX = nullptr); + GetOffsetInTableForRow(int64_t iRow, + vsi_l_offset *pnOffsetInTableX = nullptr); int HasDeletedFeaturesListed() const { @@ -686,20 +706,20 @@ class FileGDBTable /* Next call to SelectRow() or GetFieldValue() invalidates previously * returned values */ - int SelectRow(int iRow); - int GetAndSelectNextNonEmptyRow(int iRow); + bool SelectRow(int64_t iRow); + int64_t GetAndSelectNextNonEmptyRow(int64_t iRow); int HasGotError() const { return m_bError; } - int GetCurRow() const + int64_t GetCurRow() const { return m_nCurRow; } - int IsCurRowDeleted() const + bool IsCurRowDeleted() const { return m_bIsDeleted; } @@ -731,9 +751,9 @@ class FileGDBTable bool CreateFeature(const std::vector &asRawFields, const OGRGeometry *poGeom, int *pnFID = nullptr); - bool UpdateFeature(int nFID, const std::vector &asRawFields, + bool UpdateFeature(int64_t nFID, const std::vector &asRawFields, const OGRGeometry *poGeom); - bool DeleteFeature(int nFID); + bool DeleteFeature(int64_t nFID); bool CheckFreeListConsistency(); void DeleteFreeList(); @@ -766,18 +786,18 @@ class FileGDBIterator virtual FileGDBTable *GetTable() = 0; virtual void Reset() = 0; - virtual int GetNextRowSortedByFID() = 0; - virtual int GetRowCount(); + virtual int64_t GetNextRowSortedByFID() = 0; + virtual int64_t GetRowCount(); /* Only available on a BuildIsNotNull() iterator */ virtual const OGRField *GetMinValue(int &eOutOGRFieldType); virtual const OGRField *GetMaxValue(int &eOutOGRFieldType); /* will reset the iterator */ - virtual int GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, - int &nCount); + virtual bool GetMinMaxSumCount(double &dfMin, double &dfMax, double &dfSum, + int &nCount); /* Only available on a BuildIsNotNull() or Build() iterator */ - virtual int GetNextRowSortedByValue(); + virtual int64_t GetNextRowSortedByValue(); static FileGDBIterator *Build(FileGDBTable *poParent, int nFieldIdx, int bAscending, FileGDBSQLOp op, diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp index aed5fe2a9b3b..befd9bc90993 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp @@ -33,6 +33,7 @@ #include "filegdbtable.h" #include +#include #include #include #include @@ -73,6 +74,7 @@ bool FileGDBTable::Create(const char *pszFilename, int nTablxOffsetSize, { CPLAssert(m_fpTable == nullptr); + m_eGDBTableVersion = GDBTableVersion::V3; m_bUpdate = true; m_eTableGeomType = eTableGeomType; m_nTablxOffsetSize = nTablxOffsetSize; @@ -88,6 +90,7 @@ bool FileGDBTable::Create(const char *pszFilename, int nTablxOffsetSize, } m_osFilename = pszFilename; + m_osFilenameWithLayerName = m_osFilename; m_fpTable = VSIFOpenL(pszFilename, "wb+"); if (m_fpTable == nullptr) { @@ -152,8 +155,9 @@ bool FileGDBTable::WriteHeader(VSILFILE *fpTable) VSIFSeekL(fpTable, 0, SEEK_SET); bool bRet = - WriteUInt32(fpTable, 3) && // version number - WriteUInt32(fpTable, m_nValidRecordCount) && // number of valid rows + WriteUInt32(fpTable, 3) && // version number + // number of valid rows + WriteUInt32(fpTable, static_cast(m_nValidRecordCount)) && WriteUInt32(fpTable, m_nHeaderBufferMaxSize) && // largest size of a feature // record / field description @@ -190,8 +194,8 @@ bool FileGDBTable::WriteHeaderX(VSILFILE *fpTableX) { VSIFSeekL(fpTableX, 0, SEEK_SET); if (!WriteUInt32(fpTableX, 3) || // version number - !WriteUInt32(fpTableX, m_n1024BlocksPresent) || - !WriteUInt32(fpTableX, m_nTotalRecordCount) || + !WriteUInt32(fpTableX, static_cast(m_n1024BlocksPresent)) || + !WriteUInt32(fpTableX, static_cast(m_nTotalRecordCount)) || !WriteUInt32(fpTableX, m_nTablxOffsetSize)) { CPLError(CE_Failure, CPLE_FileIO, "Cannot write .gdbtablx header"); @@ -267,7 +271,8 @@ bool FileGDBTable::Sync(VSILFILE *fpTable, VSILFILE *fpTableX) if (m_bDirtyHeader && fpTable) { VSIFSeekL(fpTable, 4, SEEK_SET); - bRet &= WriteUInt32(fpTable, m_nValidRecordCount); + bRet &= + WriteUInt32(fpTable, static_cast(m_nValidRecordCount)); m_nHeaderBufferMaxSize = std::max(m_nHeaderBufferMaxSize, std::max(m_nRowBufferMaxSize, m_nFieldDescLength)); @@ -285,8 +290,10 @@ bool FileGDBTable::Sync(VSILFILE *fpTable, VSILFILE *fpTableX) if (m_bDirtyTableXHeader && fpTableX) { VSIFSeekL(fpTableX, 4, SEEK_SET); - bRet &= WriteUInt32(fpTableX, m_n1024BlocksPresent); - bRet &= WriteUInt32(fpTableX, m_nTotalRecordCount); + bRet &= + WriteUInt32(fpTableX, static_cast(m_n1024BlocksPresent)); + bRet &= + WriteUInt32(fpTableX, static_cast(m_nTotalRecordCount)); m_bDirtyTableXHeader = false; } @@ -297,8 +304,8 @@ bool FileGDBTable::Sync(VSILFILE *fpTable, VSILFILE *fpTableX) m_nTablxOffsetSize * TABLX_FEATURES_PER_PAGE * static_cast(m_n1024BlocksPresent); VSIFSeekL(fpTableX, m_nOffsetTableXTrailer, SEEK_SET); - const uint32_t n1024BlocksTotal = - DIV_ROUND_UP(m_nTotalRecordCount, TABLX_FEATURES_PER_PAGE); + const uint32_t n1024BlocksTotal = static_cast( + DIV_ROUND_UP(m_nTotalRecordCount, TABLX_FEATURES_PER_PAGE)); if (!m_abyTablXBlockMap.empty()) { CPLAssert(m_abyTablXBlockMap.size() >= (n1024BlocksTotal + 7) / 8); @@ -314,7 +321,8 @@ bool FileGDBTable::Sync(VSILFILE *fpTable, VSILFILE *fpTableX) m_abyTablXBlockMap.resize(nBitmapInt32Words * 4); bRet &= WriteUInt32(fpTableX, nBitmapInt32Words); bRet &= WriteUInt32(fpTableX, n1024BlocksTotal); - bRet &= WriteUInt32(fpTableX, m_n1024BlocksPresent); + bRet &= + WriteUInt32(fpTableX, static_cast(m_n1024BlocksPresent)); uint32_t nTrailingZero32BitWords = 0; for (int i = static_cast(m_abyTablXBlockMap.size() / 4) - 1; i >= 0; --i) @@ -339,10 +347,10 @@ bool FileGDBTable::Sync(VSILFILE *fpTable, VSILFILE *fpTableX) nCountBlocks += TEST_BIT(m_abyTablXBlockMap.data(), i) != 0; if (nCountBlocks != m_n1024BlocksPresent) { - CPLError( - CE_Failure, CPLE_AppDefined, - "Sync(): nCountBlocks(=%u) != m_n1024BlocksPresent(=%u)", - nCountBlocks, m_n1024BlocksPresent); + CPLError(CE_Failure, CPLE_AppDefined, + "Sync(): nCountBlocks(=%u) != " + "m_n1024BlocksPresent(=%" PRIu64 ")", + nCountBlocks, m_n1024BlocksPresent); } #endif bRet &= VSIFWriteL(m_abyTablXBlockMap.data(), 1, @@ -1602,20 +1610,21 @@ bool FileGDBTable::SeekIntoTableXForNewFeature(int nObjectID) int iCorrectedRow; bool bWriteEmptyPageAtEnd = false; const uint32_t nPageSize = TABLX_FEATURES_PER_PAGE * m_nTablxOffsetSize; + const int nTotalRecordCount = static_cast(m_nTotalRecordCount); if (m_abyTablXBlockMap.empty()) { // Is the OID to write in the current allocated pages, or in the next // page ? if ((nObjectID - 1) / TABLX_FEATURES_PER_PAGE <= - ((m_nTotalRecordCount == 0) + ((nTotalRecordCount == 0) ? 0 - : (1 + (m_nTotalRecordCount - 1) / TABLX_FEATURES_PER_PAGE))) + : (1 + (nTotalRecordCount - 1) / TABLX_FEATURES_PER_PAGE))) { iCorrectedRow = nObjectID - 1; const auto n1024BlocksPresentBefore = m_n1024BlocksPresent; m_n1024BlocksPresent = - DIV_ROUND_UP(std::max(m_nTotalRecordCount, nObjectID), + DIV_ROUND_UP(std::max(nTotalRecordCount, nObjectID), TABLX_FEATURES_PER_PAGE); bWriteEmptyPageAtEnd = m_n1024BlocksPresent > n1024BlocksPresentBefore; @@ -1626,13 +1635,13 @@ bool FileGDBTable::SeekIntoTableXForNewFeature(int nObjectID) m_abyTablXBlockMap.resize( (DIV_ROUND_UP(nObjectID, TABLX_FEATURES_PER_PAGE) + 7) / 8); for (int i = 0; - i < DIV_ROUND_UP(m_nTotalRecordCount, TABLX_FEATURES_PER_PAGE); + i < DIV_ROUND_UP(nTotalRecordCount, TABLX_FEATURES_PER_PAGE); ++i) m_abyTablXBlockMap[i / 8] |= (1 << (i % 8)); const int iBlock = (nObjectID - 1) / TABLX_FEATURES_PER_PAGE; m_abyTablXBlockMap[iBlock / 8] |= (1 << (iBlock % 8)); iCorrectedRow = - DIV_ROUND_UP(m_nTotalRecordCount, TABLX_FEATURES_PER_PAGE) * + DIV_ROUND_UP(nTotalRecordCount, TABLX_FEATURES_PER_PAGE) * TABLX_FEATURES_PER_PAGE + ((nObjectID - 1) % TABLX_FEATURES_PER_PAGE); m_n1024BlocksPresent++; @@ -1643,7 +1652,7 @@ bool FileGDBTable::SeekIntoTableXForNewFeature(int nObjectID) { const int iBlock = (nObjectID - 1) / TABLX_FEATURES_PER_PAGE; - if (nObjectID <= m_nTotalRecordCount) + if (nObjectID <= nTotalRecordCount) { CPLAssert(iBlock / 8 < static_cast(m_abyTablXBlockMap.size())); if (TEST_BIT(m_abyTablXBlockMap.data(), iBlock) == 0) @@ -1657,9 +1666,8 @@ bool FileGDBTable::SeekIntoTableXForNewFeature(int nObjectID) std::vector abyTmp(nPageSize); uint64_t nOffset = - TABLX_HEADER_SIZE + - static_cast(m_n1024BlocksPresent) * nPageSize; - for (int i = m_n1024BlocksPresent - 1; + TABLX_HEADER_SIZE + m_n1024BlocksPresent * nPageSize; + for (int i = static_cast(m_n1024BlocksPresent - 1); i >= static_cast(nCountBlocksBefore); --i) { nOffset -= nPageSize; @@ -1702,7 +1710,7 @@ bool FileGDBTable::SeekIntoTableXForNewFeature(int nObjectID) } } else if (DIV_ROUND_UP(nObjectID, TABLX_FEATURES_PER_PAGE) > - DIV_ROUND_UP(m_nTotalRecordCount, TABLX_FEATURES_PER_PAGE)) + DIV_ROUND_UP(nTotalRecordCount, TABLX_FEATURES_PER_PAGE)) { m_abyTablXBlockMap.resize( (DIV_ROUND_UP(nObjectID, TABLX_FEATURES_PER_PAGE) + 7) / 8); @@ -1817,7 +1825,7 @@ bool FileGDBTable::CreateFeature(const std::vector &asRawFields, "Maximum number of records per table reached"); return false; } - nObjectID = m_nTotalRecordCount + 1; + nObjectID = static_cast(m_nTotalRecordCount + 1); } try @@ -1882,7 +1890,8 @@ bool FileGDBTable::CreateFeature(const std::vector &asRawFields, m_nFileSize += sizeof(uint32_t) + m_nRowBlobLength; } - m_nTotalRecordCount = std::max(m_nTotalRecordCount, nObjectID); + m_nTotalRecordCount = + std::max(m_nTotalRecordCount, static_cast(nObjectID)); m_nValidRecordCount++; m_bDirtyHeader = true; @@ -1897,7 +1906,7 @@ bool FileGDBTable::CreateFeature(const std::vector &asRawFields, /* UpdateFeature() */ /************************************************************************/ -bool FileGDBTable::UpdateFeature(int nFID, +bool FileGDBTable::UpdateFeature(int64_t nFID, const std::vector &asRawFields, const OGRGeometry *poGeom) { @@ -2050,7 +2059,7 @@ bool FileGDBTable::UpdateFeature(int nFID, /* DeleteFeature() */ /************************************************************************/ -bool FileGDBTable::DeleteFeature(int nFID) +bool FileGDBTable::DeleteFeature(int64_t nFID) { if (!m_bUpdate) return false; @@ -2627,7 +2636,7 @@ void FileGDBTable::RecomputeExtent() // Scan all features OGREnvelope sLayerEnvelope; OGREnvelope sFeatureEnvelope; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp index c8e61c7e6f3c..894e9ff23a3d 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp @@ -837,7 +837,7 @@ bool FileGDBTable::DeleteField(int iField) m_apoFields[m_iGeomField]->m_eType = FGFT_BINARY; m_iGeomField = -1; - for (int iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < m_nTotalRecordCount; ++iCurFeat) { iCurFeat = GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) diff --git a/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp b/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp index 6a0119005011..7e337ebb0910 100644 --- a/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp @@ -117,10 +117,14 @@ bool OGROpenFileGDBDataSource::OpenRaster(const GDALOpenInfo *poOpenInfo, return false; } - int iRow = 0; + int64_t iRow = 0; while (iRow < oTable.GetTotalRecordCount() && (iRow = oTable.GetAndSelectNextNonEmptyRow(iRow)) >= 0) { + if (iRow >= INT32_MAX) + { + return false; + } auto psField = oTable.GetFieldValue(i_raster_id); if (!psField) { @@ -137,7 +141,7 @@ bool OGROpenFileGDBDataSource::OpenRaster(const GDALOpenInfo *poOpenInfo, continue; } - const int nGDBRasterBandId = iRow + 1; + const int nGDBRasterBandId = static_cast(iRow) + 1; psField = oTable.GetFieldValue(i_sequence_nbr); if (!psField) diff --git a/ogr/ogrsf_frmts/openfilegdb/ogr_openfilegdb.h b/ogr/ogrsf_frmts/openfilegdb/ogr_openfilegdb.h index 2dd3a8a2a654..edc16d750672 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogr_openfilegdb.h +++ b/ogr/ogrsf_frmts/openfilegdb/ogr_openfilegdb.h @@ -138,7 +138,7 @@ class OGROpenFileGDBLayer final : public OGRLayer int m_iGeomFieldIdx = -1; int m_iAreaField = -1; // index of Shape_Area field int m_iLengthField = -1; // index of Shape_Length field - int m_iCurFeat = 0; + int64_t m_iCurFeat = 0; int m_iFIDAsRegularColumnIndex = -1; std::string m_osDefinition{}; std::string m_osDocumentation{}; @@ -239,7 +239,7 @@ class OGROpenFileGDBLayer final : public OGRLayer int &eOutType); int GetMinMaxSumCount(OGRFieldDefn *poFieldDefn, double &dfMin, double &dfMax, double &dfSum, int &nCount); - int HasIndexForField(const char *pszFieldName); + bool HasIndexForField(const char *pszFieldName); FileGDBIterator *BuildIndex(const char *pszFieldName, int bAscending, int op, swq_expr_node *poValue); diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp index 77dae116b5b7..16d0c20092eb 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp @@ -1504,7 +1504,7 @@ OGRFeature *OGROpenFileGDBSimpleSQLLayer::GetNextFeature() if (m_nLimit >= 0 && m_nIterated == m_nLimit) return nullptr; - int nRow = poIter->GetNextRowSortedByValue(); + const int64_t nRow = poIter->GetNextRowSortedByValue(); if (nRow < 0) return nullptr; OGRFeature *poFeature = GetFeature(nRow + 1); diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp index 6cf35031911b..411bbbb368c3 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp @@ -77,7 +77,7 @@ bool OGROpenFileGDBDataSource::GetExistingSpatialRef( FETCH_FIELD_IDX(iZTolerance, "ZTolerance", FGFT_FLOAT64); FETCH_FIELD_IDX(iMTolerance, "MTolerance", FGFT_FLOAT64); - int iCurFeat = 0; + int64_t iCurFeat = 0; while (iCurFeat < oTable.GetTotalRecordCount()) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -246,7 +246,8 @@ bool OGROpenFileGDBDataSource::RemoveRelationshipFromItemRelationships( FETCH_FIELD_IDX_WITH_RET(iOriginID, "OriginID", FGFT_GUID, false); FETCH_FIELD_IDX_WITH_RET(iDestID, "DestID", FGFT_GUID, false); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -290,7 +291,7 @@ bool OGROpenFileGDBDataSource::LinkDomainToTable( FETCH_FIELD_IDX(iOriginID, "OriginID", FGFT_GUID); FETCH_FIELD_IDX(iDestID, "DestID", FGFT_GUID); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -331,7 +332,8 @@ bool OGROpenFileGDBDataSource::UnlinkDomainToTable( FETCH_FIELD_IDX(iOriginID, "OriginID", FGFT_GUID); FETCH_FIELD_IDX(iDestID, "DestID", FGFT_GUID); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -365,7 +367,8 @@ bool OGROpenFileGDBDataSource::UpdateXMLDefinition( FETCH_FIELD_IDX(iName, "Name", FGFT_STRING); FETCH_FIELD_IDX(iDefinition, "Definition", FGFT_XML); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -406,7 +409,8 @@ bool OGROpenFileGDBDataSource::FindUUIDFromName(const std::string &osName, FETCH_FIELD_IDX(iUUID, "UUID", FGFT_GLOBALID); FETCH_FIELD_IDX(iName, "Name", FGFT_STRING); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1262,9 +1266,10 @@ OGROpenFileGDBDataSource::ICreateLayer(const char *pszLayerName, auto eType = poGeomFieldDefn ? poGeomFieldDefn->GetType() : wkbNone; FileGDBTable oTable; - if (!oTable.Open(m_osGDBSystemCatalogFilename.c_str(), false)) + if (!oTable.Open(m_osGDBSystemCatalogFilename.c_str(), false) || + oTable.GetTotalRecordCount() >= INT32_MAX) return nullptr; - const int nTableNum = 1 + oTable.GetTotalRecordCount(); + const int nTableNum = static_cast(1 + oTable.GetTotalRecordCount()); oTable.Close(); const std::string osFilename(CPLFormFilename( @@ -1317,7 +1322,7 @@ OGRErr OGROpenFileGDBDataSource::DeleteLayer(int iLayer) FETCH_FIELD_IDX_WITH_RET(iName, "Name", FGFT_STRING, OGRERR_FAILURE); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -1342,7 +1347,7 @@ OGRErr OGROpenFileGDBDataSource::DeleteLayer(int iLayer) FETCH_FIELD_IDX_WITH_RET(iUUID, "UUID", FGFT_GLOBALID, OGRERR_FAILURE); FETCH_FIELD_IDX_WITH_RET(iName, "Name", FGFT_STRING, OGRERR_FAILURE); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -1374,7 +1379,7 @@ OGRErr OGROpenFileGDBDataSource::DeleteLayer(int iLayer) OGRERR_FAILURE); FETCH_FIELD_IDX_WITH_RET(iDestID, "DestID", FGFT_GUID, OGRERR_FAILURE); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -1561,7 +1566,7 @@ bool OGROpenFileGDBDataSource::DeleteFieldDomain( FETCH_FIELD_IDX(iType, "Type", FGFT_GUID); FETCH_FIELD_IDX(iName, "Name", FGFT_STRING); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -1601,7 +1606,7 @@ bool OGROpenFileGDBDataSource::DeleteFieldDomain( FETCH_FIELD_IDX(iDestID, "DestID", FGFT_GUID); FETCH_FIELD_IDX(iType, "Type", FGFT_GUID); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -1673,7 +1678,8 @@ bool OGROpenFileGDBDataSource::UpdateFieldDomain( FETCH_FIELD_IDX(iDefinition, "Definition", FGFT_XML); bool bMatchFound = false; - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) @@ -1818,12 +1824,13 @@ bool OGROpenFileGDBDataSource::AddRelationship( const std::string osThisGUID = OFGDBGenerateUUID(); FileGDBTable oTable; - if (!oTable.Open(m_osGDBItemsFilename.c_str(), true)) + if (!oTable.Open(m_osGDBItemsFilename.c_str(), true) || + oTable.GetTotalRecordCount() >= INT32_MAX) return false; // hopefully this just needs to be a unique value. Seems to autoincrement // when created from ArcMap at least! - const int iDsId = oTable.GetTotalRecordCount() + 1; + const int iDsId = static_cast(oTable.GetTotalRecordCount() + 1); std::string osMappingTableOidName; if (relationship->GetCardinality() == @@ -2002,7 +2009,7 @@ bool OGROpenFileGDBDataSource::DeleteRelationship(const std::string &name, FETCH_FIELD_IDX_WITH_RET(iType, "Type", FGFT_GUID, false); FETCH_FIELD_IDX_WITH_RET(iName, "Name", FGFT_STRING, false); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -2102,12 +2109,15 @@ bool OGROpenFileGDBDataSource::UpdateRelationship( } FileGDBTable oTable; - if (!oTable.Open(m_osGDBItemsFilename.c_str(), true)) + if (!oTable.Open(m_osGDBItemsFilename.c_str(), true) || + oTable.GetTotalRecordCount() >= INT32_MAX) + { return false; + } // hopefully this just needs to be a unique value. Seems to autoincrement // when created from ArcMap at least! - const int iDsId = oTable.GetTotalRecordCount() + 1; + const int iDsId = static_cast(oTable.GetTotalRecordCount()) + 1; std::string osMappingTableOidName; if (relationship->GetCardinality() == @@ -2145,7 +2155,8 @@ bool OGROpenFileGDBDataSource::UpdateRelationship( bool bMatchFound = false; std::string osUUID; - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); if (iCurFeat < 0) diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp index c98a978a49fa..1c33b099cc7e 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp @@ -330,7 +330,7 @@ void OGROpenFileGDBLayer::TryToDetectMultiPatchKind() if (m_poLyrTable->GetTotalRecordCount() == 0) return; - int nFirstIdx = m_poLyrTable->GetAndSelectNextNonEmptyRow(0); + const int64_t nFirstIdx = m_poLyrTable->GetAndSelectNextNonEmptyRow(0); if (nFirstIdx < 0) return; @@ -343,7 +343,7 @@ void OGROpenFileGDBLayer::TryToDetectMultiPatchKind() const OGRwkbGeometryType eType = poGeom->getGeometryType(); delete poGeom; - int nLastIdx = m_poLyrTable->GetTotalRecordCount() - 1; + int64_t nLastIdx = m_poLyrTable->GetTotalRecordCount() - 1; const GUInt32 nErrorCount = CPLGetErrorCounter(); while (nLastIdx > nFirstIdx && m_poLyrTable->GetOffsetInTableForRow(nLastIdx) == 0 && @@ -449,7 +449,8 @@ int OGROpenFileGDBLayer::BuildLayerDefinition() } #endif - if (!(m_poLyrTable->HasSpatialIndex() && + if (!(m_poLyrTable->CanUseIndices() && + m_poLyrTable->HasSpatialIndex() && CPLTestBool(CPLGetConfigOption("OPENFILEGDB_USE_SPATIAL_INDEX", "YES"))) && CPLTestBool(CPLGetConfigOption("OPENFILEGDB_IN_MEMORY_SPI", "YES"))) @@ -460,9 +461,11 @@ int OGROpenFileGDBLayer::BuildLayerDefinition() sGlobalBounds.maxx = poGDBGeomField->GetXMax(); sGlobalBounds.maxy = poGDBGeomField->GetYMax(); m_pQuadTree = CPLQuadTreeCreate(&sGlobalBounds, nullptr); - CPLQuadTreeSetMaxDepth(m_pQuadTree, - CPLQuadTreeGetAdvisedMaxDepth( - m_poLyrTable->GetValidRecordCount())); + CPLQuadTreeSetMaxDepth( + m_pQuadTree, + CPLQuadTreeGetAdvisedMaxDepth( + static_cast(std::min( + INT_MAX, m_poLyrTable->GetValidRecordCount())))); } else { @@ -957,7 +960,7 @@ void OGROpenFileGDBLayer::SetSpatialFilter(OGRGeometry *poGeom) if (poGeom != nullptr) { if (m_poSpatialIndexIterator == nullptr && - m_poLyrTable->HasSpatialIndex() && + m_poLyrTable->CanUseIndices() && m_poLyrTable->HasSpatialIndex() && CPLTestBool( CPLGetConfigOption("OPENFILEGDB_USE_SPATIAL_INDEX", "YES"))) { @@ -1703,7 +1706,7 @@ OGRFeature *OGROpenFileGDBLayer::GetCurrentFeature() { OGRFeature *poFeature = nullptr; int iOGRIdx = 0; - int iRow = m_poLyrTable->GetCurRow(); + int64_t iRow = m_poLyrTable->GetCurRow(); for (int iGDBIdx = 0; iGDBIdx < m_poLyrTable->GetFieldCount(); iGDBIdx++) { if (iOGRIdx == m_iFIDAsRegularColumnIndex) @@ -1727,16 +1730,27 @@ OGRFeature *OGROpenFileGDBLayer::GetCurrentFeature() if (m_poLyrTable->GetFeatureExtent(psField, &sFeatureEnvelope)) { - CPLRectObj sBounds; - sBounds.minx = sFeatureEnvelope.MinX; - sBounds.miny = sFeatureEnvelope.MinY; - sBounds.maxx = sFeatureEnvelope.MaxX; - sBounds.maxy = sFeatureEnvelope.MaxY; - CPLQuadTreeInsertWithBounds( - m_pQuadTree, - reinterpret_cast( - static_cast(iRow)), - &sBounds); +#if SIZEOF_VOIDP < 8 + if (iRow > INT32_MAX) + { + // m_pQuadTree stores iRow values as void* + // This would overflow here. + m_eSpatialIndexState = SPI_INVALID; + } + else +#endif + { + CPLRectObj sBounds; + sBounds.minx = sFeatureEnvelope.MinX; + sBounds.miny = sFeatureEnvelope.MinY; + sBounds.maxx = sFeatureEnvelope.MaxX; + sBounds.maxy = sFeatureEnvelope.MaxY; + CPLQuadTreeInsertWithBounds( + m_pQuadTree, + reinterpret_cast( + static_cast(iRow)), + &sBounds); + } } } @@ -1863,8 +1877,9 @@ OGRFeature *OGROpenFileGDBLayer::GetNextFeature() { return nullptr; } - int iRow = static_cast(reinterpret_cast( - m_pahFilteredFeatures[m_iCurFeat++])); + const auto iRow = + static_cast(reinterpret_cast( + m_pahFilteredFeatures[m_iCurFeat++])); if (m_poLyrTable->SelectRow(iRow)) { poFeature = GetCurrentFeature(); @@ -1882,7 +1897,7 @@ OGRFeature *OGROpenFileGDBLayer::GetNextFeature() { while (true) { - int iRow = poIterator->GetNextRowSortedByFID(); + const auto iRow = poIterator->GetNextRowSortedByFID(); if (iRow < 0) return nullptr; if (m_poLyrTable->SelectRow(iRow)) @@ -1954,7 +1969,7 @@ OGRFeature *OGROpenFileGDBLayer::GetFeature(GIntBig nFeatureId) if (nFeatureId < 1 || nFeatureId > m_poLyrTable->GetTotalRecordCount()) return nullptr; - if (!m_poLyrTable->SelectRow(static_cast(nFeatureId) - 1)) + if (!m_poLyrTable->SelectRow(nFeatureId - 1)) return nullptr; /* Temporarily disable spatial filter */ @@ -1993,7 +2008,7 @@ OGRErr OGROpenFileGDBLayer::SetNextByIndex(GIntBig nIndex) { if (nIndex < 0 || nIndex >= m_nFilteredFeatureCount) return OGRERR_FAILURE; - m_iCurFeat = static_cast(nIndex); + m_iCurFeat = nIndex; return OGRERR_NONE; } else if (m_poLyrTable->GetValidRecordCount() == @@ -2001,7 +2016,7 @@ OGRErr OGROpenFileGDBLayer::SetNextByIndex(GIntBig nIndex) { if (nIndex < 0 || nIndex >= m_poLyrTable->GetValidRecordCount()) return OGRERR_FAILURE; - m_iCurFeat = static_cast(nIndex); + m_iCurFeat = nIndex; return OGRERR_NONE; } else @@ -2105,7 +2120,7 @@ GIntBig OGROpenFileGDBLayer::GetFeatureCount(int bForce) int nCount = 0; while (true) { - const int nRowIdx = + const auto nRowIdx = m_poSpatialIndexIterator->GetNextRowSortedByFID(); if (nRowIdx < 0) break; @@ -2149,7 +2164,7 @@ GIntBig OGROpenFileGDBLayer::GetFeatureCount(int bForce) m_nFilteredFeatureCount = 0; } - for (int i = 0; i < m_poLyrTable->GetTotalRecordCount(); i++) + for (int64_t i = 0; i < m_poLyrTable->GetTotalRecordCount(); i++) { if (!m_poLyrTable->SelectRow(i)) { @@ -2158,6 +2173,15 @@ GIntBig OGROpenFileGDBLayer::GetFeatureCount(int bForce) else continue; } +#if SIZEOF_VOIDP < 8 + if (i > INT32_MAX) + { + // CPLQuadTreeInsertWithBounds stores row index values as void* + // This would overflow here. + m_eSpatialIndexState = SPI_INVALID; + break; + } +#endif const OGRField *psField = m_poLyrTable->GetFieldValue(m_iGeomFieldIdx); @@ -2307,7 +2331,8 @@ int OGROpenFileGDBLayer::TestCapability(const char *pszCap) else if (EQUAL(pszCap, OLCFastSpatialFilter)) { return m_eSpatialIndexState == SPI_COMPLETED || - m_poLyrTable->HasSpatialIndex(); + (m_poLyrTable->CanUseIndices() && + m_poLyrTable->HasSpatialIndex()); } return FALSE; @@ -2317,11 +2342,12 @@ int OGROpenFileGDBLayer::TestCapability(const char *pszCap) /* HasIndexForField() */ /***********************************************************************/ -int OGROpenFileGDBLayer::HasIndexForField(const char *pszFieldName) +bool OGROpenFileGDBLayer::HasIndexForField(const char *pszFieldName) { if (!BuildLayerDefinition()) - return FALSE; - + return false; + if (!m_poLyrTable->CanUseIndices()) + return false; int nTableColIdx = m_poLyrTable->GetFieldIdx(pszFieldName); return (nTableColIdx >= 0 && m_poLyrTable->GetField(nTableColIdx)->HasIndex()); @@ -2393,6 +2419,8 @@ const OGRField *OGROpenFileGDBLayer::GetMinMaxValue(OGRFieldDefn *poFieldDefn, eOutType = -1; if (!BuildLayerDefinition()) return nullptr; + if (!m_poLyrTable->CanUseIndices()) + return nullptr; const int nTableColIdx = m_poLyrTable->GetFieldIdx(poFieldDefn->GetNameRef()); @@ -2427,21 +2455,21 @@ int OGROpenFileGDBLayer::GetMinMaxSumCount(OGRFieldDefn *poFieldDefn, dfSum = 0.0; nCount = 0; if (!BuildLayerDefinition()) - return FALSE; + return false; + if (!m_poLyrTable->CanUseIndices()) + return false; int nTableColIdx = m_poLyrTable->GetFieldIdx(poFieldDefn->GetNameRef()); if (nTableColIdx >= 0 && m_poLyrTable->GetField(nTableColIdx)->HasIndex()) { - FileGDBIterator *poIter = - FileGDBIterator::BuildIsNotNull(m_poLyrTable, nTableColIdx, TRUE); - if (poIter != nullptr) + auto poIter = std::unique_ptr( + FileGDBIterator::BuildIsNotNull(m_poLyrTable, nTableColIdx, TRUE)); + if (poIter) { - int nRet = poIter->GetMinMaxSumCount(dfMin, dfMax, dfSum, nCount); - delete poIter; - return nRet; + return poIter->GetMinMaxSumCount(dfMin, dfMax, dfSum, nCount); } } - return FALSE; + return false; } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp index 819b24554db3..a12daa4c93cd 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp @@ -30,6 +30,7 @@ #include "ogr_openfilegdb.h" #include "filegdb_gdbtoogrfieldtype.h" +#include #include #include #include @@ -314,7 +315,8 @@ bool OGROpenFileGDBLayer::CreateFeatureDataset(const char *pszFeatureDataset) if (!oTable.Open(m_poDS->m_osGDBItemsFilename.c_str(), false)) return false; CPLCreateXMLElementAndValue( - psRoot, "DSID", CPLSPrintf("%d", 1 + oTable.GetTotalRecordCount())); + psRoot, "DSID", + CPLSPrintf("%" PRId64, 1 + oTable.GetTotalRecordCount())); } CPLCreateXMLElementAndValue(psRoot, "Versioned", "false"); @@ -458,7 +460,7 @@ bool OGROpenFileGDBLayer::Create(const OGRGeomFieldDefn *poSrcGeomFieldDefn) FETCH_FIELD_IDX(iName, "Name", FGFT_STRING); FETCH_FIELD_IDX(iDefinition, "Definition", FGFT_XML); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -2635,7 +2637,8 @@ void OGROpenFileGDBLayer::RefreshXMLDefinitionInMemory() if (!oTable.Open(m_poDS->m_osGDBItemsFilename.c_str(), false)) return; CPLCreateXMLElementAndValue( - psRoot, "DSID", CPLSPrintf("%d", 1 + oTable.GetTotalRecordCount())); + psRoot, "DSID", + CPLSPrintf("%" PRId64, 1 + oTable.GetTotalRecordCount())); } CPLCreateXMLElementAndValue(psRoot, "Versioned", "false"); @@ -3186,7 +3189,7 @@ OGRErr OGROpenFileGDBLayer::Rename(const char *pszDstTableName) FETCH_FIELD_IDX_WITH_RET(iName, "Name", FGFT_STRING, OGRERR_FAILURE); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); @@ -3224,7 +3227,7 @@ OGRErr OGROpenFileGDBLayer::Rename(const char *pszDstTableName) FETCH_FIELD_IDX_WITH_RET(iDefinition, "Definition", FGFT_XML, OGRERR_FAILURE); - for (int iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); + for (int64_t iCurFeat = 0; iCurFeat < oTable.GetTotalRecordCount(); ++iCurFeat) { iCurFeat = oTable.GetAndSelectNextNonEmptyRow(iCurFeat); From 8f324ef9a795ca9cb0f3e218066c23df6a16b5ed Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 00:11:50 +0200 Subject: [PATCH 02/31] scripts/cppcheck.sh: ignore stlIfStrFind warning --- scripts/cppcheck.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/cppcheck.sh b/scripts/cppcheck.sh index 43cc9518cd05..474e87d79697 100755 --- a/scripts/cppcheck.sh +++ b/scripts/cppcheck.sh @@ -168,6 +168,10 @@ mv ${LOG_FILE}.tmp ${LOG_FILE} grep -v -e "duplInheritedMember" ${LOG_FILE} > ${LOG_FILE}.tmp mv ${LOG_FILE}.tmp ${LOG_FILE} +# Ignore stlIfStrFind warning "Inefficient usage of string::find() in condition; string::starts_with() could be faster" (requires C++20) +grep -v -e "stlIfStrFind" ${LOG_FILE} > ${LOG_FILE}.tmp +mv ${LOG_FILE}.tmp ${LOG_FILE} + if grep "null pointer" ${LOG_FILE} ; then echo "Null pointer check failed" ret_code=1 From 6982b57d4690535af26e3178cf2eaec8086455fa Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 24 May 2024 18:45:39 +0200 Subject: [PATCH 03/31] ESRIJSON: make it able to parse response of some CadastralSpecialServices ESRI APIs Fixes #9996 --- autotest/ogr/data/esrijson/GetLatLon.json | 139 ++++++++++++++++++ autotest/ogr/ogr_esrijson.py | 17 +++ ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp | 127 +++++++++++++--- ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp | 18 ++- 4 files changed, 276 insertions(+), 25 deletions(-) create mode 100644 autotest/ogr/data/esrijson/GetLatLon.json diff --git a/autotest/ogr/data/esrijson/GetLatLon.json b/autotest/ogr/data/esrijson/GetLatLon.json new file mode 100644 index 000000000000..e40a4ae2b9a1 --- /dev/null +++ b/autotest/ogr/data/esrijson/GetLatLon.json @@ -0,0 +1,139 @@ +{ + "trs": "WA330160N0260E0SN070", + "generatedplss": [ + "WA330160N0260E0SN070" + ], + "coordinates": [ + { + "plssid": "WA330160N0260E0SN070", + "lat": 46.889846925914661, + "lon": -119.61030783431359 + } + ], + "features": [ + { + "attributes": { + "landdescription": "WA330160N0260E0SN070" + }, + "geometry": { + "rings": [ + [ + [ + -119.60204327043593, + 46.886243867876424 + ], + [ + -119.60206398468807, + 46.882628224420934 + ], + [ + -119.60732767297685, + 46.882649819203635 + ], + [ + -119.6125944631483, + 46.882671183730082 + ], + [ + -119.6126044856519, + 46.882671170222224 + ], + [ + -119.61785486360311, + 46.882664173764368 + ], + [ + -119.61875770280301, + 46.882662936567044 + ], + [ + -119.61874405829215, + 46.883568936820438 + ], + [ + -119.61868552316993, + 46.886246722756596 + ], + [ + -119.61868498238411, + 46.886271509249347 + ], + [ + -119.61867486286243, + 46.886734870031582 + ], + [ + -119.61866382256761, + 46.887240158406691 + ], + [ + -119.6186061309634, + 46.889880080462206 + ], + [ + -119.61859592700011, + 46.890346766295536 + ], + [ + -119.61858348713005, + 46.890915726238376 + ], + [ + -119.61858325446639, + 46.89092222006439 + ], + [ + -119.61847427175444, + 46.893957399712157 + ], + [ + -119.61845184261844, + 46.894581451904436 + ], + [ + -119.61836141820194, + 46.897097152613171 + ], + [ + -119.61782949338284, + 46.897096441799754 + ], + [ + -119.61263370425482, + 46.897089367425643 + ], + [ + -119.61256327453992, + 46.8970893336651 + ], + [ + -119.60729432661518, + 46.897086677014869 + ], + [ + -119.60202277267778, + 46.897083939339161 + ], + [ + -119.60202262984565, + 46.893471725151699 + ], + [ + -119.60202248521686, + 46.8898595819661 + ], + [ + -119.60204327043593, + 46.886243867876424 + ] + ] + ], + "spatialReference": { + "wkid": 4326, + "latestWkid": 4326 + } + } + } + ], + "status": "success" +} \ No newline at end of file diff --git a/autotest/ogr/ogr_esrijson.py b/autotest/ogr/ogr_esrijson.py index 0f5eda1c2e48..3c00e5d61e23 100755 --- a/autotest/ogr/ogr_esrijson.py +++ b/autotest/ogr/ogr_esrijson.py @@ -682,3 +682,20 @@ def test_ogr_esrijson_identify_srs(): sr = lyr.GetSpatialRef() assert sr assert sr.GetAuthorityCode(None) == "2223" + + +############################################################################### +# Test for https://github.com/OSGeo/gdal/issues/9996 + + +def test_ogr_esrijson_read_CadastralSpecialServies(): + + ds = ogr.Open("data/esrijson/GetLatLon.json") + lyr = ds.GetLayer(0) + sr = lyr.GetSpatialRef() + assert sr + assert sr.GetAuthorityCode(None) == "4326" + assert lyr.GetGeomType() != ogr.wkbNone + f = lyr.GetNextFeature() + assert f["landdescription"] == "WA330160N0260E0SN070" + assert f.GetGeometryRef().GetGeometryType() == ogr.wkbPolygon diff --git a/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp b/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp index 1de41008c0e9..311087b4f66e 100644 --- a/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp +++ b/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp @@ -121,9 +121,39 @@ void OGRESRIJSONReader::ReadLayers(OGRGeoJSONDataSource *poDS, } auto eGeomType = OGRESRIJSONGetGeometryType(poGJObject_); - if (eGeomType == wkbNone && poSRS != nullptr) + if (eGeomType == wkbNone) { - eGeomType = wkbUnknown; + if (poSRS) + { + eGeomType = wkbUnknown; + } + else + { + json_object *poObjFeatures = + OGRGeoJSONFindMemberByName(poGJObject_, "features"); + if (poObjFeatures && + json_type_array == json_object_get_type(poObjFeatures)) + { + const auto nFeatures = json_object_array_length(poObjFeatures); + for (auto i = decltype(nFeatures){0}; i < nFeatures; ++i) + { + json_object *poObjFeature = + json_object_array_get_idx(poObjFeatures, i); + if (poObjFeature != nullptr && + json_object_get_type(poObjFeature) == json_type_object) + { + if (auto poObjGeometry = OGRGeoJSONFindMemberByName( + poObjFeature, "geometry")) + { + eGeomType = wkbUnknown; + poSRS = + OGRESRIJSONReadSpatialReference(poObjGeometry); + break; + } + } + } + } + } } poLayer_ = new OGRGeoJSONLayer(pszName, poSRS, eGeomType, poDS, nullptr); @@ -185,28 +215,87 @@ bool OGRESRIJSONReader::GenerateLayerDefn() } } } + else if ((poFields = OGRGeoJSONFindMemberByName( + poGJObject_, "fieldAliases")) != nullptr && + json_object_get_type(poFields) == json_type_object) + { + json_object_iter it; + it.key = nullptr; + it.val = nullptr; + it.entry = nullptr; + json_object_object_foreachC(poFields, it) + { + OGRFieldDefn fldDefn(it.key, OFTString); + poDefn->AddFieldDefn(&fldDefn); + } + } else { - poFields = OGRGeoJSONFindMemberByName(poGJObject_, "fieldAliases"); - if (nullptr != poFields && - json_object_get_type(poFields) == json_type_object) + // Guess the fields' schema from the content of the features' "attributes" + // element + json_object *poObjFeatures = + OGRGeoJSONFindMemberByName(poGJObject_, "features"); + if (poObjFeatures && + json_type_array == json_object_get_type(poObjFeatures)) { - json_object_iter it; - it.key = nullptr; - it.val = nullptr; - it.entry = nullptr; - json_object_object_foreachC(poFields, it) + gdal::DirectedAcyclicGraph dag; + std::vector> apoFieldDefn{}; + std::map oMapFieldNameToIdx{}; + std::vector anCurFieldIndices; + std::set aoSetUndeterminedTypeFields; + + const auto nFeatures = json_object_array_length(poObjFeatures); + for (auto i = decltype(nFeatures){0}; i < nFeatures; ++i) { - OGRFieldDefn fldDefn(it.key, OFTString); - poDefn->AddFieldDefn(&fldDefn); + json_object *poObjFeature = + json_object_array_get_idx(poObjFeatures, i); + if (poObjFeature != nullptr && + json_object_get_type(poObjFeature) == json_type_object) + { + int nPrevFieldIdx = -1; + + json_object *poObjProps = + OGRGeoJSONFindMemberByName(poObjFeature, "attributes"); + if (nullptr != poObjProps && + json_object_get_type(poObjProps) == json_type_object) + { + json_object_iter it; + it.key = nullptr; + it.val = nullptr; + it.entry = nullptr; + json_object_object_foreachC(poObjProps, it) + { + anCurFieldIndices.clear(); + OGRGeoJSONReaderAddOrUpdateField( + anCurFieldIndices, oMapFieldNameToIdx, + apoFieldDefn, it.key, it.val, + /*bFlattenNestedAttributes = */ true, + /* chNestedAttributeSeparator = */ '.', + /* bArrayAsString =*/false, + /* bDateAsString = */ false, + aoSetUndeterminedTypeFields); + for (int idx : anCurFieldIndices) + { + dag.addNode(idx, + apoFieldDefn[idx]->GetNameRef()); + if (nPrevFieldIdx != -1) + { + dag.addEdge(nPrevFieldIdx, idx); + } + nPrevFieldIdx = idx; + } + } + } + } + } + + const auto sortedFields = dag.getTopologicalOrdering(); + CPLAssert(sortedFields.size() == apoFieldDefn.size()); + for (int idx : sortedFields) + { + // cppcheck-suppress containerOutOfBounds + poDefn->AddFieldDefn(apoFieldDefn[idx].get()); } - } - else - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid FeatureCollection object. " - "Missing \'fields\' member."); - bSuccess = false; } } diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp index 4ec6ba2b3ae6..0f80c4524ff5 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp @@ -38,9 +38,12 @@ #include #include -const char szESRIJSonPotentialStart1[] = +const char szESRIJSonFeaturesGeometryRings[] = "{\"features\":[{\"geometry\":{\"rings\":["; +// Cf https://github.com/OSGeo/gdal/issues/9996#issuecomment-2129845692 +const char szESRIJSonFeaturesAttributes[] = "{\"features\":[{\"attributes\":{"; + /************************************************************************/ /* IsJSONObject() */ /************************************************************************/ @@ -181,9 +184,10 @@ static bool IsGeoJSONLikeObject(const char *pszText, bool &bMightBeSequence, return true; } - CPLString osWithoutSpace = GetCompactJSon(pszText, strlen(pszText)); + const std::string osWithoutSpace = GetCompactJSon(pszText, strlen(pszText)); if (osWithoutSpace.find("{\"features\":[") == 0 && - osWithoutSpace.find(szESRIJSonPotentialStart1) != 0) + osWithoutSpace.find(szESRIJSonFeaturesGeometryRings) != 0 && + osWithoutSpace.find(szESRIJSonFeaturesAttributes) != 0) { return true; } @@ -262,9 +266,11 @@ bool ESRIJSONIsObject(const char *pszText) return true; } - CPLString osWithoutSpace = - GetCompactJSon(pszText, strlen(szESRIJSonPotentialStart1)); - if (osWithoutSpace.find(szESRIJSonPotentialStart1) == 0) + const std::string osWithoutSpace = GetCompactJSon(pszText, strlen(pszText)); + if (osWithoutSpace.find(szESRIJSonFeaturesGeometryRings) == 0 || + osWithoutSpace.find(szESRIJSonFeaturesAttributes) == 0 || + osWithoutSpace.find("\"spatialReference\":{\"wkid\":") != + std::string::npos) { return true; } From 91af0177d07a73b2c08249b5b5edc76dc42146ce Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 19:13:16 +0200 Subject: [PATCH 04/31] OpenFileGDB writer: correctly set flag on non-editable Shape_Length/Shape_Area special fields We fix both the flag attribute of the field declaration in .gdbtable, and the XML declaration of the field. --- ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp | 25 +- ogr/ogrsf_frmts/openfilegdb/filegdbtable.h | 31 +- .../openfilegdb/filegdbtable_write_fields.cpp | 66 ++-- .../ogropenfilegdbdatasource_write.cpp | 318 ++++++++++++------ .../openfilegdb/ogropenfilegdblayer_write.cpp | 58 +++- .../openfilegdb/test_ofgdb_write.cpp | 41 ++- 6 files changed, 347 insertions(+), 192 deletions(-) diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp index a4a7a235891c..29d4dbb54ab3 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp @@ -965,7 +965,7 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, OGRField sDefault; OGR_RawField_SetUnset(&sDefault); - if ((flags & 4) != 0) + if (flags & FileGDBField::MASK_EDITABLE) { /* Default value */ /* Found on PreNIS.gdb/a0000000d.gdbtable */ @@ -1043,7 +1043,7 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, if (eType == FGFT_OBJECTID) { - returnErrorIf(flags != 2); + returnErrorIf(flags != FileGDBField::MASK_REQUIRED); returnErrorIf(m_iObjectIdField >= 0); m_iObjectIdField = static_cast(m_apoFields.size()); } @@ -1052,7 +1052,9 @@ bool FileGDBTable::Open(const char *pszFilename, bool bUpdate, poField->m_osName = osName; poField->m_osAlias = osAlias; poField->m_eType = eType; - poField->m_bNullable = (flags & 1) != 0; + poField->m_bNullable = (flags & FileGDBField::MASK_NULLABLE) != 0; + poField->m_bRequired = (flags & FileGDBField::MASK_REQUIRED) != 0; + poField->m_bEditable = (flags & FileGDBField::MASK_EDITABLE) != 0; poField->m_nMaxWidth = nMaxWidth; poField->m_sDefault = sDefault; m_apoFields.emplace_back(std::move(poField)); @@ -2834,11 +2836,19 @@ FileGDBField::FileGDBField(FileGDBTable *poParentIn) : m_poParent(poParentIn) FileGDBField::FileGDBField(const std::string &osName, const std::string &osAlias, FileGDBFieldType eType, - bool bNullable, int nMaxWidth, - const OGRField &sDefault) + bool bNullable, bool bRequired, bool bEditable, + int nMaxWidth, const OGRField &sDefault) : m_osName(osName), m_osAlias(osAlias), m_eType(eType), - m_bNullable(bNullable), m_nMaxWidth(nMaxWidth) + m_bNullable(bNullable), m_bRequired(bRequired), m_bEditable(bEditable), + m_nMaxWidth(nMaxWidth) { + if (m_eType == FGFT_OBJECTID || m_eType == FGFT_GLOBALID) + { + CPLAssert(!m_bNullable); + CPLAssert(m_bRequired); + CPLAssert(!m_bEditable); + } + if (m_eType == FGFT_STRING && !OGR_RawField_IsUnset(&sDefault) && !OGR_RawField_IsNull(&sDefault)) { @@ -2918,7 +2928,8 @@ FileGDBGeomField::FileGDBGeomField( const std::string &osWKT, double dfXOrigin, double dfYOrigin, double dfXYScale, double dfXYTolerance, const std::vector &adfSpatialIndexGridResolution) - : FileGDBField(osName, osAlias, FGFT_GEOMETRY, bNullable, 0, + : FileGDBField(osName, osAlias, FGFT_GEOMETRY, bNullable, + /* bRequired = */ true, /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD), m_osWKT(osWKT), m_dfXOrigin(dfXOrigin), m_dfYOrigin(dfYOrigin), m_dfXYScale(dfXYScale), m_dfXYTolerance(dfXYTolerance), diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h index a5a4fc67f126..c55082b44041 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h @@ -102,8 +102,11 @@ class FileGDBField std::string m_osAlias{}; FileGDBFieldType m_eType = FGFT_UNDEFINED; - bool m_bNullable = false; - bool m_bHighPrecsion = false; // for FGFT_DATETIME + bool m_bNullable = false; // Bit 1 of flag field + bool m_bRequired = + false; // Bit 2 of flag field. Set for ObjectID, geometry field and Shape_Area/Shape_Length + bool m_bEditable = false; // Bit 3 of flag field. + bool m_bHighPrecision = false; // for FGFT_DATETIME bool m_bReadAsDouble = false; // used by FileGDBTable::CreateAttributeIndex() int m_nMaxWidth = 0; /* for string */ @@ -117,11 +120,17 @@ class FileGDBField public: static const OGRField UNSET_FIELD; + static constexpr int BIT_NULLABLE = 0; + static constexpr int BIT_REQUIRED = 1; + static constexpr int BIT_EDITABLE = 2; + static constexpr int MASK_NULLABLE = 1 << BIT_NULLABLE; + static constexpr int MASK_REQUIRED = 1 << BIT_REQUIRED; + static constexpr int MASK_EDITABLE = 1 << BIT_EDITABLE; explicit FileGDBField(FileGDBTable *m_poParent); FileGDBField(const std::string &osName, const std::string &osAlias, - FileGDBFieldType eType, bool bNullable, int nMaxWidth, - const OGRField &sDefault); + FileGDBFieldType eType, bool bNullable, bool bRequired, + bool bEditable, int nMaxWidth, const OGRField &sDefault); virtual ~FileGDBField(); void SetParent(FileGDBTable *poParent) @@ -149,6 +158,16 @@ class FileGDBField return m_bNullable; } + bool IsRequired() const + { + return m_bRequired; + } + + bool IsEditable() const + { + return m_bEditable; + } + int GetMaxWidth() const { return m_nMaxWidth; @@ -161,12 +180,12 @@ class FileGDBField void SetHighPrecision() { - m_bHighPrecsion = true; + m_bHighPrecision = true; } bool IsHighPrecision() const { - return m_bHighPrecsion; + return m_bHighPrecision; } int HasIndex(); diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp index c8e61c7e6f3c..29d7a0cdba7c 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write_fields.cpp @@ -383,8 +383,17 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, WriteUTF16String(abyBuffer, psField->GetAlias().c_str(), NUMBER_OF_CHARS_ON_UINT8); WriteUInt8(abyBuffer, static_cast(psField->GetType())); - constexpr int UNKNOWN_FIELD_FLAG = 4; + const auto &sDefault = *(psField->GetDefault()); + + uint8_t nFlag = 0; + if (psField->IsNullable()) + nFlag = static_cast(nFlag | FileGDBField::MASK_NULLABLE); + if (psField->IsRequired()) + nFlag = static_cast(nFlag | FileGDBField::MASK_REQUIRED); + if (psField->IsEditable()) + nFlag = static_cast(nFlag | FileGDBField::MASK_EDITABLE); + switch (psField->GetType()) { case FGFT_UNDEFINED: @@ -396,9 +405,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_INT16: { WriteUInt8(abyBuffer, 2); // sizeof(int16) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -415,9 +422,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_INT32: { WriteUInt8(abyBuffer, 4); // sizeof(int32) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -434,9 +439,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_FLOAT32: { WriteUInt8(abyBuffer, 4); // sizeof(float32) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -453,9 +456,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_FLOAT64: { WriteUInt8(abyBuffer, 8); // sizeof(float64) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -472,9 +473,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_STRING: { WriteUInt32(abyBuffer, psField->GetMaxWidth()); - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -505,9 +504,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_DATE: { WriteUInt8(abyBuffer, 8); // sizeof(float64) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -536,9 +533,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, const auto *geomField = cpl::down_cast(psField); WriteUInt8(abyBuffer, 0); // unknown role - WriteUInt8(abyBuffer, static_cast( - 2 | UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); WriteUTF16String(abyBuffer, geomField->GetWKT().c_str(), NUMBER_OF_BYTES_ON_UINT16); WriteUInt8( @@ -600,9 +595,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_BINARY: { WriteUInt8(abyBuffer, 0); // unknown role - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); break; } @@ -617,27 +610,21 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_GLOBALID: { WriteUInt8(abyBuffer, 38); // size - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); break; } case FGFT_XML: { WriteUInt8(abyBuffer, 0); // unknown role - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); break; } case FGFT_INT64: { WriteUInt8(abyBuffer, 8); // sizeof(int64) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -654,9 +641,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_TIME: { WriteUInt8(abyBuffer, 8); // sizeof(float64) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -673,9 +658,7 @@ WriteFieldDescriptor(std::vector &abyBuffer, const FileGDBField *psField, case FGFT_DATETIME_WITH_OFFSET: { WriteUInt8(abyBuffer, 8 + 2); // sizeof(float64) + sizeof(int16) - WriteUInt8(abyBuffer, static_cast( - UNKNOWN_FIELD_FLAG | - static_cast(psField->IsNullable()))); + WriteUInt8(abyBuffer, nFlag); if (!OGR_RawField_IsNull(&sDefault) && !OGR_RawField_IsUnset(&sDefault)) { @@ -978,7 +961,8 @@ bool FileGDBTable::AlterField(int iField, const std::string &osName, auto poIndex = m_apoFields[iField]->m_poIndex; m_apoFields[iField] = std::make_unique( - osName, osAlias, eType, bNullable, nMaxWidth, sDefault); + osName, osAlias, eType, bNullable, m_apoFields[iField]->IsRequired(), + m_apoFields[iField]->IsEditable(), nMaxWidth, sDefault); m_apoFields[iField]->SetParent(this); m_apoFields[iField]->m_poIndex = poIndex; if (poIndex && bRenameField) diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp index 6cf35031911b..8b880ecc2903 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource_write.cpp @@ -566,14 +566,20 @@ bool OGROpenFileGDBDataSource::CreateGDBSystemCatalog() if (!oTable.Create(m_osGDBSystemCatalogFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "ID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Name", std::string(), FGFT_STRING, false, 160, - FileGDBField::UNSET_FIELD)) || + "Name", std::string(), FGFT_STRING, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 160, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "FileFormat", std::string(), FGFT_INT32, false, 0, - FileGDBField::UNSET_FIELD))) + "FileFormat", std::string(), FGFT_INT32, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD))) { return false; } @@ -616,14 +622,20 @@ bool OGROpenFileGDBDataSource::CreateGDBDBTune() FileGDBTable oTable; if (!oTable.Create(osFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "Keyword", std::string(), FGFT_STRING, false, 32, - FileGDBField::UNSET_FIELD)) || + "Keyword", std::string(), FGFT_STRING, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 32, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ParameterName", std::string(), FGFT_STRING, false, 32, - FileGDBField::UNSET_FIELD)) || + "ParameterName", std::string(), FGFT_STRING, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 32, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ConfigString", std::string(), FGFT_STRING, true, 2048, - FileGDBField::UNSET_FIELD))) + "ConfigString", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 2048, FileGDBField::UNSET_FIELD))) { return false; } @@ -709,41 +721,65 @@ bool OGROpenFileGDBDataSource::CreateGDBSpatialRefs() if (!oTable.Create(m_osGDBSpatialRefsFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "ID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "SRTEXT", std::string(), FGFT_STRING, false, 2048, - FileGDBField::UNSET_FIELD)) || + "SRTEXT", std::string(), FGFT_STRING, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 2048, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "FalseX", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "FalseX", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "FalseY", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "FalseY", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "XYUnits", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "XYUnits", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "FalseZ", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "FalseZ", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ZUnits", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "ZUnits", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "FalseM", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "FalseM", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "MUnits", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "MUnits", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "XYTolerance", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "XYTolerance", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ZTolerance", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD)) || + "ZTolerance", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "MTolerance", std::string(), FGFT_FLOAT64, true, 0, - FileGDBField::UNSET_FIELD))) + "MTolerance", std::string(), FGFT_FLOAT64, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD))) { return false; } @@ -789,53 +825,85 @@ bool OGROpenFileGDBDataSource::CreateGDBItems() if (!oTable.Create(m_osGDBItemsFilename.c_str(), 4, FGTGT_POLYGON, false, false) || !oTable.CreateField(std::make_unique( - "ObjectID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ObjectID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "UUID", std::string(), FGFT_GLOBALID, false, 0, - FileGDBField::UNSET_FIELD)) || + "UUID", std::string(), FGFT_GLOBALID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Type", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "Type", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Name", std::string(), FGFT_STRING, true, 160, - FileGDBField::UNSET_FIELD)) || + "Name", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 160, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "PhysicalName", std::string(), FGFT_STRING, true, 160, - FileGDBField::UNSET_FIELD)) || + "PhysicalName", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 160, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Path", std::string(), FGFT_STRING, true, 260, - FileGDBField::UNSET_FIELD)) || + "Path", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 260, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DatasetSubtype1", std::string(), FGFT_INT32, true, 0, - FileGDBField::UNSET_FIELD)) || + "DatasetSubtype1", std::string(), FGFT_INT32, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DatasetSubtype2", std::string(), FGFT_INT32, true, 0, - FileGDBField::UNSET_FIELD)) || + "DatasetSubtype2", std::string(), FGFT_INT32, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DatasetInfo1", std::string(), FGFT_STRING, true, 255, - FileGDBField::UNSET_FIELD)) || + "DatasetInfo1", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 255, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DatasetInfo2", std::string(), FGFT_STRING, true, 255, - FileGDBField::UNSET_FIELD)) || + "DatasetInfo2", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 255, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "URL", std::string(), FGFT_STRING, true, 255, - FileGDBField::UNSET_FIELD)) || + "URL", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 255, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Definition", std::string(), FGFT_XML, true, 0, - FileGDBField::UNSET_FIELD)) || + "Definition", std::string(), FGFT_XML, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Documentation", std::string(), FGFT_XML, true, 0, - FileGDBField::UNSET_FIELD)) || + "Documentation", std::string(), FGFT_XML, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ItemInfo", std::string(), FGFT_XML, true, 0, - FileGDBField::UNSET_FIELD)) || + "ItemInfo", std::string(), FGFT_XML, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Properties", std::string(), FGFT_INT32, true, 0, - FileGDBField::UNSET_FIELD)) || + "Properties", std::string(), FGFT_INT32, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Defaults", std::string(), FGFT_BINARY, true, 0, - FileGDBField::UNSET_FIELD)) || + "Defaults", std::string(), FGFT_BINARY, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::move(poGeomField))) { return false; @@ -899,17 +967,25 @@ bool OGROpenFileGDBDataSource::CreateGDBItemTypes() FileGDBTable oTable; if (!oTable.Create(osFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "ObjectID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ObjectID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "UUID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "UUID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ParentTypeID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ParentTypeID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Name", std::string(), FGFT_STRING, false, 160, - FileGDBField::UNSET_FIELD))) + "Name", std::string(), FGFT_STRING, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 160, FileGDBField::UNSET_FIELD))) { return false; } @@ -1014,26 +1090,40 @@ bool OGROpenFileGDBDataSource::CreateGDBItemRelationships() if (!oTable.Create(m_osGDBItemRelationshipsFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "ObjectID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ObjectID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "UUID", std::string(), FGFT_GLOBALID, false, 0, - FileGDBField::UNSET_FIELD)) || + "UUID", std::string(), FGFT_GLOBALID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "OriginID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "OriginID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DestID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "DestID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Type", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "Type", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Attributes", std::string(), FGFT_XML, true, 0, - FileGDBField::UNSET_FIELD)) || + "Attributes", std::string(), FGFT_XML, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Properties", std::string(), FGFT_INT32, true, 0, - FileGDBField::UNSET_FIELD))) + "Properties", std::string(), FGFT_INT32, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD))) { return false; } @@ -1057,29 +1147,45 @@ bool OGROpenFileGDBDataSource::CreateGDBItemRelationshipTypes() FileGDBTable oTable; if (!oTable.Create(osFilename.c_str(), 4, FGTGT_NONE, false, false) || !oTable.CreateField(std::make_unique( - "ObjectID", std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)) || + "ObjectID", std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "UUID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "UUID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "OrigItemTypeID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "OrigItemTypeID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "DestItemTypeID", std::string(), FGFT_GUID, false, 0, - FileGDBField::UNSET_FIELD)) || + "DestItemTypeID", std::string(), FGFT_GUID, + /* bNullable = */ false, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "Name", std::string(), FGFT_STRING, true, 160, - FileGDBField::UNSET_FIELD)) || + "Name", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 160, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "ForwardLabel", std::string(), FGFT_STRING, true, 255, - FileGDBField::UNSET_FIELD)) || + "ForwardLabel", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 255, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "BackwardLabel", std::string(), FGFT_STRING, true, 255, - FileGDBField::UNSET_FIELD)) || + "BackwardLabel", std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 255, FileGDBField::UNSET_FIELD)) || !oTable.CreateField(std::make_unique( - "IsContainment", std::string(), FGFT_INT16, true, 0, - FileGDBField::UNSET_FIELD))) + "IsContainment", std::string(), FGFT_INT16, + /* bNullable = */ true, + /* bRequired = */ false, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD))) { return false; } diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp index 819b24554db3..5cc636fc4477 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer_write.cpp @@ -584,8 +584,10 @@ bool OGROpenFileGDBLayer::Create(const OGRGeomFieldDefn *poSrcGeomFieldDefn) { OGRFieldDefn oField("field_before_geom", OFTString); m_poLyrTable->CreateField(std::make_unique( - oField.GetNameRef(), std::string(), FGFT_STRING, true, 0, - FileGDBField::UNSET_FIELD)); + oField.GetNameRef(), std::string(), FGFT_STRING, + /* bNullable = */ true, + /* bRequired = */ true, + /* bEditable = */ true, 0, FileGDBField::UNSET_FIELD)); m_poFeatureDefn->AddFieldDefn(&oField); } @@ -699,9 +701,11 @@ bool OGROpenFileGDBLayer::Create(const OGRGeomFieldDefn *poSrcGeomFieldDefn) const std::string osFIDName = m_aosCreationOptions.FetchNameValueDef("FID", "OBJECTID"); - if (!m_poLyrTable->CreateField(std::unique_ptr( - new FileGDBField(osFIDName, std::string(), FGFT_OBJECTID, false, 0, - FileGDBField::UNSET_FIELD)))) + if (!m_poLyrTable->CreateField(std::make_unique( + osFIDName, std::string(), FGFT_OBJECTID, + /* bNullable = */ false, + /* bRequired = */ true, + /* bEditable = */ false, 0, FileGDBField::UNSET_FIELD))) { Close(); return false; @@ -954,9 +958,18 @@ static CPLXMLNode *CreateXMLFieldDefinition(const OGRFieldDefn *poFieldDefn, CPLAddXMLAttributeAndValue(psFieldType, "xmlns:typens", "http://www.esri.com/schemas/ArcGIS/10.3"); } - CPLCreateXMLElementAndValue(GPFieldInfoEx, "IsNullable", - poGDBFieldDefn->IsNullable() ? "true" - : "false"); + if (poGDBFieldDefn->IsNullable()) + { + CPLCreateXMLElementAndValue(GPFieldInfoEx, "IsNullable", "true"); + } + if (poGDBFieldDefn->IsRequired()) + { + CPLCreateXMLElementAndValue(GPFieldInfoEx, "Required", "true"); + } + if (!poGDBFieldDefn->IsEditable()) + { + CPLCreateXMLElementAndValue(GPFieldInfoEx, "Editable", "false"); + } if (poGDBFieldDefn->IsHighPrecision()) { CPLCreateXMLElementAndValue(GPFieldInfoEx, "HighPrecision", "true"); @@ -1357,23 +1370,36 @@ OGRErr OGROpenFileGDBLayer::CreateField(const OGRFieldDefn *poFieldIn, } } - const char *pszAlias = poField->GetAlternativeNameRef(); - if (!m_poLyrTable->CreateField(std::make_unique( - poField->GetNameRef(), - pszAlias ? std::string(pszAlias) : std::string(), eType, - CPL_TO_BOOL(poField->IsNullable()), nWidth, sDefault))) - { - return OGRERR_FAILURE; - } + const bool bNullable = + CPL_TO_BOOL(poField->IsNullable()) && eType != FGFT_GLOBALID; + bool bRequired = (eType == FGFT_GLOBALID); + bool bEditable = (eType != FGFT_GLOBALID); if (poField->GetType() == OFTReal) { const char *pszDefault = poField->GetDefault(); if (pszDefault && EQUAL(pszDefault, "FILEGEODATABASE_SHAPE_AREA")) + { m_iAreaField = m_poFeatureDefn->GetFieldCount(); + bRequired = true; + bEditable = false; + } else if (pszDefault && EQUAL(pszDefault, "FILEGEODATABASE_SHAPE_LENGTH")) + { m_iLengthField = m_poFeatureDefn->GetFieldCount(); + bRequired = true; + bEditable = false; + } + } + + const char *pszAlias = poField->GetAlternativeNameRef(); + if (!m_poLyrTable->CreateField(std::make_unique( + poField->GetNameRef(), + pszAlias ? std::string(pszAlias) : std::string(), eType, bNullable, + bRequired, bEditable, nWidth, sDefault))) + { + return OGRERR_FAILURE; } whileUnsealing(m_poFeatureDefn)->AddFieldDefn(poField); diff --git a/ogr/ogrsf_frmts/openfilegdb/test_ofgdb_write.cpp b/ogr/ogrsf_frmts/openfilegdb/test_ofgdb_write.cpp index 268a0bb741d7..fdd50e1225b6 100644 --- a/ogr/ogrsf_frmts/openfilegdb/test_ofgdb_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/test_ofgdb_write.cpp @@ -40,24 +40,33 @@ int main() const int nTablxOffsetSize = 4; oTable.Create("test_ofgdb.gdbtable", nTablxOffsetSize, eTableGeomType, bGeomTypeHasZ, bGeomTypeHasM); + oTable.CreateField(std::unique_ptr(new FileGDBField( + "OBJECTID", "OBJECTID", FGFT_OBJECTID, + /* nullable = */ false, + /* required = */ true, + /* editable = */ false, 0, FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( - new FileGDBField("OBJECTID", "OBJECTID", FGFT_OBJECTID, false, 0, + new FileGDBField("int16", "", FGFT_INT16, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("int32", "", FGFT_INT32, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("float32", "", FGFT_FLOAT32, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("float64", "", FGFT_FLOAT64, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("str", "", FGFT_STRING, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("datetime", "", FGFT_DATETIME, true, false, true, 0, + FileGDBField::UNSET_FIELD))); + oTable.CreateField(std::unique_ptr( + new FileGDBField("binary", "", FGFT_BINARY, true, false, true, 0, FileGDBField::UNSET_FIELD))); - - oTable.CreateField(std::unique_ptr(new FileGDBField( - "int16", "", FGFT_INT16, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "int32", "", FGFT_INT32, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "float32", "", FGFT_FLOAT32, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "float64", "", FGFT_FLOAT64, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "str", "", FGFT_STRING, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "datetime", "", FGFT_DATETIME, true, 0, FileGDBField::UNSET_FIELD))); - oTable.CreateField(std::unique_ptr(new FileGDBField( - "binary", "", FGFT_BINARY, true, 0, FileGDBField::UNSET_FIELD))); auto poGeomField = std::unique_ptr(new FileGDBGeomField( "SHAPE", "", true, "{B286C06B-0879-11D2-AACA-00C04FA33C20}", -400, -400, From eeb0626cd2764a5292e1867f208387d2cfd874b5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 19:33:19 +0200 Subject: [PATCH 05/31] OpenFileGDB writer: .gdbtable header must be rewritten when updating an existing feature at the end of file The only consequence of the omission of that fix is that the file size in the .gdbtable could be incorrect. A wrong file size in the header has no consequence on OpenFileGDB reader/writer. It also doesn't seem to affect Esri reader side, but it could *potentially* affect the writing side, if that field was trusted to seek to the end of the file. --- ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp index aed5fe2a9b3b..945f710e9817 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_write.cpp @@ -2016,6 +2016,7 @@ bool FileGDBTable::UpdateFeature(int nFID, m_nRowBufferMaxSize = std::max(m_nRowBufferMaxSize, m_nRowBlobLength); if (nFreeOffset == OFFSET_MINUS_ONE) { + m_bDirtyHeader = true; m_nFileSize += sizeof(uint32_t) + m_nRowBlobLength; } From 87ee6b7ca602a9615d49a585992b605641ffb30f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 21:08:50 +0200 Subject: [PATCH 06/31] OpenFileGDB: BuildSRS(): do not use CPLErrorReset() --- ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp index 77dae116b5b7..f0b433a81162 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp @@ -2151,7 +2151,8 @@ OGROpenFileGDBDataSource::BuildSRS(const CPLXMLNode *psInfo) [](OGRSpatialReference &oSRS, int nLatestCode, int nCode) { bool bSuccess = false; - CPLPushErrorHandler(CPLQuietErrorHandler); + CPLErrorStateBackuper oQuietError(CPLQuietErrorHandler); + // Try first with nLatestWKID as there is a higher chance it is a // EPSG code and not an ESRI one. if (nLatestCode > 0) @@ -2193,8 +2194,7 @@ OGROpenFileGDBDataSource::BuildSRS(const CPLXMLNode *psInfo) CPLDebug("OpenFileGDB", "Cannot import SRID %d", nCode); } } - CPLPopErrorHandler(); - CPLErrorReset(); + return bSuccess; }; From a9d2913285fea2c62524a6fa498d4000a43ddcfe Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 21:10:08 +0200 Subject: [PATCH 07/31] OpenFileGDB: detect and try to repair corruption of .gdbtable header Versions of the driver before commit fdf39012788b1110b3bf0ae6b8422a528f0ae8b6 didn't properly update the m_nHeaderBufferMaxSize field when updating an existing feature when the new version takes more space than the previous version. OpenFileGDB doesn't care but Esri software (FileGDB SDK or ArcMap/ArcGis) do, leading to issues such as https://github.com/qgis/QGIS/issues/57536 --- autotest/ogr/ogr_openfilegdb_write.py | 55 +++++++++++++++++++ ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp | 53 ++++++++++++++++++ ogr/ogrsf_frmts/openfilegdb/filegdbtable.h | 5 ++ .../openfilegdb/ogropenfilegdbdatasource.cpp | 8 ++- 4 files changed, 120 insertions(+), 1 deletion(-) diff --git a/autotest/ogr/ogr_openfilegdb_write.py b/autotest/ogr/ogr_openfilegdb_write.py index 628f4b75a74c..08ef72a8221f 100755 --- a/autotest/ogr/ogr_openfilegdb_write.py +++ b/autotest/ogr/ogr_openfilegdb_write.py @@ -4510,3 +4510,58 @@ def test_ogr_openfilegdb_write_geom_coord_precision(tmp_vsimem): "ZTolerance": 0.0001, "HighPrecision": "true", } + + +############################################################################### +# Test repairing a corrupted header +# Scenario similar to https://github.com/qgis/QGIS/issues/57536 + + +def test_ogr_openfilegdb_repair_corrupted_header(tmp_vsimem): + + filename = str(tmp_vsimem / "out.gdb") + ds = ogr.GetDriverByName("OpenFileGDB").CreateDataSource(filename) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + lyr = ds.CreateLayer("test", srs, ogr.wkbLineString) + f = ogr.Feature(lyr.GetLayerDefn()) + g = ogr.Geometry(ogr.wkbLineString) + g.SetPoint_2D(10, 0, 0) + f.SetGeometry(g) + lyr.CreateFeature(f) + ds = None + + # Corrupt m_nHeaderBufferMaxSize field + corrupted_filename = filename + "/a00000004.gdbtable" + f = gdal.VSIFOpenL(corrupted_filename, "r+b") + assert f + gdal.VSIFSeekL(f, 8, 0) + gdal.VSIFWriteL(b"\x00" * 4, 4, 1, f) + gdal.VSIFCloseL(f) + + with gdal.config_option( + "OGR_OPENFILEGDB_ERROR_ON_INCONSISTENT_BUFFER_MAX_SIZE", "NO" + ), gdal.quiet_errors(): + ds = ogr.Open(filename) + assert ( + gdal.GetLastErrorMsg() + == f"A corruption in the header of {corrupted_filename} has been detected. It would need to be repaired to be properly read by other software, either by using ogr2ogr to generate a new dataset, or by opening this dataset in update mode and reading all its records." + ) + assert ds.GetLayerCount() == 1 + + with gdal.config_option( + "OGR_OPENFILEGDB_ERROR_ON_INCONSISTENT_BUFFER_MAX_SIZE", "NO" + ), gdal.quiet_errors(): + ds = ogr.Open(filename, update=1) + assert ( + gdal.GetLastErrorMsg() + == f"A corruption in the header of {corrupted_filename} has been detected. It is going to be repaired to be properly read by other software." + ) + assert ds.GetLayerCount() == 1 + + with gdal.config_option( + "OGR_OPENFILEGDB_ERROR_ON_INCONSISTENT_BUFFER_MAX_SIZE", "NO" + ), gdal.quiet_errors(): + ds = ogr.Open(filename) + assert gdal.GetLastErrorMsg() == "" + assert ds.GetLayerCount() == 1 diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp index 29d4dbb54ab3..284dc2a16242 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp @@ -1550,11 +1550,64 @@ int FileGDBTable::SelectRow(int iRow) } else { + // Versions of the driver before commit + // fdf39012788b1110b3bf0ae6b8422a528f0ae8b6 didn't + // properly update the m_nHeaderBufferMaxSize field + // when updating an existing feature when the new version + // takes more space than the previous version. + // OpenFileGDB doesn't care but Esri software (FileGDB SDK + // or ArcMap/ArcGis) do, leading to issues such as + // https://github.com/qgis/QGIS/issues/57536 + CPLDebug("OpenFileGDB", "Invalid row length (%u) on feature %u compared " "to the maximum size in the header (%u)", m_nRowBlobLength, iRow + 1, m_nHeaderBufferMaxSize); + + if (m_bUpdate) + { + if (!m_bHasWarnedAboutHeaderRepair) + { + m_bHasWarnedAboutHeaderRepair = true; + CPLError(CE_Warning, CPLE_AppDefined, + "A corruption in the header of %s has " + "been detected. It is going to be " + "repaired to be properly read by other " + "software.", + m_osFilename.c_str()); + + m_bDirtyHeader = true; + + // Invalidate existing indices, as the corrupted + // m_nHeaderBufferMaxSize value may have cause + // Esri software to generate corrupted indices. + m_bDirtyIndices = true; + + // Compute file size + VSIFSeekL(m_fpTable, 0, SEEK_END); + m_nFileSize = VSIFTellL(m_fpTable); + VSIFSeekL(m_fpTable, nOffsetTable + 4, SEEK_SET); + } + } + else + { + if (!m_bHasWarnedAboutHeaderRepair) + { + m_bHasWarnedAboutHeaderRepair = true; + CPLError(CE_Warning, CPLE_AppDefined, + "A corruption in the header of %s has " + "been detected. It would need to be " + "repaired to be properly read by other " + "software, either by using ogr2ogr to " + "generate a new dataset, or by opening " + "this dataset in update mode and reading " + "all its records.", + m_osFilename.c_str()); + } + } + + m_nHeaderBufferMaxSize = m_nRowBlobLength; } } diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h index c55082b44041..11a27363b626 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h @@ -450,6 +450,11 @@ class FileGDBTable vsi_l_offset m_nFileSize = 0; /* only read when needed */ bool m_bUpdate = false; + //! This flag is set when we detect that a corruption of m_nHeaderBufferMaxSize + // prior to fix needs to fdf39012788b1110b3bf0ae6b8422a528f0ae8b6 to be + // repaired + bool m_bHasWarnedAboutHeaderRepair = false; + std::string m_osFilename{}; bool m_bIsV9 = false; std::vector> m_apoFields{}; diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp index f0b433a81162..0fa057fa2987 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdbdatasource.cpp @@ -681,7 +681,13 @@ bool OGROpenFileGDBDataSource::OpenFileGDBv10( CPLString osFilename(CPLFormFilename( m_osDirName, CPLSPrintf("a%08x.gdbtable", iGDBItems + 1), nullptr)); - if (!oTable.Open(osFilename, false)) + + // Normally we don't need to update in update mode the GDB_Items table, + // but this may help repairing it, if we have corrupted it with past + // GDAL versions. + const bool bOpenInUpdateMode = + (poOpenInfo->nOpenFlags & GDAL_OF_UPDATE) != 0; + if (!oTable.Open(osFilename, bOpenInUpdateMode)) return false; const int iUUID = oTable.GetFieldIdx("UUID"); From 48f78793d8af9cb78eb129c05252026cea15bb4d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 27 May 2024 19:03:23 +0200 Subject: [PATCH 08/31] Add GDALArgumentParser::add_extra_usage_hint() --- apps/gdalargumentparser.cpp | 26 ++++++++++++++++++++++++++ apps/gdalargumentparser.h | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/apps/gdalargumentparser.cpp b/apps/gdalargumentparser.cpp index 1d70e28ea28c..c70a939be599 100644 --- a/apps/gdalargumentparser.cpp +++ b/apps/gdalargumentparser.cpp @@ -103,6 +103,32 @@ void GDALArgumentParser::display_error_and_usage(const std::exception &err) << _(" --long-usage for full help.") << std::endl; } +/************************************************************************/ +/* usage() */ +/************************************************************************/ + +std::string GDALArgumentParser::usage() const +{ + std::string ret(ArgumentParser::usage()); + if (!m_osExtraUsageHint.empty()) + { + ret += '\n'; + ret += '\n'; + ret += m_osExtraUsageHint; + } + return ret; +} + +/************************************************************************/ +/* add_extra_usage_hint() */ +/************************************************************************/ + +void GDALArgumentParser::add_extra_usage_hint( + const std::string &osExtraUsageHint) +{ + m_osExtraUsageHint = osExtraUsageHint; +} + /************************************************************************/ /* add_quiet_argument() */ /************************************************************************/ diff --git a/apps/gdalargumentparser.h b/apps/gdalargumentparser.h index 5a6f162109d8..bb4ebb522534 100644 --- a/apps/gdalargumentparser.h +++ b/apps/gdalargumentparser.h @@ -61,6 +61,12 @@ class GDALArgumentParser : public ArgumentParser explicit GDALArgumentParser(const std::string &program_name, bool bForBinary); + //! Return usage message + std::string usage() const; + + //! Adds an extra usage hint. + void add_extra_usage_hint(const std::string &osExtraUsageHint); + //! Format an exception as an error message and display the program usage void display_error_and_usage(const std::exception &err); @@ -141,6 +147,7 @@ class GDALArgumentParser : public ArgumentParser std::map::iterator find_argument(const std::string &name); std::vector> aoSubparsers; + std::string m_osExtraUsageHint{}; }; #endif /* GDALARGUMENTPARSER_H */ From d9db075865097dea6e3e06c044f949ccea59a5e0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 27 May 2024 19:03:43 +0200 Subject: [PATCH 09/31] gdal_contour: add help hint that one and only of -i, -fl or -e must be used --- apps/gdal_contour.cpp | 10 ++++++++++ autotest/utilities/test_gdal_contour.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/apps/gdal_contour.cpp b/apps/gdal_contour.cpp index 8b57c62dd0df..f5d70e592694 100644 --- a/apps/gdal_contour.cpp +++ b/apps/gdal_contour.cpp @@ -81,6 +81,9 @@ GDALContourAppOptionsGetParser(GDALContourOptions *psOptions) "For more details, consult the full documentation for the gdal_contour " "utility: http://gdal.org/gdal_contour.html")); + argParser->add_extra_usage_hint( + _("One and only one of -i, -fl or -e must be specified.")); + argParser->add_argument("-b") .metavar("") .default_value(1) @@ -250,6 +253,13 @@ MAIN_START(argc, argv) { auto argParser = GDALContourAppOptionsGetParser(&sOptions); argParser->parse_args_without_binary_name(argv + 1); + + if (sOptions.dfInterval == 0.0 && sOptions.adfFixedLevels.empty() && + sOptions.dfExpBase == 0.0) + { + fprintf(stderr, "%s\n", argParser->usage().c_str()); + exit(1); + } } catch (const std::exception &error) { diff --git a/autotest/utilities/test_gdal_contour.py b/autotest/utilities/test_gdal_contour.py index c4af38485bd2..772c503d1581 100755 --- a/autotest/utilities/test_gdal_contour.py +++ b/autotest/utilities/test_gdal_contour.py @@ -388,3 +388,17 @@ def test_gdal_contour_5(gdal_contour_path, tmp_path): i = i + 1 feat = lyr.GetNextFeature() + + +############################################################################### +# Test missing -fl, -i or -e + + +def test_gdal_contour_missing_fl_i_or_e(gdal_contour_path, testdata_tif, tmp_path): + + contour_shp = str(tmp_path / "contour.shp") + + _, err = gdaltest.runexternal_out_and_err( + gdal_contour_path + f" {testdata_tif} {contour_shp}" + ) + assert "One and only one of -i, -fl or -e must be specified." in err From 16ade8253f26200246abb5ab24d17e18216e7a11 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 29 May 2024 00:52:05 +0200 Subject: [PATCH 10/31] HDF5: add support for libhdf5 >= 1.14.4.2 when built with Float16 Fixes https://github.com/HDFGroup/hdf5/issues/4527 --- autotest/gdrivers/hdf5multidim.py | 64 +++++++++++++++++++++ frmts/hdf5/gh5_convenience.cpp | 11 ++++ frmts/hdf5/hdf5_api.h | 4 ++ frmts/hdf5/hdf5dataset.cpp | 68 ++++++++++++++++++++++ frmts/hdf5/hdf5dataset.h | 4 +- frmts/hdf5/hdf5imagedataset.cpp | 95 +++++++++++++++++++++++++++---- frmts/hdf5/hdf5multidim.cpp | 63 +++++++++++++++++++- 7 files changed, 293 insertions(+), 16 deletions(-) diff --git a/autotest/gdrivers/hdf5multidim.py b/autotest/gdrivers/hdf5multidim.py index 6f1f003c326f..31260e9ab766 100755 --- a/autotest/gdrivers/hdf5multidim.py +++ b/autotest/gdrivers/hdf5multidim.py @@ -840,3 +840,67 @@ def test_hdf5_multidim_block_size_structural_info(): var = rg.OpenMDArray("Band1") assert var.GetBlockSize() == [1, 2] assert var.GetStructuralInfo() == {"COMPRESSION": "DEFLATE", "FILTER": "SHUFFLE"} + + +############################################################################### +# Test reading a compound data type made of 2 Float16 values + + +def test_hdf5_multidim_read_cfloat16(): + + ds = gdal.OpenEx("data/hdf5/complex.h5", gdal.OF_MULTIDIM_RASTER) + rg = ds.GetRootGroup() + var = rg.OpenMDArray("f16") + assert var.GetDataType().GetNumericDataType() == gdal.GDT_CFloat32 + assert struct.unpack("f" * (5 * 5 * 2), var.Read()) == ( + 0.0, + 0.0, + 1.0, + 1.0, + 2.0, + 2.0, + 3.0, + 3.0, + 4.0, + 4.0, + 5.0, + 5.0, + 6.0, + 6.0, + 7.0, + 7.0, + 8.0, + 8.0, + 9.0, + 9.0, + 10.0, + 10.0, + 11.0, + 11.0, + 12.0, + 12.0, + 13.0, + 13.0, + 14.0, + 14.0, + 15.0, + 15.0, + 16.0, + 16.0, + 17.0, + 17.0, + 18.0, + 18.0, + 19.0, + 19.0, + 20.0, + 20.0, + 21.0, + 21.0, + 22.0, + 22.0, + 23.0, + 23.0, + 24.0, + 24.0, + ) diff --git a/frmts/hdf5/gh5_convenience.cpp b/frmts/hdf5/gh5_convenience.cpp index 304032e08e3f..e48b5c630d21 100644 --- a/frmts/hdf5/gh5_convenience.cpp +++ b/frmts/hdf5/gh5_convenience.cpp @@ -26,6 +26,7 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ +#include "cpl_float.h" #include "gh5_convenience.h" /************************************************************************/ @@ -209,6 +210,16 @@ bool GH5_FetchAttribute(hid_t loc_id, const char *pszAttrName, double &dfResult, pszAttrName, static_cast(nVal), dfResult); } } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, hAttrNativeType)) + { + const uint16_t nVal16 = *((uint16_t *)buf); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + dfResult = fVal; + } +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, hAttrNativeType)) dfResult = *((float *)buf); else if (H5Tequal(H5T_NATIVE_DOUBLE, hAttrNativeType)) diff --git a/frmts/hdf5/hdf5_api.h b/frmts/hdf5/hdf5_api.h index 013534310488..0b3110d2cc2e 100644 --- a/frmts/hdf5/hdf5_api.h +++ b/frmts/hdf5/hdf5_api.h @@ -46,6 +46,10 @@ #include "hdf5.h" +#if defined(H5T_NATIVE_FLOAT16) && defined(H5_HAVE__FLOAT16) +#define HDF5_HAVE_FLOAT16 +#endif + #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/frmts/hdf5/hdf5dataset.cpp b/frmts/hdf5/hdf5dataset.cpp index 499b8a86c877..64c1fea2e92b 100644 --- a/frmts/hdf5/hdf5dataset.cpp +++ b/frmts/hdf5/hdf5dataset.cpp @@ -44,6 +44,7 @@ #include "cpl_conv.h" #include "cpl_error.h" +#include "cpl_float.h" #include "cpl_string.h" #include "gdal.h" #include "gdal_frmts.h" @@ -201,6 +202,10 @@ GDALDataType HDF5Dataset::GetDataType(hid_t TypeID) return GDT_Unknown; #endif } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, TypeID)) + return GDT_Float32; +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, TypeID)) return GDT_Float32; else if (H5Tequal(H5T_NATIVE_DOUBLE, TypeID)) @@ -258,6 +263,10 @@ GDALDataType HDF5Dataset::GetDataType(hid_t TypeID) eDataType = GDT_Unknown; #endif } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, ElemTypeID)) + eDataType = GDT_CFloat32; +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, ElemTypeID)) eDataType = GDT_CFloat32; else if (H5Tequal(H5T_NATIVE_DOUBLE, ElemTypeID)) @@ -272,6 +281,32 @@ GDALDataType HDF5Dataset::GetDataType(hid_t TypeID) return GDT_Unknown; } +/************************************************************************/ +/* IsNativeCFloat16() */ +/************************************************************************/ + +/* static*/ bool HDF5Dataset::IsNativeCFloat16(hid_t hDataType) +{ +#ifdef HDF5_HAVE_FLOAT16 + // For complex the compound type must contain 2 elements + if (H5Tget_class(hDataType) != H5T_COMPOUND || + H5Tget_nmembers(hDataType) != 2) + return false; + + // For complex the native types of both elements should be the same + hid_t ElemTypeID = H5Tget_member_type(hDataType, 0); + hid_t Elem2TypeID = H5Tget_member_type(hDataType, 1); + const bool bRet = H5Tequal(ElemTypeID, H5T_NATIVE_FLOAT16) > 0 && + H5Tequal(Elem2TypeID, H5T_NATIVE_FLOAT16) > 0; + H5Tclose(ElemTypeID); + H5Tclose(Elem2TypeID); + return bRet; +#else + CPL_IGNORE_RET_VAL(hDataType); + return false; +#endif +} + /************************************************************************/ /* GetDataTypeName() */ /* */ @@ -304,6 +339,10 @@ const char *HDF5Dataset::GetDataTypeName(hid_t TypeID) return "32/64-bit integer"; else if (H5Tequal(H5T_NATIVE_ULONG, TypeID)) return "32/64-bit unsigned integer"; +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, TypeID)) + return "16-bit floating-point"; +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, TypeID)) return "32-bit floating-point"; else if (H5Tequal(H5T_NATIVE_DOUBLE, TypeID)) @@ -348,6 +387,13 @@ const char *HDF5Dataset::GetDataTypeName(hid_t TypeID) H5Tclose(ElemTypeID); return "complex, 32/64-bit integer"; } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, ElemTypeID)) + { + H5Tclose(ElemTypeID); + return "complex, 16-bit floating-point"; + } +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, ElemTypeID)) { H5Tclose(ElemTypeID); @@ -1132,6 +1178,28 @@ static herr_t HDF5AttrIterate(hid_t hH5ObjID, const char *pszAttrName, psContext->m_osValue += szData; } } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(H5T_NATIVE_FLOAT16, hAttrNativeType) > 0) + { + for (hsize_t i = 0; i < nAttrElmts; i++) + { + const uint16_t nVal16 = static_cast(buf)[i]; + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + CPLsnprintf(szData, nDataLen, "%.8g", fVal); + if (psContext->m_osValue.size() > MAX_METADATA_LEN) + { + CPLError(CE_Warning, CPLE_OutOfMemory, + "Header data too long. Truncated"); + break; + } + if (i > 0) + psContext->m_osValue += ' '; + psContext->m_osValue += szData; + } + } +#endif else if (H5Tequal(H5T_NATIVE_FLOAT, hAttrNativeType) > 0) { for (hsize_t i = 0; i < nAttrElmts; i++) diff --git a/frmts/hdf5/hdf5dataset.h b/frmts/hdf5/hdf5dataset.h index dc9245b4ca31..646f7c6b6297 100644 --- a/frmts/hdf5/hdf5dataset.h +++ b/frmts/hdf5/hdf5dataset.h @@ -240,8 +240,6 @@ class HDF5Dataset CPL_NON_FINAL : public GDALPamDataset char *CreatePath(HDF5GroupObjects *); static void DestroyH5Objects(HDF5GroupObjects *); - static const char *GetDataTypeName(hid_t); - /** * Reads an array of double attributes from the HDF5 metadata. * It reads the attributes directly on its binary form directly, @@ -275,6 +273,8 @@ class HDF5Dataset CPL_NON_FINAL : public GDALPamDataset static std::shared_ptr OpenGroup( const std::shared_ptr &poSharedResources); + static bool IsNativeCFloat16(hid_t hDataType); + static const char *GetDataTypeName(hid_t); static GDALDataType GetDataType(hid_t); }; diff --git a/frmts/hdf5/hdf5imagedataset.cpp b/frmts/hdf5/hdf5imagedataset.cpp index 64a2c33917ea..4d140c523f28 100644 --- a/frmts/hdf5/hdf5imagedataset.cpp +++ b/frmts/hdf5/hdf5imagedataset.cpp @@ -29,6 +29,7 @@ #include "hdf5_api.h" +#include "cpl_float.h" #include "cpl_string.h" #include "gdal_frmts.h" #include "gdal_pam.h" @@ -72,9 +73,10 @@ class HDF5ImageDataset final : public HDF5Dataset int dimensions; hid_t dataset_id; hid_t dataspace_id; - hsize_t size; - hid_t datatype; hid_t native; +#ifdef HDF5_HAVE_FLOAT16 + bool m_bConvertFromFloat16 = false; +#endif Hdf5ProductType iSubdatasetType; HDF5CSKProductEnum iCSKProductType; double adfGeoTransform[6]; @@ -209,9 +211,9 @@ class HDF5ImageDataset final : public HDF5Dataset /************************************************************************/ HDF5ImageDataset::HDF5ImageDataset() : dims(nullptr), maxdims(nullptr), poH5Objects(nullptr), ndims(0), - dimensions(0), dataset_id(-1), dataspace_id(-1), size(0), datatype(-1), - native(-1), iSubdatasetType(UNKNOWN_PRODUCT), - iCSKProductType(PROD_UNKNOWN), bHasGeoTransform(false) + dimensions(0), dataset_id(-1), dataspace_id(-1), native(-1), + iSubdatasetType(UNKNOWN_PRODUCT), iCSKProductType(PROD_UNKNOWN), + bHasGeoTransform(false) { m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); m_oGCPSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); @@ -236,8 +238,6 @@ HDF5ImageDataset::~HDF5ImageDataset() H5Dclose(dataset_id); if (dataspace_id > 0) H5Sclose(dataspace_id); - if (datatype > 0) - H5Tclose(datatype); if (native > 0) H5Tclose(native); @@ -465,6 +465,42 @@ CPLErr HDF5ImageRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, return CE_Failure; } +#ifdef HDF5_HAVE_FLOAT16 + if (eDataType == GDT_Float32 && poGDS->m_bConvertFromFloat16) + { + for (size_t i = static_cast(nBlockXSize) * nBlockYSize; i > 0; + /* do nothing */) + { + --i; + uint16_t nVal16; + memcpy(&nVal16, static_cast(pImage) + i, + sizeof(nVal16)); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + *(static_cast(pImage) + i) = fVal; + } + } + else if (eDataType == GDT_CFloat32 && poGDS->m_bConvertFromFloat16) + { + for (size_t i = static_cast(nBlockXSize) * nBlockYSize; i > 0; + /* do nothing */) + { + --i; + for (int j = 1; j >= 0; --j) + { + uint16_t nVal16; + memcpy(&nVal16, static_cast(pImage) + 2 * i + j, + sizeof(nVal16)); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + *(static_cast(pImage) + 2 * i + j) = fVal; + } + } + } +#endif + return CE_None; } @@ -482,6 +518,15 @@ CPLErr HDF5ImageRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, { HDF5ImageDataset *poGDS = static_cast(poDS); +#ifdef HDF5_HAVE_FLOAT16 + if (poGDS->m_bConvertFromFloat16) + { + return GDALPamRasterBand::IRasterIO( + eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, + eBufType, nPixelSpace, nLineSpace, psExtraArg); + } +#endif + const bool bIsBandInterleavedData = poGDS->ndims == 3 && poGDS->m_nOtherDimIndex == 0 && poGDS->GetYIndex() == 1 && poGDS->GetXIndex() == 2; @@ -761,6 +806,16 @@ CPLErr HDF5ImageDataset::IRasterIO( GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg) { +#ifdef HDF5_HAVE_FLOAT16 + if (m_bConvertFromFloat16) + { + return HDF5Dataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, eBufType, + nBandCount, panBandMap, nPixelSpace, + nLineSpace, nBandSpace, psExtraArg); + } +#endif + const auto IsConsecutiveBands = [](const int *panVals, int nCount) { for (int i = 1; i < nCount; ++i) @@ -1021,9 +1076,25 @@ GDALDataset *HDF5ImageDataset::Open(GDALOpenInfo *poOpenInfo) static_cast(CPLCalloc(poDS->ndims, sizeof(hsize_t))); poDS->dimensions = H5Sget_simple_extent_dims(poDS->dataspace_id, poDS->dims, poDS->maxdims); - poDS->datatype = H5Dget_type(poDS->dataset_id); - poDS->size = H5Tget_size(poDS->datatype); - poDS->native = H5Tget_native_type(poDS->datatype, H5T_DIR_ASCEND); + auto datatype = H5Dget_type(poDS->dataset_id); + poDS->native = H5Tget_native_type(datatype, H5T_DIR_ASCEND); + H5Tclose(datatype); + + const auto eGDALDataType = poDS->GetDataType(poDS->native); + if (eGDALDataType == GDT_Unknown) + { + CPLError(CE_Failure, CPLE_AppDefined, "Unhandled HDF5 data type"); + delete poDS; + return nullptr; + } + +#ifdef HDF5_HAVE_FLOAT16 + if (H5Tequal(H5T_NATIVE_FLOAT16, poDS->native) || + IsNativeCFloat16(poDS->native)) + { + poDS->m_bConvertFromFloat16 = true; + } +#endif // CSK code in IdentifyProductType() and CreateProjections() // uses dataset metadata. @@ -1239,8 +1310,8 @@ GDALDataset *HDF5ImageDataset::Open(GDALOpenInfo *poOpenInfo) for (int i = 0; i < nBands; i++) { - HDF5ImageRasterBand *const poBand = new HDF5ImageRasterBand( - poDS, i + 1, poDS->GetDataType(poDS->native)); + HDF5ImageRasterBand *const poBand = + new HDF5ImageRasterBand(poDS, i + 1, eGDALDataType); poDS->SetBand(i + 1, poBand); diff --git a/frmts/hdf5/hdf5multidim.cpp b/frmts/hdf5/hdf5multidim.cpp index fa27ff79a051..f5a0704c458f 100644 --- a/frmts/hdf5/hdf5multidim.cpp +++ b/frmts/hdf5/hdf5multidim.cpp @@ -29,6 +29,8 @@ #include "hdf5eosparser.h" #include "s100.h" +#include "cpl_float.h" + #include #include #include @@ -159,7 +161,16 @@ BuildDataType(hid_t hDataType, bool &bHasString, bool &bNonNativeDataType, const auto klass = H5Tget_class(hDataType); GDALDataType eDT = ::HDF5Dataset::GetDataType(hDataType); if (eDT != GDT_Unknown) + { +#ifdef HDF5_HAVE_FLOAT16 + if (H5Tequal(hDataType, H5T_NATIVE_FLOAT16) || + HDF5Dataset::IsNativeCFloat16(hDataType)) + { + bNonNativeDataType = true; + } +#endif return GDALExtendedDataType::Create(eDT); + } else if (klass == H5T_STRING) { bHasString = true; @@ -2326,9 +2337,44 @@ static void CopyValue(const GByte *pabySrcBuffer, hid_t hSrcDataType, { if (dstDataType.GetClass() != GEDTC_COMPOUND) { + const auto eSrcDataType = ::HDF5Dataset::GetDataType(hSrcDataType); // Typically source is complex data type - auto srcDataType(GDALExtendedDataType::Create( - ::HDF5Dataset::GetDataType(hSrcDataType))); +#ifdef HDF5_HAVE_FLOAT16 + if (eSrcDataType == GDT_CFloat32 && + ::HDF5Dataset::IsNativeCFloat16(hSrcDataType)) + { + if (dstDataType.GetNumericDataType() == GDT_CFloat32) + { + for (int j = 0; j <= 1; ++j) + { + uint16_t nVal16; + memcpy(&nVal16, pabySrcBuffer + j * sizeof(nVal16), + sizeof(nVal16)); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + memcpy(pabyDstBuffer + j * sizeof(float), &nVal32, + sizeof(nVal32)); + } + } + else if (dstDataType.GetNumericDataType() == GDT_CFloat64) + { + for (int j = 0; j <= 1; ++j) + { + uint16_t nVal16; + memcpy(&nVal16, pabySrcBuffer + j * sizeof(nVal16), + sizeof(nVal16)); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + double dfVal = fVal; + memcpy(pabyDstBuffer + j * sizeof(double), &dfVal, + sizeof(dfVal)); + } + } + return; + } + +#endif + auto srcDataType(GDALExtendedDataType::Create(eSrcDataType)); if (srcDataType.GetClass() == GEDTC_NUMERIC && srcDataType.GetNumericDataType() != GDT_Unknown) { @@ -2364,6 +2410,19 @@ static void CopyValue(const GByte *pabySrcBuffer, hid_t hSrcDataType, CopyValue(pabySrcBuffer, hParent, pabyDstBuffer, dstDataType, {}); H5Tclose(hParent); } +#ifdef HDF5_HAVE_FLOAT16 + else if (H5Tequal(hSrcDataType, H5T_NATIVE_FLOAT16)) + { + uint16_t nVal16; + memcpy(&nVal16, pabySrcBuffer, sizeof(nVal16)); + const uint32_t nVal32 = CPLHalfToFloat(nVal16); + float fVal; + memcpy(&fVal, &nVal32, sizeof(fVal)); + GDALExtendedDataType::CopyValue( + &fVal, GDALExtendedDataType::Create(GDT_Float32), pabyDstBuffer, + dstDataType); + } +#endif else { GDALDataType eDT = ::HDF5Dataset::GetDataType(hSrcDataType); From 27119019fd0901e4f78a3cc324e0424c8768b263 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 29 May 2024 16:54:47 +0200 Subject: [PATCH 11/31] GML: GML_SKIP_RESOLVE_ELEMS=HUGE: keep gml:id in .resolved.gml file --- ogr/ogrsf_frmts/gml/hugefileresolver.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp index 1dac15fbb498..23f9ee2c63f1 100644 --- a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp +++ b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp @@ -1701,7 +1701,15 @@ static bool gmlHugeFileWriteResolved(huge_helper *helper, const int iPropCount = poClass->GetPropertyCount(); bool b_has_geom = false; - VSIFPrintfL(fp, " <%s>\n", poClass->GetElementName()); + VSIFPrintfL(fp, " <%s", poClass->GetElementName()); + const char *pszGmlId = poFeature->GetFID(); + if (pszGmlId) + { + char *gmlText = CPLEscapeString(pszGmlId, -1, CPLES_XML); + VSIFPrintfL(fp, " gml:id=\"%s\"", gmlText); + CPLFree(gmlText); + } + VSIFPrintfL(fp, ">\n"); for (int iProp = 0; iProp < iPropCount; iProp++) { From 3d111b732fb5ccceb821858c57614e1ca3f15ecf Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 28 May 2024 00:37:23 +0200 Subject: [PATCH 12/31] GML: avoid assertion due to trying to load existing .gfs file after reading .resolved.gml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #10015 --- ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp index cb33c90c291c..4ef9d4f6b47a 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp @@ -850,7 +850,8 @@ bool OGRGMLDataSource::Open(GDALOpenInfo *poOpenInfo) } // Can we find a GML Feature Schema (.gfs) for the input file? - if (!osGFSFilename.empty() && !bHaveSchema && osXSDFilename.empty()) + if (!osGFSFilename.empty() && !bHaveSchema && !bSchemaDone && + osXSDFilename.empty()) { VSIStatBufL sGFSStatBuf; if (bCheckAuxFile && VSIStatL(osGFSFilename, &sGFSStatBuf) == 0) From d74d338f98e0f00b842f1b1f65f46204c848a178 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 28 May 2024 00:38:26 +0200 Subject: [PATCH 13/31] GML: fix issue with nested elements with identical names in GML_SKIP_RESOLVE_ELEMS=HUGE mode Fixes #10015 --- .../data/gml/same_nested_property_name.gml | 14 +++++++++ autotest/ogr/ogr_gml.py | 29 +++++++++++++++++++ ogr/ogrsf_frmts/gml/hugefileresolver.cpp | 23 +++++++++++++-- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 autotest/ogr/data/gml/same_nested_property_name.gml diff --git a/autotest/ogr/data/gml/same_nested_property_name.gml b/autotest/ogr/data/gml/same_nested_property_name.gml new file mode 100644 index 000000000000..11dffc32bc8d --- /dev/null +++ b/autotest/ogr/data/gml/same_nested_property_name.gml @@ -0,0 +1,14 @@ + + + + + 0 0 + foo + bar + + + diff --git a/autotest/ogr/ogr_gml.py b/autotest/ogr/ogr_gml.py index 0d592827c032..eadda1826340 100755 --- a/autotest/ogr/ogr_gml.py +++ b/autotest/ogr/ogr_gml.py @@ -1362,6 +1362,35 @@ def test_ogr_gml_38(tmp_path, resolver): ds = None +############################################################################### +# Test GML_SKIP_RESOLVE_ELEMS=HUGE with a file with 2 nested identical property +# names + + +@pytest.mark.require_driver("SQLite") +@pytest.mark.require_geos +def test_ogr_gml_huge_resolver_same_nested_property_name(tmp_path): + + shutil.copy( + "data/gml/same_nested_property_name.gml", + tmp_path, + ) + + def check_ds(ds): + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f["gml_id"] == "test.0" + assert f["test"] == "foo" + assert f["bar|test"] == "bar" + + ds = ogr.Open(tmp_path / "same_nested_property_name.gml") + check_ds(ds) + + with gdal.config_option("GML_SKIP_RESOLVE_ELEMS", "HUGE"): + ds = ogr.Open(tmp_path / "same_nested_property_name.gml") + check_ds(ds) + + ############################################################################### # Test parsing XSD where simpleTypes not inlined, but defined elsewhere in the .xsd (#4328) diff --git a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp index 23f9ee2c63f1..971a9e57e4df 100644 --- a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp +++ b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp @@ -1723,8 +1723,27 @@ static bool gmlHugeFileWriteResolved(huge_helper *helper, { char *gmlText = CPLEscapeString( poProp->papszSubProperties[iSub], -1, CPLES_XML); - VSIFPrintfL(fp, " <%s>%s\n", pszPropName, gmlText, - pszPropName); + if (strchr(pszPropName, '|')) + { + const CPLStringList aosPropNameComps( + CSLTokenizeString2(pszPropName, "|", 0)); + VSIFPrintfL(fp, " "); + for (int i = 0; i < aosPropNameComps.size(); ++i) + { + VSIFPrintfL(fp, "<%s>", aosPropNameComps[i]); + } + VSIFPrintfL(fp, "%s", gmlText); + for (int i = aosPropNameComps.size() - 1; i >= 0; --i) + { + VSIFPrintfL(fp, "", aosPropNameComps[i]); + } + VSIFPrintfL(fp, "\n"); + } + else + { + VSIFPrintfL(fp, " <%s>%s\n", pszPropName, + gmlText, pszPropName); + } CPLFree(gmlText); } } From 8234d9d07d4eaac586a053da0d7443ff12a10de7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 29 May 2024 15:02:39 +0200 Subject: [PATCH 14/31] GML: make sure SRS is detected when using GML_SKIP_RESOLVE_ELEMS=HUGE on a AIXM file, otherwise GML_SWAP_COORDINATES might not work correctly --- ogr/ogrsf_frmts/gml/gmlreaderp.h | 5 +++++ ogr/ogrsf_frmts/gml/hugefileresolver.cpp | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ogr/ogrsf_frmts/gml/gmlreaderp.h b/ogr/ogrsf_frmts/gml/gmlreaderp.h index 39e80b1a84c2..9bba194cdf45 100644 --- a/ogr/ogrsf_frmts/gml/gmlreaderp.h +++ b/ogr/ogrsf_frmts/gml/gmlreaderp.h @@ -226,6 +226,11 @@ class GMLHandler const char *pszAttributeName) = 0; virtual char *GetAttributeByIdx(void *attr, unsigned int idx, char **ppszKey) = 0; + + GMLAppSchemaType GetAppSchemaType() const + { + return eAppSchemaType; + } }; #if defined(HAVE_XERCES) diff --git a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp index 971a9e57e4df..b5e204ad794c 100644 --- a/ogr/ogrsf_frmts/gml/hugefileresolver.cpp +++ b/ogr/ogrsf_frmts/gml/hugefileresolver.cpp @@ -1586,6 +1586,7 @@ static bool gmlHugeResolveEdges(CPL_UNUSED huge_helper *helper, static bool gmlHugeFileWriteResolved(huge_helper *helper, const char *pszOutputFilename, GMLReader *pReader, + GMLAppSchemaType eAppSchemaType, int *m_nHasSequentialLayers) { // Open the resolved GML file for writing. @@ -1612,10 +1613,20 @@ static bool gmlHugeFileWriteResolved(huge_helper *helper, return false; } + const char *pszTopElement = "ResolvedTopoFeatureMembers"; + // For some specific application schema, GMLHandler has specific behavior, + // so re-use the root XML element it recognizes. + if (eAppSchemaType == APPSCHEMA_AIXM) + pszTopElement = "AIXMBasicMessage"; + else if (eAppSchemaType == APPSCHEMA_CITYGML) + pszTopElement = "CityModel"; + VSIFPrintfL(fp, "\n"); - VSIFPrintfL(fp, "\n"); + VSIFPrintfL(fp, + "<%s " + "xmlns:xlink=\"http://www.w3.org/1999/xlink\" " + "xmlns:gml=\"http://www.opengis.net/gml\">\n", + pszTopElement); VSIFPrintfL(fp, " \n"); int iOutCount = 0; @@ -1818,7 +1829,7 @@ static bool gmlHugeFileWriteResolved(huge_helper *helper, } VSIFPrintfL(fp, " \n"); - VSIFPrintfL(fp, "\n"); + VSIFPrintfL(fp, "\n", pszTopElement); VSIFCloseL(fp); @@ -1991,6 +2002,9 @@ bool GMLReader::ParseXMLHugeFile(const char *pszOutputFilename, return false; } + CPLAssert(m_poGMLHandler); + const GMLAppSchemaType eAppSchemaType = m_poGMLHandler->GetAppSchemaType(); + // Restarting the GML parser. if (!SetupParser()) { @@ -2000,6 +2014,7 @@ bool GMLReader::ParseXMLHugeFile(const char *pszOutputFilename, // Output: writing the revolved GML file. if (gmlHugeFileWriteResolved(&helper, pszOutputFilename, this, + eAppSchemaType, &m_nHasSequentialLayers) == false) { gmlHugeFileCleanUp(&helper); From 9201eec84a774d659a49dd53e0dabb32f18bfb84 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 29 May 2024 15:02:53 +0200 Subject: [PATCH 15/31] gml.rst: add note about deleting .gfs file --- doc/source/drivers/vector/gml.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/drivers/vector/gml.rst b/doc/source/drivers/vector/gml.rst index fa0c465308ba..a33969974d93 100644 --- a/doc/source/drivers/vector/gml.rst +++ b/doc/source/drivers/vector/gml.rst @@ -626,6 +626,14 @@ The following open options are supported: Whether to use gml:boundedBy at feature level as feature geometry, if there are no other geometry. + +.. note:: + + When changing the value of most of the above options, it is recommended to + delete the ``.gfs`` file if it pre-exists, otherwise mis-behavior might be + observed. + + Creation Issues --------------- From 3674ec7570f324ea43eef08b6f5ce3a29de2741d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 31 May 2024 02:31:27 +0200 Subject: [PATCH 16/31] ogr2ogr: do not call CreateLayer() with a geom field defn of type wkbNone Fixes #10071 --- apps/ogr2ogr_lib.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index c2f4305a9d10..2a97dd36963c 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -4550,8 +4550,10 @@ SetupTargetLayer::Setup(OGRLayer *poSrcLayer, const char *pszNewLayerName, oGeomFieldDefn.SetSpatialRef(poOutputSRS); oGeomFieldDefn.SetCoordinatePrecision(oCoordPrec); oGeomFieldDefn.SetNullable(bGeomFieldNullable); - poDstLayer = m_poDstDS->CreateLayer(pszNewLayerName, &oGeomFieldDefn, - papszLCOTemp); + poDstLayer = m_poDstDS->CreateLayer( + pszNewLayerName, + eGCreateLayerType == wkbNone ? nullptr : &oGeomFieldDefn, + papszLCOTemp); CSLDestroy(papszLCOTemp); if (poDstLayer == nullptr) From e2b64be975d1c2e99f2bb2a0fc6624a261ee1890 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 31 May 2024 02:32:26 +0200 Subject: [PATCH 17/31] FileGDB: be robust to be called with a geometry field definition of type wkbNone Fixes https://github.com/OSGeo/gdal/issues/10071 --- autotest/ogr/ogr_fgdb.py | 27 +++++++++++++++++++++++++++ ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp | 2 ++ 2 files changed, 29 insertions(+) diff --git a/autotest/ogr/ogr_fgdb.py b/autotest/ogr/ogr_fgdb.py index 24e7af30bf3d..12c47f51e3bb 100755 --- a/autotest/ogr/ogr_fgdb.py +++ b/autotest/ogr/ogr_fgdb.py @@ -3143,3 +3143,30 @@ def test_ogr_filegdb_write_geom_coord_precision(tmp_path): "HighPrecision": "true", } } + + +############################################################################### +# Test dummy use of CreateLayerFromGeomFieldDefn() with a geometry field +# definition of type wkbNone + + +def test_ogr_filegdb_CreateLayerFromGeomFieldDefn_geom_type_none(tmp_path): + + filename = str(tmp_path / "test.gdb") + ds = gdal.GetDriverByName("FileGDB").Create(filename, 0, 0, 0, gdal.GDT_Unknown) + geom_fld = ogr.GeomFieldDefn("geometry", ogr.wkbNone) + ds.CreateLayerFromGeomFieldDefn("test", geom_fld) + ds.Close() + + ds = ogr.Open(filename) + lyr = ds.GetLayer(0) + assert lyr.GetGeomType() == ogr.wkbNone + ds.Close() + + filename2 = str(tmp_path / "test2.gdb") + gdal.VectorTranslate(filename2, filename, format="FileGDB") + + ds = ogr.Open(filename2) + lyr = ds.GetLayer(0) + assert lyr.GetGeomType() == ogr.wkbNone + ds.Close() diff --git a/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp b/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp index 4ed3c46cc4f3..7c7480f88b7a 100644 --- a/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp +++ b/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp @@ -2399,6 +2399,8 @@ bool FGdbLayer::Create(FGdbDataSource *pParentDataSource, const auto eType = poSrcGeomFieldDefn ? poSrcGeomFieldDefn->GetType() : wkbNone; + if (eType == wkbNone) + poSrcGeomFieldDefn = nullptr; #ifdef EXTENT_WORKAROUND m_bLayerJustCreated = true; From 6ab0684bc749e67442c03448ad9fdb90ce577835 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 25 May 2024 02:07:44 +0200 Subject: [PATCH 18/31] PG: avoid errors related to ogr_system_tables.metadata when user has not enough permissions Fixes #9994 --- autotest/ogr/ogr_pg.py | 68 +++++++++- doc/source/drivers/vector/pg.rst | 9 ++ ogr/ogrsf_frmts/pg/ogr_pg.h | 9 +- ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp | 171 ++++++++++++++++++++++++- ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp | 52 ++++---- 5 files changed, 279 insertions(+), 30 deletions(-) diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index b3700c8ab97f..3d139fdda891 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -4855,7 +4855,8 @@ def test_ogr_pg_84(pg_ds): @only_without_postgis -def test_ogr_pg_metadata(pg_ds): +@pytest.mark.parametrize("run_number", [1, 2]) +def test_ogr_pg_metadata(pg_ds, run_number): pg_ds = reconnect(pg_ds, update=1) pg_ds.StartTransaction() @@ -4889,6 +4890,71 @@ def test_ogr_pg_metadata(pg_ds): assert lyr.GetMetadata_Dict() == {} +############################################################################### +# Test reading/writing metadata with a user with limited rights + + +@only_without_postgis +def test_ogr_pg_metadata_restricted_user(pg_ds): + + lyr = pg_ds.CreateLayer( + "test_ogr_pg_metadata_restricted_user", + geom_type=ogr.wkbPoint, + options=["OVERWRITE=YES"], + ) + lyr.SetMetadata({"foo": "bar"}) + + pg_ds = reconnect(pg_ds, update=1) + + try: + + pg_ds.ExecuteSQL("CREATE ROLE test_ogr_pg_metadata_restricted_user") + with pg_ds.ExecuteSQL("SELECT current_schema()") as lyr: + f = lyr.GetNextFeature() + current_schema = f.GetField(0) + pg_ds.ExecuteSQL( + f"GRANT ALL PRIVILEGES ON SCHEMA {current_schema} TO test_ogr_pg_metadata_restricted_user" + ) + pg_ds.ExecuteSQL("SET ROLE test_ogr_pg_metadata_restricted_user") + + lyr = pg_ds.GetLayerByName("test_ogr_pg_metadata_restricted_user") + gdal.ErrorReset() + with gdal.quiet_errors(): + assert lyr.GetMetadata() == {} + assert ( + gdal.GetLastErrorMsg() + == "Table ogr_system_tables.metadata exists but user lacks USAGE privilege on ogr_system_tables schema" + ) + + pg_ds = reconnect(pg_ds, update=1) + pg_ds.ExecuteSQL("SET ROLE test_ogr_pg_metadata_restricted_user") + + lyr = pg_ds.CreateLayer( + "test_ogr_pg_metadata_restricted_user_bis", + geom_type=ogr.wkbPoint, + options=["OVERWRITE=YES"], + ) + with gdal.quiet_errors(): + lyr.SetMetadata({"foo": "bar"}) + + gdal.ErrorReset() + pg_ds = reconnect(pg_ds, update=1) + assert gdal.GetLastErrorMsg() == "" + + finally: + pg_ds = reconnect(pg_ds, update=1) + pg_ds.ExecuteSQL("DELLAYER:test_ogr_pg_metadata_restricted_user") + pg_ds.ExecuteSQL("DELLAYER:test_ogr_pg_metadata_restricted_user_bis") + with pg_ds.ExecuteSQL("SELECT CURRENT_USER") as lyr: + f = lyr.GetNextFeature() + current_user = f.GetField(0) + pg_ds.ExecuteSQL( + f"REASSIGN OWNED BY test_ogr_pg_metadata_restricted_user TO {current_user}" + ) + pg_ds.ExecuteSQL("DROP OWNED BY test_ogr_pg_metadata_restricted_user") + pg_ds.ExecuteSQL("DROP ROLE test_ogr_pg_metadata_restricted_user") + + ############################################################################### # Test append of several layers in PG_USE_COPY mode (#6411) diff --git a/doc/source/drivers/vector/pg.rst b/doc/source/drivers/vector/pg.rst index b5b2f44b703b..7c709f76e475 100644 --- a/doc/source/drivers/vector/pg.rst +++ b/doc/source/drivers/vector/pg.rst @@ -455,6 +455,15 @@ The following configuration options are available: be destroyed. Typical use case: ``ogr2ogr -append PG:dbname=foo abc.shp --config OGR_TRUNCATE YES``. +- .. config:: OGR_PG_ENABLE_METADATA + :choices: YES, NO + :default: YES + :since: 3.9 + + If set to "YES" (the default), the driver will try to use (and potentially + create) the ``ogr_system_tables.metadata`` table to retrieve and store + layer metadata. + Examples ~~~~~~~~ diff --git a/ogr/ogrsf_frmts/pg/ogr_pg.h b/ogr/ogrsf_frmts/pg/ogr_pg.h index 014f0826602c..393afb5f9dec 100644 --- a/ogr/ogrsf_frmts/pg/ogr_pg.h +++ b/ogr/ogrsf_frmts/pg/ogr_pg.h @@ -641,6 +641,12 @@ class OGRPGDataSource final : public OGRDataSource bool m_bOgrSystemTablesMetadataTableExistenceTested = false; bool m_bOgrSystemTablesMetadataTableFound = false; + bool m_bCreateMetadataTableIfNeededRun = false; + bool m_bCreateMetadataTableIfNeededSuccess = false; + + bool m_bHasWritePermissionsOnMetadataTableRun = false; + bool m_bHasWritePermissionsOnMetadataTableSuccess = false; + void LoadTables(); CPLString osDebugLastTransactionCommand{}; @@ -749,8 +755,9 @@ class OGRPGDataSource final : public OGRDataSource return bUserTransactionActive; } - void CreateOgrSystemTablesMetadataTableIfNeeded(); + bool CreateMetadataTableIfNeeded(); bool HasOgrSystemTablesMetadataTable(); + bool HasWritePermissionsOnMetadataTable(); }; #endif /* ndef OGR_PG_H_INCLUDED */ diff --git a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp index dbb0014bc71f..62415a574683 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp @@ -3161,13 +3161,69 @@ OGRErr OGRPGDataSource::EndCopy() } /************************************************************************/ -/* CreateOgrSystemTablesMetadataTableIfNeeded() */ +/* CreateMetadataTableIfNeeded() */ /************************************************************************/ -void OGRPGDataSource::CreateOgrSystemTablesMetadataTableIfNeeded() +bool OGRPGDataSource::CreateMetadataTableIfNeeded() { - PGresult *hResult = + if (m_bCreateMetadataTableIfNeededRun) + return m_bCreateMetadataTableIfNeededSuccess; + + m_bCreateMetadataTableIfNeededRun = true; + + PGresult *hResult; + + hResult = OGRPG_PQexec( + hPGConn, + "SELECT c.oid FROM pg_class c " + "JOIN pg_namespace n ON c.relnamespace=n.oid " + "WHERE c.relname = 'metadata' AND n.nspname = 'ogr_system_tables'"); + const bool bFound = + (hResult && PQntuples(hResult) == 1 && !PQgetisnull(hResult, 0, 0)); + OGRPGClearResult(hResult); + + hResult = OGRPG_PQexec( + hPGConn, + "SELECT has_database_privilege((select current_database()), 'CREATE')"); + const bool bCanCreateSchema = + (hResult && PQntuples(hResult) == 1 && !PQgetisnull(hResult, 0, 0) && + strcmp(PQgetvalue(hResult, 0, 0), "t") == 0); + OGRPGClearResult(hResult); + + if (!bFound) + { + if (!bCanCreateSchema) + { + CPLError(CE_Warning, CPLE_AppDefined, + "User lacks CREATE SCHEMA privilege to be able to create " + "ogr_system_tables.metadata table"); + return false; + } + } + else + { + if (!HasWritePermissionsOnMetadataTable()) + { + return false; + } + if (!bCanCreateSchema) + { + CPLError(CE_Warning, CPLE_AppDefined, + "User lacks CREATE SCHEMA privilege. Assuming " + "ogr_system_tables.metadata table has correct structure"); + m_bCreateMetadataTableIfNeededSuccess = true; + return true; + } + } + + hResult = OGRPG_PQexec(hPGConn, "CREATE SCHEMA IF NOT EXISTS ogr_system_tables"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); hResult = OGRPG_PQexec( @@ -3177,12 +3233,24 @@ void OGRPGDataSource::CreateOgrSystemTablesMetadataTableIfNeeded() "table_name TEXT NOT NULL, " "metadata TEXT," "UNIQUE(schema_name, table_name))"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); hResult = OGRPG_PQexec( hPGConn, "DROP FUNCTION IF EXISTS " "ogr_system_tables.event_trigger_function_for_metadata() CASCADE"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); hResult = OGRPG_PQexec( @@ -3193,6 +3261,9 @@ void OGRPGDataSource::CreateOgrSystemTablesMetadataTableIfNeeded() "DECLARE\n" " obj record;\n" "BEGIN\n" + " IF has_schema_privilege('ogr_system_tables', 'USAGE') THEN\n" + " IF has_table_privilege('ogr_system_tables.metadata', 'DELETE') " + "THEN\n" " FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects()\n" " LOOP\n" " IF obj.object_type = 'table' THEN\n" @@ -3201,13 +3272,27 @@ void OGRPGDataSource::CreateOgrSystemTablesMetadataTableIfNeeded() "obj.object_name;\n" " END IF;\n" " END LOOP;\n" + " END IF;\n" + " END IF;\n" "END;\n" "$$;"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); hResult = OGRPG_PQexec(hPGConn, "DROP EVENT TRIGGER IF EXISTS " "ogr_system_tables_event_trigger_for_metadata"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); hResult = OGRPG_PQexec( @@ -3216,7 +3301,18 @@ void OGRPGDataSource::CreateOgrSystemTablesMetadataTableIfNeeded() "ON sql_drop " "EXECUTE FUNCTION " "ogr_system_tables.event_trigger_function_for_metadata()"); + if (!hResult || (PQresultStatus(hResult) != PGRES_COMMAND_OK && + PQresultStatus(hResult) != PGRES_TUPLES_OK)) + { + OGRPGClearResult(hResult); + return false; + } OGRPGClearResult(hResult); + + m_bCreateMetadataTableIfNeededSuccess = true; + m_bOgrSystemTablesMetadataTableExistenceTested = true; + m_bOgrSystemTablesMetadataTableFound = true; + return true; } /************************************************************************/ @@ -3236,9 +3332,76 @@ bool OGRPGDataSource::HasOgrSystemTablesMetadataTable() "SELECT c.oid FROM pg_class c " "JOIN pg_namespace n ON c.relnamespace=n.oid " "WHERE c.relname = 'metadata' AND n.nspname = 'ogr_system_tables'"); - m_bOgrSystemTablesMetadataTableFound = + const bool bFound = (hResult && PQntuples(hResult) == 1 && !PQgetisnull(hResult, 0, 0)); OGRPGClearResult(hResult); + if (!bFound) + return false; + + hResult = OGRPG_PQexec( + hPGConn, + "SELECT has_schema_privilege('ogr_system_tables', 'USAGE')"); + const bool bHasSchemaPrivilege = + (hResult && PQntuples(hResult) == 1 && + !PQgetisnull(hResult, 0, 0) && + strcmp(PQgetvalue(hResult, 0, 0), "t") == 0); + OGRPGClearResult(hResult); + if (!bHasSchemaPrivilege) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Table ogr_system_tables.metadata exists but user lacks " + "USAGE privilege on ogr_system_tables schema"); + return false; + } + + hResult = OGRPG_PQexec( + hPGConn, "SELECT has_table_privilege('ogr_system_tables.metadata', " + "'SELECT')"); + m_bOgrSystemTablesMetadataTableFound = + (hResult && PQntuples(hResult) == 1 && + !PQgetisnull(hResult, 0, 0) && + strcmp(PQgetvalue(hResult, 0, 0), "t") == 0); + OGRPGClearResult(hResult); + if (!m_bOgrSystemTablesMetadataTableFound) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Table ogr_system_tables.metadata exists but user lacks " + "SELECT privilege on it"); + } } return m_bOgrSystemTablesMetadataTableFound; } + +/************************************************************************/ +/* HasWritePermissionsOnMetadataTable() */ +/************************************************************************/ + +bool OGRPGDataSource::HasWritePermissionsOnMetadataTable() +{ + if (!m_bHasWritePermissionsOnMetadataTableRun) + { + m_bHasWritePermissionsOnMetadataTableRun = true; + + if (HasOgrSystemTablesMetadataTable()) + { + PGresult *hResult = OGRPG_PQexec( + hPGConn, + "SELECT has_table_privilege('ogr_system_tables.metadata', " + "'INSERT') " + "AND has_table_privilege('ogr_system_tables.metadata', " + "'DELETE')"); + m_bHasWritePermissionsOnMetadataTableSuccess = + (hResult && PQntuples(hResult) == 1 && + !PQgetisnull(hResult, 0, 0) && + strcmp(PQgetvalue(hResult, 0, 0), "t") == 0); + OGRPGClearResult(hResult); + if (!m_bHasWritePermissionsOnMetadataTableSuccess) + { + CPLError(CE_Warning, CPLE_AppDefined, + "User lacks INSERT and/OR DELETE privilege on " + "ogr_system_tables.metadata table"); + } + } + } + return m_bHasWritePermissionsOnMetadataTableSuccess; +} diff --git a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp index 2ac569c03c3d..da4a8ff1533a 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp @@ -355,34 +355,38 @@ void OGRPGTableLayer::SerializeMetadata() if (psMD) { - poDS->CreateOgrSystemTablesMetadataTableIfNeeded(); - - CPLString osCommand; - osCommand.Printf("DELETE FROM ogr_system_tables.metadata WHERE " - "schema_name = %s AND table_name = %s", - OGRPGEscapeString(hPGConn, pszSchemaName).c_str(), - OGRPGEscapeString(hPGConn, pszTableName).c_str()); - PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - OGRPGClearResult(hResult); + if (poDS->CreateMetadataTableIfNeeded() && + poDS->HasWritePermissionsOnMetadataTable()) + { + CPLString osCommand; + osCommand.Printf("DELETE FROM ogr_system_tables.metadata WHERE " + "schema_name = %s AND table_name = %s", + OGRPGEscapeString(hPGConn, pszSchemaName).c_str(), + OGRPGEscapeString(hPGConn, pszTableName).c_str()); + PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); + OGRPGClearResult(hResult); - CPLXMLNode *psRoot = - CPLCreateXMLNode(nullptr, CXT_Element, "GDALMetadata"); - CPLAddXMLChild(psRoot, psMD); - char *pszXML = CPLSerializeXMLTree(psRoot); - // CPLDebug("PG", "Serializing %s", pszXML); + CPLXMLNode *psRoot = + CPLCreateXMLNode(nullptr, CXT_Element, "GDALMetadata"); + CPLAddXMLChild(psRoot, psMD); + char *pszXML = CPLSerializeXMLTree(psRoot); + // CPLDebug("PG", "Serializing %s", pszXML); - osCommand.Printf("INSERT INTO ogr_system_tables.metadata (schema_name, " - "table_name, metadata) VALUES (%s, %s, %s)", - OGRPGEscapeString(hPGConn, pszSchemaName).c_str(), - OGRPGEscapeString(hPGConn, pszTableName).c_str(), - OGRPGEscapeString(hPGConn, pszXML).c_str()); - hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - OGRPGClearResult(hResult); + osCommand.Printf( + "INSERT INTO ogr_system_tables.metadata (schema_name, " + "table_name, metadata) VALUES (%s, %s, %s)", + OGRPGEscapeString(hPGConn, pszSchemaName).c_str(), + OGRPGEscapeString(hPGConn, pszTableName).c_str(), + OGRPGEscapeString(hPGConn, pszXML).c_str()); + hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); + OGRPGClearResult(hResult); - CPLDestroyXMLNode(psRoot); - CPLFree(pszXML); + CPLDestroyXMLNode(psRoot); + CPLFree(pszXML); + } } - else if (poDS->HasOgrSystemTablesMetadataTable()) + else if (poDS->HasOgrSystemTablesMetadataTable() && + poDS->HasWritePermissionsOnMetadataTable()) { CPLString osCommand; osCommand.Printf("DELETE FROM ogr_system_tables.metadata WHERE " From e46e6eaa567b362e0276f360784abe0d239935a9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 31 May 2024 20:55:58 +0200 Subject: [PATCH 19/31] PG: really honor OGR_PG_ENABLE_METADATA=NO in SerializeMetadata() --- autotest/ogr/ogr_pg.py | 28 ++++++++++++++++++++++++++ ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp | 4 ++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index 3d139fdda891..37d03c4a95b0 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -4868,6 +4868,12 @@ def test_ogr_pg_metadata(pg_ds, run_number): lyr.SetMetadataItem("DESCRIPTION", "my_desc") pg_ds.CommitTransaction() + pg_ds = reconnect(pg_ds, update=1) + + with gdal.config_option("OGR_PG_ENABLE_METADATA", "NO"): + lyr = pg_ds.GetLayerByName("test_ogr_pg_metadata") + assert lyr.GetMetadata_Dict() == {"DESCRIPTION": "my_desc"} + pg_ds = reconnect(pg_ds, update=1) with pg_ds.ExecuteSQL( "SELECT * FROM ogr_system_tables.metadata WHERE table_name = 'test_ogr_pg_metadata'" @@ -4955,6 +4961,28 @@ def test_ogr_pg_metadata_restricted_user(pg_ds): pg_ds.ExecuteSQL("DROP ROLE test_ogr_pg_metadata_restricted_user") +############################################################################### +# Test disabling writing metadata + + +@only_without_postgis +def test_ogr_pg_write_metadata_disabled(pg_ds): + + with gdal.config_option("OGR_PG_ENABLE_METADATA", "NO"): + + pg_ds = reconnect(pg_ds, update=1) + lyr = pg_ds.CreateLayer( + "test_ogr_pg_metadata", geom_type=ogr.wkbPoint, options=["OVERWRITE=YES"] + ) + lyr.SetMetadata({"foo": "bar"}) + lyr.SetMetadataItem("bar", "baz") + + pg_ds = reconnect(pg_ds, update=1) + + lyr = pg_ds.GetLayerByName("test_ogr_pg_metadata") + assert lyr.GetMetadata_Dict() == {} + + ############################################################################### # Test append of several layers in PG_USE_COPY mode (#6411) diff --git a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp index da4a8ff1533a..f46f8b393c8e 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp @@ -289,8 +289,8 @@ void OGRPGTableLayer::LoadMetadata() void OGRPGTableLayer::SerializeMetadata() { - if (!m_bMetadataModified && - CPLTestBool(CPLGetConfigOption("OGR_PG_ENABLE_METADATA", "YES"))) + if (!m_bMetadataModified || + !CPLTestBool(CPLGetConfigOption("OGR_PG_ENABLE_METADATA", "YES"))) { return; } From c11cb59b508d6116df1bd1c863808e01a6f34e7a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 2 Jun 2024 00:59:59 +0200 Subject: [PATCH 20/31] netCDF: try to better round geotransform values when read from single-precsion lon/lat float arrays Related to https://lists.osgeo.org/pipermail/gdal-dev/2024-May/059050.html --- autotest/gdrivers/netcdf.py | 14 ++------ frmts/netcdf/netcdfdataset.cpp | 62 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/autotest/gdrivers/netcdf.py b/autotest/gdrivers/netcdf.py index 07e6ca616fb4..ff21d67235b5 100755 --- a/autotest/gdrivers/netcdf.py +++ b/autotest/gdrivers/netcdf.py @@ -1359,18 +1359,8 @@ def test_netcdf_36(): gt = ds.GetGeoTransform() assert gt is not None, "got no GeoTransform" - gt_expected = ( - -3.498749944898817, - 0.0025000042385525173, - 0.0, - 46.61749818589952, - 0.0, - -0.001666598849826389, - ) - assert gt == gt_expected, "got GeoTransform %s, expected %s" % ( - str(gt), - str(gt_expected), - ) + gt_expected = (-3.49875, 0.0025, 0.0, 46.61749818589952, 0.0, -0.001666598849826389) + assert gt == pytest.approx(gt_expected, rel=1e-8) ############################################################################### diff --git a/frmts/netcdf/netcdfdataset.cpp b/frmts/netcdf/netcdfdataset.cpp index 69e030de649c..6ee312b5213f 100644 --- a/frmts/netcdf/netcdfdataset.cpp +++ b/frmts/netcdf/netcdfdataset.cpp @@ -3901,6 +3901,56 @@ void netCDFDataset::SetProjectionFromVar( double xMinMax[2] = {0.0, 0.0}; double yMinMax[2] = {0.0, 0.0}; + const auto RoundMinMaxForFloatVals = + [](double &dfMin, double &dfMax, int nIntervals) + { + // Helps for a case where longitudes range from + // -179.99 to 180.0 with a 0.01 degree spacing. + // However as this is encoded in a float array, + // -179.99 is actually read as -179.99000549316406 as + // a double. Try to detect that and correct the rounding + + const auto IsAlmostInteger = [](double dfVal) + { + constexpr double THRESHOLD_INTEGER = 1e-3; + return std::fabs(dfVal - std::round(dfVal)) <= + THRESHOLD_INTEGER; + }; + + const double dfSpacing = (dfMax - dfMin) / nIntervals; + if (dfSpacing > 0) + { + const double dfInvSpacing = 1.0 / dfSpacing; + if (IsAlmostInteger(dfInvSpacing)) + { + const double dfRoundedSpacing = + 1.0 / std::round(dfInvSpacing); + const double dfMinDivRoundedSpacing = + dfMin / dfRoundedSpacing; + const double dfMaxDivRoundedSpacing = + dfMax / dfRoundedSpacing; + if (IsAlmostInteger(dfMinDivRoundedSpacing) && + IsAlmostInteger(dfMaxDivRoundedSpacing)) + { + const double dfRoundedMin = + std::round(dfMinDivRoundedSpacing) * + dfRoundedSpacing; + const double dfRoundedMax = + std::round(dfMaxDivRoundedSpacing) * + dfRoundedSpacing; + if (static_cast(dfMin) == + static_cast(dfRoundedMin) && + static_cast(dfMax) == + static_cast(dfRoundedMax)) + { + dfMin = dfRoundedMin; + dfMax = dfRoundedMax; + } + } + } + } + }; + if (!nc_get_att_double(nGroupDimXID, nVarDimXID, "actual_range", adfActualRange)) { @@ -3920,6 +3970,12 @@ void netCDFDataset::SetProjectionFromVar( xMinMax[0] = pdfXCoord[0]; xMinMax[1] = pdfXCoord[xdim - 1]; node_offset = 0; + + if (nc_var_dimx_datatype == NC_FLOAT) + { + RoundMinMaxForFloatVals(xMinMax[0], xMinMax[1], + poDS->nRasterXSize - 1); + } } if (!nc_get_att_double(nGroupDimYID, nVarDimYID, "actual_range", @@ -3941,6 +3997,12 @@ void netCDFDataset::SetProjectionFromVar( yMinMax[0] = pdfYCoord[0]; yMinMax[1] = pdfYCoord[ydim - 1]; node_offset = 0; + + if (nc_var_dimy_datatype == NC_FLOAT) + { + RoundMinMaxForFloatVals(yMinMax[0], yMinMax[1], + poDS->nRasterYSize - 1); + } } double dfCoordOffset = 0.0; From 0420e5ab85c7260b9be8338174a02724c9765c5e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 2 Jun 2024 01:32:36 +0200 Subject: [PATCH 21/31] test_gdalwarp_lib.py: add test for warping an image with [-180,180] longitude to [180 - X, 180 + X] --- autotest/utilities/test_gdalwarp_lib.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index fd3aeb2985a4..7deca9eb621b 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4186,3 +4186,23 @@ def test_target_extent_consistent_size(): assert ds.RasterXSize == 4793 assert ds.RasterYSize == 4143 + + +############################################################################### +# Test warping an image with [-180,180] longitude to [180 - X, 180 + X] + + +def test_gdalwarp_lib_minus_180_plus_180_to_span_over_180(tmp_vsimem): + + dst_filename = str(tmp_vsimem / "out.tif") + src_ds = gdal.Open("../gdrivers/data/small_world.tif") + out_ds = gdal.Warp(dst_filename, src_ds, outputBounds=[0, -90, 360, 90]) + # Check that east/west hemispheres have been switched + assert out_ds.ReadRaster( + 0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize + ) == src_ds.ReadRaster( + src_ds.RasterXSize // 2, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize + ) + assert out_ds.ReadRaster( + src_ds.RasterXSize // 2, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize + ) == src_ds.ReadRaster(0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize) From 501fda81026adbcae282da8008fffe360327ec90 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 2 Jun 2024 02:07:47 +0200 Subject: [PATCH 22/31] Warper: relax longitude extend check to decide whether we can insert a CENTER_LONG wrapping longitude Related to https://lists.osgeo.org/pipermail/gdal-dev/2024-May/059050.html --- alg/gdaltransformer.cpp | 7 ++++- autotest/utilities/test_gdalwarp_lib.py | 34 +++++++++++++++++++++---- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/alg/gdaltransformer.cpp b/alg/gdaltransformer.cpp index bff7c93cc3bc..263fbc2fa08d 100644 --- a/alg/gdaltransformer.cpp +++ b/alg/gdaltransformer.cpp @@ -1482,7 +1482,12 @@ static void InsertCenterLong(GDALDatasetH hDS, OGRSpatialReference *poSRS, adfGeoTransform[0] + nXSize * adfGeoTransform[1] + nYSize * adfGeoTransform[2])); - if (dfMaxLong - dfMinLong > 360.0) + const double dfEpsilon = + std::max(std::fabs(adfGeoTransform[1]), std::fabs(adfGeoTransform[2])); + // If the raster covers more than 360 degree (allow an extra pixel), + // give up + constexpr double RELATIVE_EPSILON = 0.05; // for numeric precision issues + if (dfMaxLong - dfMinLong > 360.0 + dfEpsilon * (1 + RELATIVE_EPSILON)) return; /* -------------------------------------------------------------------- */ diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 7deca9eb621b..a3f7b300a64d 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4192,17 +4192,41 @@ def test_target_extent_consistent_size(): # Test warping an image with [-180,180] longitude to [180 - X, 180 + X] -def test_gdalwarp_lib_minus_180_plus_180_to_span_over_180(tmp_vsimem): +@pytest.mark.parametrize("extra_column", [False, True]) +def test_gdalwarp_lib_minus_180_plus_180_to_span_over_180(tmp_vsimem, extra_column): dst_filename = str(tmp_vsimem / "out.tif") src_ds = gdal.Open("../gdrivers/data/small_world.tif") + if extra_column: + tmp_ds = gdal.GetDriverByName("MEM").Create( + "", src_ds.RasterXSize + 1, src_ds.RasterYSize + ) + tmp_ds.SetGeoTransform(src_ds.GetGeoTransform()) + tmp_ds.SetSpatialRef(src_ds.GetSpatialRef()) + tmp_ds.WriteRaster( + 0, + 0, + src_ds.RasterXSize, + src_ds.RasterYSize, + src_ds.GetRasterBand(1).ReadRaster(), + ) + tmp_ds.WriteRaster( + src_ds.RasterXSize, + 0, + 1, + src_ds.RasterYSize, + src_ds.GetRasterBand(1).ReadRaster(0, 0, 1, src_ds.RasterYSize), + ) + src_ds = tmp_ds out_ds = gdal.Warp(dst_filename, src_ds, outputBounds=[0, -90, 360, 90]) # Check that east/west hemispheres have been switched - assert out_ds.ReadRaster( + assert out_ds.GetRasterBand(1).ReadRaster( 0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize - ) == src_ds.ReadRaster( + ) == src_ds.GetRasterBand(1).ReadRaster( src_ds.RasterXSize // 2, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize ) - assert out_ds.ReadRaster( + assert out_ds.GetRasterBand(1).ReadRaster( src_ds.RasterXSize // 2, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize - ) == src_ds.ReadRaster(0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize) + ) == src_ds.GetRasterBand(1).ReadRaster( + 0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize + ) From 03a0ee915117383228f3c9dd79f4cce7518f6059 Mon Sep 17 00:00:00 2001 From: Thomas Jurk Date: Mon, 3 Jun 2024 12:17:11 +0200 Subject: [PATCH 23/31] Update OAPIF Samples The OAPIF examples contain outdated URLs --- doc/source/drivers/vector/oapif.rst | 180 ++++++++++++++++------------ 1 file changed, 102 insertions(+), 78 deletions(-) diff --git a/doc/source/drivers/vector/oapif.rst b/doc/source/drivers/vector/oapif.rst index a5c98b3243c7..370f90559f22 100644 --- a/doc/source/drivers/vector/oapif.rst +++ b/doc/source/drivers/vector/oapif.rst @@ -143,106 +143,130 @@ Examples :: - $ ogrinfo OAPIF:https://www.ldproxy.nrw.de/rest/services/kataster - - INFO: Open of `OAPIF:https://www.ldproxy.nrw.de/rest/services/kataster' + $ ogrinfo OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr + + INFO: Open of `OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr' using driver `OAPIF' successful. - 1: flurstueck (Multi Polygon) - 2: gebaeudebauwerk (Multi Polygon) - 3: verwaltungseinheit (Multi Polygon) + 1: governmentalservice (title: Feuerwehrleitstellen) (Point) - Listing the summary information of a OGC API - Features layer : :: - $ ogrinfo -al -so OAPIF:https://www.ldproxy.nrw.de/rest/services/kataster flurstueck - - Layer name: flurstueck + $ ogrinfo OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr governmentalservice -al -so + + INFO: Open of `OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr' + using driver `OAPIF' successful. + + Layer name: governmentalservice Metadata: - TITLE=Flurstück - Geometry: Multi Polygon - Feature Count: 9308456 - Extent: (5.612726, 50.237351) - (9.589634, 52.528630) + DESCRIPTION=Staatliche Verwaltungs- und Sozialdienste wie öffentliche Verwaltung, Katastrophenschutz, Schulen und Krankenhäuser, die von öffentlichen oder privaten Einrichtungen erbracht werden, soweit sie in den Anwendungsbereich der Richtlinie 2007/2/EG fallen. Dieser Datensatz enthält Informationen zu Feuerwehrleitstellen. + TITLE=Feuerwehrleitstellen + Geometry: Point + Feature Count: 52 + Extent: (6.020720, 50.654901) - (9.199363, 52.300806) Layer SRS WKT: - GEOGCS["WGS 84", - DATUM["WGS_1984", - SPHEROID["WGS 84",6378137,298.257223563, - AUTHORITY["EPSG","7030"]], - AUTHORITY["EPSG","6326"]], + GEOGCRS["WGS 84", + DATUM["World Geodetic System 1984", + ELLIPSOID["WGS 84",6378137,298.257223563, + LENGTHUNIT["metre",1]]], PRIMEM["Greenwich",0, - AUTHORITY["EPSG","8901"]], - UNIT["degree",0.0174532925199433, - AUTHORITY["EPSG","9122"]], - AUTHORITY["EPSG","4326"]] + ANGLEUNIT["degree",0.0174532925199433]], + CS[ellipsoidal,2], + AXIS["geodetic latitude (Lat)",north, + ORDER[1], + ANGLEUNIT["degree",0.0174532925199433]], + AXIS["geodetic longitude (Lon)",east, + ORDER[2], + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4326]] + Data axis to CRS axis mapping: 2,1 id: String (0.0) - aktualit: Date (0.0) - flaeche: Real (0.0) - flstkennz: String (0.0) - land: String (0.0) - gemarkung: String (0.0) - flur: String (0.0) - flurstnr: String (0.0) - gmdschl: String (0.0) - regbezirk: String (0.0) - kreis: String (0.0) - gemeinde: String (0.0) - lagebeztxt: String (0.0) - tntxt: String (0.0) + name: String (0.0) + inspireId: String (0.0) + serviceType.title: String (0.0) + serviceType.href: String (0.0) + areaOfResponsibility.1.title: String (0.0) + areaOfResponsibility.1.href: String (0.0) + pointOfContact.address.thoroughfare: String (0.0) + pointOfContact.address.locatorDesignator: String (0.0) + pointOfContact.address.postCode: String (0.0) + pointOfContact.address.adminUnit: String (0.0) + pointOfContact.address.text: String (0.0) + pointOfContact.telephoneVoice: String (0.0) + pointOfContact.telephoneFacsimile: String (0.0) + pointOfContact.telephoneFacsimileEmergency: String (0.0) + inDistrict.title: String (0.0) + inDistrict.href: String (0.0) + inDistrictFreeTown.title: String (0.0) + inDistrictFreeTown.href: String (0.0) + inGovernmentalDistrict.title: String (0.0) + inGovernmentalDistrict.href: String (0.0) - Filtering on a property (depending on if the server exposes filtering capabilities of the properties, part or totally of the filter might be evaluated on client side) :: - - $ ogrinfo OAPIF:https://www.ldproxy.nrw.de/rest/services/kataster flurstueck -al -q -where "flur = '028'" - Layer name: flurstueck + $ ogrinfo OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr governmentalservice -al -q -where "name = 'Schwelm'" + + Layer name: governmentalservice Metadata: - TITLE=Flurstück - OGRFeature(flurstueck):1 - id (String) = DENW19AL0000geMFFL - aktualit (Date) = 2017/04/26 - flaeche (Real) = 1739 - flstkennz (String) = 05297001600193______ - land (String) = Nordrhein-Westfalen - gemarkung (String) = Wünnenberg - flur (String) = 016 - flurstnr (String) = 193 - gmdschl (String) = 05774040 - regbezirk (String) = Detmold - kreis (String) = Paderborn - gemeinde (String) = Bad Wünnenberg - lagebeztxt (String) = Bleiwäscher Straße - tntxt (String) = Platz / Parkplatz;1739 - MULTIPOLYGON (((8.71191 51.491084,8.7123 51.491067,8.712385 51.491645,8.712014 51.491666,8.711993 51.491603,8.71196 51.491396,8.711953 51.491352,8.71191 51.491084))) - - [...] + DESCRIPTION=Staatliche Verwaltungs- und Sozialdienste wie öffentliche Verwaltung, Katastrophenschutz, Schulen und Krankenhäuser, die von öffentlichen oder privaten Einrichtungen erbracht werden, soweit sie in den Anwendungsbereich der Richtlinie 2007/2/EG fallen. Dieser Datensatz enthält Informationen zu Feuerwehrleitstellen. + TITLE=Feuerwehrleitstellen + OGRFeature(governmentalservice):1 + id (String) = LtS01 + name (String) = Schwelm + inspireId (String) = https://geodaten.nrw.de/id/inspire-us-feuerwehr/governmentalservice/LtS01 + serviceType.title (String) = Brandschutzdienst + serviceType.href (String) = http://inspire.ec.europa.eu/codelist/ServiceTypeValue/fireProtectionService + areaOfResponsibility.1.title (String) = Breckerfeld + areaOfResponsibility.1.href (String) = https://registry.gdi-de.org/id/de.nw.inspire.au.basis-dlm/AdministrativeUnit_05954004 + pointOfContact.address.thoroughfare (String) = Hauptstr. + pointOfContact.address.locatorDesignator (String) = 92 + pointOfContact.address.postCode (String) = 58332 + pointOfContact.address.adminUnit (String) = Schwelm + pointOfContact.address.text (String) = Hauptstr. 92, 58332 Schwelm + pointOfContact.telephoneVoice (String) = +49233644400 + pointOfContact.telephoneFacsimile (String) = +4923364440400 + pointOfContact.telephoneFacsimileEmergency (String) = +49233644407100 + inDistrict.title (String) = Ennepe-Ruhr + inDistrict.href (String) = Ennepe-Ruhr + inGovernmentalDistrict.title (String) = Arnsberg + inGovernmentalDistrict.href (String) = https://registry.gdi-de.org/id/de.nw.inspire.au.basis-dlm/AdministrativeUnit_059 + POINT (7.29854802787082 51.2855116825595) + - Spatial filtering :: - $ ogrinfo OAPIF:https://www.ldproxy.nrw.de/rest/services/kataster flurstueck -al -q -spat 8.7 51.4 8.8 51.5 - - Layer name: flurstueck + $ ogrinfo OAPIF:https://ogc-api.nrw.de/inspire-us-feuerwehr governmentalservice -al -q -spat 7.1 51.2 7.2 51.5 + + Layer name: governmentalservice Metadata: - TITLE=Flurstück - OGRFeature(flurstueck):1 - id (String) = DENW19AL0000ht7LFL - aktualit (Date) = 2013/02/19 - flaeche (Real) = 18 - flstkennz (String) = 05292602900206______ - land (String) = Nordrhein-Westfalen - gemarkung (String) = Fürstenberg - flur (String) = 029 - flurstnr (String) = 206 - gmdschl (String) = 05774040 - regbezirk (String) = Detmold - kreis (String) = Paderborn - gemeinde (String) = Bad Wünnenberg - lagebeztxt (String) = Karpke - tntxt (String) = Fließgewässer / Bach;18 - MULTIPOLYGON (((8.768521 51.494915,8.768535 51.494882,8.768569 51.494908,8.768563 51.494925,8.768521 51.494915))) - [...] + DESCRIPTION=Staatliche Verwaltungs- und Sozialdienste wie öffentliche Verwaltung, Katastrophenschutz, Schulen und Krankenhäuser, die von öffentlichen oder privaten Einrichtungen erbracht werden, soweit sie in den Anwendungsbereich der Richtlinie 2007/2/EG fallen. Dieser Datensatz enthält Informationen zu Feuerwehrleitstellen. + TITLE=Feuerwehrleitstellen + OGRFeature(governmentalservice):1 + id (String) = LtS33 + name (String) = Wuppertal-Solingen + inspireId (String) = https://geodaten.nrw.de/id/inspire-us-feuerwehr/governmentalservice/LtS33 + serviceType.title (String) = Brandschutzdienst + serviceType.href (String) = http://inspire.ec.europa.eu/codelist/ServiceTypeValue/fireProtectionService + areaOfResponsibility.1.title (String) = Wuppertal + areaOfResponsibility.1.href (String) = https://registry.gdi-de.org/id/de.nw.inspire.au.basis-dlm/AdministrativeUnit_05124000 + pointOfContact.address.thoroughfare (String) = August-Bebel-Str. + pointOfContact.address.locatorDesignator (String) = 55 + pointOfContact.address.postCode (String) = 42109 + pointOfContact.address.adminUnit (String) = Wuppertal + pointOfContact.address.text (String) = August-Bebel-Str. 55, 42109 Wuppertal + pointOfContact.telephoneVoice (String) = +492025631111 + pointOfContact.telephoneFacsimile (String) = +49202445331 + pointOfContact.telephoneFacsimileEmergency (String) = 112 + inDistrictFreeTown.title (String) = Wuppertal + inDistrictFreeTown.href (String) = Wuppertal + inGovernmentalDistrict.title (String) = Düsseldorf + inGovernmentalDistrict.href (String) = https://registry.gdi-de.org/id/de.nw.inspire.au.basis-dlm/AdministrativeUnit_051 + POINT (7.13806554104892 51.2674471939457) See Also -------- From 99ed79f69161bcf6adf8eb89c3c52a88d8c89f80 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Mon, 3 Jun 2024 13:51:37 -0400 Subject: [PATCH 24/31] gdal_viewshed: multi-threaded implementation (#9991) --- alg/viewshed.cpp | 1107 +++++++++++----------- alg/viewshed.h | 89 +- autotest/utilities/test_gdal_viewshed.py | 94 +- 3 files changed, 697 insertions(+), 593 deletions(-) diff --git a/alg/viewshed.cpp b/alg/viewshed.cpp index e5f81db19815..22f322bb1a3b 100644 --- a/alg/viewshed.cpp +++ b/alg/viewshed.cpp @@ -27,10 +27,10 @@ #include #include -#include +#include +#include #include "gdal_alg.h" -#include "gdal_priv.h" #include "gdal_priv_templates.hpp" #include "viewshed.h" @@ -197,13 +197,12 @@ GDALDatasetH GDALViewshedGenerate( "dfOutOfRangeVal out of range. Must be [0, 255]."); return nullptr; } - oOpts.visibleVal = static_cast(dfVisibleVal); - oOpts.invisibleVal = static_cast(dfInvisibleVal); - oOpts.outOfRangeVal = static_cast(dfOutOfRangeVal); + oOpts.visibleVal = dfVisibleVal; + oOpts.invisibleVal = dfInvisibleVal; + oOpts.outOfRangeVal = dfOutOfRangeVal; gdal::Viewshed v(oOpts); - //ABELL - Make a function for progress that captures the progress argument. v.run(hBand, pfnProgress, pProgressArg); return GDALDataset::FromHandle(v.output().release()); @@ -215,198 +214,253 @@ namespace gdal namespace { -bool AdjustHeightInRange(const double *adfGeoTransform, int iPixel, int iLine, - double &dfHeight, double dfDistance2, - double dfCurvCoeff, double dfSphereDiameter) +// Calculate the height adjustment factor. +double CalcHeightAdjFactor(const GDALDataset *poDataset, double dfCurveCoeff) { - if (dfDistance2 <= 0 && dfCurvCoeff == 0) - return true; + const OGRSpatialReference *poDstSRS = poDataset->GetSpatialRef(); - double dfX = adfGeoTransform[1] * iPixel + adfGeoTransform[2] * iLine; - double dfY = adfGeoTransform[4] * iPixel + adfGeoTransform[5] * iLine; - double dfR2 = dfX * dfX + dfY * dfY; + if (poDstSRS) + { + OGRErr eSRSerr; - /* calc adjustment */ - if (dfCurvCoeff != 0 && - dfSphereDiameter != std::numeric_limits::infinity()) - dfHeight -= dfCurvCoeff * dfR2 / dfSphereDiameter; + // If we can't get a SemiMajor axis from the SRS, it will be SRS_WGS84_SEMIMAJOR + double dfSemiMajor = poDstSRS->GetSemiMajor(&eSRSerr); - if (dfDistance2 > 0 && dfR2 > dfDistance2) - return false; + /* If we fetched the axis from the SRS, use it */ + if (eSRSerr != OGRERR_FAILURE) + return dfCurveCoeff / (dfSemiMajor * 2.0); - return true; + CPLDebug("GDALViewshedGenerate", + "Unable to fetch SemiMajor axis from spatial reference"); + } + return 0; } -double CalcHeightLine(int i, double Za, double Zo) +// Calculate the height at nDistance units along a line through the origin given the height +// at nDistance - 1 units along the line. +double CalcHeightLine(int nDistance, double Za) { - if (i == 1) + nDistance = std::abs(nDistance); + if (nDistance == 1) return Za; else - return (Za - Zo) / (i - 1) + Za; + return Za * nDistance / (nDistance - 1); } -double CalcHeightDiagonal(int i, int j, double Za, double Zb, double Zo) +// Calulate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) +// and passing through the line connecting (i - 1, j, Za) and (i, j - 1, Zb). +// In other words, the origin and the two points form a plane and we're calculating Zc +// of the point (i, j, Zc), also on the plane. +double CalcHeightDiagonal(int i, int j, double Za, double Zb) { - return ((Za - Zo) * i + (Zb - Zo) * j) / (i + j - 1) + Zo; + return (Za * i + Zb * j) / (i + j - 1); } -double CalcHeightEdge(int i, int j, double Za, double Zb, double Zo) +// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) +// and through the line connecting (i -1, j - 1, Za) and (i - 1, j, Zb). In other words, +// the origin and the other two points form a plane and we're calculating Zc of the +// point (i, j, Zc), also on the plane. +double CalcHeightEdge(int i, int j, double Za, double Zb) { if (i == j) - return CalcHeightLine(i, Za, Zo); + return CalcHeightLine(i, Za); else - return ((Za - Zo) * i + (Zb - Zo) * (j - i)) / (j - 1) + Zo; + return (Za * i + Zb * (j - i)) / (j - 1); } } // unnamed namespace -void Viewshed::setVisibility(int iPixel, double dfZ, double *padfZVal, - std::vector &vResult) -{ - if (padfZVal[iPixel] + oOpts.targetHeight < dfZ) - vResult[iPixel] = oOpts.invisibleVal; - else - vResult[iPixel] = oOpts.visibleVal; - - if (padfZVal[iPixel] < dfZ) - padfZVal[iPixel] = dfZ; -} - -double Viewshed::calcHeight(double dfDiagZ, double dfEdgeZ) +/// Calculate the output extent of the output raster in terms of the input raster. +/// +/// @param nX observer X position in the input raster +/// @param nY observer Y position in the input raster +/// @return false on error, true otherwise +bool Viewshed::calcOutputExtent(int nX, int nY) { - double dfHeight = dfEdgeZ; + // We start with the assumption that the output size matches the input. + oOutExtent.xStop = GDALGetRasterBandXSize(pSrcBand); + oOutExtent.yStop = GDALGetRasterBandYSize(pSrcBand); - switch (oOpts.cellMode) + if (nX < 0 || nX >= oOutExtent.xStop || nY < 0 || nY >= oOutExtent.yStop) { - case Viewshed::CellMode::Max: - dfHeight = std::max(dfDiagZ, dfEdgeZ); - break; - case Viewshed::CellMode::Min: - dfHeight = std::min(dfDiagZ, dfEdgeZ); - break; - case Viewshed::CellMode::Diagonal: - dfHeight = dfDiagZ; - break; - default: // Edge case set in initialization. - break; + CPLError(CE_Failure, CPLE_AppDefined, + "The observer location falls outside of the DEM area"); + return false; } - return dfHeight; -} -bool Viewshed::run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress, - void *pProgressArg) -{ - if (!pfnProgress) - pfnProgress = GDALDummyProgress; - - if (!pfnProgress(0.0, "", pProgressArg)) + constexpr double EPSILON = 1e-8; + if (oOpts.maxDistance > 0) { - CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - return false; - } + //ABELL - This assumes that the transformation is only a scaling. Should be fixed. + // Find the distance in the direction of the transformed unit vector in the X and Y + // directions and use those factors to determine the limiting values in the raster space. + int nXStart = static_cast( + std::floor(nX - adfInvTransform[1] * oOpts.maxDistance + EPSILON)); + int nXStop = static_cast( + std::ceil(nX + adfInvTransform[1] * oOpts.maxDistance - EPSILON) + + 1); + int nYStart = + static_cast(std::floor( + nY - std::fabs(adfInvTransform[5]) * oOpts.maxDistance + + EPSILON)) - + (adfInvTransform[5] > 0 ? 1 : 0); + int nYStop = static_cast( + std::ceil(nY + std::fabs(adfInvTransform[5]) * oOpts.maxDistance - + EPSILON) + + (adfInvTransform[5] < 0 ? 1 : 0)); - /* set up geotransformation */ - std::array adfGeoTransform{{0.0, 1.0, 0.0, 0.0, 0.0, 1.0}}; - GDALDatasetH hSrcDS = GDALGetBandDataset(hBand); - if (hSrcDS != nullptr) - GDALGetGeoTransform(hSrcDS, adfGeoTransform.data()); + oOutExtent.xStart = std::max(nXStart, 0); + oOutExtent.yStart = std::max(nYStart, 0); + oOutExtent.xStop = std::min(nXStop, oOutExtent.xStop); + oOutExtent.yStop = std::min(nYStop, oOutExtent.yStop); + } - double adfInvGeoTransform[6]; - if (!GDALInvGeoTransform(adfGeoTransform.data(), adfInvGeoTransform)) + if (oOutExtent.xSize() == 0 || oOutExtent.ySize() == 0) { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot invert geotransform"); + CPLError(CE_Failure, CPLE_AppDefined, "Invalid target raster size"); return false; } + return true; +} - /* calculate observer position */ - double dfX, dfY; - GDALApplyGeoTransform(adfInvGeoTransform, oOpts.observer.x, - oOpts.observer.y, &dfX, &dfY); - int nX = static_cast(dfX); - int nY = static_cast(dfY); - - int nXSize = GDALGetRasterBandXSize(hBand); - int nYSize = GDALGetRasterBandYSize(hBand); +/// Read a line of raster data. +/// +/// @param nLine Line number to read. +/// @param data Pointer to location in which to store data. +/// @return Success or failure. +bool Viewshed::readLine(int nLine, double *data) +{ + std::lock_guard g(iMutex); - if (nX < 0 || nX > nXSize || nY < 0 || nY > nYSize) + if (GDALRasterIO(pSrcBand, GF_Read, oOutExtent.xStart, nLine, + oOutExtent.xSize(), 1, data, oOutExtent.xSize(), 1, + GDT_Float64, 0, 0)) { CPLError(CE_Failure, CPLE_AppDefined, - "The observer location falls outside of the DEM area"); + "RasterIO error when reading DEM at position (%d,%d), " + "size (%d,%d)", + oOutExtent.xStart, nLine, oOutExtent.xSize(), 1); return false; } + return true; +} - /* calculate the area of interest */ - constexpr double EPSILON = 1e-8; +/// Write an output line of either visibility or height data. +/// +/// @param nLine Line number being written. +/// @param vResult Result line to write. +/// @return True on success, false otherwise. +bool Viewshed::writeLine(int nLine, std::vector &vResult) +{ + // GDALRasterIO isn't thread-safe. + std::lock_guard g(oMutex); - int nXStart = 0; - int nYStart = 0; - int nXStop = nXSize; - int nYStop = nYSize; - if (oOpts.maxDistance > 0) + if (GDALRasterIO(pDstBand, GF_Write, 0, nLine - oOutExtent.yStart, + oOutExtent.xSize(), 1, vResult.data(), oOutExtent.xSize(), + 1, GDT_Float64, 0, 0)) { - nXStart = static_cast(std::floor( - nX - adfInvGeoTransform[1] * oOpts.maxDistance + EPSILON)); - nXStop = static_cast( - std::ceil(nX + adfInvGeoTransform[1] * oOpts.maxDistance - - EPSILON) + - 1); - nYStart = - static_cast(std::floor( - nY - std::fabs(adfInvGeoTransform[5]) * oOpts.maxDistance + - EPSILON)) - - (adfInvGeoTransform[5] > 0 ? 1 : 0); - nYStop = static_cast( - std::ceil(nY + - std::fabs(adfInvGeoTransform[5]) * oOpts.maxDistance - - EPSILON) + - (adfInvGeoTransform[5] < 0 ? 1 : 0)); + CPLError(CE_Failure, CPLE_AppDefined, + "RasterIO error when writing target raster at position " + "(%d,%d), size (%d,%d)", + 0, nLine - oOutExtent.yStart, oOutExtent.xSize(), 1); + return false; } - nXStart = std::max(nXStart, 0); - nYStart = std::max(nYStart, 0); - nXStop = std::min(nXStop, nXSize); - nYStop = std::min(nYStop, nYSize); - - /* normalize horizontal index (0 - nXSize) */ - nXSize = nXStop - nXStart; - nX -= nXStart; + return true; +} - nYSize = nYStop - nYStart; +/// Emit progress information saying that a line has been written to output. +/// +/// @return True on success, false otherwise. +bool Viewshed::lineProgress() +{ + if (nLineCount < oOutExtent.ySize()) + nLineCount++; + return emitProgress(nLineCount / static_cast(oOutExtent.ySize())); +} - if (nXSize == 0 || nYSize == 0) +/// Emit progress information saying that a fraction of work has been completed. +/// +/// @return True on success, false otherwise. +bool Viewshed::emitProgress(double fraction) +{ + // Call the progress function. + if (!oProgress(fraction, "")) { - CPLError(CE_Failure, CPLE_AppDefined, "Invalid target raster size"); + CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); return false; } + return true; +} - std::vector vFirstLineVal; - std::vector vLastLineVal; - std::vector vThisLineVal; - std::vector vResult; - std::vector vHeightResult; +/// Adjust the height of the line of data by the observer height and the curvature of the +/// earth. +/// +/// @param nYOffset Y offset of the line being adjusted. +/// @param nX X location of the observer. +/// @param pdfNx Pointer to the data at the nX location of the line being adjusted +/// @return [left, right) Leftmost and one past the rightmost cell in the line within +/// the max distance +std::pair Viewshed::adjustHeight(int nYOffset, int nX, + double *const pdfNx) +{ + int nLeft = 0; + int nRight = oOutExtent.xSize(); - try + // If there is a height adjustment factor other than zero or a max distance, + // calculate the adjusted height of the cell, stopping if we've exceeded the max + // distance. + if (static_cast(dfHeightAdjFactor) || dfMaxDistance2 > 0) { - vFirstLineVal.resize(nXSize); - vLastLineVal.resize(nXSize); - vThisLineVal.resize(nXSize); - vResult.resize(nXSize); + // Hoist invariants from the loops. + const double dfLineX = adfTransform[2] * nYOffset; + const double dfLineY = adfTransform[5] * nYOffset; - if (oOpts.outputMode != OutputMode::Normal) - vHeightResult.resize(nXSize); + double *pdfHeight = pdfNx; + for (int nXOffset = 0; nXOffset >= -nX; nXOffset--, pdfHeight--) + { + double dfX = adfTransform[1] * nXOffset + dfLineX; + double dfY = adfTransform[4] * nXOffset + dfLineY; + double dfR2 = dfX * dfX + dfY * dfY; + if (dfR2 > dfMaxDistance2) + { + nLeft = nXOffset + nX + 1; + break; + } + *pdfHeight -= dfHeightAdjFactor * dfR2 + dfZObserver; + } + + pdfHeight = pdfNx + 1; + for (int nXOffset = 1; nXOffset < oOutExtent.xSize() - nX; + nXOffset++, pdfHeight++) + { + double dfX = adfTransform[1] * nXOffset + dfLineX; + double dfY = adfTransform[4] * nXOffset + dfLineY; + double dfR2 = dfX * dfX + dfY * dfY; + if (dfR2 > dfMaxDistance2) + { + nRight = nXOffset + nX; + break; + } + *pdfHeight -= dfHeightAdjFactor * dfR2 + dfZObserver; + } } - catch (...) + else { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot allocate vectors for viewshed"); - return false; + double *pdfHeight = pdfNx - nX; + for (int i = 0; i < oOutExtent.xSize(); ++i) + { + *pdfHeight -= dfZObserver; + pdfHeight++; + } } + return {nLeft, nRight}; +} - double *padfFirstLineVal = vFirstLineVal.data(); - double *padfLastLineVal = vLastLineVal.data(); - double *padfThisLineVal = vThisLineVal.data(); - GByte *pabyResult = vResult.data(); - double *dfHeightResult = vHeightResult.data(); - +/// Create the output dataset. +/// +/// @return True on success, false otherwise. +bool Viewshed::createOutputDataset() +{ GDALDriverManager *hMgr = GetGDALDriverManager(); GDALDriver *hDriver = hMgr->GetDriverByName(oOpts.outputFormat.c_str()); if (!hDriver) @@ -417,7 +471,7 @@ bool Viewshed::run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress, /* create output raster */ poDstDS.reset(hDriver->Create( - oOpts.outputFilename.c_str(), nXSize, nYSize, 1, + oOpts.outputFilename.c_str(), oOutExtent.xSize(), oOutExtent.ySize(), 1, oOpts.outputMode == OutputMode::Normal ? GDT_Byte : GDT_Float64, const_cast(oOpts.creationOpts.List()))); if (!poDstDS) @@ -426,24 +480,26 @@ bool Viewshed::run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress, oOpts.outputFilename.c_str()); return false; } + /* copy srs */ + GDALDatasetH hSrcDS = GDALGetBandDataset(pSrcBand); if (hSrcDS) poDstDS->SetSpatialRef( GDALDataset::FromHandle(hSrcDS)->GetSpatialRef()); - std::array adfDstGeoTransform; - adfDstGeoTransform[0] = adfGeoTransform[0] + adfGeoTransform[1] * nXStart + - adfGeoTransform[2] * nYStart; - adfDstGeoTransform[1] = adfGeoTransform[1]; - adfDstGeoTransform[2] = adfGeoTransform[2]; - adfDstGeoTransform[3] = adfGeoTransform[3] + adfGeoTransform[4] * nXStart + - adfGeoTransform[5] * nYStart; - adfDstGeoTransform[4] = adfGeoTransform[4]; - adfDstGeoTransform[5] = adfGeoTransform[5]; - poDstDS->SetGeoTransform(adfDstGeoTransform.data()); - - auto hTargetBand = poDstDS->GetRasterBand(1); - if (hTargetBand == nullptr) + std::array adfDstTransform; + adfDstTransform[0] = adfTransform[0] + adfTransform[1] * oOutExtent.xStart + + adfTransform[2] * oOutExtent.yStart; + adfDstTransform[1] = adfTransform[1]; + adfDstTransform[2] = adfTransform[2]; + adfDstTransform[3] = adfTransform[3] + adfTransform[4] * oOutExtent.xStart + + adfTransform[5] * oOutExtent.yStart; + adfDstTransform[4] = adfTransform[4]; + adfDstTransform[5] = adfTransform[5]; + poDstDS->SetGeoTransform(adfDstTransform.data()); + + pDstBand = poDstDS->GetRasterBand(1); + if (!pDstBand) { CPLError(CE_Failure, CPLE_AppDefined, "Cannot get band for %s", oOpts.outputFilename.c_str()); @@ -451,474 +507,367 @@ bool Viewshed::run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress, } if (oOpts.nodataVal >= 0) - GDALSetRasterNoDataValue(hTargetBand, oOpts.nodataVal); + GDALSetRasterNoDataValue(pDstBand, oOpts.nodataVal); + return true; +} - /* process first line */ - if (GDALRasterIO(hBand, GF_Read, nXStart, nY, nXSize, 1, padfFirstLineVal, - nXSize, 1, GDT_Float64, 0, 0)) - { - CPLError( - CE_Failure, CPLE_AppDefined, - "RasterIO error when reading DEM at position(%d, %d), size(%d, %d)", - nXStart, nY, nXSize, 1); - return false; - } +namespace +{ - const double dfZObserver = oOpts.observer.z + padfFirstLineVal[nX]; - const double dfDistance2 = oOpts.maxDistance * oOpts.maxDistance; +double doDiagonal(int nXOffset, [[maybe_unused]] int nYOffset, + double dfThisPrev, double dfLast, + [[maybe_unused]] double dfLastPrev) +{ + return CalcHeightDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast); +} - /* If we can't get a SemiMajor axis from the SRS, it will be - * SRS_WGS84_SEMIMAJOR - */ - double dfSphereDiameter(std::numeric_limits::infinity()); - const OGRSpatialReference *poDstSRS = poDstDS->GetSpatialRef(); - if (poDstSRS) - { - OGRErr eSRSerr; - double dfSemiMajor = poDstSRS->GetSemiMajor(&eSRSerr); +double doEdge(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + if (nXOffset >= nYOffset) + return CalcHeightEdge(nYOffset, nXOffset, dfLastPrev, dfThisPrev); + else + return CalcHeightEdge(nXOffset, nYOffset, dfLastPrev, dfLast); +} - /* If we fetched the axis from the SRS, use it */ - if (eSRSerr != OGRERR_FAILURE) - dfSphereDiameter = dfSemiMajor * 2.0; - else - CPLDebug("GDALViewshedGenerate", - "Unable to fetch SemiMajor axis from spatial reference"); - } +double doMin(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + double dfDiagonal = + doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + return std::min(dfEdge, dfDiagonal); +} - /* mark the observer point as visible */ - double dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfFirstLineVal[nX]; +double doMax(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + double dfDiagonal = + doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + return std::max(dfEdge, dfDiagonal); +} - pabyResult[nX] = oOpts.visibleVal; +double doLine(int nXOffset, [[maybe_unused]] int nYOffset, double dfThisPrev, + [[maybe_unused]] double dfLast, + [[maybe_unused]] double dfLastPrev) +{ + return CalcHeightLine(nXOffset, dfThisPrev); +} - //ABELL - Do we care about this conditional? - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX] = dfGroundLevel; +} // unnamed namespace - dfGroundLevel = 0; - if (nX > 0) - { - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfFirstLineVal[nX - 1]; - CPL_IGNORE_RET_VAL(AdjustHeightInRange( - adfGeoTransform.data(), 1, 0, padfFirstLineVal[nX - 1], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)); - pabyResult[nX - 1] = oOpts.visibleVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX - 1] = dfGroundLevel; - } - if (nX < nXSize - 1) +/// Process a line to the left of the observer. +/// +/// @param nX X coordinate of the observer. +/// @param nYOffset Offset of the line being processed from the observer +/// @param iStart X coordinate of the first cell to the left of the observer to be procssed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +/// @param vLastLineVal Observable height of each cell in the previous line processed. +void Viewshed::processLineLeft(int nX, int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal) +{ + double *pThis = vThisLineVal.data() + iStart; + double *pLast = vLastLineVal.data() + iStart; + + nYOffset = std::abs(nYOffset); + + // Go from the observer to the left, calculating Z as we go. + for (int iPixel = iStart; iPixel > iEnd; iPixel--, pThis--, pLast--) { - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfFirstLineVal[nX + 1]; - CPL_IGNORE_RET_VAL(AdjustHeightInRange( - adfGeoTransform.data(), 1, 0, padfFirstLineVal[nX + 1], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)); - pabyResult[nX + 1] = oOpts.visibleVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX + 1] = dfGroundLevel; + int nXOffset = std::abs(iPixel - nX); + double dfZ = + oZcalc(nXOffset, nYOffset, *(pThis + 1), *pLast, *(pLast + 1)); + setOutput(vResult[iPixel], *pThis, dfZ); } - /* process left direction */ - for (int iPixel = nX - 2; iPixel >= 0; iPixel--) - { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfFirstLineVal[iPixel]; + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin(), vResult.begin() + iEnd + 1, oOpts.outOfRangeVal); +} - if (AdjustHeightInRange(adfGeoTransform.data(), nX - iPixel, 0, - padfFirstLineVal[iPixel], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)) - { - double dfZ = CalcHeightLine( - nX - iPixel, padfFirstLineVal[iPixel + 1], dfZObserver); +/// Process a line to the right of the observer. +/// +/// @param nX X coordinate of the observer. +/// @param nYOffset Offset of the line being processed from the observer +/// @param iStart X coordinate of the first cell to the right of the observer to be procssed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +/// @param vLastLineVal Observable height of each cell in the previous line processed. +void Viewshed::processLineRight(int nX, int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal) +{ + double *pThis = vThisLineVal.data() + iStart; + double *pLast = vLastLineVal.data() + iStart; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfFirstLineVal[iPixel] + dfGroundLevel)); + nYOffset = std::abs(nYOffset); - setVisibility(iPixel, dfZ, padfFirstLineVal, vResult); - } - else - { - for (; iPixel >= 0; iPixel--) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } + // Go from the observer to the right, calculating Z as we go. + for (int iPixel = iStart; iPixel < iEnd; iPixel++, pThis++, pLast++) + { + int nXOffset = std::abs(iPixel - nX); + double dfZ = + oZcalc(nXOffset, nYOffset, *(pThis - 1), *pLast, *(pLast - 1)); + setOutput(vResult[iPixel], *pThis, dfZ); } - /* process right direction */ - for (int iPixel = nX + 2; iPixel < nXSize; iPixel++) + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin() + iEnd, vResult.end(), oOpts.outOfRangeVal); +} + +/// Set the output Z value depending o the observable height and computation mode. +/// +/// dfResult Reference to the result cell +/// dfCellVal Reference to the current cell height. Replace with observable height. +/// dfZ Observable height at cell. +void Viewshed::setOutput(double &dfResult, double &dfCellVal, double dfZ) +{ + if (oOpts.outputMode != OutputMode::Normal) { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfFirstLineVal[iPixel]; - if (AdjustHeightInRange(adfGeoTransform.data(), iPixel - nX, 0, - padfFirstLineVal[iPixel], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)) - { - double dfZ = CalcHeightLine( - iPixel - nX, padfFirstLineVal[iPixel - 1], dfZObserver); + dfResult += (dfZ - dfCellVal); + dfResult = std::max(0.0, dfResult); + } + else + dfResult = (dfCellVal + oOpts.targetHeight < dfZ) ? oOpts.invisibleVal + : oOpts.visibleVal; + dfCellVal = std::max(dfCellVal, dfZ); +} - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfFirstLineVal[iPixel] + dfGroundLevel)); +/// Process the first line (the one with the X coordinate the same as the observer). +/// +/// @param nX X location of the observer +/// @param nY Y location of the observer +/// @param nLine Line number being processed (should always be the same as nY) +/// @param vLastLineVal Vector in which to store the read line. Becomes the last line +/// in further processing. +/// @return True on success, false otherwise. +bool Viewshed::processFirstLine(int nX, int nY, int nLine, + std::vector &vLastLineVal) +{ + int nYOffset = nLine - nY; // Should be zero. + std::vector vResult(oOutExtent.xSize()); + std::vector vThisLineVal(oOutExtent.xSize()); - setVisibility(iPixel, dfZ, padfFirstLineVal, vResult); - } - else - { - for (; iPixel < nXSize; iPixel++) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } - } - /* write result line */ + if (!readLine(nLine, vThisLineVal.data())) + return false; - void *data; - GDALDataType dataType; + // This bit is only relevant for the first line. + dfZObserver = oOpts.observer.z + vThisLineVal[nX]; + dfHeightAdjFactor = CalcHeightAdjFactor(poDstDS.get(), oOpts.curveCoeff); if (oOpts.outputMode == OutputMode::Normal) { - data = static_cast(pabyResult); - dataType = GDT_Byte; + vResult[nX] = oOpts.visibleVal; + if (nX - 1 >= 0) + vResult[nX - 1] = oOpts.visibleVal; + if (nX + 1 < oOutExtent.xSize()) + vResult[nX + 1] = oOpts.visibleVal; } - else + + // In DEM mode the base is the pre-adjustment value. + // In ground mode the base is zero. + if (oOpts.outputMode == OutputMode::DEM) + vResult = vThisLineVal; + + const auto [iLeft, iRight] = + adjustHeight(nYOffset, nX, vThisLineVal.data() + nX); + + auto t1 = + std::async(std::launch::async, + [&, left = iLeft]() + { + processLineLeft(nX, nYOffset, nX - 2, left - 1, vResult, + vThisLineVal, vLastLineVal); + }); + + auto t2 = + std::async(std::launch::async, + [&, right = iRight]() + { + processLineRight(nX, nYOffset, nX + 2, right, vResult, + vThisLineVal, vLastLineVal); + }); + t1.wait(); + t2.wait(); + + // Make the current line the last line. + vLastLineVal = std::move(vThisLineVal); + + // Create the output writer. + if (!writeLine(nY, vResult)) + return false; + + if (!lineProgress()) + return false; + return true; +} + +/// Process a line above or below the observer. +/// +/// @param nX X location of the observer +/// @param nY Y location of the observer +/// @param nLine Line number being processed. +/// @param vLastLineVal Vector in which to store the read line. Becomes the last line +/// in further processing. +/// @return True on success, false otherwise. +bool Viewshed::processLine(int nX, int nY, int nLine, + std::vector &vLastLineVal) +{ + int nYOffset = nLine - nY; + std::vector vResult(oOutExtent.xSize()); + std::vector vThisLineVal(oOutExtent.xSize()); + + if (!readLine(nLine, vThisLineVal.data())) + return false; + + // In DEM mode the base is the input DEM value. + // In ground mode the base is zero. + if (oOpts.outputMode == OutputMode::DEM) + vResult = vThisLineVal; + + // Adjust height of the read line. + const auto [iLeft, iRight] = + adjustHeight(nYOffset, nX, vThisLineVal.data() + nX); + + // Handle the initial position on the line. + if (iLeft < iRight) { - data = static_cast(dfHeightResult); - dataType = GDT_Float64; + double dfZ = CalcHeightLine(nYOffset, vLastLineVal[nX]); + setOutput(vResult[nX], vThisLineVal[nX], dfZ); } - if (GDALRasterIO(hTargetBand, GF_Write, 0, nY - nYStart, nXSize, 1, data, - nXSize, 1, dataType, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when writing target raster at position " - "(%d,%d), size (%d,%d)", - 0, nY - nYStart, nXSize, 1); + else + vResult[nX] = oOpts.outOfRangeVal; + + // process left half then right half of line + auto t1 = + std::async(std::launch::async, + [&, left = iLeft]() + { + processLineLeft(nX, nYOffset, nX - 1, left - 1, vResult, + vThisLineVal, vLastLineVal); + }); + + auto t2 = + std::async(std::launch::async, + [&, right = iRight]() + { + processLineRight(nX, nYOffset, nX + 1, right, vResult, + vThisLineVal, vLastLineVal); + }); + t1.wait(); + t2.wait(); + + // Make the current line the last line. + vLastLineVal = std::move(vThisLineVal); + + if (!writeLine(nLine, vResult)) return false; - } - /* scan upwards */ - std::copy(vFirstLineVal.begin(), vFirstLineVal.end(), vLastLineVal.begin()); - for (int iLine = nY - 1; iLine >= nYStart; iLine--) - { - if (GDALRasterIO(hBand, GF_Read, nXStart, iLine, nXSize, 1, - padfThisLineVal, nXSize, 1, GDT_Float64, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when reading DEM at position (%d,%d), " - "size (%d,%d)", - nXStart, iLine, nXSize, 1); - return false; - } + if (!lineProgress()) + return false; + return true; +} - /* set up initial point on the scanline */ - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[nX]; - if (AdjustHeightInRange(adfGeoTransform.data(), 0, nY - iLine, - padfThisLineVal[nX], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)) - { - double dfZ = - CalcHeightLine(nY - iLine, padfLastLineVal[nX], dfZObserver); +/// Compute the viewshed of a raster band. +/// +/// @param band Pointer to the raster band to be processed. +/// @param pfnProgress Pointer to the progress function. Can be null. +/// @param pProgressArg Argument passed to the progress function +/// @return True on success, false otherwise. +bool Viewshed::run(GDALRasterBandH band, GDALProgressFunc pfnProgress, + void *pProgressArg) +{ + using namespace std::placeholders; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX] = - std::max(0.0, (dfZ - padfThisLineVal[nX] + dfGroundLevel)); + nLineCount = 0; + pSrcBand = static_cast(band); - setVisibility(nX, dfZ, padfThisLineVal, vResult); - } - else - { - pabyResult[nX] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX] = oOpts.outOfRangeVal; - } - - /* process left direction */ - for (int iPixel = nX - 1; iPixel >= 0; iPixel--) - { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[iPixel]; - if (AdjustHeightInRange(adfGeoTransform.data(), nX - iPixel, - nY - iLine, padfThisLineVal[iPixel], - dfDistance2, oOpts.curveCoeff, - dfSphereDiameter)) - { - double dfDiagZ = 0; - double dfEdgeZ = 0; - if (oOpts.cellMode != CellMode::Edge) - dfDiagZ = CalcHeightDiagonal( - nX - iPixel, nY - iLine, padfThisLineVal[iPixel + 1], - padfLastLineVal[iPixel], dfZObserver); - - if (oOpts.cellMode != CellMode::Diagonal) - dfEdgeZ = nX - iPixel >= nY - iLine - ? CalcHeightEdge(nY - iLine, nX - iPixel, - padfLastLineVal[iPixel + 1], - padfThisLineVal[iPixel + 1], - dfZObserver) - : CalcHeightEdge(nX - iPixel, nY - iLine, - padfLastLineVal[iPixel + 1], - padfLastLineVal[iPixel], - dfZObserver); - - double dfZ = calcHeight(dfDiagZ, dfEdgeZ); - - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfThisLineVal[iPixel] + dfGroundLevel)); - - setVisibility(iPixel, dfZ, padfThisLineVal, vResult); - } - else - { - for (; iPixel >= 0; iPixel--) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } - } - /* process right direction */ - for (int iPixel = nX + 1; iPixel < nXSize; iPixel++) - { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[iPixel]; - - if (AdjustHeightInRange(adfGeoTransform.data(), iPixel - nX, - nY - iLine, padfThisLineVal[iPixel], - dfDistance2, oOpts.curveCoeff, - dfSphereDiameter)) - { - double dfDiagZ = 0; - double dfEdgeZ = 0; - if (oOpts.cellMode != CellMode::Edge) - dfDiagZ = CalcHeightDiagonal( - iPixel - nX, nY - iLine, padfThisLineVal[iPixel - 1], - padfLastLineVal[iPixel], dfZObserver); - if (oOpts.cellMode != CellMode::Diagonal) - dfEdgeZ = iPixel - nX >= nY - iLine - ? CalcHeightEdge(nY - iLine, iPixel - nX, - padfLastLineVal[iPixel - 1], - padfThisLineVal[iPixel - 1], - dfZObserver) - : CalcHeightEdge(iPixel - nX, nY - iLine, - padfLastLineVal[iPixel - 1], - padfLastLineVal[iPixel], - dfZObserver); - - double dfZ = calcHeight(dfDiagZ, dfEdgeZ); - - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfThisLineVal[iPixel] + dfGroundLevel)); - - setVisibility(iPixel, dfZ, padfThisLineVal, vResult); - } - else - { - for (; iPixel < nXSize; iPixel++) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } - } + if (!pfnProgress) + pfnProgress = GDALDummyProgress; + oProgress = std::bind(pfnProgress, _1, _2, pProgressArg); - /* write result line */ - if (GDALRasterIO(hTargetBand, GF_Write, 0, iLine - nYStart, nXSize, 1, - data, nXSize, 1, dataType, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when writing target raster at position " - "(%d,%d), size (%d,%d)", - 0, iLine - nYStart, nXSize, 1); - return false; - } + if (!emitProgress(0)) + return false; - std::swap(padfLastLineVal, padfThisLineVal); + // set up geotransformation + GDALDatasetH hSrcDS = GDALGetBandDataset(pSrcBand); + if (hSrcDS != nullptr) + GDALGetGeoTransform(hSrcDS, adfTransform.data()); - if (!pfnProgress((nY - iLine) / static_cast(nYSize), "", - pProgressArg)) - { - CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - return false; - } + if (!GDALInvGeoTransform(adfTransform.data(), adfInvTransform.data())) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot invert geotransform"); + return false; } - /* scan downwards */ - memcpy(padfLastLineVal, padfFirstLineVal, nXSize * sizeof(double)); - for (int iLine = nY + 1; iLine < nYStop; iLine++) - { - if (GDALRasterIO(hBand, GF_Read, nXStart, iLine, nXSize, 1, - padfThisLineVal, nXSize, 1, GDT_Float64, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when reading DEM at position (%d,%d), " - "size (%d,%d)", - nXStart, iLine, nXStop - nXStart, 1); - return false; - } + // calculate observer position + double dfX, dfY; + GDALApplyGeoTransform(adfInvTransform.data(), oOpts.observer.x, + oOpts.observer.y, &dfX, &dfY); + int nX = static_cast(dfX); + int nY = static_cast(dfY); - /* set up initial point on the scanline */ - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[nX]; + // calculate the area of interest + if (!calcOutputExtent(nX, nY)) + return false; - if (AdjustHeightInRange(adfGeoTransform.data(), 0, iLine - nY, - padfThisLineVal[nX], dfDistance2, - oOpts.curveCoeff, dfSphereDiameter)) - { - double dfZ = - CalcHeightLine(iLine - nY, padfLastLineVal[nX], dfZObserver); + // normalize horizontal index to [ 0, oOutExtent.xSize() ) + nX -= oOutExtent.xStart; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX] = - std::max(0.0, (dfZ - padfThisLineVal[nX] + dfGroundLevel)); + // create the output dataset + if (!createOutputDataset()) + return false; - setVisibility(nX, dfZ, padfThisLineVal, vResult); - } - else - { - pabyResult[nX] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[nX] = oOpts.outOfRangeVal; - } + oZcalc = doLine; - /* process left direction */ - for (int iPixel = nX - 1; iPixel >= 0; iPixel--) - { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[iPixel]; - - if (AdjustHeightInRange(adfGeoTransform.data(), nX - iPixel, - iLine - nY, padfThisLineVal[iPixel], - dfDistance2, oOpts.curveCoeff, - dfSphereDiameter)) - { - double dfDiagZ = 0; - double dfEdgeZ = 0; - if (oOpts.cellMode != CellMode::Edge) - dfDiagZ = CalcHeightDiagonal( - nX - iPixel, iLine - nY, padfThisLineVal[iPixel + 1], - padfLastLineVal[iPixel], dfZObserver); - - if (oOpts.cellMode != CellMode::Diagonal) - dfEdgeZ = nX - iPixel >= iLine - nY - ? CalcHeightEdge(iLine - nY, nX - iPixel, - padfLastLineVal[iPixel + 1], - padfThisLineVal[iPixel + 1], - dfZObserver) - : CalcHeightEdge(nX - iPixel, iLine - nY, - padfLastLineVal[iPixel + 1], - padfLastLineVal[iPixel], - dfZObserver); - - double dfZ = calcHeight(dfDiagZ, dfEdgeZ); - - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfThisLineVal[iPixel] + dfGroundLevel)); - - setVisibility(iPixel, dfZ, padfThisLineVal, vResult); - } - else - { - for (; iPixel >= 0; iPixel--) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } - } - /* process right direction */ - for (int iPixel = nX + 1; iPixel < nXSize; iPixel++) - { - dfGroundLevel = 0; - if (oOpts.outputMode == OutputMode::DEM) - dfGroundLevel = padfThisLineVal[iPixel]; - - if (AdjustHeightInRange(adfGeoTransform.data(), iPixel - nX, - iLine - nY, padfThisLineVal[iPixel], - dfDistance2, oOpts.curveCoeff, - dfSphereDiameter)) - { - double dfDiagZ = 0; - double dfEdgeZ = 0; - if (oOpts.cellMode != CellMode::Edge) - dfDiagZ = CalcHeightDiagonal( - iPixel - nX, iLine - nY, padfThisLineVal[iPixel - 1], - padfLastLineVal[iPixel], dfZObserver); - - if (oOpts.cellMode != CellMode::Diagonal) - dfEdgeZ = iPixel - nX >= iLine - nY - ? CalcHeightEdge(iLine - nY, iPixel - nX, - padfLastLineVal[iPixel - 1], - padfThisLineVal[iPixel - 1], - dfZObserver) - : CalcHeightEdge(iPixel - nX, iLine - nY, - padfLastLineVal[iPixel - 1], - padfLastLineVal[iPixel], - dfZObserver); - - double dfZ = calcHeight(dfDiagZ, dfEdgeZ); - - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = std::max( - 0.0, (dfZ - padfThisLineVal[iPixel] + dfGroundLevel)); - - setVisibility(iPixel, dfZ, padfThisLineVal, vResult); - } - else - { - for (; iPixel < nXSize; iPixel++) - { - pabyResult[iPixel] = oOpts.outOfRangeVal; - if (oOpts.outputMode != OutputMode::Normal) - dfHeightResult[iPixel] = oOpts.outOfRangeVal; - } - } - } + std::vector vFirstLineVal(oOutExtent.xSize()); + + if (!processFirstLine(nX, nY, nY, vFirstLineVal)) + return false; - /* write result line */ - if (GDALRasterIO(hTargetBand, GF_Write, 0, iLine - nYStart, nXSize, 1, - data, nXSize, 1, dataType, 0, 0)) + if (oOpts.cellMode == CellMode::Edge) + oZcalc = doEdge; + else if (oOpts.cellMode == CellMode::Diagonal) + oZcalc = doDiagonal; + else if (oOpts.cellMode == CellMode::Min) + oZcalc = doMin; + else if (oOpts.cellMode == CellMode::Max) + oZcalc = doMax; + + // scan upwards + std::atomic err(false); + auto tUp = std::async(std::launch::async, + [&]() + { + std::vector vLastLineVal = vFirstLineVal; + + for (int nLine = nY - 1; + nLine >= oOutExtent.yStart && !err; nLine--) + if (!processLine(nX, nY, nLine, vLastLineVal)) + err = true; + }); + + // scan downwards + auto tDown = std::async( + std::launch::async, + [&]() { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when writing target raster at position " - "(%d,%d), size (%d,%d)", - 0, iLine - nYStart, nXSize, 1); - return false; - } + std::vector vLastLineVal = vFirstLineVal; - std::swap(padfLastLineVal, padfThisLineVal); + for (int nLine = nY + 1; nLine < oOutExtent.yStop && !err; nLine++) + if (!processLine(nX, nY, nLine, vLastLineVal)) + err = true; + }); - if (!pfnProgress((iLine - nYStart) / static_cast(nYSize), "", - pProgressArg)) - { - CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - return false; - } - } + tUp.wait(); + tDown.wait(); - if (!pfnProgress(1.0, "", pProgressArg)) - { - CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); + if (!emitProgress(1)) return false; - } return true; } diff --git a/alg/viewshed.h b/alg/viewshed.h index 1dc1120806e5..857ec6225096 100644 --- a/alg/viewshed.h +++ b/alg/viewshed.h @@ -25,8 +25,12 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ +#include #include +#include +#include #include +#include #include #include "cpl_progress.h" @@ -72,16 +76,39 @@ class Viewshed double z; //!< Z value }; + /** + * A window in a raster including pixels in [xStart, xEnd) and [yStart, yEnd). + */ + struct Window + { + int xStart{}; //!< X start position + int xStop{}; //!< X end position + int yStart{}; //!< Y start position + int yStop{}; //!< Y end position + + /// \brief Window size in the X direction. + int xSize() + { + return xStop - xStart; + } + + /// \brief Window size in the Y direction. + int ySize() + { + return yStop - yStart; + } + }; + /** * Options for viewshed generation. */ struct Options { Point observer{0, 0, 0}; //!< x, y, and z of the observer - uint8_t visibleVal{255}; //!< raster output value for visible pixels. - uint8_t invisibleVal{ + double visibleVal{255}; //!< raster output value for visible pixels. + double invisibleVal{ 0}; //!< raster output value for non-visible pixels. - uint8_t outOfRangeVal{ + double outOfRangeVal{ 0}; //!< raster output value for pixels outside of max distance. double nodataVal{-1}; //!< raster output value for pixels with no data double targetHeight{0.0}; //!< target height above the DEM surface @@ -102,17 +129,20 @@ class Viewshed * * @param opts Options to use when calculating viewshed. */ - CPL_DLL explicit Viewshed(const Options &opts) : oOpts{opts}, poDstDS{} + CPL_DLL explicit Viewshed(const Options &opts) + : oOpts{opts}, oOutExtent{}, dfMaxDistance2{opts.maxDistance * + opts.maxDistance}, + dfZObserver{0}, poDstDS{}, pSrcBand{}, pDstBand{}, + dfHeightAdjFactor{0}, nLineCount{0}, adfTransform{0, 1, 0, 0, 0, 1}, + adfInvTransform{}, oProgress{}, oZcalc{}, oMutex{}, iMutex{} { + if (dfMaxDistance2 == 0) + dfMaxDistance2 = std::numeric_limits::max(); } - /** - * Create the viewshed for the provided raster band. - * - * @param hBand Handle to the raster band. - * @param pfnProgress Progress reporting callback function. - * @param pProgressArg Argument to pass to the progress callback. - */ + Viewshed(const Viewshed &) = delete; + Viewshed &operator=(const Viewshed &) = delete; + CPL_DLL bool run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress, void *pProgressArg = nullptr); @@ -128,11 +158,44 @@ class Viewshed private: Options oOpts; + Window oOutExtent; + double dfMaxDistance2; + double dfZObserver; std::unique_ptr poDstDS; + GDALRasterBand *pSrcBand; + GDALRasterBand *pDstBand; + double dfHeightAdjFactor; + int nLineCount; + std::array adfTransform; + std::array adfInvTransform; + using ProgressFunc = std::function; + ProgressFunc oProgress; + using ZCalc = std::function; + ZCalc oZcalc; + std::mutex oMutex; + std::mutex iMutex; - void setVisibility(int iPixel, double dfZ, double *padfZVal, - std::vector &vResult); + void setOutput(double &dfResult, double &dfCellVal, double dfZ); double calcHeight(double dfZ, double dfZ2); + bool readLine(int nLine, double *data); + bool writeLine(int nLine, std::vector &vResult); + bool processLine(int nX, int nY, int nLine, + std::vector &vLastLineVal); + bool processFirstLine(int nX, int nY, int nLine, + std::vector &vLastLineVal); + void processLineLeft(int nX, int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal); + void processLineRight(int nX, int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal); + std::pair adjustHeight(int iLine, int nX, double *const pdfNx); + bool calcOutputExtent(int nX, int nY); + bool createOutputDataset(); + bool lineProgress(); + bool emitProgress(double fraction); }; } // namespace gdal diff --git a/autotest/utilities/test_gdal_viewshed.py b/autotest/utilities/test_gdal_viewshed.py index 5773a0b0279e..eda228c25c5a 100755 --- a/autotest/utilities/test_gdal_viewshed.py +++ b/autotest/utilities/test_gdal_viewshed.py @@ -68,7 +68,7 @@ def viewshed_input(tmp_path): gdaltest.runexternal( test_cli_utilities.get_gdalwarp_path() + " -t_srs EPSG:32617 -overwrite ../gdrivers/data/n43.tif " - + fname + + fname, ) return fname @@ -215,6 +215,98 @@ def test_gdal_viewshed_all_options(gdal_viewshed_path, tmp_path, viewshed_input) ############################################################################### +def test_gdal_viewshed_value_options(gdal_viewshed_path, tmp_path, viewshed_input): + + viewshed_out = str(tmp_path / "test_gdal_viewshed_out.tif") + + _, err = gdaltest.runexternal_out_and_err( + gdal_viewshed_path + + " -om NORMAL -f GTiff -oz {} -ox {} -oy {} -b 1 -a_nodata 0 -iv 127 -vv 254 -ov 0 {} {}".format( + oz[1], ox[0], oy[0], viewshed_input, viewshed_out + ) + ) + assert err is None or err == "" + ds = gdal.Open(viewshed_out) + assert ds + cs = ds.GetRasterBand(1).Checksum() + nodata = ds.GetRasterBand(1).GetNoDataValue() + ds = None + assert cs == 35091 + assert nodata == 0 + + +############################################################################### + + +def test_gdal_viewshed_tz_option(gdal_viewshed_path, tmp_path, viewshed_input): + + viewshed_out = str(tmp_path / "test_gdal_viewshed_out.tif") + + _, err = gdaltest.runexternal_out_and_err( + gdal_viewshed_path + + " -om NORMAL -f GTiff -oz {} -ox {} -oy {} -b 1 -a_nodata 0 -tz 5 {} {}".format( + oz[1], ox[0], oy[0], viewshed_input, viewshed_out + ) + ) + assert err is None or err == "" + ds = gdal.Open(viewshed_out) + assert ds + cs = ds.GetRasterBand(1).Checksum() + nodata = ds.GetRasterBand(1).GetNoDataValue() + ds = None + assert cs == 33725 + assert nodata == 0 + + +############################################################################### + + +def test_gdal_viewshed_cc_option(gdal_viewshed_path, tmp_path, viewshed_input): + + viewshed_out = str(tmp_path / "test_gdal_viewshed_out.tif") + + _, err = gdaltest.runexternal_out_and_err( + gdal_viewshed_path + + " -om NORMAL -f GTiff -oz {} -ox {} -oy {} -b 1 -a_nodata 0 -cc 0 {} {}".format( + oz[1], ox[0], oy[0], viewshed_input, viewshed_out + ) + ) + assert err is None or err == "" + ds = gdal.Open(viewshed_out) + assert ds + cs = ds.GetRasterBand(1).Checksum() + nodata = ds.GetRasterBand(1).GetNoDataValue() + ds = None + assert cs == 17241 + assert nodata == 0 + + +############################################################################### + + +def test_gdal_viewshed_md_option(gdal_viewshed_path, tmp_path, viewshed_input): + + viewshed_out = str(tmp_path / "test_gdal_viewshed_out.tif") + + _, err = gdaltest.runexternal_out_and_err( + gdal_viewshed_path + + " -om NORMAL -f GTiff -oz {} -ox {} -oy {} -b 1 -a_nodata 0 -tz 5 -md 20000 {} {}".format( + oz[1], ox[0], oy[0], viewshed_input, viewshed_out + ) + ) + assert err is None or err == "" + ds = gdal.Open(viewshed_out) + assert ds + cs = ds.GetRasterBand(1).Checksum() + nodata = ds.GetRasterBand(1).GetNoDataValue() + ds = None + assert cs == 22617 + assert nodata == 0 + + +############################################################################### + + def test_gdal_viewshed_missing_source(gdal_viewshed_path): _, err = gdaltest.runexternal_out_and_err(gdal_viewshed_path + " -ox 0 -oy 0") From 901ca4dbe15d9440f59c56b58f085c2dac6fa81a Mon Sep 17 00:00:00 2001 From: coim32 Date: Mon, 3 Jun 2024 12:56:13 -0600 Subject: [PATCH 25/31] Fix OPTIMALPADDING documentation --- doc/source/drivers/raster/georaster.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/drivers/raster/georaster.rst b/doc/source/drivers/raster/georaster.rst index a1c35d5efd0b..b221c6d2d5ba 100644 --- a/doc/source/drivers/raster/georaster.rst +++ b/doc/source/drivers/raster/georaster.rst @@ -127,7 +127,7 @@ The following creation options are supported: - .. co:: BLOCKING Decline the use of blocking (NO) or request an - automatic blocking size (OPTIMUM). + automatic blocking size (OPTIMALPADDING). - .. co:: SRID From 7758a2a1097304fd331b1e41a6cea82011c24211 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 4 Jun 2024 00:01:05 +0200 Subject: [PATCH 26/31] typo fixes [ci skip] --- alg/viewshed.cpp | 2 +- autotest/ogr/ogr_esrijson.py | 2 +- scripts/typos_allowlist.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/alg/viewshed.cpp b/alg/viewshed.cpp index 22f322bb1a3b..1c95701278f3 100644 --- a/alg/viewshed.cpp +++ b/alg/viewshed.cpp @@ -247,7 +247,7 @@ double CalcHeightLine(int nDistance, double Za) return Za * nDistance / (nDistance - 1); } -// Calulate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) +// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) // and passing through the line connecting (i - 1, j, Za) and (i, j - 1, Zb). // In other words, the origin and the two points form a plane and we're calculating Zc // of the point (i, j, Zc), also on the plane. diff --git a/autotest/ogr/ogr_esrijson.py b/autotest/ogr/ogr_esrijson.py index 1924acac2438..3b93528c5141 100755 --- a/autotest/ogr/ogr_esrijson.py +++ b/autotest/ogr/ogr_esrijson.py @@ -693,7 +693,7 @@ def test_ogr_esrijson_identify_srs(): # Test for https://github.com/OSGeo/gdal/issues/9996 -def test_ogr_esrijson_read_CadastralSpecialServies(): +def test_ogr_esrijson_read_CadastralSpecialServices(): ds = ogr.Open("data/esrijson/GetLatLon.json") lyr = ds.GetLayer(0) diff --git a/scripts/typos_allowlist.txt b/scripts/typos_allowlist.txt index 212047039ab2..8708492d5792 100644 --- a/scripts/typos_allowlist.txt +++ b/scripts/typos_allowlist.txt @@ -342,3 +342,4 @@ either 2 or 4 comma separated values. The same rules apply for the source and de "Invalid value for MITRE_LIMIT: %s", pszValue); *
  • JOIN_STYLE=ROUND/MITRE/BEVEL
    • *
  • MITRE_LIMIT=double
    • + DESCRIPTION=Staatliche Verwaltungs- und Sozialdienste wie öffentliche Verwaltung, Katastrophenschutz, Schulen und Krankenhäuser, die von öffentlichen oder privaten Einrichtungen erbracht werden, soweit sie in den Anwendungsbereich der Richtlinie 2007/2/EG fallen. Dieser Datensatz enthält Informationen zu Feuerwehrleitstellen. From 33681b215a3ae14e826c40bac34d03c897db81a1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 4 Jun 2024 01:23:35 +0200 Subject: [PATCH 27/31] OpenFileGDB: fix CodeQL cpp/integer-multiplication-cast-to-long warning Fixe https://github.com/OSGeo/gdal/security/code-scanning/569 --- ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp index 6f7fec8e6c48..d6fee21adf7c 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp @@ -1324,8 +1324,8 @@ bool FileGDBIndexIterator::FindPages(int iLevel, uint64_t nPage) returnErrorIf(nSubPagesCount[iLevel] == 0 || nSubPagesCount[iLevel] > nMaxPerPages); if (nIndexDepth == 2) - returnErrorIf(m_nValueCountInIdx > - nMaxPerPages * (nSubPagesCount[0] + 1)); + returnErrorIf(m_nValueCountInIdx > static_cast(nMaxPerPages) * + (nSubPagesCount[0] + 1)); if (eOp == FGSO_ISNOTNULL) { From 5573b18034fc8c6b84c709bef92a4658c7d738af Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Tue, 4 Jun 2024 22:58:00 +1000 Subject: [PATCH 28/31] User configuration docs: mention default User-Agent (#10124) --- doc/source/user/configoptions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index 8c34ba88fcd0..1878022b9d23 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -928,8 +928,9 @@ Networking options - .. config:: GDAL_HTTP_USERAGENT - When set this string will be used to set the ``User-Agent`` header in the http + This string will be used to set the ``User-Agent`` header in the HTTP request sent to the remote server. + Defaults to "GDAL/x.y.z" where x.y.z is the GDAL build version. - .. config:: GDAL_HTTP_UNSAFESSL :choices: YES, NO From 802b23c6cac6a892704b85a09a1c876b4545d707 Mon Sep 17 00:00:00 2001 From: Alessandro Pasotti Date: Tue, 4 Jun 2024 17:03:22 +0200 Subject: [PATCH 29/31] [gdalmanage] Use GDALArgumentParser --- apps/gdalmanage.cpp | 306 ++++++++++++++++++++++++++------------------ 1 file changed, 182 insertions(+), 124 deletions(-) diff --git a/apps/gdalmanage.cpp b/apps/gdalmanage.cpp index 1860cf9ab349..46f965cb3809 100644 --- a/apps/gdalmanage.cpp +++ b/apps/gdalmanage.cpp @@ -33,22 +33,22 @@ #include "gdal_version.h" #include "gdal.h" #include "commonutils.h" +#include "gdalargumentparser.h" /************************************************************************/ -/* Usage() */ +/* GDALManageOptions() */ /************************************************************************/ -static void Usage(bool bIsError) - +struct GDALManageOptions { - fprintf(bIsError ? stderr : stdout, - "Usage: gdalmanage [--help] [--help-general]\n" - " or gdalmanage identify [-r|-fr] [-u] *\n" - " or gdalmanage copy [-f ] \n" - " or gdalmanage rename [-f ] \n" - " or gdalmanage delete [-f ] \n"); - exit(bIsError ? 1 : 0); -} + bool bRecursive = false; + bool bForceRecurse = false; + bool bReportFailures = false; + std::string osNewName; + std::string osDatasetName; + std::vector aosDatasetNames; + std::string osDriverName; +}; /************************************************************************/ /* ProcessIdentifyTarget() */ @@ -92,73 +92,111 @@ static void ProcessIdentifyTarget(const char *pszTarget, } /************************************************************************/ -/* Identify() */ +/* GDALManageAppOptionsGetParser() */ /************************************************************************/ -static void Identify(int nArgc, char **papszArgv) - +static std::unique_ptr +GDALManageAppOptionsGetParser(GDALManageOptions *psOptions) { - /* -------------------------------------------------------------------- */ - /* Scan for command line switches */ - /* -------------------------------------------------------------------- */ - bool bRecursive = false; - bool bForceRecurse = false; - bool bReportFailures = false; + auto argParser = std::make_unique( + "gdalmanage", /* bForBinary */ true); - int i = 0; - for (; i < nArgc && papszArgv[i][0] == '-'; ++i) - { - if (EQUAL(papszArgv[i], "-r")) - bRecursive = true; - else if (EQUAL(papszArgv[i], "-fr")) - { - bForceRecurse = true; - bRecursive = true; - } - else if (EQUAL(papszArgv[i], "-u")) - bReportFailures = true; - else - Usage(true); - } + argParser->add_description( + _("Identify, delete, rename and copy raster data files.")); + argParser->add_epilog(_("For more details, consult the full documentation " + "for the gdalmanage utility " + "https://gdal.org/programs/gdalmanage.html")); - /* -------------------------------------------------------------------- */ - /* Process given files. */ - /* -------------------------------------------------------------------- */ - for (; i < nArgc; ++i) + auto addCommonOptions = [psOptions](GDALArgumentParser *subParser) { - ProcessIdentifyTarget(papszArgv[i], nullptr, bRecursive, - bReportFailures, bForceRecurse); - } -} - -/************************************************************************/ -/* Delete() */ -/************************************************************************/ - -static void Delete(GDALDriverH hDriver, int nArgc, char **papszArgv) - -{ - if (nArgc != 1) - Usage(true); - - GDALDeleteDataset(hDriver, papszArgv[0]); -} - -/************************************************************************/ -/* Copy() */ -/************************************************************************/ - -static void Copy(GDALDriverH hDriver, int nArgc, char **papszArgv, - const char *pszOperation) - -{ - if (nArgc != 2) - Usage(true); - - if (EQUAL(pszOperation, "copy")) - GDALCopyDatasetFiles(hDriver, papszArgv[1], papszArgv[0]); - else - GDALRenameDataset(hDriver, papszArgv[1], papszArgv[0]); + subParser->add_argument("-f") + .metavar("") + .store_into(psOptions->osDriverName) + .help(_("Specify format of raster file if unknown by the " + "application.")); + + subParser->add_argument("newdatasetname") + .metavar("") + .store_into(psOptions->osNewName) + .help(_("Name of the new file.")); + }; + + // Identify + + auto identifyParser = + argParser->add_subparser("identify", /* bForBinary */ true); + identifyParser->add_description(_("List data format of file(s).")); + + identifyParser->add_argument("-r") + .flag() + .store_into(psOptions->bRecursive) + .help(_("Recursively scan files/folders for raster files.")); + + identifyParser->add_argument("-fr") + .flag() + .store_into(psOptions->bRecursive) + .store_into(psOptions->bForceRecurse) + .help(_("Recursively scan folders for raster files, forcing " + "recursion in folders recognized as valid formats.")); + + identifyParser->add_argument("-u") + .flag() + .store_into(psOptions->bReportFailures) + .help(_("Report failures if file type is unidentified.")); + + // Note: this accepts multiple files + identifyParser->add_argument("datasetname") + .metavar("") + .store_into(psOptions->aosDatasetNames) + .remaining() + .help(_("Name(s) of the file(s) to identify.")); + + // Copy + + auto copyParser = argParser->add_subparser("copy", /* bForBinary */ true); + copyParser->add_description( + _("Create a copy of the raster file with a new name.")); + + addCommonOptions(copyParser); + + copyParser->add_argument("datasetname") + .metavar("") + .store_into(psOptions->osDatasetName) + .help(_("Name of the file to copy.")); + + // Rename + + auto renameParser = + argParser->add_subparser("rename", /* bForBinary */ true); + renameParser->add_description(_("Change the name of the raster file.")); + + addCommonOptions(renameParser); + + renameParser->add_argument("datasetname") + .metavar("") + .store_into(psOptions->osDatasetName) + .help(_("Name of the file to rename.")); + + // Delete + + auto deleteParser = + argParser->add_subparser("delete", /* bForBinary */ true); + deleteParser->add_description(_("Delete the raster file(s).")); + + // Note: this accepts multiple files + deleteParser->add_argument("datasetname") + .metavar("") + .store_into(psOptions->aosDatasetNames) + .remaining() + .help(_("Name(s) of the file(s) to delete.")); + + deleteParser->add_argument("-f") + .metavar("") + .store_into(psOptions->osDriverName) + .help( + _("Specify format of raster file if unknown by the application.")); + + return argParser; } /************************************************************************/ @@ -168,64 +206,70 @@ static void Copy(GDALDriverH hDriver, int nArgc, char **papszArgv, MAIN_START(argc, argv) { - char *pszDriver = nullptr; - GDALDriverH hDriver = nullptr; - /* Check that we are running against at least GDAL 1.5 */ - /* Note to developers : if we use newer API, please change the requirement - */ - if (atoi(GDALVersionInfo("VERSION_NUM")) < 1500) - { - fprintf(stderr, - "At least, GDAL >= 1.5.0 is required for this version of %s, " - "which was compiled against GDAL %s\n", - argv[0], GDAL_RELEASE_NAME); - exit(1); - } - - GDALAllRegister(); + EarlySetConfigOptions(argc, argv); argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); if (argc < 1) exit(-argc); - for (int i = 0; i < argc; i++) + /* -------------------------------------------------------------------- */ + /* Parse arguments. */ + /* -------------------------------------------------------------------- */ + + if (argc < 2) { - if (EQUAL(argv[i], "--utility_version")) + try { - printf("%s was compiled against GDAL %s and is running against " - "GDAL %s\n", - argv[0], GDAL_RELEASE_NAME, GDALVersionInfo("RELEASE_NAME")); - return 0; + GDALManageOptions sOptions; + auto argParser = GDALManageAppOptionsGetParser(&sOptions); + fprintf(stderr, "%s\n", argParser->usage().c_str()); } - else if (EQUAL(argv[i], "--help")) + catch (const std::exception &err) { - Usage(false); + CPLError(CE_Failure, CPLE_AppDefined, "Unexpected exception: %s", + err.what()); } + CSLDestroy(argv); + exit(1); } - if (argc < 3) - Usage(true); + GDALAllRegister(); - /* -------------------------------------------------------------------- */ - /* Do we have a driver specifier? */ - /* -------------------------------------------------------------------- */ - char **papszRemainingArgv = argv + 2; - int nRemainingArgc = argc - 2; + GDALManageOptions psOptions; + auto argParser = GDALManageAppOptionsGetParser(&psOptions); - if (EQUAL(papszRemainingArgv[0], "-f") && nRemainingArgc > 1) + try { - pszDriver = papszRemainingArgv[1]; - papszRemainingArgv += 2; - nRemainingArgc -= 2; + argParser->parse_args_without_binary_name(argv + 1); + CSLDestroy(argv); + } + catch (const std::exception &error) + { + argParser->display_error_and_usage(error); + CSLDestroy(argv); + exit(1); } - if (pszDriver != nullptr) + // For some obscure reason datasetname is parsed as mandatory + // if used with remaining() in a subparser + if (psOptions.aosDatasetNames.empty() && psOptions.osDatasetName.empty()) { - hDriver = GDALGetDriverByName(pszDriver); + std::invalid_argument error( + _("No dataset name provided. At least one dataset " + "name is required.")); + argParser->display_error_and_usage(error); + exit(1); + } + + GDALDriverH hDriver = nullptr; + if (!psOptions.osDriverName.empty()) + { + hDriver = GDALGetDriverByName(psOptions.osDriverName.c_str()); if (hDriver == nullptr) { - fprintf(stderr, "Unable to find driver named '%s'.\n", pszDriver); + CPLError(CE_Failure, CPLE_AppDefined, "Failed to find driver '%s'.", + psOptions.osDriverName.c_str()); exit(1); } } @@ -233,26 +277,40 @@ MAIN_START(argc, argv) /* -------------------------------------------------------------------- */ /* Split out based on operation. */ /* -------------------------------------------------------------------- */ - if (STARTS_WITH_CI(argv[1], "ident" /* identify" */)) - Identify(nRemainingArgc, papszRemainingArgv); - - else if (EQUAL(argv[1], "copy")) - Copy(hDriver, nRemainingArgc, papszRemainingArgv, "copy"); - - else if (EQUAL(argv[1], "rename")) - Copy(hDriver, nRemainingArgc, papszRemainingArgv, "rename"); - - else if (EQUAL(argv[1], "delete")) - Delete(hDriver, nRemainingArgc, papszRemainingArgv); - else - Usage(true); + if (argParser->is_subcommand_used("identify")) + { + // Process all files in aosDatasetName + for (const auto &datasetName : psOptions.aosDatasetNames) + { + ProcessIdentifyTarget( + datasetName.c_str(), nullptr, psOptions.bRecursive, + psOptions.bReportFailures, psOptions.bForceRecurse); + } + } + else if (argParser->is_subcommand_used("copy")) + { + GDALCopyDatasetFiles(hDriver, psOptions.osDatasetName.c_str(), + psOptions.osNewName.c_str()); + } + else if (argParser->is_subcommand_used("rename")) + { + GDALRenameDataset(hDriver, psOptions.osDatasetName.c_str(), + psOptions.osNewName.c_str()); + } + else if (argParser->is_subcommand_used("delete")) + { + // Process all files in aosDatasetName + for (const auto &datasetName : psOptions.aosDatasetNames) + { + GDALDeleteDataset(hDriver, datasetName.c_str()); + } + } /* -------------------------------------------------------------------- */ /* Cleanup */ /* -------------------------------------------------------------------- */ - CSLDestroy(argv); - GDALDestroyDriverManager(); + GDALDestroy(); exit(0); } From 1d390796cf46386f54a968378c33ac7b3f25026c Mon Sep 17 00:00:00 2001 From: Alessandro Pasotti Date: Tue, 4 Jun 2024 17:03:59 +0200 Subject: [PATCH 30/31] [gdalmanage] Update/fix documentation --- doc/source/programs/gdalmanage.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/source/programs/gdalmanage.rst b/doc/source/programs/gdalmanage.rst index 8c8b995f088c..7929c08c2406 100644 --- a/doc/source/programs/gdalmanage.rst +++ b/doc/source/programs/gdalmanage.rst @@ -16,7 +16,7 @@ Synopsis .. code-block:: Usage: gdalmanage [--help] [--help-general] - [-r] [-u] [-f ] + [-r] [-fr] [-u] [-f ] [] Description @@ -31,18 +31,22 @@ data types and deleting, renaming or copying the files. Mode of operation **identify** **: - List data format of file. + List data format of file(s). **copy** ** **: Create a copy of the raster file with a new name. **rename** ** **: Change the name of the raster file. **delete** **: - Delete raster file. + Delete raster file(s). .. option:: -r Recursively scan files/folders for raster files. +.. option:: -fr + + Recursively scan folders for raster files, forcing recursion in folders recognized as valid formats. + .. option:: -u Report failures if file type is unidentified. @@ -54,7 +58,7 @@ data types and deleting, renaming or copying the files. .. option:: - Raster file to operate on. + Raster file to operate on. With **identify** may be repeated for multiple files. .. option:: From 2c5b5090f67a844be4615479a8cac4ff2046cce0 Mon Sep 17 00:00:00 2001 From: Alessandro Pasotti Date: Tue, 4 Jun 2024 17:04:29 +0200 Subject: [PATCH 31/31] [gdalmanage] Tests --- autotest/pymod/test_cli_utilities.py | 8 + autotest/utilities/test_gdalmanage.py | 275 ++++++++++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 autotest/utilities/test_gdalmanage.py diff --git a/autotest/pymod/test_cli_utilities.py b/autotest/pymod/test_cli_utilities.py index 5c9257d0153f..9a8251d4ff4e 100755 --- a/autotest/pymod/test_cli_utilities.py +++ b/autotest/pymod/test_cli_utilities.py @@ -104,6 +104,14 @@ def get_gdalmdiminfo_path(): # +def get_gdalmanage_path(): + return get_cli_utility_path("gdalmanage") + + +############################################################################### +# + + def get_gdal_translate_path(): return get_cli_utility_path("gdal_translate") diff --git a/autotest/utilities/test_gdalmanage.py b/autotest/utilities/test_gdalmanage.py new file mode 100644 index 000000000000..22ef87c4c40c --- /dev/null +++ b/autotest/utilities/test_gdalmanage.py @@ -0,0 +1,275 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: gdalmanage testing +# Author: Alessandro Pasotti +# +############################################################################### +# Copyright (c) 2024, Alessandro Pasotti +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import os + +import gdaltest +import pytest +import test_cli_utilities + +pytestmark = pytest.mark.skipif( + test_cli_utilities.get_gdalmanage_path() is None, + reason="gdalmanage not available", +) + + +@pytest.fixture() +def gdalmanage_path(): + return test_cli_utilities.get_gdalmanage_path() + + +############################################################################### +# Simple identify test + + +def test_gdalmanage_identify(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify data/utmsmall.tif" + ) + assert err == "" + assert "GTiff" in ret + + +############################################################################### +# Test -r option + + +def test_gdalmanage_identify_recursive_option(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err(gdalmanage_path + " identify -r data") + assert err == "" + assert "ESRI Shapefile" in ret + assert len(ret.split("\n")) == 2 + + +############################################################################### +# Test -fr option + + +def test_gdalmanage_identify_force_recursive_option(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify -fr data" + ) + assert err == "" + ret = ret.replace("\\", "/") + assert len(ret.split("\n")) > 10 + assert "whiteblackred.tif: GTiff" in ret + assert "utmsmall.tif: GTiff" in ret + assert "ESRI Shapefile" in ret + assert "data/path.cpg: unrecognized" not in ret + + # Test both the -r and -fr options (shouldn't change the output) + (ret2, err2) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify -r -fr data" + ) + ret2 = ret2.replace("\\", "/") + assert ret2 == ret and err2 == err + + +############################################################################### +# Test -u option + + +def test_gdalmanage_identify_report_failures_option(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify -fr -u data" + ) + assert err == "" + ret = ret.replace("\\", "/") + assert "whiteblackred.tif: GTiff" in ret + assert "utmsmall.tif: GTiff" in ret + assert "ESRI Shapefile" in ret + assert "data/path.cpg: unrecognized" in ret + + +############################################################################### +# Test identify multiple files + + +def test_gdalmanage_identify_multiple_files(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify data/utmsmall.tif data/whiteblackred.tif" + ) + assert err == "" + assert len(ret.split("\n")) == 3 + assert "whiteblackred.tif: GTiff" in ret + assert "utmsmall.tif: GTiff" in ret + + +############################################################################### +# Test copy file + + +def test_gdalmanage_copy_file(tmp_path, gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" copy data/utmsmall.tif {tmp_path}/utmsmall.tif" + ) + assert err == "" + # Verify the file was created + assert os.path.exists(f"{tmp_path}/utmsmall.tif") + + +############################################################################### +# Test copy file with -f option + + +def test_gdalmanage_copy_file_format(tmp_path, gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" copy -f GTiff data/utmsmall.tif {tmp_path}/utmsmall2.tif" + ) + assert err == "" + # Verify the file was created + assert os.path.exists(f"{tmp_path}/utmsmall2.tif") + + # Wrong format + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + + f" copy -f WRONGFORMAT data/utmsmall.tif {tmp_path}/utmsmall3.tif" + ) + assert "Failed to find driver 'WRONGFORMAT'" in err + + +############################################################################### +# Test rename file + + +def test_gdalmanage_rename_file(tmp_path, gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" copy data/utmsmall.tif {tmp_path}/utmsmall_to_rename.tif" + ) + assert err == "" + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + + f" rename {tmp_path}/utmsmall_to_rename.tif {tmp_path}/utmsmall_renamed.tif" + ) + assert err == "" + # Verify the file was renamed + assert os.path.exists(f"{tmp_path}/utmsmall_renamed.tif") + assert not os.path.exists(f"{tmp_path}/utmsmall_to_rename.tif") + + +############################################################################### +# Test delete file + + +def test_gdalmanage_delete_file(tmp_path, gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" copy data/utmsmall.tif {tmp_path}/utmsmall_to_delete.tif" + ) + assert err == "" + assert os.path.exists(f"{tmp_path}/utmsmall_to_delete.tif") + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" delete {tmp_path}/utmsmall_to_delete.tif" + ) + assert err == "" + # Verify the file was deleted + assert not os.path.exists(f"{tmp_path}/utmsmall_to_delete.tif") + + +############################################################################### +# Test delete multiple files + + +def test_gdalmanage_delete_multiple_files(tmp_path, gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + f" copy data/utmsmall.tif {tmp_path}/utmsmall_to_delete.tif" + ) + assert err == "" + assert os.path.exists(f"{tmp_path}/utmsmall_to_delete.tif") + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + + f" copy data/whiteblackred.tif {tmp_path}/whiteblackred_to_delete.tif" + ) + assert err == "" + assert os.path.exists(f"{tmp_path}/whiteblackred_to_delete.tif") + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + + f" delete {tmp_path}/utmsmall_to_delete.tif {tmp_path}/whiteblackred_to_delete.tif" + ) + assert err == "" + # Verify the files were deleted + assert not os.path.exists(f"{tmp_path}/utmsmall_to_delete.tif") + assert not os.path.exists(f"{tmp_path}/whiteblackred_to_delete.tif") + + +############################################################################### +# Test no arguments + + +def test_gdalmanage_no_arguments(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err(gdalmanage_path) + assert "Usage: gdalmanage" in err + + +############################################################################### +# Test invalid command + + +def test_gdalmanage_invalid_command(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err(gdalmanage_path + " invalidcommand") + assert "Usage: gdalmanage" in err + + +############################################################################### +# Test invalid argument + + +def test_gdalmanage_invalid_argument(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err( + gdalmanage_path + " identify -WTF data/utmsmall.tif" + ) + assert "Usage: gdalmanage" in err + assert "Unknown argument: -WTF" in err + + +############################################################################### +# Test valid command with no required argument + + +def test_gdalmanage_valid_command_no_argument(gdalmanage_path): + + (ret, err) = gdaltest.runexternal_out_and_err(gdalmanage_path + " identify") + assert "Usage: gdalmanage" in err + assert ( + "Error: No dataset name provided. At least one dataset name is required" in err + )