Merge commit 'b677f3363baacce2b1e58aed9107ca079aaf6b1d' into dt/appen…

…d_truncate

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 ## v4.0
 
+### v4.1.0-alpha.1
+- feat(zip): cd header and zip64 info generation implemented (#2792)
+- fix: wms example (#2814)
+- fix(tile-converter): trailing slash Removing from path in i3s-server (#2813)
+- docs: fgb and tilejson docs, streamline fgbloader exports (#2811)
+- chore: upgrade to vite 5 (#2812)
+- chore: typescript 5.3 (#2810)
+- fix(GeoArrow): handle tessellation error & improve mean centers (#2803)
+- chore(parquet): GeoParquetLoader, collect the parquet loader zoo in one file (#2809)
+- GLTF: inline emod function to remove loaders.gl/math dependency (#2808)
+- chore(website): polish (#2807)
+- feat(website): Add Shapefile and KML examples (#2806)
+- chore(website): minor touches (#2805)
+- docs(las): Mark LAS v1.4 as not supported (#2804)
+- feat(website): Add GeoArrow example (#2802)
+- feat(flatgeobuf): Extract arrow compatible schema (#2801)
+- chore: update package.json keywords (#2797)
+- Convert GetImageParameters to WMSGetMapParameters (#2799)
+- chore(arrow): Support WKT and WKB geoarrow encodings (v4.1) (#2798)
+
 ### v4.0.4
 - feat(arrow): GeoArrowLoader (#2796)
 - fix(draco): revert --format=esm for the worker (#2795)

lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "2.9.1",
-  "version": "4.0.4",
+  "version": "4.1.0-alpha.1",
   "command": {
     "publish": {},
     "bootstrap": {}

modules/3d-tiles/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/3d-tiles",
-  "version": "4.0.4",
+  "version": "4.1.0-alpha.1",
   "description": "3D Tiles, an open standard for streaming massive heterogeneous 3D geospatial datasets.",
   "license": "MIT",
   "type": "module",
@@ -45,12 +45,12 @@
     "build-bundle": "ocular-bundle ./src/index.ts"
   },
   "dependencies": {
-    "@loaders.gl/draco": "4.0.4",
-    "@loaders.gl/gltf": "4.0.4",
-    "@loaders.gl/loader-utils": "4.0.4",
-    "@loaders.gl/math": "4.0.4",
-    "@loaders.gl/tiles": "4.0.4",
-    "@loaders.gl/zip": "4.0.4",
+    "@loaders.gl/draco": "4.1.0-alpha.1",
+    "@loaders.gl/gltf": "4.1.0-alpha.1",
+    "@loaders.gl/loader-utils": "4.1.0-alpha.1",
+    "@loaders.gl/math": "4.1.0-alpha.1",
+    "@loaders.gl/tiles": "4.1.0-alpha.1",
+    "@loaders.gl/zip": "4.1.0-alpha.1",
     "@math.gl/core": "^4.0.0",
     "@math.gl/geospatial": "^4.0.0",
     "@probe.gl/log": "^4.0.4",

modules/arrow/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.0.4",
+  "version": "4.1.0-alpha.1",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "type": "module",
@@ -52,10 +52,10 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.0.4",
-    "@loaders.gl/loader-utils": "4.0.4",
-    "@loaders.gl/schema": "4.0.4",
-    "@loaders.gl/wkt": "4.0.4",
+    "@loaders.gl/gis": "4.1.0-alpha.1",
+    "@loaders.gl/loader-utils": "4.1.0-alpha.1",
+    "@loaders.gl/schema": "4.1.0-alpha.1",
+    "@loaders.gl/wkt": "4.1.0-alpha.1",
     "@math.gl/polygon": "4.0.0",
     "apache-arrow": "^13.0.0"
   },

modules/arrow/src/geoarrow/convert-geoarrow-to-binary-geometry.ts

@@ -45,7 +45,7 @@ type BinaryGeometryContent = {
   geomOffset: Int32Array;
   /** Array of geometry indicies: the start index of each geometry */
   geometryIndicies: Uint16Array;
-  /** (Optional) indices of triangels returned from polygon tessellation (Polygon only) */
+  /** (Optional) indices of triangels returned from polygon triangulation (Polygon only) */
   triangles?: Uint32Array;
   /** (Optional) array of mean center of each geometry */
   meanCenters?: Float64Array;
@@ -54,17 +54,21 @@ type BinaryGeometryContent = {
 /**
  * binary geometry template, see deck.gl BinaryGeometry
  */
-export const BINARY_GEOMETRY_TEMPLATE = {
-  globalFeatureIds: {value: new Uint32Array(0), size: 1},
-  positions: {value: new Float32Array(0), size: 2},
-  properties: [],
-  numericProps: {},
-  featureIds: {value: new Uint32Array(0), size: 1}
-};
+export function getBinaryGeometryTemplate() {
+  return {
+    globalFeatureIds: {value: new Uint32Array(0), size: 1},
+    positions: {value: new Float32Array(0), size: 2},
+    properties: [],
+    numericProps: {},
+    featureIds: {value: new Uint32Array(0), size: 1}
+  };
+}
 
 export type BinaryGeometriesFromArrowOptions = {
   /** option to specify which chunk to get binary geometries from, for progressive rendering */
   chunkIndex?: number;
+  /** The offset (beginning index of rows) of input chunk. Used for reconstructing globalFeatureIds in web workers */
+  chunkOffset?: number;
   /** option to get mean centers from geometries, for polygon filtering */
   calculateMeanCenters?: boolean;
   /** option to compute the triangle indices by tesselating polygons */
@@ -90,9 +94,12 @@ export function getBinaryGeometriesFromArrow(
     line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
   };
 
-  const chunks = options?.chunkIndex ? [geoColumn.data[options?.chunkIndex]] : geoColumn.data;
+  const chunks =
+    options?.chunkIndex !== undefined && options?.chunkIndex >= 0
+      ? [geoColumn.data[options?.chunkIndex]]
+      : geoColumn.data;
   let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
-  let globalFeatureIdOffset = 0;
+  let globalFeatureIdOffset = options?.chunkOffset || 0;
   const binaryGeometries: BinaryFeatures[] = [];
 
   chunks.forEach((chunk) => {
@@ -111,6 +118,7 @@ export function getBinaryGeometriesFromArrow(
         size: nDim
       },
       featureIds: {value: featureIds, size: 1},
+      // eslint-disable-next-line no-loop-func
       properties: [...Array(chunk.length).keys()].map((i) => ({
         index: i + globalFeatureIdOffset
       }))
@@ -123,18 +131,18 @@ export function getBinaryGeometriesFromArrow(
       shape: 'binary-feature-collection',
       points: {
         type: 'Point',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.point ? binaryContent : {})
       },
       lines: {
         type: 'LineString',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.line ? binaryContent : {}),
         pathIndices: {value: featureTypes.line ? geomOffset : new Uint16Array(0), size: 1}
       },
       polygons: {
         type: 'Polygon',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.polygon ? binaryContent : {}),
         polygonIndices: {
           // use geomOffset as polygonIndices same as primitivePolygonIndices since we are using earcut to get triangule indices
@@ -281,7 +289,7 @@ function getBinaryGeometriesFromChunk(
  * @param primitivePolygonIndices Indices within positions of the start of each primitive Polygon/ring
  * @param flatCoordinateArray Array of x, y or x, y, z positions
  * @param nDim - number of dimensions per position
- * @returns
+ * @returns triangle indices or null if invalid polygon and earcut fails
  */
 export function getTriangleIndices(
   polygonIndices: Uint16Array,
@@ -306,13 +314,14 @@ export function getTriangleIndices(
         }
         primitiveIndex++;
       }
+      // TODO check if each ring is closed
       const triangleIndices = earcut(
         slicedFlatCoords,
         holeIndices.length > 0 ? holeIndices : undefined,
         nDim
       );
       if (triangleIndices.length === 0) {
-        throw Error('can not tesselate invalid polygon');
+        throw Error('earcut failed e.g. invalid polygon');
       }
       for (let j = 0; j < triangleIndices.length; j++) {
         triangles.push(triangleIndices[j] + startIdx);
@@ -325,9 +334,7 @@ export function getTriangleIndices(
     }
     return trianglesUint32;
   } catch (error) {
-    // TODO - add logging
-    // there is an expection when tesselating invalid polygon, e.g. polygon with self-intersection
-    // return null to skip tesselating
+    // if earcut fails, return null
     return null;
   }
 }
@@ -379,8 +386,8 @@ function getBinaryPolygonsFromChunk(
 
   return {
     featureIds,
-    flatCoordinateArray,
     nDim,
+    flatCoordinateArray,
     geomOffset,
     geometryIndicies,
     ...(options?.triangulate && triangles ? {triangles} : {})

modules/arrow/src/index.ts

@@ -52,7 +52,7 @@ export type {
   BinaryGeometriesFromArrowOptions
 } from './geoarrow/convert-geoarrow-to-binary-geometry';
 export {
-  BINARY_GEOMETRY_TEMPLATE,
+  getBinaryGeometryTemplate,
   getBinaryGeometriesFromArrow,
   getTriangleIndices,
   getMeanCentersFromBinaryGeometries
@@ -65,5 +65,11 @@ export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson-geo
 export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
 
 // EXPERIMENTAL WORKER
+export {hardClone} from './workers/hard-clone';
 
-export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';
+export type {ParseGeoArrowInput, ParseGeoArrowResult} from './triangulate-on-worker';
+export {
+  TriangulationWorker,
+  triangulateOnWorker,
+  parseGeoArrowOnWorker
+} from './triangulate-on-worker';

modules/arrow/src/triangulate-on-worker.ts

@@ -1,19 +1,52 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
+import * as arrow from 'apache-arrow';
 import type {WorkerOptions} from '@loaders.gl/worker-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
+import {BinaryDataFromGeoArrow, GeoArrowEncoding} from '@loaders.gl/arrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export type TriangulationWorkerInput = TriangulateInput | {operation: 'test'; data: any};
-export type TriangulationWorkerOutput = TriangulateResult | {operation: 'test'; data: any};
+export type TriangulationWorkerInput =
+  | ({operation: 'triangulate'} & TriangulateInput)
+  | ParseGeoArrowInput
+  | {operation: 'test'; data: any};
+
+export type TriangulationWorkerOutput =
+  | ({operation: 'triangulate'} & TriangulateResult)
+  | ({operation: 'parse-geoarrow'} & ParseGeoArrowResult)
+  | {operation: 'test'; data: any};
+
+type GeoArrowChunkData = {
+  type: arrow.DataType;
+  offset: number;
+  length: number;
+  nullCount: number;
+  buffers: any;
+  children: arrow.Data[];
+  dictionary?: arrow.Vector;
+};
+
+export type ParseGeoArrowInput = {
+  operation: 'parse-geoarrow';
+  chunkData: GeoArrowChunkData;
+  chunkIndex: number;
+  chunkOffset: number;
+  geometryEncoding: GeoArrowEncoding;
+  calculateMeanCenters: boolean;
+  triangle: boolean;
+};
+
+export type ParseGeoArrowResult = {
+  chunkIndex: number;
+  binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null;
+};
 
 /** Input data for operation: 'triangulate' */
 export type TriangulateInput = {
-  operation: 'triangulate';
   polygonIndices: Uint16Array;
   primitivePolygonIndices: Int32Array;
   flatCoordinateArray: Float64Array;
@@ -37,11 +70,21 @@ export const TriangulationWorker = {
 };
 
 /**
- * Provide type safety
+ * Triangulate a set of polygons on worker, type safe API
  */
 export function triangulateOnWorker(
-  data: TriangulationWorkerInput,
+  data: TriangulateInput,
+  options: WorkerOptions = {}
+): Promise<TriangulateResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'triangulate'}, options);
+}
+
+/**
+ * Parse GeoArrow geometry colum on worker, type safe API
+ */
+export function parseGeoArrowOnWorker(
+  data: ParseGeoArrowInput,
   options: WorkerOptions = {}
-): Promise<TriangulationWorkerOutput> {
-  return processOnWorker(TriangulationWorker, data, options);
+): Promise<ParseGeoArrowResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'parse-geoarrow'}, options);
 }