feat(arrow): Add Polygon column support (GeoArrow compatible) (#2280)

modules/arrow/package.json

@@ -43,6 +43,7 @@
   "dependencies": {
     "@luma.gl/core": "9.2.0-alpha.0",
     "@math.gl/polygon": "^4.1.0",
+    "@math.gl/types": "^4.1.0",
     "apache-arrow": "^17.0.0"
   },
   "gitHead": "c636c34b8f1581eed163e94543a8eb1f4382ba8e"

modules/arrow/src/arrow/arrow-utils.ts

@@ -0,0 +1,15 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+
+/** Count number of nested top level Arrow Lists */
+export function getArrowListNestingLevel(data: arrow.Data): number {
+  let nestingLevel = 0;
+  if (arrow.DataType.isList(data.type)) {
+    nestingLevel += 1;
+    data = data.children[0];
+  }
+  return nestingLevel;
+}

modules/arrow/src/attribute-utils/attribute-utils.ts

@@ -0,0 +1,63 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+
+/**
+ * Expand an array from "one element per geometry" to "one element per coordinate"
+ *
+ * @param input: the input array to expand
+ * @param size : the number of nested elements in the input array per geometry. So for example, for RGB data this would be 3, for RGBA this would be 4. For radius, this would be 1.
+ * @param geomOffsets : an offsets array mapping from the geometry to the coordinate indexes. So in the case of a LineStringArray, this is retrieved directly from the GeoArrow storage. In the case of a PolygonArray, this comes from the resolved indexes that need to be given to the SolidPolygonLayer anyways.
+ *
+ * @return values expanded to be per-coordinate
+ */
+export function expandArrayToCoords<T extends TypedArray>(
+  input: T,
+  size: number,
+  geomOffsets: Int32Array
+): T {
+  const numCoords = geomOffsets[geomOffsets.length - 1];
+  // @ts-expect-error
+  const outputArray: T = new input.constructor(numCoords * size);
+
+  // geomIdx is an index into the geomOffsets array
+  // geomIdx is also the geometry/table index
+  for (let geomIdx = 0; geomIdx < geomOffsets.length - 1; geomIdx++) {
+    // geomOffsets maps from the geometry index to the coord index
+    // So here we get the range of coords that this geometry covers
+    const lastCoordIdx = geomOffsets[geomIdx];
+    const nextCoordIdx = geomOffsets[geomIdx + 1];
+
+    // Iterate over this range of coord indices
+    for (let coordIdx = lastCoordIdx; coordIdx < nextCoordIdx; coordIdx++) {
+      // Iterate over size
+      for (let i = 0; i < size; i++) {
+        // Copy from the geometry index in `input` to the coord index in
+        // `output`
+        outputArray[coordIdx * size + i] = input[geomIdx * size + i];
+      }
+    }
+  }
+
+  return outputArray;
+}
+
+/**
+ * Invert offsets so that lookup can go in the opposite direction
+ */
+export function invertOffsets(offsets: Int32Array): Uint32Array {
+  const largestOffset = offsets[offsets.length - 1];
+
+  const invertedOffsets = new Uint32Array(largestOffset);
+  for (let arrayIdx = 0; arrayIdx < offsets.length - 1; arrayIdx++) {
+    const thisOffset = offsets[arrayIdx];
+    const nextOffset = offsets[arrayIdx + 1];
+    for (let offset = thisOffset; offset < nextOffset; offset++) {
+      invertedOffsets[offset] = arrayIdx;
+    }
+  }
+
+  return invertedOffsets;
+}

modules/arrow/src/geoarrow/earcut.ts

@@ -0,0 +1,60 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {earcut} from '@math.gl/polygon';
+import * as arrow from 'apache-arrow';
+import {ArrowPolygon} from './geoarrow-types';
+import {getLineStringChild, getPointChild, getPolygonChild} from './geoarrow';
+
+export function earcutPolygonArray(data: arrow.Data<ArrowPolygon>): Uint32Array {
+  const trianglesResults: number[][] = [];
+  let outputSize = 0;
+  for (let geometryIndex = 0; geometryIndex < data.length; geometryIndex++) {
+    const triangles = earcutSinglePolygon(data, geometryIndex);
+    trianglesResults.push(triangles);
+    outputSize += triangles.length;
+  }
+
+  const outputArray = new Uint32Array(outputSize);
+  let idx = 0;
+  for (const triangles of trianglesResults) {
+    for (const value of triangles) {
+      outputArray[idx] = value;
+      idx += 1;
+    }
+  }
+
+  return outputArray;
+}
+
+function earcutSinglePolygon(data: arrow.Data<ArrowPolygon>, geometryIndex: number): number[] {
+  const geometryOffsets = data.valueOffsets;
+  const rings = getPolygonChild(data);
+  const ringOffsets = rings.valueOffsets;
+
+  const coords = getLineStringChild(rings);
+  const dim = coords.type.listSize;
+  const flatCoords = getPointChild(coords);
+
+  const ringBegin = geometryOffsets[geometryIndex];
+  const ringEnd = geometryOffsets[geometryIndex + 1];
+
+  const coordsBegin = ringOffsets[ringBegin];
+  const coordsEnd = ringOffsets[ringEnd];
+
+  const slicedFlatCoords = flatCoords.values.subarray(coordsBegin * dim, coordsEnd * dim);
+
+  const initialCoordIndex = ringOffsets[ringBegin];
+  const holeIndices = [];
+  for (let holeRingIdx = ringBegin + 1; holeRingIdx < ringEnd; holeRingIdx++) {
+    holeIndices.push(ringOffsets[holeRingIdx] - initialCoordIndex);
+  }
+  const triangles = earcut(slicedFlatCoords, holeIndices, dim);
+
+  for (let i = 0; i < triangles.length; i++) {
+    triangles[i] += initialCoordIndex;
+  }
+
+  return triangles;
+}

modules/arrow/src/geoarrow/geoarrow-transform.ts

@@ -0,0 +1,55 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import {ArrowMultiLineString, ArrowPolygon, ArrowMultiPolygon} from './geoarrow-types';
+import {getMultiLineStringChild, getPolygonChild, getMultiPolygonChild} from './geoarrow';
+
+export function getMultiLineStringResolvedOffsets(
+  data: arrow.Data<ArrowMultiLineString>
+): Int32Array {
+  const geomOffsets = data.valueOffsets;
+  const lineStringData = getMultiLineStringChild(data);
+  const ringOffsets = lineStringData.valueOffsets;
+
+  const resolvedRingOffsets = new Int32Array(geomOffsets.length);
+  for (let i = 0; i < resolvedRingOffsets.length; ++i) {
+    // Perform the lookup into the ringIndices array using the geomOffsets
+    // array
+    resolvedRingOffsets[i] = ringOffsets[geomOffsets[i]];
+  }
+
+  return resolvedRingOffsets;
+}
+
+export function getPolygonResolvedOffsets(data: arrow.Data<ArrowPolygon>): Int32Array {
+  const geomOffsets = data.valueOffsets;
+  const ringData = getPolygonChild(data);
+  const ringOffsets = ringData.valueOffsets;
+
+  const resolvedRingOffsets = new Int32Array(geomOffsets.length);
+  for (let i = 0; i < resolvedRingOffsets.length; ++i) {
+    // Perform the lookup into the ringIndices array using the geomOffsets
+    // array
+    resolvedRingOffsets[i] = ringOffsets[geomOffsets[i]];
+  }
+
+  return resolvedRingOffsets;
+}
+
+export function getMultiPolygonResolvedOffsets(data: arrow.Data<ArrowMultiPolygon>): Int32Array {
+  const polygonData = getMultiPolygonChild(data);
+  const ringData = getPolygonChild(polygonData);
+
+  const geomOffsets = data.valueOffsets;
+  const polygonOffsets = polygonData.valueOffsets;
+  const ringOffsets = ringData.valueOffsets;
+
+  const resolvedRingOffsets = new Int32Array(geomOffsets.length);
+  for (let i = 0; i < resolvedRingOffsets.length; ++i) {
+    resolvedRingOffsets[i] = ringOffsets[polygonOffsets[geomOffsets[i]]];
+  }
+
+  return resolvedRingOffsets;
+}

modules/arrow/src/geoarrow/geoarrow-types.ts

@@ -0,0 +1,49 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+
+/**
+ * Enum holding GeoArrow extension type names
+ */
+export enum EXTENSION_NAME {
+  POINT = 'geoarrow.point',
+  LINESTRING = 'geoarrow.linestring',
+  POLYGON = 'geoarrow.polygon',
+  MULTIPOINT = 'geoarrow.multipoint',
+  MULTILINESTRING = 'geoarrow.multilinestring',
+  MULTIPOLYGON = 'geoarrow.multipolygon'
+}
+
+export type ArrowInterleavedCoord = arrow.FixedSizeList<arrow.Float64>;
+export type ArrowSeparatedCoord = arrow.Struct<{
+  x: arrow.Float64;
+  y: arrow.Float64;
+}>;
+// TODO: support separated coords
+export type ArrowCoord = ArrowInterleavedCoord; // | SeparatedCoord;
+export type ArrowPoint = ArrowCoord;
+export type ArrowLineString = arrow.List<ArrowCoord>;
+export type ArrowPolygon = arrow.List<arrow.List<ArrowCoord>>;
+export type ArrowMultiPoint = arrow.List<ArrowCoord>;
+export type ArrowMultiLineString = arrow.List<arrow.List<ArrowCoord>>;
+export type ArrowMultiPolygon = arrow.List<arrow.List<arrow.List<ArrowCoord>>>;
+
+// export type PointVector = arrow.Vector<ArrowCoord>;
+// export type LineStringVector = arrow.Vector<ArrowLineString>;
+// export type PolygonVector = arrow.Vector<ArrowPolygon>;
+// export type MultiPointVector = arrow.Vector<ArrowMultiPoint>;
+// export type MultiLineStringVector = arrow.Vector<ArrowMultiLineString>;
+// export type MultiPolygonVector = arrow.Vector<ArrowMultiPolygon>;
+
+// export type PointData = arrow.Data<Point>;
+// export type LineStringData = arrow.Data<LineString>;
+// export type PolygonData = arrow.Data<Polygon>;
+// export type MultiPointData = arrow.Data<MultiPoint>;
+// export type MultiLineStringData = arrow.Data<MultiLineString>;
+// export type MultiPolygonData = arrow.Data<MultiPolygon>;
+
+export type GeoArrowPickingInfo = {
+  object: arrow.StructRowProxy;
+};