diff --git a/modules/gis/README.md b/modules/gis/README.md new file mode 100644 index 0000000000..95bf2ebb7f --- /dev/null +++ b/modules/gis/README.md @@ -0,0 +1,5 @@ +# @loaders.gl/gis + +This module contains helper classes for the GIS category of loaders. + +[loaders.gl](https://loaders.gl/docs) is a collection of framework independent visualization-focused loaders (parsers). diff --git a/modules/gis/docs/README.md b/modules/gis/docs/README.md new file mode 100644 index 0000000000..1b710f3aca --- /dev/null +++ b/modules/gis/docs/README.md @@ -0,0 +1,15 @@ +# @loaders.gl/gis + +This module contains helper classes for the GIS category of loaders. + +## Installation + +```bash +npm install @loaders.gl/gis +``` + +## Utility Functions + +| Utility Functions | +| ----------------------------------------------------------------------- | +| [`geojson-to-binary`](modules/gis/docs/api-reference/geojson-to-binary) | diff --git a/modules/gis/docs/api-reference/geojson-to-binary.md b/modules/gis/docs/api-reference/geojson-to-binary.md new file mode 100644 index 0000000000..87b352cee2 --- /dev/null +++ b/modules/gis/docs/api-reference/geojson-to-binary.md @@ -0,0 +1,69 @@ +# GeoJSON to TypedArrays + +Helper function to transform an array of GeoJSON `Feature`s into binary typed +arrays. This is designed to speed up geospatial loaders by removing the need for +serialization and deserialization of data transferred by the worker back to the +main process. + +## Usage + +```js +import {load} from '@loaders.gl/core'; +import {MVTLoader} from '@loaders.gl/mvt'; +import {geojsonToBinary} from '@loaders.gl/gis'; + +// See MVTLoader docs for loader options +const geoJSONfeatures = await load(url, MVTLoader, loaderOptions); + +/* +* Default options are: +* +* { +* PositionDataType: Float32Array +* } +*/ +const binaryArrays = geojsonToBinary(geoJSONfeatures, options); +``` + +## Outputs + +### TypedArrays + +`geojsonToBinary` returns an object containing typed arrays sorted by geometry +type. `positions` corresponds to 2D or 3D coordinates; `objectIds` returns the +index of the vertex in the initial `features` array. + +```js +{ + points: { + // Array of x, y or x, y, z positions + positions: {value: Float32Array, size: coordLength}, + // Array of original feature indexes by vertex + objectIds: {value: Uint32Array, size: 1}, + }, + lines: { + // Indices within positions of the start of each individual LineString + pathIndices: {value: Uint32Array, size: 1}, + // Array of x, y or x, y, z positions + positions: {value: Float32Array, size: coordLength}, + // Array of original feature indexes by vertex + objectIds: {value: Uint32Array, size: 1}, + }, + polygons: { + // Indices within positions of the start of each complex Polygon + polygonIndices: {value: Uint32Array, size: 1}, + // Indices within positions of the start of each primitive Polygon/ring + primitivePolygonIndices: {value: Uint32Array, size: 1}, + // Array of x, y or x, y, z positions + positions: {value: Float32Array, size: coordLength}, + // Array of original feature indexes by vertex + objectIds: {value: Uint32Array, size: 1}, + } +} +``` + +## Options + +| Option | Type | Default | Description | +| ---------------- | -------------------------------- | -------------- | ----------------------------------- | +| PositionDataType | `Float32Array` or `Float64Array` | `Float32Array` | Data type used for positions arrays | diff --git a/modules/gis/package.json b/modules/gis/package.json new file mode 100644 index 0000000000..f240376e95 --- /dev/null +++ b/modules/gis/package.json @@ -0,0 +1,31 @@ +{ + "name": "@loaders.gl/gis", + "description": "Helpers for GIS category data", + "version": "2.1.0-beta.2", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/uber-web/loaders.gl" + }, + "keywords": [ + "geometry", + "GeoJSON" + ], + "main": "dist/es5/index.js", + "module": "dist/esm/index.js", + "esnext": "dist/es6/index.js", + "sideEffects": false, + "files": [ + "src", + "dist", + "README.md" + ], + "dependencies": { + "@loaders.gl/loader-utils": "2.1.0-beta.2", + "@mapbox/vector-tile": "^1.3.1", + "pbf": "^3.2.1" + } +} diff --git a/modules/gis/src/bundle.js b/modules/gis/src/bundle.js new file mode 100644 index 0000000000..7316a80287 --- /dev/null +++ b/modules/gis/src/bundle.js @@ -0,0 +1,7 @@ +/* global window, global */ +const moduleExports = require('./index'); +const _global = typeof window === 'undefined' ? global : window; +// @ts-ignore +_global.loaders = _global.loaders || {}; +// @ts-ignore +module.exports = Object.assign(_global.loaders, moduleExports); diff --git a/modules/gis/src/index.js b/modules/gis/src/index.js new file mode 100644 index 0000000000..a71bc8ad1e --- /dev/null +++ b/modules/gis/src/index.js @@ -0,0 +1 @@ +export {geojsonToBinary} from './lib/geojson-to-binary'; diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js new file mode 100644 index 0000000000..c80293bb54 --- /dev/null +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -0,0 +1,247 @@ +// Convert GeoJSON features to flat binary arrays +export function geojsonToBinary(features, options = {}) { + const firstPassData = firstPass(features); + return secondPass(features, firstPassData, options); +} + +// Initial scan over GeoJSON features +// Counts number of coordinates of each geometry type and keeps track of the max coordinate +// dimensions +// eslint-disable-next-line complexity +function firstPass(features) { + // Counts the number of _positions_, so [x, y, z] counts as one + let pointPositions = 0; + let linePositions = 0; + let linePaths = 0; + let polygonPositions = 0; + let polygonObjects = 0; + let polygonRings = 0; + let maxCoordLength = 2; + + for (const feature of features) { + const geometry = feature.geometry; + switch (geometry.type) { + case 'Point': + pointPositions++; + if (geometry.coordinates.length === 3) { + maxCoordLength = 3; + } + break; + case 'MultiPoint': + pointPositions += geometry.coordinates.length; + for (const point of geometry.coordinates) { + // eslint-disable-next-line max-depth + if (point.length === 3) maxCoordLength = 3; + } + break; + case 'LineString': + linePositions += geometry.coordinates.length; + linePaths++; + + for (const coord of geometry.coordinates) { + // eslint-disable-next-line max-depth + if (coord.length === 3) maxCoordLength = 3; + } + break; + case 'MultiLineString': + for (const line of geometry.coordinates) { + linePositions += line.length; + linePaths++; + + // eslint-disable-next-line max-depth + for (const coord of line) { + // eslint-disable-next-line max-depth + if (coord.length === 3) maxCoordLength = 3; + } + } + break; + case 'Polygon': + polygonObjects++; + polygonRings += geometry.coordinates.length; + polygonPositions += flatten(geometry.coordinates).length; + + for (const coord of flatten(geometry.coordinates)) { + // eslint-disable-next-line max-depth + if (coord.length === 3) maxCoordLength = 3; + } + break; + case 'MultiPolygon': + for (const polygon of geometry.coordinates) { + polygonObjects++; + polygonRings += polygon.length; + polygonPositions += flatten(polygon).length; + + // eslint-disable-next-line max-depth + for (const coord of flatten(polygon)) { + // eslint-disable-next-line max-depth + if (coord.length === 3) maxCoordLength = 3; + } + } + break; + default: + throw new Error(`Unsupported geometry type: ${geometry.type}`); + } + } + + return { + pointPositions, + linePositions, + linePaths, + coordLength: maxCoordLength, + polygonPositions, + polygonObjects, + polygonRings + }; +} + +// Second scan over GeoJSON features +// Fills coordinates into pre-allocated typed arrays +function secondPass(features, firstPassData, options = {}) { + const { + pointPositions, + linePositions, + linePaths, + coordLength, + polygonPositions, + polygonObjects, + polygonRings + } = firstPassData; + const {PositionDataType = Float32Array} = options; + const points = { + positions: new PositionDataType(pointPositions * coordLength), + objectIds: new Uint32Array(pointPositions) + }; + const lines = { + pathIndices: new Uint32Array(linePaths), + positions: new PositionDataType(linePositions * coordLength), + objectIds: new Uint32Array(linePositions) + }; + const polygons = { + polygonIndices: new Uint32Array(polygonObjects), + primitivePolygonIndices: new Uint32Array(polygonRings), + positions: new PositionDataType(polygonPositions * coordLength), + objectIds: new Uint32Array(polygonPositions) + }; + + const indexMap = { + pointPosition: 0, + linePosition: 0, + linePath: 0, + polygonPosition: 0, + polygonObject: 0, + polygonRing: 0, + feature: 0 + }; + + for (const feature of features) { + const geometry = feature.geometry; + + switch (geometry.type) { + case 'Point': + handlePoint(geometry.coordinates, points, indexMap, coordLength); + break; + case 'MultiPoint': + handleMultiPoint(geometry.coordinates, points, indexMap, coordLength); + break; + case 'LineString': + handleLineString(geometry.coordinates, lines, indexMap, coordLength); + break; + case 'MultiLineString': + handleMultiLineString(geometry.coordinates, lines, indexMap, coordLength); + break; + case 'Polygon': + handlePolygon(geometry.coordinates, polygons, indexMap, coordLength); + break; + case 'MultiPolygon': + handleMultiPolygon(geometry.coordinates, polygons, indexMap, coordLength); + break; + default: + throw new Error('Invalid geometry type'); + } + + indexMap.feature++; + } + + // Wrap each array in an accessor object with value and size keys + return { + points: { + positions: {value: points.positions, size: coordLength}, + objectIds: {value: points.objectIds, size: 1} + }, + lines: { + pathIndices: {value: lines.pathIndices, size: 1}, + positions: {value: lines.positions, size: coordLength}, + objectIds: {value: lines.objectIds, size: 1} + }, + polygons: { + polygonIndices: {value: polygons.polygonIndices, size: 1}, + primitivePolygonIndices: {value: polygons.primitivePolygonIndices, size: 1}, + positions: {value: polygons.positions, size: coordLength}, + objectIds: {value: polygons.objectIds, size: 1} + } + }; +} + +// Fills Point coordinates into points object of arrays +function handlePoint(coords, points, indexMap, coordLength) { + points.positions.set(coords, indexMap.pointPosition * coordLength); + points.objectIds[indexMap.pointPosition] = indexMap.feature; + indexMap.pointPosition++; +} + +// Fills MultiPoint coordinates into points object of arrays +function handleMultiPoint(coords, points, indexMap, coordLength) { + for (const point of coords) { + handlePoint(point, points, indexMap, coordLength); + } +} + +// Fills LineString coordinates into lines object of arrays +function handleLineString(coords, lines, indexMap, coordLength) { + lines.pathIndices[indexMap.linePath] = indexMap.linePosition; + indexMap.linePath++; + + lines.positions.set(flatten(coords), indexMap.linePosition * coordLength); + + const nPositions = coords.length; + lines.objectIds.set(new Uint32Array(nPositions).fill(indexMap.feature), indexMap.linePosition); + indexMap.linePosition += nPositions; +} + +// Fills MultiLineString coordinates into lines object of arrays +function handleMultiLineString(coords, lines, indexMap, coordLength) { + for (const line of coords) { + handleLineString(line, lines, indexMap, coordLength); + } +} + +// Fills Polygon coordinates into polygons object of arrays +function handlePolygon(coords, polygons, indexMap, coordLength) { + polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition; + indexMap.polygonObject++; + + for (const ring of coords) { + polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition; + indexMap.polygonRing++; + + polygons.positions.set(flatten(ring), indexMap.polygonPosition * coordLength); + + const nPositions = ring.length; + polygons.objectIds.set( + new Uint32Array(nPositions).fill(indexMap.feature), + indexMap.polygonPosition + ); + indexMap.polygonPosition += nPositions; + } +} + +// Fills MultiPolygon coordinates into polygons object of arrays +function handleMultiPolygon(coords, polygons, indexMap, coordLength) { + for (const polygon of coords) { + handlePolygon(polygon, polygons, indexMap, coordLength); + } +} + +function flatten(arrays) { + return [].concat(...arrays); +} diff --git a/modules/gis/test/data/2d_features.json b/modules/gis/test/data/2d_features.json new file mode 100644 index 0000000000..51e7c33389 --- /dev/null +++ b/modules/gis/test/data/2d_features.json @@ -0,0 +1,70 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [100.0, 0.0] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPoint", + "coordinates": [[100.0, 0.0], [101.0, 1.0]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[100.0, 0.0], [101.0, 1.0]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [[[100.0, 0.0], [101.0, 1.0]], [[102.0, 2.0], [103.0, 3.0]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], + [[100.8, 0.8], [100.8, 0.2], [100.2, 0.2], [100.2, 0.8], [100.8, 0.8]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]], + [ + [[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], + [[100.2, 0.2], [100.2, 0.8], [100.8, 0.8], [100.8, 0.2], [100.2, 0.2]] + ] + ] + }, + "properties": {} + } + ] +} diff --git a/modules/gis/test/data/3d_features.json b/modules/gis/test/data/3d_features.json new file mode 100644 index 0000000000..8092e997aa --- /dev/null +++ b/modules/gis/test/data/3d_features.json @@ -0,0 +1,98 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [100.0, 0.0, 1] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPoint", + "coordinates": [[100.0, 0.0, 2], [101.0, 1.0, 3]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[100.0, 0.0, 4], [101.0, 1.0, 5]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [[[100.0, 0.0, 6], [101.0, 1.0, 7]], [[102.0, 2.0, 8], [103.0, 3.0, 9]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[100.0, 0.0, 10], [101.0, 0.0, 11], [101.0, 1.0, 12], [100.0, 1.0, 13], [100.0, 0.0, 14]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [100.0, 0.0, 15], + [101.0, 0.0, 16], + [101.0, 1.0, 17], + [100.0, 1.0, 18], + [100.0, 0.0, 19] + ], + [[100.8, 0.8, 20], [100.8, 0.2, 21], [100.2, 0.2, 22], [100.2, 0.8, 23], [100.8, 0.8, 24]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [102.0, 2.0, 25], + [103.0, 2.0, 26], + [103.0, 3.0, 27], + [102.0, 3.0, 28], + [102.0, 2.0, 29] + ] + ], + [ + [ + [100.0, 0.0, 30], + [101.0, 0.0, 31], + [101.0, 1.0, 32], + [100.0, 1.0, 33], + [100.0, 0.0, 34] + ], + [ + [100.2, 0.2, 35], + [100.2, 0.8, 36], + [100.8, 0.8, 37], + [100.8, 0.2, 38], + [100.2, 0.2, 39] + ] + ] + ] + }, + "properties": {} + } + ] +} diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js new file mode 100644 index 0000000000..c9f1623ae1 --- /dev/null +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -0,0 +1,287 @@ +import test from 'tape-promise/tape'; +import {fetchFile} from '@loaders.gl/core'; +import {geojsonToBinary} from '@loaders.gl/gis'; + +// Sample GeoJSON data derived from examples in GeoJSON specification +// https://tools.ietf.org/html/rfc7946#appendix-A +// All features have 2D coordinates +const FEATURES_2D = '@loaders.gl/gis/test/data/2d_features.json'; +// All features have 3D coordinates +const FEATURES_3D = '@loaders.gl/gis/test/data/3d_features.json'; + +test('gis#geojson-to-binary 2D features', async t => { + const response = await fetchFile(FEATURES_2D); + const {features} = await response.json(); + const {points, lines, polygons} = geojsonToBinary(features); + + // 2D size + t.equal(points.positions.size, 2); + t.equal(lines.positions.size, 2); + t.equal(polygons.positions.size, 2); + + // Other arrays have coordinate size 1 + t.equal(points.objectIds.size, 1); + t.equal(lines.pathIndices.size, 1); + t.equal(lines.objectIds.size, 1); + t.equal(polygons.polygonIndices.size, 1); + t.equal(polygons.primitivePolygonIndices.size, 1); + t.equal(polygons.objectIds.size, 1); + + // Point value equality + t.deepEqual(points.positions.value, [100, 0, 100, 0, 101, 1]); + t.deepEqual(points.objectIds.value, [0, 1, 1]); + + // LineString value equality + t.deepEqual(lines.pathIndices.value, [0, 2, 4]); + t.deepEqual(lines.positions.value, [100, 0, 101, 1, 100, 0, 101, 1, 102, 2, 103, 3]); + t.deepEqual(lines.objectIds.value, [2, 2, 3, 3, 3, 3]); + + // Polygon value equality + t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); + t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); + t.deepEqual( + polygons.positions.value, + Float32Array.from([ + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 102, + 2, + 103, + 2, + 103, + 3, + 102, + 3, + 102, + 2, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2 + ]) + ); + t.deepEqual(polygons.objectIds.value, [ + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6 + ]); + t.end(); +}); + +test('gis#geojson-to-binary 3D features', async t => { + const response = await fetchFile(FEATURES_3D); + const {features} = await response.json(); + const {points, lines, polygons} = geojsonToBinary(features); + + // 3D size + t.equal(points.positions.size, 3); + t.equal(lines.positions.size, 3); + t.equal(polygons.positions.size, 3); + + // Other arrays have coordinate size 1 + t.equal(points.objectIds.size, 1); + t.equal(lines.pathIndices.size, 1); + t.equal(lines.objectIds.size, 1); + t.equal(polygons.polygonIndices.size, 1); + t.equal(polygons.primitivePolygonIndices.size, 1); + t.equal(polygons.objectIds.size, 1); + + // Point value equality + t.deepEqual(points.positions.value, [100, 0, 1, 100, 0, 2, 101, 1, 3]); + t.deepEqual(points.objectIds.value, [0, 1, 1]); + + // LineString value equality + t.deepEqual(lines.pathIndices.value, [0, 2, 4]); + t.deepEqual(lines.positions.value, [ + 100, + 0, + 4, + 101, + 1, + 5, + 100, + 0, + 6, + 101, + 1, + 7, + 102, + 2, + 8, + 103, + 3, + 9 + ]); + t.deepEqual(lines.objectIds.value, [2, 2, 3, 3, 3, 3]); + + // Polygon value equality + t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); + t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); + t.deepEqual( + polygons.positions.value, + Float32Array.from([ + 100, + 0, + 10, + 101, + 0, + 11, + 101, + 1, + 12, + 100, + 1, + 13, + 100, + 0, + 14, + 100, + 0, + 15, + 101, + 0, + 16, + 101, + 1, + 17, + 100, + 1, + 18, + 100, + 0, + 19, + 100.8, + 0.8, + 20, + 100.8, + 0.2, + 21, + 100.2, + 0.2, + 22, + 100.2, + 0.8, + 23, + 100.8, + 0.8, + 24, + 102, + 2, + 25, + 103, + 2, + 26, + 103, + 3, + 27, + 102, + 3, + 28, + 102, + 2, + 29, + 100, + 0, + 30, + 101, + 0, + 31, + 101, + 1, + 32, + 100, + 1, + 33, + 100, + 0, + 34, + 100.2, + 0.2, + 35, + 100.2, + 0.8, + 36, + 100.8, + 0.8, + 37, + 100.8, + 0.2, + 38, + 100.2, + 0.2, + 39 + ]) + ); + t.end(); +}); diff --git a/modules/gis/test/index.js b/modules/gis/test/index.js new file mode 100644 index 0000000000..879fd3ecd0 --- /dev/null +++ b/modules/gis/test/index.js @@ -0,0 +1 @@ +import './geojson-to-binary.spec'; diff --git a/test/aliases.js b/test/aliases.js index c2101f1045..724d91fdbf 100644 --- a/test/aliases.js +++ b/test/aliases.js @@ -31,6 +31,7 @@ const makeAliases = () => ({ '@loaders.gl/csv/test': path.resolve(__dirname, '../modules/csv/test'), '@loaders.gl/draco/test': path.resolve(__dirname, '../modules/draco/test'), '@loaders.gl/images/test': path.resolve(__dirname, '../modules/images/test'), + '@loaders.gl/gis/test': path.resolve(__dirname, '../modules/gis/test'), '@loaders.gl/gltf/test': path.resolve(__dirname, '../modules/gltf/test'), '@loaders.gl/json/test': path.resolve(__dirname, '../modules/json/test'), '@loaders.gl/kml/test': path.resolve(__dirname, '../modules/kml/test'), diff --git a/test/modules.js b/test/modules.js index 1abb4349a5..8c56657dd3 100644 --- a/test/modules.js +++ b/test/modules.js @@ -40,6 +40,7 @@ require('@loaders.gl/tiles/test'); require('@loaders.gl/kml/test'); require('@loaders.gl/wkt/test'); require('@loaders.gl/mvt/test'); +require('@loaders.gl/gis/test') // Table Formats require('@loaders.gl/tables/test');