From db62dad8ffabc75f6f863ff2cce4df9667eb36d1 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 13:38:40 -0600 Subject: [PATCH 01/27] Start of scaffolding --- modules/gis/README.md | 5 ++++ modules/gis/docs/README.md | 15 ++++++++++ modules/gis/package.json | 36 ++++++++++++++++++++++++ modules/gis/src/bundle.js | 7 +++++ modules/gis/src/lib/geojson-to-binary.js | 0 5 files changed, 63 insertions(+) create mode 100644 modules/gis/README.md create mode 100644 modules/gis/docs/README.md create mode 100644 modules/gis/package.json create mode 100644 modules/gis/src/bundle.js create mode 100644 modules/gis/src/lib/geojson-to-binary.js diff --git a/modules/gis/README.md b/modules/gis/README.md new file mode 100644 index 0000000000..95bf2ebb7f --- /dev/null +++ b/modules/gis/README.md @@ -0,0 +1,5 @@ +# @loaders.gl/gis + +This module contains helper classes for the GIS category of loaders. + +[loaders.gl](https://loaders.gl/docs) is a collection of framework independent visualization-focused loaders (parsers). diff --git a/modules/gis/docs/README.md b/modules/gis/docs/README.md new file mode 100644 index 0000000000..98bacd55a7 --- /dev/null +++ b/modules/gis/docs/README.md @@ -0,0 +1,15 @@ +# @loaders.gl/gis + +This module contains helper classes for the GIS category of loaders. + +## Installation + +```bash +npm install @loaders.gl/gis +``` + +## Loaders and Writers + +| Loader | +| --------------------------------------------------------- | +| [`MVTLoader`](modules/mvts/docs/api-reference/mvt-loader) | diff --git a/modules/gis/package.json b/modules/gis/package.json new file mode 100644 index 0000000000..18b16a5aa0 --- /dev/null +++ b/modules/gis/package.json @@ -0,0 +1,36 @@ +{ + "name": "@loaders.gl/gis", + "description": "Helpers for GIS category data", + "version": "2.1.0-beta.2", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/uber-web/loaders.gl" + }, + "keywords": [ + "geometry", + "GeoJSON" + ], + "main": "dist/es5/index.js", + "module": "dist/esm/index.js", + "esnext": "dist/es6/index.js", + "sideEffects": false, + "files": [ + "src", + "dist", + "README.md" + ], + "scripts": { + "pre-build": "npm run build-worker && npm run build-worker --env.dev && npm run build-bundle && npm run build-bundle -- --env.dev", + "build-worker": "webpack --entry ./src/workers/mvt-loader.worker.js --output ./dist/mvt-loader.worker.js --config ../../scripts/worker-webpack-config.js", + "build-bundle": "webpack --display=minimal --config ../../scripts/bundle.config.js" + }, + "dependencies": { + "@loaders.gl/loader-utils": "2.1.0-beta.2", + "@mapbox/vector-tile": "^1.3.1", + "pbf": "^3.2.1" + } +} diff --git a/modules/gis/src/bundle.js b/modules/gis/src/bundle.js new file mode 100644 index 0000000000..7316a80287 --- /dev/null +++ b/modules/gis/src/bundle.js @@ -0,0 +1,7 @@ +/* global window, global */ +const moduleExports = require('./index'); +const _global = typeof window === 'undefined' ? global : window; +// @ts-ignore +_global.loaders = _global.loaders || {}; +// @ts-ignore +module.exports = Object.assign(_global.loaders, moduleExports); diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js new file mode 100644 index 0000000000..e69de29bb2 From 5212c2bd77bc073593e46714bdb8eba0c71c650a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 16:03:35 -0600 Subject: [PATCH 02/27] WIP LineString support --- modules/gis/src/lib/geojson-to-binary.js | 142 +++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index e69de29bb2..b700c8864a 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -0,0 +1,142 @@ +/* eslint-disable no-var */ + +// var features = require('../../test/data/roads.json').features; +// var features = require("../../test/data/vancouver-blocks.json").features; + +// TODO: add data type to options to customize whether positions data type is float 32 or float 64 +export function featuresToBinary(features, options = {}) { + var options = firstPass(features); + return secondPass(features, firstPassObject); +} + + +// detecting if any 3D coordinates are present +// detecting which properties are present so that you can add columns for these (may not be in all features) +// counting the lengths of various arrays so you can allocate the typed arrays up front instead of building up temporary JS arrays. +// etc... + +function firstPass(features, options = {}) { + var {} = options; + + // Counts the number of _positions_, so [x, y, z] counts as one + var pointPositions = 0; + var linePositions = 0; + var linePaths = 0; + var maxCoordLength = 2; + + for (var feature of features) { + var geometry = feature.geometry; + + switch (geometry.type) { + case 'Point': + pointPositions++; + if (geometry.coordinates.length === 3) maxCoordLength = 3; + break; + case 'MultiPoint': + pointPositions += geometry.coordinates.length; + + break; + case 'LineString': + linePositions += geometry.coordinates.length; + linePaths++; + break; + case 'MultiLineString': + for (const line of geometry.coordinates) { + linePositions += line.length; + linePaths++; + } + break; + case 'Polygon': + break; + case 'MultiPolygon': + break; + default: + throw new Error('Invalid geometry type'); + } + } + + return {pointPositions, linePositions, linePaths, maxCoordLength}; +} + +function secondPass(features, options) { + var {pointPositions, linePositions, linePaths, maxCoordLength} = options; + var points = { + positions: new Float32Array(pointPositions * maxCoordLength), + objectIds: new Uint32Array(pointPositions) + }; + var lines = { + // NOTE: does pathIndices need to be linePathsCounter + 1? + pathIndices: new Uint32Array(linePaths), + positions: new Float32Array(linePositions * maxCoordLength), + objectIds: new Uint32Array(linePositions) + }; + var polygons = { + polygonIndices: [0], + primitivePolygonIndices: [0], + positions: [], + objectIds: [] + }; + + var index = { + pointPosition: 0, + linePosition: 0, + linePath: 0, + feature: 0, + } + + for (var feature of features) { + var geometry = feature.geometry; + + switch (geometry.type) { + case 'Point': + + break; + case 'MultiPoint': + + break; + case 'LineString': + // coords = geometry.coordinates + handleLineString({coords: geometry.coordinates, lines, index, maxCoordLength}) + break; + case 'MultiLineString': + handleMultiLineString({coords: geometry.coordinates, lines, index, maxCoordLength}) + break; + case 'Polygon': + break; + case 'MultiPolygon': + break; + default: + throw new Error('Invalid geometry type'); + } + + index.feature++ + } + + return { + points, + lines, + polygons + }; +} + +// NOTE: Functions are impure + +function handleLineString({coords, lines, index, maxCoordLength}) { + lines.pathIndices[index.linePath] = index.linePosition * maxCoordLength + index.linePath++ + + // TODO: if maxCoordLength is 3, check length of each geometry.coordinates array, filling + // 3rd value if necessary? + lines.positions.set(coords.flat(), index.linePosition * maxCoordLength) + + var nCoords = coords.length; + lines.objectIds.set(new Uint32Array(nCoords).fill(index.feature), index.linePosition) + index.linePosition += nCoords +} + +function handleMultiLineString({coords, lines, index, maxCoordLength}) { + for (var line of coords) { + handleLineString({coords: line, lines, index, maxCoordLength}) + } +} + From cb0632edfa66c6fba77abc93ed8cf90212a6a283 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 16:53:51 -0600 Subject: [PATCH 03/27] WIP Polygon support --- modules/gis/src/lib/geojson-to-binary.js | 108 +++++++++++++++++------ 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index b700c8864a..4765656bda 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -9,7 +9,6 @@ export function featuresToBinary(features, options = {}) { return secondPass(features, firstPassObject); } - // detecting if any 3D coordinates are present // detecting which properties are present so that you can add columns for these (may not be in all features) // counting the lengths of various arrays so you can allocate the typed arrays up front instead of building up temporary JS arrays. @@ -22,6 +21,9 @@ function firstPass(features, options = {}) { var pointPositions = 0; var linePositions = 0; var linePaths = 0; + var polygonPositions = 0; + var polygonObjects = 0; + var polygonRings = 0; var maxCoordLength = 2; for (var feature of features) { @@ -47,69 +49,94 @@ function firstPass(features, options = {}) { } break; case 'Polygon': + polygonObjects++ + polygonRings += geometry.coordinates.length + polygonPositions += geometry.coordinates.flat(1).length break; case 'MultiPolygon': + for (const polygon of geometry.coordinates) { + polygonObjects++ + polygonRings += polygon.length + polygonPositions += polygon.flat(1).length + } break; default: throw new Error('Invalid geometry type'); } } - return {pointPositions, linePositions, linePaths, maxCoordLength}; + return { + pointPositions, + linePositions, + linePaths, + coordLength: maxCoordLength, + polygonPositions, + polygonObjects, + polygonRings + }; } function secondPass(features, options) { - var {pointPositions, linePositions, linePaths, maxCoordLength} = options; + var { + pointPositions, + linePositions, + linePaths, + coordLength, + polygonPositions, + polygonObjects, + polygonRings + } = options; var points = { - positions: new Float32Array(pointPositions * maxCoordLength), + positions: new Float32Array(pointPositions * coordLength), objectIds: new Uint32Array(pointPositions) }; var lines = { - // NOTE: does pathIndices need to be linePathsCounter + 1? pathIndices: new Uint32Array(linePaths), - positions: new Float32Array(linePositions * maxCoordLength), + positions: new Float32Array(linePositions * coordLength), objectIds: new Uint32Array(linePositions) }; var polygons = { - polygonIndices: [0], - primitivePolygonIndices: [0], - positions: [], - objectIds: [] + polygonIndices: new Uint32Array(polygonObjects), + primitivePolygonIndices: new Uint32Array(polygonRings), + positions: new Float32Array(polygonPositions * coordLength), + objectIds: new Uint32Array(polygonPositions) }; var index = { pointPosition: 0, linePosition: 0, linePath: 0, - feature: 0, - } + polygonPosition: 0, + polygonObject: 0, + polygonRing: 0, + feature: 0 + }; for (var feature of features) { var geometry = feature.geometry; switch (geometry.type) { case 'Point': - break; case 'MultiPoint': - break; case 'LineString': - // coords = geometry.coordinates - handleLineString({coords: geometry.coordinates, lines, index, maxCoordLength}) + handleLineString({coords: geometry.coordinates, lines, index, coordLength}); break; case 'MultiLineString': - handleMultiLineString({coords: geometry.coordinates, lines, index, maxCoordLength}) + handleMultiLineString({coords: geometry.coordinates, lines, index, coordLength}); break; case 'Polygon': + handlePolygon({coords: geometry.coordinates, polygons, index, coordLength}) break; case 'MultiPolygon': + handleMultiPolygon({coords: geometry.coordinates, polygons, index, coordLength}) break; default: throw new Error('Invalid geometry type'); } - index.feature++ + index.feature++; } return { @@ -120,23 +147,46 @@ function secondPass(features, options) { } // NOTE: Functions are impure +function handleLineString({coords, lines, index, coordLength}) { + lines.pathIndices[index.linePath] = index.linePosition * coordLength; + index.linePath++; -function handleLineString({coords, lines, index, maxCoordLength}) { - lines.pathIndices[index.linePath] = index.linePosition * maxCoordLength - index.linePath++ - - // TODO: if maxCoordLength is 3, check length of each geometry.coordinates array, filling + // TODO: if coordLength is 3, check length of each geometry.coordinates array, filling // 3rd value if necessary? - lines.positions.set(coords.flat(), index.linePosition * maxCoordLength) + lines.positions.set(coords.flat(), index.linePosition * coordLength); - var nCoords = coords.length; - lines.objectIds.set(new Uint32Array(nCoords).fill(index.feature), index.linePosition) - index.linePosition += nCoords + var nPositions = coords.length; + lines.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.linePosition); + index.linePosition += nPositions; } -function handleMultiLineString({coords, lines, index, maxCoordLength}) { +function handleMultiLineString({coords, lines, index, coordLength}) { for (var line of coords) { - handleLineString({coords: line, lines, index, maxCoordLength}) + handleLineString({coords: line, lines, index, coordLength}); } } +function handlePolygon({coords, polygons, index, coordLength}) { + // index within polygon positions array of where each polygon starts + polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength + index.polygonObject++ + + for (var ring of coords) { + polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength + index.polygonRing++ + } + + polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength) + + var nPositions = coords.flat(1).length; + polygons.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.polygonPosition); + index.polygonPosition += nPositions; +} + +function handleMultiPolygon({coords, lines, index, coordLength}) { + for (var polygon of coords) { + handleLineString({coords: polygon, lines, index, coordLength}); + } +} + + From 66bd8a63b5841c1b7dedf02a711228e7fc4e0bd2 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:02:18 -0600 Subject: [PATCH 04/27] WIP Point support --- modules/gis/src/lib/geojson-to-binary.js | 47 +++++++++++++++--------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 4765656bda..eca48bbfe0 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -6,7 +6,7 @@ // TODO: add data type to options to customize whether positions data type is float 32 or float 64 export function featuresToBinary(features, options = {}) { var options = firstPass(features); - return secondPass(features, firstPassObject); + return secondPass(features, options); } // detecting if any 3D coordinates are present @@ -36,7 +36,6 @@ function firstPass(features, options = {}) { break; case 'MultiPoint': pointPositions += geometry.coordinates.length; - break; case 'LineString': linePositions += geometry.coordinates.length; @@ -49,19 +48,19 @@ function firstPass(features, options = {}) { } break; case 'Polygon': - polygonObjects++ - polygonRings += geometry.coordinates.length - polygonPositions += geometry.coordinates.flat(1).length + polygonObjects++; + polygonRings += geometry.coordinates.length; + polygonPositions += geometry.coordinates.flat(1).length; break; case 'MultiPolygon': for (const polygon of geometry.coordinates) { - polygonObjects++ - polygonRings += polygon.length - polygonPositions += polygon.flat(1).length + polygonObjects++; + polygonRings += polygon.length; + polygonPositions += polygon.flat(1).length; } break; default: - throw new Error('Invalid geometry type'); + throw new Error(`Unsupported geometry type: ${geometry.type}`); } } @@ -117,8 +116,10 @@ function secondPass(features, options) { switch (geometry.type) { case 'Point': + handlePoint({coords: geometry.coordinates, lines, index, coordLength}); break; case 'MultiPoint': + handleMultiPoint({coords: geometry.coordinates, lines, index, coordLength}); break; case 'LineString': handleLineString({coords: geometry.coordinates, lines, index, coordLength}); @@ -127,10 +128,10 @@ function secondPass(features, options) { handleMultiLineString({coords: geometry.coordinates, lines, index, coordLength}); break; case 'Polygon': - handlePolygon({coords: geometry.coordinates, polygons, index, coordLength}) + handlePolygon({coords: geometry.coordinates, polygons, index, coordLength}); break; case 'MultiPolygon': - handleMultiPolygon({coords: geometry.coordinates, polygons, index, coordLength}) + handleMultiPolygon({coords: geometry.coordinates, polygons, index, coordLength}); break; default: throw new Error('Invalid geometry type'); @@ -146,6 +147,18 @@ function secondPass(features, options) { }; } +function handlePoint({coords, points, index, coordLength}) { + points.positions.set(coords, index.pointPosition * coordLength); + points.objectIds[index.pointPosition] = index.feature; + index.pointPosition++; +} + +function handleMultiPoint({coords, points, index, coordLength}) { + for (var point of coords) { + handlePoint({coords: point, points, index, coordLength}); + } +} + // NOTE: Functions are impure function handleLineString({coords, lines, index, coordLength}) { lines.pathIndices[index.linePath] = index.linePosition * coordLength; @@ -168,15 +181,15 @@ function handleMultiLineString({coords, lines, index, coordLength}) { function handlePolygon({coords, polygons, index, coordLength}) { // index within polygon positions array of where each polygon starts - polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength - index.polygonObject++ + polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; + index.polygonObject++; for (var ring of coords) { - polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength - index.polygonRing++ + polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength; + index.polygonRing++; } - polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength) + polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength); var nPositions = coords.flat(1).length; polygons.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.polygonPosition); @@ -188,5 +201,3 @@ function handleMultiPolygon({coords, lines, index, coordLength}) { handleLineString({coords: polygon, lines, index, coordLength}); } } - - From e9835efac3c9972751d03f2bc9f9fb9a77cf37a4 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:11:50 -0600 Subject: [PATCH 05/27] Cleanup; var => const --- modules/gis/src/lib/geojson-to-binary.js | 79 +++++++++++------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index eca48bbfe0..316f669a78 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -1,5 +1,3 @@ -/* eslint-disable no-var */ - // var features = require('../../test/data/roads.json').features; // var features = require("../../test/data/vancouver-blocks.json").features; @@ -14,21 +12,18 @@ export function featuresToBinary(features, options = {}) { // counting the lengths of various arrays so you can allocate the typed arrays up front instead of building up temporary JS arrays. // etc... -function firstPass(features, options = {}) { - var {} = options; - +function firstPass(features) { // Counts the number of _positions_, so [x, y, z] counts as one - var pointPositions = 0; - var linePositions = 0; - var linePaths = 0; - var polygonPositions = 0; - var polygonObjects = 0; - var polygonRings = 0; - var maxCoordLength = 2; - - for (var feature of features) { - var geometry = feature.geometry; - + let pointPositions = 0; + let linePositions = 0; + let linePaths = 0; + let polygonPositions = 0; + let polygonObjects = 0; + let polygonRings = 0; + let maxCoordLength = 2; + + features.forEach(feature => { + const geometry = feature.geometry; switch (geometry.type) { case 'Point': pointPositions++; @@ -42,10 +37,10 @@ function firstPass(features, options = {}) { linePaths++; break; case 'MultiLineString': - for (const line of geometry.coordinates) { + geometry.coordinates.forEach(line => { linePositions += line.length; linePaths++; - } + }); break; case 'Polygon': polygonObjects++; @@ -53,16 +48,16 @@ function firstPass(features, options = {}) { polygonPositions += geometry.coordinates.flat(1).length; break; case 'MultiPolygon': - for (const polygon of geometry.coordinates) { + geometry.coordinates.forEach(polygon => { polygonObjects++; polygonRings += polygon.length; polygonPositions += polygon.flat(1).length; - } + }); break; default: throw new Error(`Unsupported geometry type: ${geometry.type}`); } - } + }); return { pointPositions, @@ -76,7 +71,7 @@ function firstPass(features, options = {}) { } function secondPass(features, options) { - var { + const { pointPositions, linePositions, linePaths, @@ -85,23 +80,23 @@ function secondPass(features, options) { polygonObjects, polygonRings } = options; - var points = { + const points = { positions: new Float32Array(pointPositions * coordLength), objectIds: new Uint32Array(pointPositions) }; - var lines = { + const lines = { pathIndices: new Uint32Array(linePaths), positions: new Float32Array(linePositions * coordLength), objectIds: new Uint32Array(linePositions) }; - var polygons = { + const polygons = { polygonIndices: new Uint32Array(polygonObjects), primitivePolygonIndices: new Uint32Array(polygonRings), positions: new Float32Array(polygonPositions * coordLength), objectIds: new Uint32Array(polygonPositions) }; - var index = { + const index = { pointPosition: 0, linePosition: 0, linePath: 0, @@ -111,15 +106,15 @@ function secondPass(features, options) { feature: 0 }; - for (var feature of features) { - var geometry = feature.geometry; + features.forEach(feature => { + const geometry = feature.geometry; switch (geometry.type) { case 'Point': - handlePoint({coords: geometry.coordinates, lines, index, coordLength}); + handlePoint({coords: geometry.coordinates, points, index, coordLength}); break; case 'MultiPoint': - handleMultiPoint({coords: geometry.coordinates, lines, index, coordLength}); + handleMultiPoint({coords: geometry.coordinates, points, index, coordLength}); break; case 'LineString': handleLineString({coords: geometry.coordinates, lines, index, coordLength}); @@ -138,7 +133,7 @@ function secondPass(features, options) { } index.feature++; - } + }); return { points, @@ -154,12 +149,11 @@ function handlePoint({coords, points, index, coordLength}) { } function handleMultiPoint({coords, points, index, coordLength}) { - for (var point of coords) { + coords.forEach(point => { handlePoint({coords: point, points, index, coordLength}); - } + }); } -// NOTE: Functions are impure function handleLineString({coords, lines, index, coordLength}) { lines.pathIndices[index.linePath] = index.linePosition * coordLength; index.linePath++; @@ -168,36 +162,35 @@ function handleLineString({coords, lines, index, coordLength}) { // 3rd value if necessary? lines.positions.set(coords.flat(), index.linePosition * coordLength); - var nPositions = coords.length; + const nPositions = coords.length; lines.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.linePosition); index.linePosition += nPositions; } function handleMultiLineString({coords, lines, index, coordLength}) { - for (var line of coords) { + coords.forEach(line => { handleLineString({coords: line, lines, index, coordLength}); - } + }); } function handlePolygon({coords, polygons, index, coordLength}) { - // index within polygon positions array of where each polygon starts polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; index.polygonObject++; - for (var ring of coords) { + coords.forEach(() => { polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength; index.polygonRing++; - } + }); polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength); - var nPositions = coords.flat(1).length; + const nPositions = coords.flat(1).length; polygons.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.polygonPosition); index.polygonPosition += nPositions; } function handleMultiPolygon({coords, lines, index, coordLength}) { - for (var polygon of coords) { + coords.forEach(polygon => { handleLineString({coords: polygon, lines, index, coordLength}); - } + }); } From 47e118334b5bda4dc112ce9a2f228fe4089b3fbd Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:16:10 -0600 Subject: [PATCH 06/27] Add option for Float64 positions --- modules/gis/src/lib/geojson-to-binary.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 316f669a78..a8f1e5f0aa 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -1,10 +1,9 @@ // var features = require('../../test/data/roads.json').features; // var features = require("../../test/data/vancouver-blocks.json").features; -// TODO: add data type to options to customize whether positions data type is float 32 or float 64 export function featuresToBinary(features, options = {}) { - var options = firstPass(features); - return secondPass(features, options); + const firstPassData = firstPass(features); + return secondPass(features, firstPassData, options); } // detecting if any 3D coordinates are present @@ -70,7 +69,7 @@ function firstPass(features) { }; } -function secondPass(features, options) { +function secondPass(features, firstPassData, options = {}) { const { pointPositions, linePositions, @@ -79,20 +78,21 @@ function secondPass(features, options) { polygonPositions, polygonObjects, polygonRings - } = options; + } = firstPassData; + const {PositionDataType = Float32Array} = options; const points = { - positions: new Float32Array(pointPositions * coordLength), + positions: new PositionDataType(pointPositions * coordLength), objectIds: new Uint32Array(pointPositions) }; const lines = { pathIndices: new Uint32Array(linePaths), - positions: new Float32Array(linePositions * coordLength), + positions: new PositionDataType(linePositions * coordLength), objectIds: new Uint32Array(linePositions) }; const polygons = { polygonIndices: new Uint32Array(polygonObjects), primitivePolygonIndices: new Uint32Array(polygonRings), - positions: new Float32Array(polygonPositions * coordLength), + positions: new PositionDataType(polygonPositions * coordLength), objectIds: new Uint32Array(polygonPositions) }; From 68299f6e43e0a9eb820aba30fb6a976e0bfc684e Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:23:07 -0600 Subject: [PATCH 07/27] Detect 3D coordinates --- modules/gis/src/lib/geojson-to-binary.js | 27 +++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index a8f1e5f0aa..0336da1d30 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -1,16 +1,8 @@ -// var features = require('../../test/data/roads.json').features; -// var features = require("../../test/data/vancouver-blocks.json").features; - export function featuresToBinary(features, options = {}) { const firstPassData = firstPass(features); return secondPass(features, firstPassData, options); } -// detecting if any 3D coordinates are present -// detecting which properties are present so that you can add columns for these (may not be in all features) -// counting the lengths of various arrays so you can allocate the typed arrays up front instead of building up temporary JS arrays. -// etc... - function firstPass(features) { // Counts the number of _positions_, so [x, y, z] counts as one let pointPositions = 0; @@ -30,27 +22,46 @@ function firstPass(features) { break; case 'MultiPoint': pointPositions += geometry.coordinates.length; + geometry.coordinates.forEach(point => { + if (point.length === 3) maxCoordLength = 3; + }); break; case 'LineString': linePositions += geometry.coordinates.length; linePaths++; + + geometry.coordinates.forEach(coord => { + if (coord.length === 3) maxCoordLength = 3; + }); break; case 'MultiLineString': geometry.coordinates.forEach(line => { linePositions += line.length; linePaths++; + + line.coordinates.forEach(coord => { + if (coord.length === 3) maxCoordLength = 3; + }); }); break; case 'Polygon': polygonObjects++; polygonRings += geometry.coordinates.length; polygonPositions += geometry.coordinates.flat(1).length; + + geometry.coordinates.flat().forEach(coord => { + if (coord.length === 3) maxCoordLength = 3; + }); break; case 'MultiPolygon': geometry.coordinates.forEach(polygon => { polygonObjects++; polygonRings += polygon.length; polygonPositions += polygon.flat(1).length; + + geometry.coordinates.flat().forEach(coord => { + if (coord.length === 3) maxCoordLength = 3; + }); }); break; default: From 78657160916b8582b96778eb0f59a5db3b8316e6 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:25:11 -0600 Subject: [PATCH 08/27] Extra scaffolding --- modules/gis/src/index.js | 1 + modules/gis/test/geojson-to-binary.spec.js | 0 modules/gis/test/index.js | 1 + 3 files changed, 2 insertions(+) create mode 100644 modules/gis/src/index.js create mode 100644 modules/gis/test/geojson-to-binary.spec.js create mode 100644 modules/gis/test/index.js diff --git a/modules/gis/src/index.js b/modules/gis/src/index.js new file mode 100644 index 0000000000..f3b988efe8 --- /dev/null +++ b/modules/gis/src/index.js @@ -0,0 +1 @@ +export {featuresToBinary} from './lib/geojson-to-binary'; diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/gis/test/index.js b/modules/gis/test/index.js new file mode 100644 index 0000000000..879fd3ecd0 --- /dev/null +++ b/modules/gis/test/index.js @@ -0,0 +1 @@ +import './geojson-to-binary.spec'; From 39f0fa0d2aa2de186d46f8b45ab7bfcbed5d2caf Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Sat, 14 Mar 2020 17:35:36 -0600 Subject: [PATCH 09/27] Remove comment --- modules/gis/src/lib/geojson-to-binary.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 0336da1d30..d7f89e0dbf 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -169,8 +169,6 @@ function handleLineString({coords, lines, index, coordLength}) { lines.pathIndices[index.linePath] = index.linePosition * coordLength; index.linePath++; - // TODO: if coordLength is 3, check length of each geometry.coordinates array, filling - // 3rd value if necessary? lines.positions.set(coords.flat(), index.linePosition * coordLength); const nPositions = coords.length; From 8044a7daf245ecdde23e0abd559180fcf5d63769 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 13:17:08 -0600 Subject: [PATCH 10/27] Remove scripts from package.json --- modules/gis/package.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/gis/package.json b/modules/gis/package.json index 18b16a5aa0..f240376e95 100644 --- a/modules/gis/package.json +++ b/modules/gis/package.json @@ -23,11 +23,6 @@ "dist", "README.md" ], - "scripts": { - "pre-build": "npm run build-worker && npm run build-worker --env.dev && npm run build-bundle && npm run build-bundle -- --env.dev", - "build-worker": "webpack --entry ./src/workers/mvt-loader.worker.js --output ./dist/mvt-loader.worker.js --config ../../scripts/worker-webpack-config.js", - "build-bundle": "webpack --display=minimal --config ../../scripts/bundle.config.js" - }, "dependencies": { "@loaders.gl/loader-utils": "2.1.0-beta.2", "@mapbox/vector-tile": "^1.3.1", From 7af35d6ecb75c84a26513ec914214438519e4677 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 13:17:54 -0600 Subject: [PATCH 11/27] forEach -> for loop --- modules/gis/src/lib/geojson-to-binary.js | 54 ++++++++++++------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index d7f89e0dbf..0b463a1e7d 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -13,7 +13,7 @@ function firstPass(features) { let polygonRings = 0; let maxCoordLength = 2; - features.forEach(feature => { + for (const feature of features) { const geometry = feature.geometry; switch (geometry.type) { case 'Point': @@ -22,52 +22,52 @@ function firstPass(features) { break; case 'MultiPoint': pointPositions += geometry.coordinates.length; - geometry.coordinates.forEach(point => { + for (const point of geometry.coordinates) { if (point.length === 3) maxCoordLength = 3; - }); + } break; case 'LineString': linePositions += geometry.coordinates.length; linePaths++; - geometry.coordinates.forEach(coord => { + for (const coord of geometry.coordinates) { if (coord.length === 3) maxCoordLength = 3; - }); + } break; case 'MultiLineString': - geometry.coordinates.forEach(line => { + for (const line of geometry.coordinates) { linePositions += line.length; linePaths++; - line.coordinates.forEach(coord => { + for (const coord of line.coordinates) { if (coord.length === 3) maxCoordLength = 3; - }); - }); + } + } break; case 'Polygon': polygonObjects++; polygonRings += geometry.coordinates.length; polygonPositions += geometry.coordinates.flat(1).length; - geometry.coordinates.flat().forEach(coord => { + for (const coord of geometry.coordinates.flat()) { if (coord.length === 3) maxCoordLength = 3; - }); + } break; case 'MultiPolygon': - geometry.coordinates.forEach(polygon => { + for (const polygon of geometry.coordinates) { polygonObjects++; polygonRings += polygon.length; polygonPositions += polygon.flat(1).length; - geometry.coordinates.flat().forEach(coord => { + for (const coord of geometry.coordinates.flat()) { if (coord.length === 3) maxCoordLength = 3; - }); - }); + } + } break; default: throw new Error(`Unsupported geometry type: ${geometry.type}`); } - }); + }; return { pointPositions, @@ -117,7 +117,7 @@ function secondPass(features, firstPassData, options = {}) { feature: 0 }; - features.forEach(feature => { + for (const feature of features) { const geometry = feature.geometry; switch (geometry.type) { @@ -144,7 +144,7 @@ function secondPass(features, firstPassData, options = {}) { } index.feature++; - }); + }; return { points, @@ -160,9 +160,9 @@ function handlePoint({coords, points, index, coordLength}) { } function handleMultiPoint({coords, points, index, coordLength}) { - coords.forEach(point => { + for (const point of coords) { handlePoint({coords: point, points, index, coordLength}); - }); + } } function handleLineString({coords, lines, index, coordLength}) { @@ -177,19 +177,19 @@ function handleLineString({coords, lines, index, coordLength}) { } function handleMultiLineString({coords, lines, index, coordLength}) { - coords.forEach(line => { + for (const line of coords) { handleLineString({coords: line, lines, index, coordLength}); - }); + } } function handlePolygon({coords, polygons, index, coordLength}) { polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; index.polygonObject++; - coords.forEach(() => { + for (const {} of coords) { polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength; index.polygonRing++; - }); + } polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength); @@ -199,7 +199,7 @@ function handlePolygon({coords, polygons, index, coordLength}) { } function handleMultiPolygon({coords, lines, index, coordLength}) { - coords.forEach(polygon => { - handleLineString({coords: polygon, lines, index, coordLength}); - }); + for (const polygon of coords) { + handlePolygon({coords: polygon, lines, index, coordLength}); + } } From c8c0d8de0af42f1e4c61f062fc68111853313aae Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 13:27:16 -0600 Subject: [PATCH 12/27] Use positional arguments for inner functions --- modules/gis/src/lib/geojson-to-binary.js | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 0b463a1e7d..fa53bd87b1 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -122,22 +122,22 @@ function secondPass(features, firstPassData, options = {}) { switch (geometry.type) { case 'Point': - handlePoint({coords: geometry.coordinates, points, index, coordLength}); + handlePoint(geometry.coordinates, points, index, coordLength); break; case 'MultiPoint': - handleMultiPoint({coords: geometry.coordinates, points, index, coordLength}); + handleMultiPoint(geometry.coordinates, points, index, coordLength); break; case 'LineString': - handleLineString({coords: geometry.coordinates, lines, index, coordLength}); + handleLineString(geometry.coordinates, lines, index, coordLength); break; case 'MultiLineString': - handleMultiLineString({coords: geometry.coordinates, lines, index, coordLength}); + handleMultiLineString(geometry.coordinates, lines, index, coordLength); break; case 'Polygon': - handlePolygon({coords: geometry.coordinates, polygons, index, coordLength}); + handlePolygon(geometry.coordinates, polygons, index, coordLength); break; case 'MultiPolygon': - handleMultiPolygon({coords: geometry.coordinates, polygons, index, coordLength}); + handleMultiPolygon(geometry.coordinates, polygons, index, coordLength); break; default: throw new Error('Invalid geometry type'); @@ -153,19 +153,19 @@ function secondPass(features, firstPassData, options = {}) { }; } -function handlePoint({coords, points, index, coordLength}) { +function handlePoint(coords, points, index, coordLength) { points.positions.set(coords, index.pointPosition * coordLength); points.objectIds[index.pointPosition] = index.feature; index.pointPosition++; } -function handleMultiPoint({coords, points, index, coordLength}) { +function handleMultiPoint(coords, points, index, coordLength) { for (const point of coords) { - handlePoint({coords: point, points, index, coordLength}); + handlePoint(point, points, index, coordLength); } } -function handleLineString({coords, lines, index, coordLength}) { +function handleLineString(coords, lines, index, coordLength) { lines.pathIndices[index.linePath] = index.linePosition * coordLength; index.linePath++; @@ -176,13 +176,13 @@ function handleLineString({coords, lines, index, coordLength}) { index.linePosition += nPositions; } -function handleMultiLineString({coords, lines, index, coordLength}) { +function handleMultiLineString(coords, lines, index, coordLength) { for (const line of coords) { - handleLineString({coords: line, lines, index, coordLength}); + handleLineString(line, lines, index, coordLength); } } -function handlePolygon({coords, polygons, index, coordLength}) { +function handlePolygon(coords, polygons, index, coordLength) { polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; index.polygonObject++; @@ -198,8 +198,8 @@ function handlePolygon({coords, polygons, index, coordLength}) { index.polygonPosition += nPositions; } -function handleMultiPolygon({coords, lines, index, coordLength}) { +function handleMultiPolygon(coords, polygons, index, coordLength) { for (const polygon of coords) { - handlePolygon({coords: polygon, lines, index, coordLength}); + handlePolygon(polygon, polygons, index, coordLength); } } From 3111cad3307fc719108a46bc67c9ff19d2d7fc10 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 13:30:34 -0600 Subject: [PATCH 13/27] Disable eslint complexity and max-depth --- modules/gis/src/lib/geojson-to-binary.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index fa53bd87b1..d4491ca3d0 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -3,6 +3,7 @@ export function featuresToBinary(features, options = {}) { return secondPass(features, firstPassData, options); } +// eslint-disable-next-line complexity function firstPass(features) { // Counts the number of _positions_, so [x, y, z] counts as one let pointPositions = 0; @@ -23,6 +24,7 @@ function firstPass(features) { case 'MultiPoint': pointPositions += geometry.coordinates.length; for (const point of geometry.coordinates) { + // eslint-disable-next-line max-depth if (point.length === 3) maxCoordLength = 3; } break; @@ -31,6 +33,7 @@ function firstPass(features) { linePaths++; for (const coord of geometry.coordinates) { + // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } break; @@ -39,7 +42,9 @@ function firstPass(features) { linePositions += line.length; linePaths++; + // eslint-disable-next-line max-depth for (const coord of line.coordinates) { + // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } } @@ -50,6 +55,7 @@ function firstPass(features) { polygonPositions += geometry.coordinates.flat(1).length; for (const coord of geometry.coordinates.flat()) { + // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } break; @@ -59,7 +65,9 @@ function firstPass(features) { polygonRings += polygon.length; polygonPositions += polygon.flat(1).length; + // eslint-disable-next-line max-depth for (const coord of geometry.coordinates.flat()) { + // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } } From 5442b97ad51cd90c4fb2ab9d97ae32cfabb7da4a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 13:52:03 -0600 Subject: [PATCH 14/27] Add header comments --- modules/gis/src/lib/geojson-to-binary.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index d4491ca3d0..42c3dbf51b 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -1,8 +1,12 @@ -export function featuresToBinary(features, options = {}) { +// Convert GeoJSON features to flat binary arrays +export function geojsonToBinary(features, options = {}) { const firstPassData = firstPass(features); return secondPass(features, firstPassData, options); } +// Initial scan over GeoJSON features +// Counts number of coordinates of each geometry type and keeps track of the max coordinate +// dimensions // eslint-disable-next-line complexity function firstPass(features) { // Counts the number of _positions_, so [x, y, z] counts as one @@ -75,7 +79,7 @@ function firstPass(features) { default: throw new Error(`Unsupported geometry type: ${geometry.type}`); } - }; + } return { pointPositions, @@ -88,6 +92,8 @@ function firstPass(features) { }; } +// Second scan over GeoJSON features +// Fills coordinates into pre-allocated typed arrays function secondPass(features, firstPassData, options = {}) { const { pointPositions, @@ -152,7 +158,7 @@ function secondPass(features, firstPassData, options = {}) { } index.feature++; - }; + } return { points, @@ -161,18 +167,21 @@ function secondPass(features, firstPassData, options = {}) { }; } +// Fills Point coordinates into points object of arrays function handlePoint(coords, points, index, coordLength) { points.positions.set(coords, index.pointPosition * coordLength); points.objectIds[index.pointPosition] = index.feature; index.pointPosition++; } +// Fills MultiPoint coordinates into points object of arrays function handleMultiPoint(coords, points, index, coordLength) { for (const point of coords) { handlePoint(point, points, index, coordLength); } } +// Fills LineString coordinates into lines object of arrays function handleLineString(coords, lines, index, coordLength) { lines.pathIndices[index.linePath] = index.linePosition * coordLength; index.linePath++; @@ -184,12 +193,14 @@ function handleLineString(coords, lines, index, coordLength) { index.linePosition += nPositions; } +// Fills MultiLineString coordinates into lines object of arrays function handleMultiLineString(coords, lines, index, coordLength) { for (const line of coords) { handleLineString(line, lines, index, coordLength); } } +// Fills Polygon coordinates into polygons object of arrays function handlePolygon(coords, polygons, index, coordLength) { polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; index.polygonObject++; @@ -206,6 +217,7 @@ function handlePolygon(coords, polygons, index, coordLength) { index.polygonPosition += nPositions; } +// Fills MultiPolygon coordinates into polygons object of arrays function handleMultiPolygon(coords, polygons, index, coordLength) { for (const polygon of coords) { handlePolygon(polygon, polygons, index, coordLength); From 9502a583be46f6b2678555e37a448da834d02519 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 14:14:32 -0600 Subject: [PATCH 15/27] Add docs --- modules/gis/docs/README.md | 8 +-- .../docs/api-reference/geojson-to-binary.md | 69 +++++++++++++++++++ modules/gis/src/index.js | 2 +- 3 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 modules/gis/docs/api-reference/geojson-to-binary.md diff --git a/modules/gis/docs/README.md b/modules/gis/docs/README.md index 98bacd55a7..1b710f3aca 100644 --- a/modules/gis/docs/README.md +++ b/modules/gis/docs/README.md @@ -8,8 +8,8 @@ This module contains helper classes for the GIS category of loaders. npm install @loaders.gl/gis ``` -## Loaders and Writers +## Utility Functions -| Loader | -| --------------------------------------------------------- | -| [`MVTLoader`](modules/mvts/docs/api-reference/mvt-loader) | +| Utility Functions | +| ----------------------------------------------------------------------- | +| [`geojson-to-binary`](modules/gis/docs/api-reference/geojson-to-binary) | diff --git a/modules/gis/docs/api-reference/geojson-to-binary.md b/modules/gis/docs/api-reference/geojson-to-binary.md new file mode 100644 index 0000000000..020466d2af --- /dev/null +++ b/modules/gis/docs/api-reference/geojson-to-binary.md @@ -0,0 +1,69 @@ +# GeoJSON to TypedArrays + +Helper function to transform an array of GeoJSON `Feature`s into binary typed +arrays. This is designed to speed up geospatial loaders by removing the need for +serialization and deserialization of data transferred by the worker back to the +main process. + +## Usage + +```js +import {load} from '@loaders.gl/core'; +import {MVTLoader} from '@loaders.gl/mvt'; +import {geojsonToBinary} from '@loaders.gl/gis'; + +// See MVTLoader docs for loader options +const geoJSONfeatures = await load(url, MVTLoader, loaderOptions); + +/* +* Default options are: +* +* { +* PositionDataType: Float32Array +* } +*/ +const binaryArrays = geojsonToBinary(geoJSONfeatures, options); +``` + +## Outputs + +### TypedArrays + +`geojsonToBinary` returns an object containing typed arrays sorted by geometry +type. `positions` corresponds to 2D or 3D coordinates; `objectIds` returns the +index of the vertex in the initial `features` array. + +```js +{ + points: { + // Array of x, y or x, y, z positions + positions: Float32Array, + // Array of original feature indexes by vertex + objectIds: Uint32Array, + }, + lines: { + // Indices within positions of the start of each individual LineString + pathIndices: Uint32Array, + // Array of x, y or x, y, z positions + positions: Float32Array, + // Array of original feature indexes by vertex + objectIds: Uint32Array, + }, + polygons: { + // Indices within positions of the start of each complex Polygon + polygonIndices: Uint32Array, + // Indices within positions of the start of each primitive Polygon/ring + primitivePolygonIndices: Uint32Array, + // Array of x, y or x, y, z positions + positions: Float32Array, + // Array of original feature indexes by vertex + objectIds: Uint32Array, + } +} +``` + +## Options + +| Option | Type | Default | Description | +| ---------------- | -------------------------------- | -------------- | ----------------------------------- | +| PositionDataType | `Float32Array` or `Float64Array` | `Float32Array` | Data type used for positions arrays | diff --git a/modules/gis/src/index.js b/modules/gis/src/index.js index f3b988efe8..a71bc8ad1e 100644 --- a/modules/gis/src/index.js +++ b/modules/gis/src/index.js @@ -1 +1 @@ -export {featuresToBinary} from './lib/geojson-to-binary'; +export {geojsonToBinary} from './lib/geojson-to-binary'; From 28a9d9df8c5c9ec213d9f2356662b8a7ed33ba3a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 16 Mar 2020 14:29:09 -0600 Subject: [PATCH 16/27] Wrap return objects in accessor objects --- .../docs/api-reference/geojson-to-binary.md | 18 +++++++++--------- modules/gis/src/lib/geojson-to-binary.js | 19 ++++++++++++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/modules/gis/docs/api-reference/geojson-to-binary.md b/modules/gis/docs/api-reference/geojson-to-binary.md index 020466d2af..87b352cee2 100644 --- a/modules/gis/docs/api-reference/geojson-to-binary.md +++ b/modules/gis/docs/api-reference/geojson-to-binary.md @@ -37,27 +37,27 @@ index of the vertex in the initial `features` array. { points: { // Array of x, y or x, y, z positions - positions: Float32Array, + positions: {value: Float32Array, size: coordLength}, // Array of original feature indexes by vertex - objectIds: Uint32Array, + objectIds: {value: Uint32Array, size: 1}, }, lines: { // Indices within positions of the start of each individual LineString - pathIndices: Uint32Array, + pathIndices: {value: Uint32Array, size: 1}, // Array of x, y or x, y, z positions - positions: Float32Array, + positions: {value: Float32Array, size: coordLength}, // Array of original feature indexes by vertex - objectIds: Uint32Array, + objectIds: {value: Uint32Array, size: 1}, }, polygons: { // Indices within positions of the start of each complex Polygon - polygonIndices: Uint32Array, + polygonIndices: {value: Uint32Array, size: 1}, // Indices within positions of the start of each primitive Polygon/ring - primitivePolygonIndices: Uint32Array, + primitivePolygonIndices: {value: Uint32Array, size: 1}, // Array of x, y or x, y, z positions - positions: Float32Array, + positions: {value: Float32Array, size: coordLength}, // Array of original feature indexes by vertex - objectIds: Uint32Array, + objectIds: {value: Uint32Array, size: 1}, } } ``` diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 42c3dbf51b..c54d17438f 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -160,10 +160,23 @@ function secondPass(features, firstPassData, options = {}) { index.feature++; } + // Wrap each array in an accessor object with value and size keys return { - points, - lines, - polygons + points: { + positions: {value: points.positions, size: coordLength}, + objectIds: {value: points.objectIds, size: 1} + }, + lines: { + pathIndices: {value: lines.pathIndices, size: 1}, + positions: {value: lines.positions, size: coordLength}, + objectIds: {value: lines.objectIds, size: 1} + }, + polygons: { + polygonIndices: {value: polygons.polygonIndices, size: 1}, + primitivePolygonIndices: {value: polygons.primitivePolygonIndices, size: 1}, + positions: {value: polygons.positions, size: coordLength}, + objectIds: {value: polygons.objectIds, size: 1} + } }; } From a7731c6166414be2e975453dc68bcd3632049edb Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 12:50:11 -0600 Subject: [PATCH 17/27] Fix iteration --- modules/gis/src/lib/geojson-to-binary.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index c54d17438f..dcb984bfb1 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -47,7 +47,7 @@ function firstPass(features) { linePaths++; // eslint-disable-next-line max-depth - for (const coord of line.coordinates) { + for (const coord of line) { // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } @@ -70,7 +70,7 @@ function firstPass(features) { polygonPositions += polygon.flat(1).length; // eslint-disable-next-line max-depth - for (const coord of geometry.coordinates.flat()) { + for (const coord of polygon.flat()) { // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } From cd12f4003d70e6d734f527433b1183523750c016 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 16:48:25 -0600 Subject: [PATCH 18/27] Sample test --- modules/gis/test/data/2d_features.json | 70 ++++++++++++++++++++++ modules/gis/test/geojson-to-binary.spec.js | 27 +++++++++ 2 files changed, 97 insertions(+) create mode 100644 modules/gis/test/data/2d_features.json diff --git a/modules/gis/test/data/2d_features.json b/modules/gis/test/data/2d_features.json new file mode 100644 index 0000000000..51e7c33389 --- /dev/null +++ b/modules/gis/test/data/2d_features.json @@ -0,0 +1,70 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [100.0, 0.0] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPoint", + "coordinates": [[100.0, 0.0], [101.0, 1.0]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[100.0, 0.0], [101.0, 1.0]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [[[100.0, 0.0], [101.0, 1.0]], [[102.0, 2.0], [103.0, 3.0]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], + [[100.8, 0.8], [100.8, 0.2], [100.2, 0.2], [100.2, 0.8], [100.8, 0.8]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]], + [ + [[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], + [[100.2, 0.2], [100.2, 0.8], [100.8, 0.8], [100.8, 0.2], [100.2, 0.2]] + ] + ] + }, + "properties": {} + } + ] +} diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index e69de29bb2..82b016322c 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -0,0 +1,27 @@ +import test from 'tape-promise/tape'; +import {fetchFile} from '@loaders.gl/core'; + +import {geojsonToBinary} from '@loaders.gl/gis'; + +// Sample GeoJSON Files +// All features have 2D coordinates +const FEATURES_2D = '@loaders.gl/gis/test/data/2d_features.json'; + +test('gis#geojson-to-binary 2D features', async t => { + const {features} = await fetchFile(FEATURES_2D); + const {points, lines, polygons} = geojsonToBinary(features); + + // 2D size + t.equal(points.positions.size, 2); + t.equal(lines.positions.size, 2); + t.equal(polygons.positions.size, 2); + + // Other arrays have coordinate size 1 + t.equal(points.objectIds.size, 2); + t.equal(lines.pathIndices.size, 2); + t.equal(lines.objectIds.size, 2); + t.equal(polygons.polygonIndices.size, 2); + t.equal(polygons.primitivePolygonIndices.size, 2); + t.equal(polygons.objectIds.size, 2); + t.end(); +}); From 3c18914b2b355d07890bae1624f5eabd8603b488 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 17:00:43 -0600 Subject: [PATCH 19/27] Failing test --- modules/gis/test/geojson-to-binary.spec.js | 15 ++++++++------- test/modules.js | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index 82b016322c..da2d87256a 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -8,7 +8,8 @@ import {geojsonToBinary} from '@loaders.gl/gis'; const FEATURES_2D = '@loaders.gl/gis/test/data/2d_features.json'; test('gis#geojson-to-binary 2D features', async t => { - const {features} = await fetchFile(FEATURES_2D); + const response = await fetchFile(FEATURES_2D); + const {features} = await response.json(); const {points, lines, polygons} = geojsonToBinary(features); // 2D size @@ -17,11 +18,11 @@ test('gis#geojson-to-binary 2D features', async t => { t.equal(polygons.positions.size, 2); // Other arrays have coordinate size 1 - t.equal(points.objectIds.size, 2); - t.equal(lines.pathIndices.size, 2); - t.equal(lines.objectIds.size, 2); - t.equal(polygons.polygonIndices.size, 2); - t.equal(polygons.primitivePolygonIndices.size, 2); - t.equal(polygons.objectIds.size, 2); + t.equal(points.objectIds.size, 1); + t.equal(lines.pathIndices.size, 1); + t.equal(lines.objectIds.size, 1); + t.equal(polygons.polygonIndices.size, 1); + t.equal(polygons.primitivePolygonIndices.size, 1); + t.equal(polygons.objectIds.size, 1); t.end(); }); diff --git a/test/modules.js b/test/modules.js index 1abb4349a5..8c56657dd3 100644 --- a/test/modules.js +++ b/test/modules.js @@ -40,6 +40,7 @@ require('@loaders.gl/tiles/test'); require('@loaders.gl/kml/test'); require('@loaders.gl/wkt/test'); require('@loaders.gl/mvt/test'); +require('@loaders.gl/gis/test') // Table Formats require('@loaders.gl/tables/test'); From d73841fac4b3cd250c1de9d70562a393dcbf60ed Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 17:08:47 -0600 Subject: [PATCH 20/27] Rename index -> indexMap --- modules/gis/src/lib/geojson-to-binary.js | 71 +++++++++++++----------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index dcb984bfb1..8d3b283984 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -23,7 +23,9 @@ function firstPass(features) { switch (geometry.type) { case 'Point': pointPositions++; - if (geometry.coordinates.length === 3) maxCoordLength = 3; + if (geometry.coordinates.length === 3) { + maxCoordLength = 3; + } break; case 'MultiPoint': pointPositions += geometry.coordinates.length; @@ -121,7 +123,7 @@ function secondPass(features, firstPassData, options = {}) { objectIds: new Uint32Array(polygonPositions) }; - const index = { + const indexMap = { pointPosition: 0, linePosition: 0, linePath: 0, @@ -136,28 +138,28 @@ function secondPass(features, firstPassData, options = {}) { switch (geometry.type) { case 'Point': - handlePoint(geometry.coordinates, points, index, coordLength); + handlePoint(geometry.coordinates, points, indexMap, coordLength); break; case 'MultiPoint': - handleMultiPoint(geometry.coordinates, points, index, coordLength); + handleMultiPoint(geometry.coordinates, points, indexMap, coordLength); break; case 'LineString': - handleLineString(geometry.coordinates, lines, index, coordLength); + handleLineString(geometry.coordinates, lines, indexMap, coordLength); break; case 'MultiLineString': - handleMultiLineString(geometry.coordinates, lines, index, coordLength); + handleMultiLineString(geometry.coordinates, lines, indexMap, coordLength); break; case 'Polygon': - handlePolygon(geometry.coordinates, polygons, index, coordLength); + handlePolygon(geometry.coordinates, polygons, indexMap, coordLength); break; case 'MultiPolygon': - handleMultiPolygon(geometry.coordinates, polygons, index, coordLength); + handleMultiPolygon(geometry.coordinates, polygons, indexMap, coordLength); break; default: throw new Error('Invalid geometry type'); } - index.feature++; + indexMap.feature++; } // Wrap each array in an accessor object with value and size keys @@ -181,58 +183,61 @@ function secondPass(features, firstPassData, options = {}) { } // Fills Point coordinates into points object of arrays -function handlePoint(coords, points, index, coordLength) { - points.positions.set(coords, index.pointPosition * coordLength); - points.objectIds[index.pointPosition] = index.feature; - index.pointPosition++; +function handlePoint(coords, points, indexMap, coordLength) { + points.positions.set(coords, indexMap.pointPosition * coordLength); + points.objectIds[indexMap.pointPosition] = indexMap.feature; + indexMap.pointPosition++; } // Fills MultiPoint coordinates into points object of arrays -function handleMultiPoint(coords, points, index, coordLength) { +function handleMultiPoint(coords, points, indexMap, coordLength) { for (const point of coords) { - handlePoint(point, points, index, coordLength); + handlePoint(point, points, indexMap, coordLength); } } // Fills LineString coordinates into lines object of arrays -function handleLineString(coords, lines, index, coordLength) { - lines.pathIndices[index.linePath] = index.linePosition * coordLength; - index.linePath++; +function handleLineString(coords, lines, indexMap, coordLength) { + lines.pathIndices[indexMap.linePath] = indexMap.linePosition * coordLength; + indexMap.linePath++; - lines.positions.set(coords.flat(), index.linePosition * coordLength); + lines.positions.set(coords.flat(), indexMap.linePosition * coordLength); const nPositions = coords.length; - lines.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.linePosition); - index.linePosition += nPositions; + lines.objectIds.set(new Uint32Array(nPositions).fill(indexMap.feature), indexMap.linePosition); + indexMap.linePosition += nPositions; } // Fills MultiLineString coordinates into lines object of arrays -function handleMultiLineString(coords, lines, index, coordLength) { +function handleMultiLineString(coords, lines, indexMap, coordLength) { for (const line of coords) { - handleLineString(line, lines, index, coordLength); + handleLineString(line, lines, indexMap, coordLength); } } // Fills Polygon coordinates into polygons object of arrays -function handlePolygon(coords, polygons, index, coordLength) { - polygons.polygonIndices[index.polygonObject] = index.polygonPosition * coordLength; - index.polygonObject++; +function handlePolygon(coords, polygons, indexMap, coordLength) { + polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition * coordLength; + indexMap.polygonObject++; for (const {} of coords) { - polygons.primitivePolygonIndices[index.polygonRing] = index.polygonPosition * coordLength; - index.polygonRing++; + polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition * coordLength; + indexMap.polygonRing++; } - polygons.positions.set(coords.flat(2), index.polygonPosition * coordLength); + polygons.positions.set(coords.flat(2), indexMap.polygonPosition * coordLength); const nPositions = coords.flat(1).length; - polygons.objectIds.set(new Uint32Array(nPositions).fill(index.feature), index.polygonPosition); - index.polygonPosition += nPositions; + polygons.objectIds.set( + new Uint32Array(nPositions).fill(indexMap.feature), + indexMap.polygonPosition + ); + indexMap.polygonPosition += nPositions; } // Fills MultiPolygon coordinates into polygons object of arrays -function handleMultiPolygon(coords, polygons, index, coordLength) { +function handleMultiPolygon(coords, polygons, indexMap, coordLength) { for (const polygon of coords) { - handlePolygon(polygon, polygons, index, coordLength); + handlePolygon(polygon, polygons, indexMap, coordLength); } } From a421188e545ddf9b0d861d5eb9c31502d3a560d1 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 18:47:59 -0600 Subject: [PATCH 21/27] Use vertex number for path/polygon index --- modules/gis/src/lib/geojson-to-binary.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 8d3b283984..338123ed45 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -198,7 +198,7 @@ function handleMultiPoint(coords, points, indexMap, coordLength) { // Fills LineString coordinates into lines object of arrays function handleLineString(coords, lines, indexMap, coordLength) { - lines.pathIndices[indexMap.linePath] = indexMap.linePosition * coordLength; + lines.pathIndices[indexMap.linePath] = indexMap.linePosition; indexMap.linePath++; lines.positions.set(coords.flat(), indexMap.linePosition * coordLength); @@ -217,11 +217,11 @@ function handleMultiLineString(coords, lines, indexMap, coordLength) { // Fills Polygon coordinates into polygons object of arrays function handlePolygon(coords, polygons, indexMap, coordLength) { - polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition * coordLength; + polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition; indexMap.polygonObject++; for (const {} of coords) { - polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition * coordLength; + polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition; indexMap.polygonRing++; } From 9ceaded09cb1bd9298018ad5eca34d0558042074 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 19:09:50 -0600 Subject: [PATCH 22/27] Fix primitivePolygonIndices calculation --- modules/gis/src/lib/geojson-to-binary.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index 338123ed45..dbfaae39a8 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -220,19 +220,19 @@ function handlePolygon(coords, polygons, indexMap, coordLength) { polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition; indexMap.polygonObject++; - for (const {} of coords) { + for (const ring of coords) { polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition; indexMap.polygonRing++; - } - polygons.positions.set(coords.flat(2), indexMap.polygonPosition * coordLength); + polygons.positions.set(ring.flat(1), indexMap.polygonPosition * coordLength); - const nPositions = coords.flat(1).length; - polygons.objectIds.set( - new Uint32Array(nPositions).fill(indexMap.feature), - indexMap.polygonPosition - ); - indexMap.polygonPosition += nPositions; + const nPositions = ring.length; + polygons.objectIds.set( + new Uint32Array(nPositions).fill(indexMap.feature), + indexMap.polygonPosition + ); + indexMap.polygonPosition += nPositions; + } } // Fills MultiPolygon coordinates into polygons object of arrays From 5d7fbd1d9e4134d05fc0ecedbe2f2922b5d00561 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 19:13:44 -0600 Subject: [PATCH 23/27] Add tests --- modules/gis/test/geojson-to-binary.spec.js | 108 ++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index da2d87256a..54f25c02ca 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -1,6 +1,5 @@ import test from 'tape-promise/tape'; import {fetchFile} from '@loaders.gl/core'; - import {geojsonToBinary} from '@loaders.gl/gis'; // Sample GeoJSON Files @@ -24,5 +23,112 @@ test('gis#geojson-to-binary 2D features', async t => { t.equal(polygons.polygonIndices.size, 1); t.equal(polygons.primitivePolygonIndices.size, 1); t.equal(polygons.objectIds.size, 1); + + // Point value equality + t.deepEqual(points.positions.value, [100, 0, 100, 0, 101, 1]); + t.deepEqual(points.objectIds.value, [0, 1, 1]); + + // LineString value equality + t.deepEqual(lines.pathIndices.value, [0, 2, 4]); + t.deepEqual(lines.positions.value, [100, 0, 101, 1, 100, 0, 101, 1, 102, 2, 103, 3]); + t.deepEqual(lines.objectIds.value, [2, 2, 3, 3, 3, 3]); + + // Polygon value equality + t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); + t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); + t.deepEqual(polygons.positions.value, [ + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 102, + 2, + 103, + 2, + 103, + 3, + 102, + 3, + 102, + 2, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2 + ]); + t.deepEqual(polygons.objectIds.value, [ + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6 + ]); t.end(); }); From 1e92d60d3a49a79401b6dd29f8fded01ea26cb21 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 19:52:00 -0600 Subject: [PATCH 24/27] Add 3D coordinates test --- modules/gis/test/data/3d_features.json | 98 ++++++++++++++ modules/gis/test/geojson-to-binary.spec.js | 145 +++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 modules/gis/test/data/3d_features.json diff --git a/modules/gis/test/data/3d_features.json b/modules/gis/test/data/3d_features.json new file mode 100644 index 0000000000..8092e997aa --- /dev/null +++ b/modules/gis/test/data/3d_features.json @@ -0,0 +1,98 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [100.0, 0.0, 1] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPoint", + "coordinates": [[100.0, 0.0, 2], [101.0, 1.0, 3]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[100.0, 0.0, 4], [101.0, 1.0, 5]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [[[100.0, 0.0, 6], [101.0, 1.0, 7]], [[102.0, 2.0, 8], [103.0, 3.0, 9]]] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[100.0, 0.0, 10], [101.0, 0.0, 11], [101.0, 1.0, 12], [100.0, 1.0, 13], [100.0, 0.0, 14]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [100.0, 0.0, 15], + [101.0, 0.0, 16], + [101.0, 1.0, 17], + [100.0, 1.0, 18], + [100.0, 0.0, 19] + ], + [[100.8, 0.8, 20], [100.8, 0.2, 21], [100.2, 0.2, 22], [100.2, 0.8, 23], [100.8, 0.8, 24]] + ] + }, + "properties": {} + }, + { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [102.0, 2.0, 25], + [103.0, 2.0, 26], + [103.0, 3.0, 27], + [102.0, 3.0, 28], + [102.0, 2.0, 29] + ] + ], + [ + [ + [100.0, 0.0, 30], + [101.0, 0.0, 31], + [101.0, 1.0, 32], + [100.0, 1.0, 33], + [100.0, 0.0, 34] + ], + [ + [100.2, 0.2, 35], + [100.2, 0.8, 36], + [100.8, 0.8, 37], + [100.8, 0.2, 38], + [100.2, 0.2, 39] + ] + ] + ] + }, + "properties": {} + } + ] +} diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index 54f25c02ca..c5a330463c 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -5,6 +5,7 @@ import {geojsonToBinary} from '@loaders.gl/gis'; // Sample GeoJSON Files // All features have 2D coordinates const FEATURES_2D = '@loaders.gl/gis/test/data/2d_features.json'; +const FEATURES_3D = '@loaders.gl/gis/test/data/3d_features.json'; test('gis#geojson-to-binary 2D features', async t => { const response = await fetchFile(FEATURES_2D); @@ -132,3 +133,147 @@ test('gis#geojson-to-binary 2D features', async t => { ]); t.end(); }); + +test('gis#geojson-to-binary 3D features', async t => { + const response = await fetchFile(FEATURES_3D); + const {features} = await response.json(); + const {points, lines, polygons} = geojsonToBinary(features); + + // 3D size + t.equal(points.positions.size, 3); + t.equal(lines.positions.size, 3); + t.equal(polygons.positions.size, 3); + + // Other arrays have coordinate size 1 + t.equal(points.objectIds.size, 1); + t.equal(lines.pathIndices.size, 1); + t.equal(lines.objectIds.size, 1); + t.equal(polygons.polygonIndices.size, 1); + t.equal(polygons.primitivePolygonIndices.size, 1); + t.equal(polygons.objectIds.size, 1); + + // Point value equality + t.deepEqual(points.positions.value, [100, 0, 1, 100, 0, 2, 101, 1, 3]); + t.deepEqual(points.objectIds.value, [0, 1, 1]); + + // LineString value equality + t.deepEqual(lines.pathIndices.value, [0, 2, 4]); + t.deepEqual(lines.positions.value, [ + 100, + 0, + 4, + 101, + 1, + 5, + 100, + 0, + 6, + 101, + 1, + 7, + 102, + 2, + 8, + 103, + 3, + 9 + ]); + t.deepEqual(lines.objectIds.value, [2, 2, 3, 3, 3, 3]); + + // Polygon value equality + t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); + t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); + t.deepEqual(polygons.positions.value, [ + 100, + 0, + 10, + 101, + 0, + 11, + 101, + 1, + 12, + 100, + 1, + 13, + 100, + 0, + 14, + 100, + 0, + 15, + 101, + 0, + 16, + 101, + 1, + 17, + 100, + 1, + 18, + 100, + 0, + 19, + 100.8, + 0.8, + 20, + 100.8, + 0.2, + 21, + 100.2, + 0.2, + 22, + 100.2, + 0.8, + 23, + 100.8, + 0.8, + 24, + 102, + 2, + 25, + 103, + 2, + 26, + 103, + 3, + 27, + 102, + 3, + 28, + 102, + 2, + 29, + 100, + 0, + 30, + 101, + 0, + 31, + 101, + 1, + 32, + 100, + 1, + 33, + 100, + 0, + 34, + 100.2, + 0.2, + 35, + 100.2, + 0.8, + 36, + 100.8, + 0.8, + 37, + 100.8, + 0.2, + 38, + 100.2, + 0.2, + 39 + ]); + t.end(); +}); From 06bf3dcb86c525912c5548652e563994a01d9153 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 19:54:07 -0600 Subject: [PATCH 25/27] Add comment to test --- modules/gis/test/geojson-to-binary.spec.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index c5a330463c..8383706f0b 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -2,9 +2,11 @@ import test from 'tape-promise/tape'; import {fetchFile} from '@loaders.gl/core'; import {geojsonToBinary} from '@loaders.gl/gis'; -// Sample GeoJSON Files +// Sample GeoJSON data derived from examples in GeoJSON specification +// https://tools.ietf.org/html/rfc7946#appendix-A // All features have 2D coordinates const FEATURES_2D = '@loaders.gl/gis/test/data/2d_features.json'; +// All features have 3D coordinates const FEATURES_3D = '@loaders.gl/gis/test/data/3d_features.json'; test('gis#geojson-to-binary 2D features', async t => { From 36e06ad146c2f38dedb2ac619a35dc2d3b2383b8 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 20:01:24 -0600 Subject: [PATCH 26/27] Fix tests --- modules/gis/test/geojson-to-binary.spec.js | 314 +++++++++++---------- test/aliases.js | 1 + 2 files changed, 161 insertions(+), 154 deletions(-) diff --git a/modules/gis/test/geojson-to-binary.spec.js b/modules/gis/test/geojson-to-binary.spec.js index 8383706f0b..c9f1623ae1 100644 --- a/modules/gis/test/geojson-to-binary.spec.js +++ b/modules/gis/test/geojson-to-binary.spec.js @@ -39,68 +39,71 @@ test('gis#geojson-to-binary 2D features', async t => { // Polygon value equality t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); - t.deepEqual(polygons.positions.value, [ - 100, - 0, - 101, - 0, - 101, - 1, - 100, - 1, - 100, - 0, - 100, - 0, - 101, - 0, - 101, - 1, - 100, - 1, - 100, - 0, - 100.8, - 0.8, - 100.8, - 0.2, - 100.2, - 0.2, - 100.2, - 0.8, - 100.8, - 0.8, - 102, - 2, - 103, - 2, - 103, - 3, - 102, - 3, - 102, - 2, - 100, - 0, - 101, - 0, - 101, - 1, - 100, - 1, - 100, - 0, - 100.2, - 0.2, - 100.2, - 0.8, - 100.8, - 0.8, - 100.8, - 0.2, - 100.2, - 0.2 - ]); + t.deepEqual( + polygons.positions.value, + Float32Array.from([ + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 102, + 2, + 103, + 2, + 103, + 3, + 102, + 3, + 102, + 2, + 100, + 0, + 101, + 0, + 101, + 1, + 100, + 1, + 100, + 0, + 100.2, + 0.2, + 100.2, + 0.8, + 100.8, + 0.8, + 100.8, + 0.2, + 100.2, + 0.2 + ]) + ); t.deepEqual(polygons.objectIds.value, [ 4, 4, @@ -185,97 +188,100 @@ test('gis#geojson-to-binary 3D features', async t => { // Polygon value equality t.deepEqual(polygons.polygonIndices.value, [0, 5, 15, 20]); t.deepEqual(polygons.primitivePolygonIndices.value, [0, 5, 10, 15, 20, 25]); - t.deepEqual(polygons.positions.value, [ - 100, - 0, - 10, - 101, - 0, - 11, - 101, - 1, - 12, - 100, - 1, - 13, - 100, - 0, - 14, - 100, - 0, - 15, - 101, - 0, - 16, - 101, - 1, - 17, - 100, - 1, - 18, - 100, - 0, - 19, - 100.8, - 0.8, - 20, - 100.8, - 0.2, - 21, - 100.2, - 0.2, - 22, - 100.2, - 0.8, - 23, - 100.8, - 0.8, - 24, - 102, - 2, - 25, - 103, - 2, - 26, - 103, - 3, - 27, - 102, - 3, - 28, - 102, - 2, - 29, - 100, - 0, - 30, - 101, - 0, - 31, - 101, - 1, - 32, - 100, - 1, - 33, - 100, - 0, - 34, - 100.2, - 0.2, - 35, - 100.2, - 0.8, - 36, - 100.8, - 0.8, - 37, - 100.8, - 0.2, - 38, - 100.2, - 0.2, - 39 - ]); + t.deepEqual( + polygons.positions.value, + Float32Array.from([ + 100, + 0, + 10, + 101, + 0, + 11, + 101, + 1, + 12, + 100, + 1, + 13, + 100, + 0, + 14, + 100, + 0, + 15, + 101, + 0, + 16, + 101, + 1, + 17, + 100, + 1, + 18, + 100, + 0, + 19, + 100.8, + 0.8, + 20, + 100.8, + 0.2, + 21, + 100.2, + 0.2, + 22, + 100.2, + 0.8, + 23, + 100.8, + 0.8, + 24, + 102, + 2, + 25, + 103, + 2, + 26, + 103, + 3, + 27, + 102, + 3, + 28, + 102, + 2, + 29, + 100, + 0, + 30, + 101, + 0, + 31, + 101, + 1, + 32, + 100, + 1, + 33, + 100, + 0, + 34, + 100.2, + 0.2, + 35, + 100.2, + 0.8, + 36, + 100.8, + 0.8, + 37, + 100.8, + 0.2, + 38, + 100.2, + 0.2, + 39 + ]) + ); t.end(); }); diff --git a/test/aliases.js b/test/aliases.js index c2101f1045..724d91fdbf 100644 --- a/test/aliases.js +++ b/test/aliases.js @@ -31,6 +31,7 @@ const makeAliases = () => ({ '@loaders.gl/csv/test': path.resolve(__dirname, '../modules/csv/test'), '@loaders.gl/draco/test': path.resolve(__dirname, '../modules/draco/test'), '@loaders.gl/images/test': path.resolve(__dirname, '../modules/images/test'), + '@loaders.gl/gis/test': path.resolve(__dirname, '../modules/gis/test'), '@loaders.gl/gltf/test': path.resolve(__dirname, '../modules/gltf/test'), '@loaders.gl/json/test': path.resolve(__dirname, '../modules/json/test'), '@loaders.gl/kml/test': path.resolve(__dirname, '../modules/kml/test'), From daa2281a46a4afc7fe435daeaeb4a6ba5ee3618a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 18 Mar 2020 23:08:04 -0600 Subject: [PATCH 27/27] Helper function to flatten coords --- modules/gis/src/lib/geojson-to-binary.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/gis/src/lib/geojson-to-binary.js b/modules/gis/src/lib/geojson-to-binary.js index dbfaae39a8..c80293bb54 100644 --- a/modules/gis/src/lib/geojson-to-binary.js +++ b/modules/gis/src/lib/geojson-to-binary.js @@ -58,9 +58,9 @@ function firstPass(features) { case 'Polygon': polygonObjects++; polygonRings += geometry.coordinates.length; - polygonPositions += geometry.coordinates.flat(1).length; + polygonPositions += flatten(geometry.coordinates).length; - for (const coord of geometry.coordinates.flat()) { + for (const coord of flatten(geometry.coordinates)) { // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } @@ -69,10 +69,10 @@ function firstPass(features) { for (const polygon of geometry.coordinates) { polygonObjects++; polygonRings += polygon.length; - polygonPositions += polygon.flat(1).length; + polygonPositions += flatten(polygon).length; // eslint-disable-next-line max-depth - for (const coord of polygon.flat()) { + for (const coord of flatten(polygon)) { // eslint-disable-next-line max-depth if (coord.length === 3) maxCoordLength = 3; } @@ -201,7 +201,7 @@ function handleLineString(coords, lines, indexMap, coordLength) { lines.pathIndices[indexMap.linePath] = indexMap.linePosition; indexMap.linePath++; - lines.positions.set(coords.flat(), indexMap.linePosition * coordLength); + lines.positions.set(flatten(coords), indexMap.linePosition * coordLength); const nPositions = coords.length; lines.objectIds.set(new Uint32Array(nPositions).fill(indexMap.feature), indexMap.linePosition); @@ -224,7 +224,7 @@ function handlePolygon(coords, polygons, indexMap, coordLength) { polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition; indexMap.polygonRing++; - polygons.positions.set(ring.flat(1), indexMap.polygonPosition * coordLength); + polygons.positions.set(flatten(ring), indexMap.polygonPosition * coordLength); const nPositions = ring.length; polygons.objectIds.set( @@ -241,3 +241,7 @@ function handleMultiPolygon(coords, polygons, indexMap, coordLength) { handlePolygon(polygon, polygons, indexMap, coordLength); } } + +function flatten(arrays) { + return [].concat(...arrays); +}