Skip to content

Commit

Permalink
CARTO: Add example for boundaries (#8269)
Browse files Browse the repository at this point in the history
  • Loading branch information
felixpalmer authored Nov 9, 2023
1 parent 79b3e9d commit 9a4375c
Show file tree
Hide file tree
Showing 13 changed files with 332 additions and 18 deletions.
50 changes: 39 additions & 11 deletions docs/api-reference/carto/data-sources.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ In addition, the following options are supported on each source:
#### vectorTableSource

```ts
type vectorTableSourceOptions = {
type VectorTableSourceOptions = {
columns?: string[];
spatialDataColumn?: string;
tableName: string;
Expand All @@ -70,8 +70,7 @@ type vectorTableSourceOptions = {
#### vectorQuerySource
```ts
type vectorQuerySourceOptions = {
columns?: string[];
type VectorQuerySourceOptions = {
spatialDataColumn?: string;
sqlQuery: string;
queryParameters: QueryParameters;
Expand All @@ -81,7 +80,7 @@ type vectorQuerySourceOptions = {
#### vectorTilesetSource
```ts
type vectorTilesetSourceOptions = {
type VectorTilesetSourceOptions = {
tableName: string;
}
```
Expand All @@ -104,7 +103,6 @@ type H3TableSourceOptions = {
type H3QuerySourceOptions = {
aggregationExp: string;
aggregationResLevel?: number;
columns?: string[];
spatialDataColumn?: string;
sqlQuery: string;
queryParameters: QueryParameters;
Expand All @@ -122,7 +120,7 @@ type H3TilesetSourceOptions = {
#### quadbinTableSource
```ts
type quadbinTableSourceOptions = {
type QuadbinTableSourceOptions = {
aggregationExp: string;
aggregationResLevel?: number;
columns?: string[];
Expand All @@ -134,10 +132,9 @@ type quadbinTableSourceOptions = {
#### quadbinQuerySource
```ts
type quadbinQuerySourceOptions = {
type QuadbinQuerySourceOptions = {
aggregationExp: string;
aggregationResLevel?: number;
columns?: string[];
spatialDataColumn?: string;
sqlQuery: string;
queryParameters: QueryParameters;
Expand All @@ -147,19 +144,50 @@ type quadbinQuerySourceOptions = {
#### quadbinTilesetSource
```ts
type quadbinTilesetSourceOptions = {
type QuadbinTilesetSourceOptions = {
tableName: string;
}
```
#### rasterTilesetSource (Experimental)
```ts
type RasterTilesetSourceOptions = {
tableName: string;
}
```
#### rasterTilesetSource
#### boundaryTableSource (Experimental)
```ts
type rasterTilesetSourceOptions = {
type BoundaryTableSourceOptions = {
boundaryId: string;
columns?: string[];
matchingColumn?: string;
tableName: string;
}
```
#### boundaryQuerySource (Experimental)
```ts
type BoundaryQuerySourceOptions = {
boundaryId: string;
matchingColumn?: string;
sqlQuery: string;
queryParameters: QueryParameters;
}
```
#### boundaryTilesetSource (Experimental)
```ts
type BoundaryTilesetSourceOptions = {
boundaryId: string;
columns?: string[];
}
```
### QueryParameters
QueryParameters are used to parametrize SQL queries. The format depends on the source's provider, some examples:
Expand Down
2 changes: 1 addition & 1 deletion modules/carto/src/api/types.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {SCALE_TYPE} from './layer-map';

export type Format = 'json' | 'geojson' | 'tilejson';
export type MapType = 'query' | 'table' | 'tileset' | 'raster';
export type MapType = 'boundary' | 'query' | 'table' | 'tileset' | 'raster';
export type RequestType = 'Map data' | 'Map instantiation' | 'Public map' | 'Tile stats' | 'SQL';
export type APIErrorContext = {
requestType: RequestType;
Expand Down
12 changes: 12 additions & 0 deletions modules/carto/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ export type {
} from './api';

import {
boundaryQuerySource,
boundaryTableSource,
boundaryTilesetSource,
h3QuerySource,
h3TableSource,
h3TilesetSource,
Expand All @@ -49,6 +52,9 @@ import {
} from './sources';

const CARTO_SOURCES = {
boundaryQuerySource,
boundaryTableSource,
boundaryTilesetSource,
h3QuerySource,
h3TableSource,
h3TilesetSource,
Expand All @@ -62,6 +68,9 @@ const CARTO_SOURCES = {
};

export {
boundaryQuerySource,
boundaryTableSource,
boundaryTilesetSource,
h3QuerySource,
h3TableSource,
h3TilesetSource,
Expand All @@ -78,6 +87,9 @@ export {

export type {
TilejsonResult,
BoundaryQuerySourceOptions,
BoundaryTableSourceOptions,
BoundaryTilesetSourceOptions,
H3QuerySourceOptions,
H3TableSourceOptions,
H3TilesetSourceOptions,
Expand Down
25 changes: 25 additions & 0 deletions modules/carto/src/layers/schema/carto-properties-tile-loader.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';

import {Tile, TileReader} from './carto-properties-tile';
import {parsePbf} from './tile-loader-utils';

const CartoPropertiesTileLoader: LoaderWithParser = {
name: 'CARTO Properties Tile',
version: '1',
id: 'cartoPropertiesTile',
module: 'carto',
extensions: ['pbf'],
mimeTypes: ['application/vnd.carto-properties-tile'],
category: 'geometry',
worker: false,
parse: async (arrayBuffer, options) => parseCartoPropertiesTile(arrayBuffer, options),
parseSync: parseCartoPropertiesTile,
options: {}
};

function parseCartoPropertiesTile(arrayBuffer: ArrayBuffer, options?: LoaderOptions): Tile | null {
if (!arrayBuffer) return null;
return parsePbf(arrayBuffer, TileReader);
}

export default CartoPropertiesTileLoader;
21 changes: 21 additions & 0 deletions modules/carto/src/layers/schema/carto-properties-tile.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import {NumericProp, NumericPropKeyValueReader, PropertiesReader} from './carto-tile';

// Tile ========================================

export interface Tile {
properties: Record<string, string>[];
numericProps: Record<string, NumericProp>;
}

export class TileReader {
static read(pbf, end?: number): Tile {
return pbf.readFields(TileReader._readField, {properties: [], numericProps: {}}, end);
}
static _readField(this: void, tag: number, obj: Tile, pbf) {
if (tag === 1) obj.properties.push(PropertiesReader.read(pbf, pbf.readVarint() + pbf.pos));
else if (tag === 2) {
const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos);
obj.numericProps[entry.key] = entry.value;
}
}
}
54 changes: 54 additions & 0 deletions modules/carto/src/layers/utils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import {Tile as PropertiesTile} from './schema/carto-properties-tile';
import {Tile as VectorTile} from './schema/carto-tile';
import {_deepEqual as deepEqual} from '@deck.gl/core';
import type {TilejsonResult} from '../sources/types';

Expand All @@ -13,6 +15,58 @@ export function injectAccessToken(loadOptions: any, accessToken: string): void {
}
}

export function mergeBoundaryData(geometry: VectorTile, properties: PropertiesTile): VectorTile {
const mapping = {};
for (const {geoid, ...rest} of properties.properties) {
if (geoid in mapping) {
throw new Error(`Duplicate geoid key in mapping: ${geoid}`);
}
mapping[geoid] = rest;
}

for (const type of ['points', 'lines', 'polygons']) {
const geom = geometry[type];
if (geom.positions.value.length === 0) {
continue;
}

geom.properties = geom.properties.map(({geoid}) => mapping[geoid]);

// numericProps need to be filled to match length of positions buffer
const {positions, globalFeatureIds} = geom;
let indices: Uint16Array | Uint32Array | null = null;
if (type === 'lines') indices = geom.pathIndices.value;
if (type === 'polygons') indices = geom.polygonIndices.value;
const length = positions.value.length / positions.size;
for (const key in properties.numericProps) {
const sourceProp = properties.numericProps[key].value;
const TypedArray = sourceProp.constructor as
| Float32ArrayConstructor
| Float64ArrayConstructor;
const destProp = new TypedArray(length);
geom.numericProps[key] = {value: destProp, size: 1};

if (!indices) {
for (let i = 0; i < length; i++) {
// points
const featureId = globalFeatureIds.value[i];
destProp[i] = sourceProp[featureId];
}
} else {
// lines|polygons
for (let i = 0; i < indices.length - 1; i++) {
const startIndex = indices[i];
const endIndex = indices[i + 1];
const featureId = globalFeatureIds.value[startIndex];
destProp.fill(sourceProp[featureId], startIndex, endIndex);
}
}
}
}

return geometry;
}

export const TilejsonPropType = {
type: 'object' as const,
value: null as null | TilejsonResult,
Expand Down
36 changes: 31 additions & 5 deletions modules/carto/src/layers/vector-tile-layer.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {registerLoaders} from '@loaders.gl/core';
import CartoPropertiesTileLoader from './schema/carto-properties-tile-loader';
import CartoVectorTileLoader from './schema/carto-vector-tile-loader';
registerLoaders([CartoVectorTileLoader]);
registerLoaders([CartoPropertiesTileLoader, CartoVectorTileLoader]);

import {DefaultProps} from '@deck.gl/core';
import {ClipExtension} from '@deck.gl/extensions';
Expand All @@ -16,8 +17,9 @@ import {GeoJsonLayer} from '@deck.gl/layers';
import {binaryToGeojson} from '@loaders.gl/gis';
import type {BinaryFeatureCollection} from '@loaders.gl/schema';
import type {Feature} from 'geojson';

import type {TilejsonResult} from '../sources/types';
import {injectAccessToken, TilejsonPropType} from './utils';
import {TilejsonPropType, injectAccessToken, mergeBoundaryData} from './utils';

const defaultProps: DefaultProps<VectorTileLayerProps> = {
...MVTLayer.defaultProps,
Expand Down Expand Up @@ -69,16 +71,40 @@ export default class VectorTileLayer<ExtraProps extends {} = {}> extends MVTLaye
return loadOptions;
}

getTileData(tile: TileLoadProps) {
const url = _getURLFromTemplate(this.state.data, tile);
async getTileData(tile: TileLoadProps) {
const tileJSON = this.props.data as TilejsonResult;
const {tiles, properties_tiles} = tileJSON;
const url = _getURLFromTemplate(tiles, tile);
if (!url) {
return Promise.reject('Invalid URL');
}

const loadOptions = this.getLoadOptions();
const {fetch} = this.props;
const {signal} = tile;
return fetch(url, {propName: 'data', layer: this, loadOptions, signal});

// Fetch geometry and attributes separately
const geometryFetch = fetch(url, {propName: 'data', layer: this, loadOptions, signal});

if (!properties_tiles) {
return await geometryFetch;
}

const propertiesUrl = _getURLFromTemplate(properties_tiles, tile);
if (!propertiesUrl) {
return Promise.reject('Invalid properties URL');
}

const attributesFetch = fetch(propertiesUrl, {
propName: 'data',
layer: this,
loadOptions,
signal
});
const [geometry, attributes] = await Promise.all([geometryFetch, attributesFetch]);
if (!geometry) return null;

return mergeBoundaryData(geometry, attributes);
}

renderSubLayers(
Expand Down
31 changes: 31 additions & 0 deletions modules/carto/src/sources/boundary-query-source.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import {QueryParameters} from '../api';
import {baseSource} from './base-source';
import type {SourceOptions, TilejsonResult} from './types';

export type BoundaryQuerySourceOptions = SourceOptions & {
boundaryId: string;
matchingColumn?: string;
sqlQuery: string;
queryParameters: QueryParameters;
};
type UrlParameters = {
boundaryId: string;
matchingColumn: string;
sqlQuery: string;
queryParameters?: string;
};

export const boundaryQuerySource = async function (
options: BoundaryQuerySourceOptions
): Promise<TilejsonResult> {
const {boundaryId, matchingColumn = 'id', sqlQuery, queryParameters} = options;
const urlParameters: UrlParameters = {
boundaryId,
matchingColumn,
sqlQuery
};
if (queryParameters) {
urlParameters.queryParameters = JSON.stringify(queryParameters);
}
return baseSource<UrlParameters>('boundary', options, urlParameters) as Promise<TilejsonResult>;
};
Loading

0 comments on commit 9a4375c

Please sign in to comment.