diff --git a/docs/api-reference/carto/data-sources.md b/docs/api-reference/carto/data-sources.md index ff5465295f1..67a42953858 100644 --- a/docs/api-reference/carto/data-sources.md +++ b/docs/api-reference/carto/data-sources.md @@ -60,7 +60,7 @@ In addition, the following options are supported on each source: #### vectorTableSource ```ts -type vectorTableSourceOptions = { +type VectorTableSourceOptions = { columns?: string[]; spatialDataColumn?: string; tableName: string; @@ -70,8 +70,7 @@ type vectorTableSourceOptions = { #### vectorQuerySource ```ts -type vectorQuerySourceOptions = { - columns?: string[]; +type VectorQuerySourceOptions = { spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; @@ -81,7 +80,7 @@ type vectorQuerySourceOptions = { #### vectorTilesetSource ```ts -type vectorTilesetSourceOptions = { +type VectorTilesetSourceOptions = { tableName: string; } ``` @@ -104,7 +103,6 @@ type H3TableSourceOptions = { type H3QuerySourceOptions = { aggregationExp: string; aggregationResLevel?: number; - columns?: string[]; spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; @@ -122,7 +120,7 @@ type H3TilesetSourceOptions = { #### quadbinTableSource ```ts -type quadbinTableSourceOptions = { +type QuadbinTableSourceOptions = { aggregationExp: string; aggregationResLevel?: number; columns?: string[]; @@ -134,10 +132,9 @@ type quadbinTableSourceOptions = { #### quadbinQuerySource ```ts -type quadbinQuerySourceOptions = { +type QuadbinQuerySourceOptions = { aggregationExp: string; aggregationResLevel?: number; - columns?: string[]; spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; @@ -147,19 +144,50 @@ type quadbinQuerySourceOptions = { #### quadbinTilesetSource ```ts -type quadbinTilesetSourceOptions = { +type QuadbinTilesetSourceOptions = { + tableName: string; +} +``` + +#### rasterTilesetSource (Experimental) + +```ts +type RasterTilesetSourceOptions = { tableName: string; } ``` -#### rasterTilesetSource +#### boundaryTableSource (Experimental) ```ts -type rasterTilesetSourceOptions = { +type BoundaryTableSourceOptions = { + boundaryId: string; + columns?: string[]; + matchingColumn?: string; tableName: string; } ``` +#### boundaryQuerySource (Experimental) + +```ts +type BoundaryQuerySourceOptions = { + boundaryId: string; + matchingColumn?: string; + sqlQuery: string; + queryParameters: QueryParameters; +} +``` + +#### boundaryTilesetSource (Experimental) + +```ts +type BoundaryTilesetSourceOptions = { + boundaryId: string; + columns?: string[]; +} +``` + ### QueryParameters QueryParameters are used to parametrize SQL queries. The format depends on the source's provider, some examples: diff --git a/modules/carto/src/api/types.ts b/modules/carto/src/api/types.ts index d941506e47b..0e1b3a6eb59 100644 --- a/modules/carto/src/api/types.ts +++ b/modules/carto/src/api/types.ts @@ -1,7 +1,7 @@ import {SCALE_TYPE} from './layer-map'; export type Format = 'json' | 'geojson' | 'tilejson'; -export type MapType = 'query' | 'table' | 'tileset' | 'raster'; +export type MapType = 'boundary' | 'query' | 'table' | 'tileset' | 'raster'; export type RequestType = 'Map data' | 'Map instantiation' | 'Public map' | 'Tile stats' | 'SQL'; export type APIErrorContext = { requestType: RequestType; diff --git a/modules/carto/src/index.ts b/modules/carto/src/index.ts index 8fcb7cee8be..b94c50d0461 100644 --- a/modules/carto/src/index.ts +++ b/modules/carto/src/index.ts @@ -35,6 +35,9 @@ export type { } from './api'; import { + boundaryQuerySource, + boundaryTableSource, + boundaryTilesetSource, h3QuerySource, h3TableSource, h3TilesetSource, @@ -49,6 +52,9 @@ import { } from './sources'; const CARTO_SOURCES = { + boundaryQuerySource, + boundaryTableSource, + boundaryTilesetSource, h3QuerySource, h3TableSource, h3TilesetSource, @@ -62,6 +68,9 @@ const CARTO_SOURCES = { }; export { + boundaryQuerySource, + boundaryTableSource, + boundaryTilesetSource, h3QuerySource, h3TableSource, h3TilesetSource, @@ -78,6 +87,9 @@ export { export type { TilejsonResult, + BoundaryQuerySourceOptions, + BoundaryTableSourceOptions, + BoundaryTilesetSourceOptions, H3QuerySourceOptions, H3TableSourceOptions, H3TilesetSourceOptions, diff --git a/modules/carto/src/layers/schema/carto-properties-tile-loader.ts b/modules/carto/src/layers/schema/carto-properties-tile-loader.ts new file mode 100644 index 00000000000..59ce9f2b347 --- /dev/null +++ b/modules/carto/src/layers/schema/carto-properties-tile-loader.ts @@ -0,0 +1,25 @@ +import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils'; + +import {Tile, TileReader} from './carto-properties-tile'; +import {parsePbf} from './tile-loader-utils'; + +const CartoPropertiesTileLoader: LoaderWithParser = { + name: 'CARTO Properties Tile', + version: '1', + id: 'cartoPropertiesTile', + module: 'carto', + extensions: ['pbf'], + mimeTypes: ['application/vnd.carto-properties-tile'], + category: 'geometry', + worker: false, + parse: async (arrayBuffer, options) => parseCartoPropertiesTile(arrayBuffer, options), + parseSync: parseCartoPropertiesTile, + options: {} +}; + +function parseCartoPropertiesTile(arrayBuffer: ArrayBuffer, options?: LoaderOptions): Tile | null { + if (!arrayBuffer) return null; + return parsePbf(arrayBuffer, TileReader); +} + +export default CartoPropertiesTileLoader; diff --git a/modules/carto/src/layers/schema/carto-properties-tile.ts b/modules/carto/src/layers/schema/carto-properties-tile.ts new file mode 100644 index 00000000000..542b9f30462 --- /dev/null +++ b/modules/carto/src/layers/schema/carto-properties-tile.ts @@ -0,0 +1,21 @@ +import {NumericProp, NumericPropKeyValueReader, PropertiesReader} from './carto-tile'; + +// Tile ======================================== + +export interface Tile { + properties: Record[]; + numericProps: Record; +} + +export class TileReader { + static read(pbf, end?: number): Tile { + return pbf.readFields(TileReader._readField, {properties: [], numericProps: {}}, end); + } + static _readField(this: void, tag: number, obj: Tile, pbf) { + if (tag === 1) obj.properties.push(PropertiesReader.read(pbf, pbf.readVarint() + pbf.pos)); + else if (tag === 2) { + const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos); + obj.numericProps[entry.key] = entry.value; + } + } +} diff --git a/modules/carto/src/layers/utils.ts b/modules/carto/src/layers/utils.ts index 46388d90715..2fbf42c2978 100644 --- a/modules/carto/src/layers/utils.ts +++ b/modules/carto/src/layers/utils.ts @@ -1,3 +1,5 @@ +import {Tile as PropertiesTile} from './schema/carto-properties-tile'; +import {Tile as VectorTile} from './schema/carto-tile'; import {_deepEqual as deepEqual} from '@deck.gl/core'; import type {TilejsonResult} from '../sources/types'; @@ -13,6 +15,58 @@ export function injectAccessToken(loadOptions: any, accessToken: string): void { } } +export function mergeBoundaryData(geometry: VectorTile, properties: PropertiesTile): VectorTile { + const mapping = {}; + for (const {geoid, ...rest} of properties.properties) { + if (geoid in mapping) { + throw new Error(`Duplicate geoid key in mapping: ${geoid}`); + } + mapping[geoid] = rest; + } + + for (const type of ['points', 'lines', 'polygons']) { + const geom = geometry[type]; + if (geom.positions.value.length === 0) { + continue; + } + + geom.properties = geom.properties.map(({geoid}) => mapping[geoid]); + + // numericProps need to be filled to match length of positions buffer + const {positions, globalFeatureIds} = geom; + let indices: Uint16Array | Uint32Array | null = null; + if (type === 'lines') indices = geom.pathIndices.value; + if (type === 'polygons') indices = geom.polygonIndices.value; + const length = positions.value.length / positions.size; + for (const key in properties.numericProps) { + const sourceProp = properties.numericProps[key].value; + const TypedArray = sourceProp.constructor as + | Float32ArrayConstructor + | Float64ArrayConstructor; + const destProp = new TypedArray(length); + geom.numericProps[key] = {value: destProp, size: 1}; + + if (!indices) { + for (let i = 0; i < length; i++) { + // points + const featureId = globalFeatureIds.value[i]; + destProp[i] = sourceProp[featureId]; + } + } else { + // lines|polygons + for (let i = 0; i < indices.length - 1; i++) { + const startIndex = indices[i]; + const endIndex = indices[i + 1]; + const featureId = globalFeatureIds.value[startIndex]; + destProp.fill(sourceProp[featureId], startIndex, endIndex); + } + } + } + } + + return geometry; +} + export const TilejsonPropType = { type: 'object' as const, value: null as null | TilejsonResult, diff --git a/modules/carto/src/layers/vector-tile-layer.ts b/modules/carto/src/layers/vector-tile-layer.ts index 7b78c23597b..8f25547046a 100644 --- a/modules/carto/src/layers/vector-tile-layer.ts +++ b/modules/carto/src/layers/vector-tile-layer.ts @@ -1,6 +1,7 @@ import {registerLoaders} from '@loaders.gl/core'; +import CartoPropertiesTileLoader from './schema/carto-properties-tile-loader'; import CartoVectorTileLoader from './schema/carto-vector-tile-loader'; -registerLoaders([CartoVectorTileLoader]); +registerLoaders([CartoPropertiesTileLoader, CartoVectorTileLoader]); import {DefaultProps} from '@deck.gl/core'; import {ClipExtension} from '@deck.gl/extensions'; @@ -16,8 +17,9 @@ import {GeoJsonLayer} from '@deck.gl/layers'; import {binaryToGeojson} from '@loaders.gl/gis'; import type {BinaryFeatureCollection} from '@loaders.gl/schema'; import type {Feature} from 'geojson'; + import type {TilejsonResult} from '../sources/types'; -import {injectAccessToken, TilejsonPropType} from './utils'; +import {TilejsonPropType, injectAccessToken, mergeBoundaryData} from './utils'; const defaultProps: DefaultProps = { ...MVTLayer.defaultProps, @@ -69,8 +71,10 @@ export default class VectorTileLayer extends MVTLaye return loadOptions; } - getTileData(tile: TileLoadProps) { - const url = _getURLFromTemplate(this.state.data, tile); + async getTileData(tile: TileLoadProps) { + const tileJSON = this.props.data as TilejsonResult; + const {tiles, properties_tiles} = tileJSON; + const url = _getURLFromTemplate(tiles, tile); if (!url) { return Promise.reject('Invalid URL'); } @@ -78,7 +82,29 @@ export default class VectorTileLayer extends MVTLaye const loadOptions = this.getLoadOptions(); const {fetch} = this.props; const {signal} = tile; - return fetch(url, {propName: 'data', layer: this, loadOptions, signal}); + + // Fetch geometry and attributes separately + const geometryFetch = fetch(url, {propName: 'data', layer: this, loadOptions, signal}); + + if (!properties_tiles) { + return await geometryFetch; + } + + const propertiesUrl = _getURLFromTemplate(properties_tiles, tile); + if (!propertiesUrl) { + return Promise.reject('Invalid properties URL'); + } + + const attributesFetch = fetch(propertiesUrl, { + propName: 'data', + layer: this, + loadOptions, + signal + }); + const [geometry, attributes] = await Promise.all([geometryFetch, attributesFetch]); + if (!geometry) return null; + + return mergeBoundaryData(geometry, attributes); } renderSubLayers( diff --git a/modules/carto/src/sources/boundary-query-source.ts b/modules/carto/src/sources/boundary-query-source.ts new file mode 100644 index 00000000000..d95a9f12b5b --- /dev/null +++ b/modules/carto/src/sources/boundary-query-source.ts @@ -0,0 +1,31 @@ +import {QueryParameters} from '../api'; +import {baseSource} from './base-source'; +import type {SourceOptions, TilejsonResult} from './types'; + +export type BoundaryQuerySourceOptions = SourceOptions & { + boundaryId: string; + matchingColumn?: string; + sqlQuery: string; + queryParameters: QueryParameters; +}; +type UrlParameters = { + boundaryId: string; + matchingColumn: string; + sqlQuery: string; + queryParameters?: string; +}; + +export const boundaryQuerySource = async function ( + options: BoundaryQuerySourceOptions +): Promise { + const {boundaryId, matchingColumn = 'id', sqlQuery, queryParameters} = options; + const urlParameters: UrlParameters = { + boundaryId, + matchingColumn, + sqlQuery + }; + if (queryParameters) { + urlParameters.queryParameters = JSON.stringify(queryParameters); + } + return baseSource('boundary', options, urlParameters) as Promise; +}; diff --git a/modules/carto/src/sources/boundary-table-source.ts b/modules/carto/src/sources/boundary-table-source.ts new file mode 100644 index 00000000000..f65d30b45ec --- /dev/null +++ b/modules/carto/src/sources/boundary-table-source.ts @@ -0,0 +1,31 @@ +import {baseSource} from './base-source'; +import type {SourceOptions, TilejsonResult} from './types'; + +export type BoundaryTableSourceOptions = SourceOptions & { + boundaryId: string; + columns?: string[]; + matchingColumn?: string; + tableName: string; +}; +type UrlParameters = { + boundaryId: string; + columns?: string; + matchingColumn: string; + tableName: string; +}; + +export const boundaryTableSource = async function ( + options: BoundaryTableSourceOptions +): Promise { + const {boundaryId, columns, matchingColumn = 'id', tableName} = options; + const urlParameters: UrlParameters = { + boundaryId, + matchingColumn, + tableName + }; + + if (columns) { + urlParameters.columns = columns.join(','); + } + return baseSource('boundary', options, urlParameters) as Promise; +}; diff --git a/modules/carto/src/sources/boundary-tileset-source.ts b/modules/carto/src/sources/boundary-tileset-source.ts new file mode 100644 index 00000000000..d5c6a93cbb3 --- /dev/null +++ b/modules/carto/src/sources/boundary-tileset-source.ts @@ -0,0 +1,14 @@ +import {baseSource} from './base-source'; +import type {SourceOptions, TilejsonResult} from './types'; + +export type BoundaryTilesetSourceOptions = SourceOptions & {boundaryId: string}; +type UrlParameters = {boundaryId: string}; + +export const boundaryTilesetSource = async function ( + options: BoundaryTilesetSourceOptions +): Promise { + const {boundaryId} = options; + const urlParameters: UrlParameters = {boundaryId}; + + return baseSource('boundary', options, urlParameters) as Promise; +}; diff --git a/modules/carto/src/sources/index.ts b/modules/carto/src/sources/index.ts index 5834a22ccfa..1c05012e393 100644 --- a/modules/carto/src/sources/index.ts +++ b/modules/carto/src/sources/index.ts @@ -1,6 +1,15 @@ export {SOURCE_DEFAULTS} from './base-source'; export type {TilejsonResult, GeojsonResult, JsonResult} from './types'; +export {boundaryQuerySource} from './boundary-query-source'; +export type {BoundaryQuerySourceOptions} from './boundary-query-source'; + +export {boundaryTableSource} from './boundary-table-source'; +export type {BoundaryTableSourceOptions} from './boundary-table-source'; + +export {boundaryTilesetSource} from './boundary-tileset-source'; +export type {BoundaryTilesetSourceOptions} from './boundary-tileset-source'; + export {h3QuerySource} from './h3-query-source'; export type {H3QuerySourceOptions} from './h3-query-source'; diff --git a/test/apps/carto-dynamic-tile/app.tsx b/test/apps/carto-dynamic-tile/app.tsx index b61e70b3672..8ed5057db17 100644 --- a/test/apps/carto-dynamic-tile/app.tsx +++ b/test/apps/carto-dynamic-tile/app.tsx @@ -30,7 +30,9 @@ function Root() { const datasource = datasets[dataset]; let layers: Layer[] = []; - if (dataset.includes('h3')) { + if (dataset.includes('boundary')) { + layers = [useBoundaryLayer(datasource)]; + } else if (dataset.includes('h3')) { layers = [useH3Layer(datasource)]; } else if (dataset.includes('raster')) { layers = [useRasterLayer(datasource)]; @@ -38,6 +40,8 @@ function Root() { layers = [useQuadbinLayer(datasource)]; } else if (dataset.includes('vector')) { layers = [useVectorLayer(datasource)]; + } else { + console.error('Unknown type of dataset', dataset); } return ( @@ -66,6 +70,28 @@ function Root() { ); } +function useBoundaryLayer(datasource) { + const {getFillColor, source, boundaryId, columns, matchingColumn, sqlQuery, tableName} = + datasource; + const tilejson = source({ + ...globalOptions, + boundaryId, + columns, + matchingColumn, + tableName, + sqlQuery + }); + + return new VectorTileLayer({ + id: 'carto', + // @ts-ignore + data: tilejson, // TODO how to correctly specify data type? + pickable: true, + pointRadiusMinPixels: 5, + getFillColor + }); +} + function useH3Layer(datasource) { const {getFillColor, source, aggregationExp, columns, spatialDataColumn, sqlQuery, tableName} = datasource; diff --git a/test/apps/carto-dynamic-tile/datasets.ts b/test/apps/carto-dynamic-tile/datasets.ts index 961b684fbe0..b53ab267585 100644 --- a/test/apps/carto-dynamic-tile/datasets.ts +++ b/test/apps/carto-dynamic-tile/datasets.ts @@ -1,4 +1,7 @@ import { + boundaryQuerySource, + boundaryTableSource, + boundaryTilesetSource, h3TilesetSource, h3TableSource, h3QuerySource, @@ -13,6 +16,40 @@ import { } from '@deck.gl/carto'; export default { + 'boundary-query': { + source: boundaryQuerySource, + boundaryId: 'usa_pos4', + matchingColumn: 'geoid', + sqlQuery: + 'select * from `cartodb-on-gcp-backend-team.juanra.geography_usa_zcta5_2019_clustered`', + getFillColor: colorBins({ + // TODO remove parseFloat, only needed as binary format encodes as strings + attr: d => parseFloat(d!.properties!['do_perimeter']), + domain: [0, 1, 5, 10, 25, 50, 100].map(n => 10000 * n), + colors: 'Peach' + }) + }, + 'boundary-table': { + source: boundaryTableSource, + boundaryId: 'usa_pos4', + matchingColumn: 'geoid', + columns: ['do_label', 'do_perimeter'], + tableName: 'cartodb-on-gcp-backend-team.juanra.geography_usa_zcta5_2019_clustered', + getFillColor: colorBins({ + attr: d => parseFloat(d!.properties!['do_perimeter']), + domain: [0, 1, 5, 10, 25, 50, 100].map(n => 10000 * n), + colors: 'Purp' + }) + }, + 'boundary-tileset': { + source: boundaryTilesetSource, + boundaryId: 'usa_pos4', + getFillColor: colorBins({ + attr: d => parseFloat(d!.properties!['do_perimeter']), + domain: [0, 1, 5, 10, 25, 50, 100].map(n => 10000 * n), + colors: 'Burg' + }) + }, 'h3-query': { source: h3QuerySource, sqlQuery: