[fix] vector tile layer fixes (#2911)

- dataset type change: vectorTile to vector-tile
- vector tile dataset metadata: type to remoteTileFormat
- support for remote examples

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

examples/demo-app/src/actions.js

@@ -36,12 +36,13 @@ export function initApp() {
   };
 }
 
-export function loadRemoteResourceSuccess(response, config, options) {
+export function loadRemoteResourceSuccess(response, config, options, remoteDatasetConfig) {
   return {
     type: LOAD_REMOTE_RESOURCE_SUCCESS,
     response,
     config,
-    options
+    options,
+    remoteDatasetConfig
   };
 }
 
@@ -191,12 +192,17 @@ export function loadSample(options, pushRoute = true) {
 function loadRemoteSampleMap(options) {
   return dispatch => {
     // Load configuration first
-    const {configUrl, dataUrl} = options;
+    const {configUrl, dataUrl, remoteDatasetConfigUrl} = options;
+    const toLoad = [loadRemoteConfig(configUrl)];
+    toLoad.push(dataUrl ? loadRemoteData(dataUrl) : null);
+    // Load remote dataset config for tiled layers
+    toLoad.push(remoteDatasetConfigUrl ? loadRemoteConfig(remoteDatasetConfigUrl) : null);
 
-    Promise.all([loadRemoteConfig(configUrl), loadRemoteData(dataUrl)]).then(
-      ([config, data]) => {
+    Promise.all(toLoad).then(
+      ([config, data, remoteDatasetConfig]) => {
         // TODO: these two actions can be merged
-        dispatch(loadRemoteResourceSuccess(data, config, options));
+        dispatch(loadRemoteResourceSuccess(data, config, options, remoteDatasetConfig));
+        // TODO: toggleModal when async dataset task is done, show the spinner until then
         dispatch(toggleModal(null));
       },
       error => {

examples/demo-app/src/app.tsx

@@ -219,7 +219,7 @@ const App = props => {
               label: 'Railroads',
               id: 'railroads.pmtiles',
               color: [255, 0, 0],
-              type: 'vectorTile'
+              type: 'vector-tile'
             },
             data: {
               rows: [],
@@ -235,7 +235,8 @@ const App = props => {
             metadata: {
               name: 'output.pmtiles',
               description: 'output.pmtiles',
-              type: 'pmtiles',
+              type: 'remote',
+              remoteTileFormat: 'pmtiles',
               tilesetDataUrl:
                 'https://4sq-studio-public.s3.us-west-2.amazonaws.com/pmtiles-test/161727fe-7952-4e57-aa05-850b3086b0b2.pmtiles',
               tilesetMetadataUrl:

examples/demo-app/src/components/map-control/map-control.js

@@ -125,7 +125,9 @@ export function SampleMapPanel(props) {
           <div className="project-links">
             <LinkButton
               label="Data"
-              href={getURL(props.currentSample.dataUrl)}
+              href={getURL(
+                props.currentSample.dataUrl || props.currentSample.remoteDatasetConfigUrl
+              )}
               iconComponent={Icons.Files}
               height="15px"
             />

examples/demo-app/src/reducers/index.js

@@ -93,26 +93,32 @@ export const loadRemoteResourceSuccess = (state, action) => {
   const datasetId = action.options.id || generateHashId(6);
   const {dataUrl} = action.options;
 
-  const {shape} = action.response;
+  const {shape} = dataUrl ? action.response : {};
   let processorMethod = processRowObject;
   let unprocessedData = action.response;
-  if (shape === 'arrow-table') {
-    processorMethod = processArrowTable;
-  } else if (shape === 'object-row-table') {
-    processorMethod = processRowObject;
-    unprocessedData = action.response.data;
-  } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
-    processorMethod = processGeojson;
-  } else {
-    throw new Error('Failed to select data processor');
+
+  if (dataUrl) {
+    if (shape === 'arrow-table') {
+      processorMethod = processArrowTable;
+    } else if (shape === 'object-row-table') {
+      processorMethod = processRowObject;
+      unprocessedData = action.response.data;
+    } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
+      processorMethod = processGeojson;
+    } else {
+      throw new Error('Failed to select data processor');
+    }
   }
 
-  const datasets = {
-    info: {
-      id: datasetId
-    },
-    data: processorMethod(unprocessedData)
-  };
+  const datasets = dataUrl
+    ? {
+        info: {
+          id: datasetId
+        },
+        data: processorMethod(unprocessedData)
+      }
+    : // remote datasets like vector tile datasets
+      action.remoteDatasetConfig;
 
   const config = action.config ? KeplerGlSchema.parseSavedConfig(action.config) : null;
 

src/components/src/hooks/use-fetch-vector-tile-metadata.ts

@@ -6,12 +6,12 @@ import {useCallback, useEffect, useState} from 'react';
 import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
 import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
 
-import {VectorTileType} from '@kepler.gl/constants';
+import {RemoteTileFormat} from '@kepler.gl/constants';
 import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';
 
 type FetchVectorTileMetadataProps = {
   url: string | null;
-  type: VectorTileType;
+  remoteTileFormat: RemoteTileFormat;
   process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;
 };
 
@@ -35,7 +35,7 @@ type FetchVectorTileMetadataReturn = {
 
 /** Hook to fetch and return mvt or pmtiles metadata. */
 export default function useFetchVectorTileMetadata({
-  type,
+  remoteTileFormat,
   url,
   process = DEFAULT_PROCESS_FUNCTION
 }: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {
@@ -67,7 +67,7 @@ export default function useFetchVectorTileMetadata({
 
         try {
           let metadata: PMTilesMetadata | TileJSON | null = null;
-          if (type === VectorTileType.MVT) {
+          if (remoteTileFormat === RemoteTileFormat.MVT) {
             metadata = await getMVTMetadata(url);
 
             // MVTSource returns messy partial metadata
@@ -94,7 +94,7 @@ export default function useFetchVectorTileMetadata({
     };
 
     getAndProcessMetadata();
-  }, [url, type, setProcessedData]);
+  }, [url, remoteTileFormat, setProcessedData]);
 
   return {data, loading, error};
 }

src/components/src/modals/tilesets-modals/tileset-vector-form.tsx

@@ -4,10 +4,16 @@
 import React, {useCallback, useEffect, useState, useMemo} from 'react';
 import styled from 'styled-components';
 
-import {DatasetType, VectorTileType} from '@kepler.gl/constants';
-import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';
+import {
+  DatasetType,
+  RemoteTileFormat,
+  VectorTileDatasetMetadata,
+  REMOTE_TILE
+} from '@kepler.gl/constants';
 import {TileJSON} from '@loaders.gl/mvt';
 import {PMTilesMetadata} from '@loaders.gl/pmtiles';
+import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';
+import {Merge} from '@kepler.gl/types';
 
 import {default as useFetchVectorTileMetadata} from '../../hooks/use-fetch-vector-tile-metadata';
 import {DatasetCreationAttributes, MetaResponse} from './common';
@@ -34,16 +40,24 @@ export type VectorTilesetFormData = {
 
 const isPMTilesUrl = (url?: string | null) => url?.includes('.pmtiles');
 
+export type VectorTileDatasetCreationAttributes = Merge<
+  DatasetCreationAttributes,
+  {
+    metadata: VectorTileDatasetMetadata;
+  }
+>;
+
 export function getDatasetAttributesFromVectorTile({
   name,
   dataUrl,
   metadataUrl
-}: VectorTilesetFormData): DatasetCreationAttributes {
+}: VectorTilesetFormData): VectorTileDatasetCreationAttributes {
   return {
     name,
     type: DatasetType.VECTOR_TILE,
     metadata: {
-      type: isPMTilesUrl(dataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,
+      type: REMOTE_TILE,
+      remoteTileFormat: isPMTilesUrl(dataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,
       tilesetDataUrl: dataUrl,
       tilesetMetadataUrl: metadataUrl
     }
@@ -114,7 +128,7 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {
     error: metaError
   } = useFetchVectorTileMetadata({
     url: metadataUrl,
-    type: isPMTilesUrl(metadataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,
+    remoteTileFormat: isPMTilesUrl(metadataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,
     process
   });
 

src/constants/src/dataset.ts

@@ -3,20 +3,19 @@
 
 export enum DatasetType {
   LOCAL = 'local',
-  VECTOR_TILE = 'vectorTile'
+  VECTOR_TILE = 'vector-tile'
 }
 
-export enum TileType {
-  VECTOR_TILE = 'vectorTile'
-}
-
-export enum VectorTileType {
+export enum RemoteTileFormat {
   MVT = 'mvt',
   PMTILES = 'pmtiles'
 }
 
+export const REMOTE_TILE = 'remote';
+
 export type VectorTileDatasetMetadata = {
-  type: VectorTileType;
+  type: typeof REMOTE_TILE;
+  remoteTileFormat: RemoteTileFormat;
   tilesetDataUrl: string;
   tilesetMetadataUrl?: string;
 };

src/layers/src/index.ts

@@ -40,7 +40,7 @@ import {default as S2GeometryLayer} from './s2-geometry-layer/s2-geometry-layer'
 export {defaultElevation as s2DefaultElevation} from './s2-geometry-layer/s2-geometry-layer';
 export {getS2Center} from './s2-geometry-layer/s2-utils';
 export {default as AggregationLayer} from './aggregation-layer';
-//import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
+// import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
 
 export {default as VectorTileIcon} from './vector-tile/vector-tile-icon';
 export {default as VectorTileLayer} from './vector-tile/vector-tile-layer';

src/layers/src/vector-tile/vector-tile-layer.ts

@@ -13,8 +13,8 @@ import GL from '@luma.gl/constants';
 import {notNullorUndefined} from '@kepler.gl/common-utils';
 import {
   DatasetType,
-  TileType,
-  VectorTileType,
+  LAYER_TYPES,
+  RemoteTileFormat,
   VectorTileDatasetMetadata,
   SCALE_TYPES,
   CHANNEL_SCALES,
@@ -210,7 +210,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
   }
 
   get type() {
-    return TileType.VECTOR_TILE;
+    return LAYER_TYPES.vectorTile;
   }
 
   get name(): string {
@@ -406,7 +406,8 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
 
     if (dataset?.type === DatasetType.VECTOR_TILE) {
       const datasetMetadata = dataset.metadata as VectorTileMetadata & VectorTileDatasetMetadata;
-      if (datasetMetadata?.type === VectorTileType.MVT) {
+      const remoteTileFormat = datasetMetadata?.remoteTileFormat;
+      if (remoteTileFormat === RemoteTileFormat.MVT) {
         const transformFetch = async (input: RequestInfo | URL, init?: RequestInit | undefined) => {
           const requestData: RequestParameters = {
             url: input as string,
@@ -428,7 +429,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
               }
             })
           : null;
-      } else if (datasetMetadata?.type === VectorTileType.PMTILES) {
+      } else if (remoteTileFormat === RemoteTileFormat.PMTILES) {
         // TODO: to render image pmtiles need to use TileLayer and BitmapLayer (https://github.com/visgl/loaders.gl/blob/master/examples/website/tiles/components/tile-source-layer.ts)
         tilesetDataUrl = datasetMetadata?.tilesetDataUrl;
         tileSource = tilesetDataUrl ? PMTilesSource.createDataSource(tilesetDataUrl, {}) : null;

src/table/src/dataset-utils.ts

@@ -6,7 +6,7 @@ import KeplerTable, {Datasets} from './kepler-table';
 import {ProtoDataset, RGBColor} from '@kepler.gl/types';
 import Task from 'react-palm/tasks';
 
-import {DatasetType, VectorTileType} from '@kepler.gl/constants';
+import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
 import {
   hexToRgb,
   validateInputData,
@@ -95,12 +95,16 @@ type CreateTableProps = {
   data: any;
 };
 
-async function createTable(dataasetInfo: CreateTableProps) {
-  const {info, color, opts, data} = dataasetInfo;
+async function createTable(datasetInfo: CreateTableProps) {
+  const {info, color, opts, data} = datasetInfo;
 
   // update metadata for remote tiled datasets
-  const refreshedMetadata = await refreshMetadata(dataasetInfo);
-  const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;
+  const refreshedMetadata = await refreshRemoteData(datasetInfo);
+  let metadata = opts.metadata;
+  if (refreshedMetadata) {
+    metadata = {...opts.metadata, ...refreshedMetadata};
+    data.fields = metadata?.fields;
+  }
 
   const TableClass = getApplicationConfig().table ?? KeplerTable;
   const table = new TableClass({
@@ -121,24 +125,25 @@ const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');
  * @param datasetInfo
  * @returns
  */
-async function refreshMetadata(datasetInfo: CreateTableProps) {
+async function refreshRemoteData(datasetInfo: CreateTableProps) {
   // so far only vector tile layers should refresh metadata
   if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
     return null;
   }
 
-  const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata || {};
+  const {remoteTileFormat, tilesetMetadataUrl} =
+    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};
 
   if (
-    !(type === VectorTileType.PMTILES || type === VectorTileType.MVT) ||
+    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
     typeof tilesetMetadataUrl !== 'string'
   ) {
     return null;
   }
 
   try {
     let rawMetadata: PMTilesMetadata | TileJSON | null = null;
-    if (type === VectorTileType.MVT) {
+    if (remoteTileFormat === RemoteTileFormat.MVT) {
       rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
     } else {
       const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});