[fix] arrow tables - save timestamps as iso date string (#2953)

- convert numerical timestamps from arrow tables into iso date strings before saving.

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

src/schemas/package.json

@@ -40,6 +40,7 @@
     "@types/keymirror": "^0.1.1",
     "@types/lodash.clonedeep": "^4.5.7",
     "@types/lodash.pick": "^4.4.6",
+    "apache-arrow": ">=15.0.0",
     "global": "^4.3.0",
     "keymirror": "^0.1.1",
     "lodash.clonedeep": "^4.0.1",

src/schemas/src/dataset-schema.ts

@@ -3,13 +3,15 @@
 
 import pick from 'lodash.pick';
 import {console as globalConsole} from 'global/window';
+import {Type as ArrowTypes} from 'apache-arrow';
 
 import {DATASET_FORMATS} from '@kepler.gl/constants';
 import {ProtoDataset, RGBColor, JsonObject} from '@kepler.gl/types';
 import {KeplerTable} from '@kepler.gl/table';
 import {VERSIONS} from './versions';
 import Schema from './schema';
 import {getFieldsFromData, getSampleForTypeAnalyze} from '@kepler.gl/common-utils';
+import {ArrowDataContainer, DataContainerInterface} from '@kepler.gl/utils';
 
 export type SavedField = {
   name: string;
@@ -100,14 +102,45 @@ export const propertiesV1 = {
   disableDataOperation: null
 };
 
+/**
+ * TODO Consider moving this cast to ArrowDataContainer?
+ * Prepare a data container for export as part of json / html.
+ * 1) Arrow tables can store Timestamps as BigInts, so convert numbers to ISOStrings compatible with Kepler.gl's TIMESTAMP.
+ * @param dataContainer A data container to flatten.
+ * @returns Row based data.
+ */
+const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] => {
+  const allData = dataContainer.flattenData();
+
+  if (dataContainer instanceof ArrowDataContainer) {
+    const numColumns = dataContainer.numColumns();
+
+    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+      const column = dataContainer.getColumn(columnIndex);
+      const typeId = column.type?.typeId;
+      if (
+        typeId === ArrowTypes.Timestamp ||
+        typeId === ArrowTypes.Date ||
+        typeId === ArrowTypes.Time
+      ) {
+        allData.forEach(row => {
+          row[columnIndex] = new Date(row[columnIndex]).toISOString();
+        });
+      }
+    }
+  }
+
+  return allData;
+};
+
 export class DatasetSchema extends Schema {
   key = 'dataset';
 
   save(dataset: KeplerTable): SavedDatasetV1['data'] {
     const datasetFlattened = dataset.dataContainer
       ? {
           ...dataset,
-          allData: dataset.dataContainer.flattenData(),
+          allData: getAllDataForSaving(dataset.dataContainer),
           // we use flattenData to save arrow tables,
           // but once flattened it's not an arrow file anymore.
           metadata: {

yarn.lock

@@ -3605,6 +3605,7 @@ __metadata:
     "@types/keymirror": "npm:^0.1.1"
     "@types/lodash.clonedeep": "npm:^4.5.7"
     "@types/lodash.pick": "npm:^4.4.6"
+    apache-arrow: "npm:>=15.0.0"
     global: "npm:^4.3.0"
     keymirror: "npm:^0.1.1"
     lodash.clonedeep: "npm:^4.0.1"