preparing datatable for unifying with kibana_datatable (#73654)

src/plugins/expressions/common/expression_types/specs/boolean.ts

@@ -41,7 +41,8 @@ export const boolean: ExpressionTypeDefinition<'boolean', boolean> = {
     },
     datatable: (value): Datatable => ({
       type: 'datatable',
-      columns: [{ name: 'value', type: name }],
+      meta: {},
+      columns: [{ id: 'value', name: 'value', meta: { type: name } }],
       rows: [{ value }],
     }),
   },

src/plugins/expressions/common/expression_types/specs/datatable.ts

@@ -23,6 +23,13 @@ import { ExpressionTypeDefinition } from '../types';
 import { PointSeries, PointSeriesColumn } from './pointseries';
 import { ExpressionValueRender } from './render';
 
+type State = string | number | boolean | null | undefined | SerializableState;
+
+/** @internal **/
+export interface SerializableState {
+  [key: string]: State | State[];
+}
+
 const name = 'datatable';
 
 /**
@@ -42,19 +49,31 @@ export type DatatableColumnType = 'string' | 'number' | 'boolean' | 'date' | 'nu
  */
 export type DatatableRow = Record<string, any>;
 
+export interface DatatableColumnMeta {
+  type: DatatableColumnType;
+  field?: string;
+  params?: SerializableState;
+}
 /**
  * This type represents the shape of a column in a `Datatable`.
  */
 export interface DatatableColumn {
+  id: string;
   name: string;
-  type: DatatableColumnType;
+  meta: DatatableColumnMeta;
+}
+
+export interface DatatableMeta {
+  type?: string;
+  source?: string;
 }
 
 /**
  * A `Datatable` in Canvas is a unique structure that represents tabulated data.
  */
 export interface Datatable {
   type: typeof name;
+  meta?: DatatableMeta;
   columns: DatatableColumn[];
   rows: DatatableRow[];
 }
@@ -103,14 +122,16 @@ export const datatable: ExpressionTypeDefinition<typeof name, Datatable, Seriali
   from: {
     null: () => ({
       type: name,
+      meta: {},
       rows: [],
       columns: [],
     }),
     pointseries: (value: PointSeries) => ({
       type: name,
+      meta: {},
       rows: value.rows,
       columns: map(value.columns, (val: PointSeriesColumn, colName) => {
-        return { name: colName, type: val.type };
+        return { id: colName, name: colName, meta: { type: val.type } };
       }),
     }),
   },
@@ -127,13 +148,13 @@ export const datatable: ExpressionTypeDefinition<typeof name, Datatable, Seriali
     }),
     pointseries: (table: Datatable): PointSeries => {
       const validFields = ['x', 'y', 'color', 'size', 'text'];
-      const columns = table.columns.filter((column) => validFields.includes(column.name));
+      const columns = table.columns.filter((column) => validFields.includes(column.id));
       const rows = table.rows.map((row) => pick(row, validFields));
       return {
         type: 'pointseries',
         columns: columns.reduce<Record<string, PointSeries['columns']>>((acc, column) => {
           acc[column.name] = {
-            type: column.type,
+            type: column.meta.type,
             expression: column.name,
             role: 'dimension',
           };

src/plugins/expressions/common/expression_types/specs/num.ts

@@ -73,7 +73,8 @@ export const num: ExpressionTypeDefinition<'num', ExpressionValueNum> = {
     },
     datatable: ({ value }): Datatable => ({
       type: 'datatable',
-      columns: [{ name: 'value', type: 'number' }],
+      meta: {},
+      columns: [{ id: 'value', name: 'value', meta: { type: 'number' } }],
       rows: [{ value }],
     }),
   },

src/plugins/expressions/common/expression_types/specs/number.ts

@@ -55,7 +55,8 @@ export const number: ExpressionTypeDefinition<typeof name, number> = {
     },
     datatable: (value): Datatable => ({
       type: 'datatable',
-      columns: [{ name: 'value', type: 'number' }],
+      meta: {},
+      columns: [{ id: 'value', name: 'value', meta: { type: 'number' } }],
       rows: [{ value }],
     }),
   },

src/plugins/expressions/common/expression_types/specs/string.ts

@@ -40,7 +40,8 @@ export const string: ExpressionTypeDefinition<typeof name, string> = {
     },
     datatable: (value): Datatable => ({
       type: 'datatable',
-      columns: [{ name: 'value', type: 'string' }],
+      meta: {},
+      columns: [{ id: 'value', name: 'value', meta: { type: 'string' } }],
       rows: [{ value }],
     }),
   },

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/location.ts

@@ -10,7 +10,10 @@ import { getFunctionHelp } from '../../../i18n';
 const noop = () => {};
 
 interface Return extends Datatable {
-  columns: [{ name: 'latitude'; type: 'number' }, { name: 'longitude'; type: 'number' }];
+  columns: [
+    { id: 'latitude'; name: 'latitude'; meta: { type: 'number' } },
+    { id: 'longitude'; name: 'longitude'; meta: { type: 'number' } }
+  ];
   rows: [{ latitude: number; longitude: number }];
 }
 
@@ -30,8 +33,8 @@ export function location(): ExpressionFunctionDefinition<'location', null, {}, P
           return resolve({
             type: 'datatable',
             columns: [
-              { name: 'latitude', type: 'number' },
-              { name: 'longitude', type: 'number' },
+              { id: 'latitude', name: 'latitude', meta: { type: 'number' } },
+              { id: 'longitude', name: 'longitude', meta: { type: 'number' } },
             ],
             rows: [{ latitude, longitude }],
           });

x-pack/plugins/canvas/canvas_plugin_src/functions/common/__tests__/fixtures/test_tables.ts

@@ -16,24 +16,29 @@ const testTable: Datatable = {
   type: 'datatable',
   columns: [
     {
+      id: 'name',
       name: 'name',
-      type: 'string',
+      meta: { type: 'string' },
     },
     {
+      id: 'time',
       name: 'time',
-      type: 'date',
+      meta: { type: 'date' },
     },
     {
+      id: 'price',
       name: 'price',
-      type: 'number',
+      meta: { type: 'number' },
     },
     {
+      id: 'quantity',
       name: 'quantity',
-      type: 'number',
+      meta: { type: 'number' },
     },
     {
+      id: 'in_stock',
       name: 'in_stock',
-      type: 'boolean',
+      meta: { type: 'boolean' },
     },
   ],
   rows: [
@@ -107,24 +112,29 @@ const stringTable: Datatable = {
   type: 'datatable',
   columns: [
     {
+      id: 'name',
       name: 'name',
-      type: 'string',
+      meta: { type: 'string' },
     },
     {
+      id: 'time',
       name: 'time',
-      type: 'string',
+      meta: { type: 'string' },
     },
     {
+      id: 'price',
       name: 'price',
-      type: 'string',
+      meta: { type: 'string' },
     },
     {
+      id: 'quantity',
       name: 'quantity',
-      type: 'string',
+      meta: { type: 'string' },
     },
     {
+      id: 'in_stock',
       name: 'in_stock',
-      type: 'string',
+      meta: { type: 'string' },
     },
   ],
   rows: [

x-pack/plugins/canvas/canvas_plugin_src/functions/common/alterColumn.test.js

@@ -38,7 +38,7 @@ describe('alterColumn', () => {
         const arbitraryRowIndex = 6;
 
         expect(newColumn.name).not.toBe(originalColumn.name);
-        expect(newColumn.type).not.toBe(originalColumn.type);
+        expect(newColumn.meta.type).not.toBe(originalColumn.meta.type);
         expect(typeof dateToString.rows[arbitraryRowIndex].timeISO).toBe('string');
         expect(new Date(dateToString.rows[arbitraryRowIndex].timeISO)).toEqual(
           new Date(testTable.rows[arbitraryRowIndex].time)
@@ -60,7 +60,7 @@ describe('alterColumn', () => {
       it('converts the column to the specified type', () => {
         const dateToString = fn(testTable, { column: 'time', type: 'string', name: 'timeISO' });
 
-        expect(typeof dateToString.columns[timeColumnIndex].type).toBe('string');
+        expect(typeof dateToString.columns[timeColumnIndex].meta.type).toBe('string');
         expect(typeof dateToString.rows[timeColumnIndex].timeISO).toBe('string');
         expect(new Date(dateToString.rows[timeColumnIndex].timeISO)).toEqual(
           new Date(testTable.rows[timeColumnIndex].time)
@@ -69,10 +69,10 @@ describe('alterColumn', () => {
 
       it('does not change column if type is not specified', () => {
         const unconvertedColumn = fn(testTable, { column: 'price', name: 'foo' });
-        const originalType = testTable.columns[priceColumnIndex].type;
+        const originalType = testTable.columns[priceColumnIndex].meta.type;
         const arbitraryRowIndex = 2;
 
-        expect(unconvertedColumn.columns[priceColumnIndex].type).toBe(originalType);
+        expect(unconvertedColumn.columns[priceColumnIndex].meta.type).toBe(originalType);
         expect(typeof unconvertedColumn.rows[arbitraryRowIndex].foo).toBe(originalType);
       });
 
@@ -99,7 +99,7 @@ describe('alterColumn', () => {
         const arbitraryRowIndex = 5;
 
         expect(newColumn.name).not.toBe(originalColumn.name);
-        expect(newColumn.type).not.toBe(originalColumn.type);
+        expect(newColumn.meta.type).not.toBe(originalColumn.meta.type);
         expect(typeof overwriteName.rows[arbitraryRowIndex].name).toBe('string');
         expect(new Date(overwriteName.rows[arbitraryRowIndex].name)).toEqual(
           new Date(testTable.rows[arbitraryRowIndex].time)
@@ -122,7 +122,7 @@ describe('alterColumn', () => {
       const numberToString = fn(testTable, { column: 'price', type: 'string' });
 
       expect(numberToString.columns[priceColumnIndex]).toHaveProperty('name', 'price');
-      expect(numberToString.columns[priceColumnIndex]).toHaveProperty('type', 'string');
+      expect(numberToString.columns[priceColumnIndex].meta).toHaveProperty('type', 'string');
 
       expect(typeof numberToString.rows[arbitraryRowIndex].price).toBe('string');
       expect(numberToString.rows[arbitraryRowIndex].price).toBe(
@@ -132,7 +132,7 @@ describe('alterColumn', () => {
       const stringToNumber = fn(numberToString, { column: 'price', type: 'number' });
 
       expect(stringToNumber.columns[priceColumnIndex]).toHaveProperty('name', 'price');
-      expect(stringToNumber.columns[priceColumnIndex]).toHaveProperty('type', 'number');
+      expect(stringToNumber.columns[priceColumnIndex].meta).toHaveProperty('type', 'number');
 
       expect(typeof stringToNumber.rows[arbitraryRowIndex].price).toBe('number');
 
@@ -146,7 +146,7 @@ describe('alterColumn', () => {
       const dateToString = fn(testTable, { column: 'time', type: 'string' });
 
       expect(dateToString.columns[timeColumnIndex]).toHaveProperty('name', 'time');
-      expect(dateToString.columns[timeColumnIndex]).toHaveProperty('type', 'string');
+      expect(dateToString.columns[timeColumnIndex].meta).toHaveProperty('type', 'string');
 
       expect(typeof dateToString.rows[arbitraryRowIndex].time).toBe('string');
       expect(new Date(dateToString.rows[arbitraryRowIndex].time)).toEqual(
@@ -156,7 +156,7 @@ describe('alterColumn', () => {
       const stringToDate = fn(dateToString, { column: 'time', type: 'date' });
 
       expect(stringToDate.columns[timeColumnIndex]).toHaveProperty('name', 'time');
-      expect(stringToDate.columns[timeColumnIndex]).toHaveProperty('type', 'date');
+      expect(stringToDate.columns[timeColumnIndex].meta).toHaveProperty('type', 'date');
       expect(new Date(stringToDate.rows[timeColumnIndex].time)).toBeInstanceOf(Date);
 
       expect(new Date(stringToDate.rows[timeColumnIndex].time)).toEqual(
@@ -169,7 +169,7 @@ describe('alterColumn', () => {
       const arbitraryRowIndex = 1;
 
       expect(dateToNumber.columns[timeColumnIndex]).toHaveProperty('name', 'time');
-      expect(dateToNumber.columns[timeColumnIndex]).toHaveProperty('type', 'number');
+      expect(dateToNumber.columns[timeColumnIndex].meta).toHaveProperty('type', 'number');
 
       expect(typeof dateToNumber.rows[arbitraryRowIndex].time).toBe('number');
       expect(dateToNumber.rows[arbitraryRowIndex].time).toEqual(
@@ -179,7 +179,7 @@ describe('alterColumn', () => {
       const numberToDate = fn(dateToNumber, { column: 'time', type: 'date' });
 
       expect(numberToDate.columns[timeColumnIndex]).toHaveProperty('name', 'time');
-      expect(numberToDate.columns[timeColumnIndex]).toHaveProperty('type', 'date');
+      expect(numberToDate.columns[timeColumnIndex].meta).toHaveProperty('type', 'date');
 
       expect(new Date(numberToDate.rows[arbitraryRowIndex].time)).toBeInstanceOf(Date);
       expect(new Date(numberToDate.rows[arbitraryRowIndex].time)).toEqual(
@@ -192,7 +192,7 @@ describe('alterColumn', () => {
       const arbitraryRowIndex = 7;
 
       expect(booleanToNumber.columns[inStockColumnIndex]).toHaveProperty('name', 'in_stock');
-      expect(booleanToNumber.columns[inStockColumnIndex]).toHaveProperty('type', 'number');
+      expect(booleanToNumber.columns[inStockColumnIndex].meta).toHaveProperty('type', 'number');
 
       expect(typeof booleanToNumber.rows[arbitraryRowIndex].in_stock).toBe('number');
       expect(booleanToNumber.rows[arbitraryRowIndex].in_stock).toEqual(
@@ -202,7 +202,7 @@ describe('alterColumn', () => {
       const numberToBoolean = fn(booleanToNumber, { column: 'in_stock', type: 'boolean' });
 
       expect(numberToBoolean.columns[inStockColumnIndex]).toHaveProperty('name', 'in_stock');
-      expect(numberToBoolean.columns[inStockColumnIndex]).toHaveProperty('type', 'boolean');
+      expect(numberToBoolean.columns[inStockColumnIndex].meta).toHaveProperty('type', 'boolean');
 
       expect(typeof numberToBoolean.rows[arbitraryRowIndex].in_stock).toBe('boolean');
       expect(numberToBoolean.rows[arbitraryRowIndex].in_stock).toEqual(
@@ -216,7 +216,7 @@ describe('alterColumn', () => {
 
       expect(stringToNull.columns[nameColumnIndex]).toHaveProperty('name', 'name');
 
-      expect(stringToNull.columns[nameColumnIndex]).toHaveProperty('type', 'null');
+      expect(stringToNull.columns[nameColumnIndex].meta).toHaveProperty('type', 'null');
 
       expect(stringToNull.rows[arbitraryRowIndex].name).toBe(null);
     });

x-pack/plugins/canvas/canvas_plugin_src/functions/common/alterColumn.ts

@@ -57,14 +57,14 @@ export function alterColumn(): ExpressionFunctionDefinition<
       }
 
       const name = args.name || column.name;
-      const type = args.type || column.type;
+      const type = args.type || column.meta.type;
 
       const columns = input.columns.reduce((all: DatatableColumn[], col) => {
         if (col.name !== args.name) {
           if (col.name !== column.name) {
             all.push(col);
           } else {
-            all.push({ name, type });
+            all.push({ id: name, name, meta: { type } });
           }
         }
         return all;
@@ -76,7 +76,7 @@ export function alterColumn(): ExpressionFunctionDefinition<
         handler = (function getHandler() {
           switch (type) {
             case 'string':
-              if (column.type === 'date') {
+              if (column.meta.type === 'date') {
                 return (v: string) => new Date(v).toISOString();
               }
               return String;

x-pack/plugins/canvas/canvas_plugin_src/functions/common/as.test.js

@@ -13,19 +13,19 @@ describe('as', () => {
   it('returns a datatable with a single column and single row', () => {
     expect(fn('foo', { name: 'bar' })).toEqual({
       type: 'datatable',
-      columns: [{ name: 'bar', type: 'string' }],
+      columns: [{ id: 'bar', name: 'bar', meta: { type: 'string' } }],
       rows: [{ bar: 'foo' }],
     });
 
     expect(fn(2, { name: 'num' })).toEqual({
       type: 'datatable',
-      columns: [{ name: 'num', type: 'number' }],
+      columns: [{ id: 'num', name: 'num', meta: { type: 'number' } }],
       rows: [{ num: 2 }],
     });
 
     expect(fn(true, { name: 'bool' })).toEqual({
       type: 'datatable',
-      columns: [{ name: 'bool', type: 'boolean' }],
+      columns: [{ id: 'bool', name: 'bool', meta: { type: 'boolean' } }],
       rows: [{ bool: true }],
     });
   });

x-pack/plugins/canvas/canvas_plugin_src/functions/common/as.ts

@@ -34,8 +34,9 @@ export function asFn(): ExpressionFunctionDefinition<'as', Input, Arguments, Dat
         type: 'datatable',
         columns: [
           {
+            id: args.name,
             name: args.name,
-            type: getType(input),
+            meta: { type: getType(input) },
           },
         ],
         rows: [