Create switch to render line charts using vega-lite (#3106)

* initial commit for vega migration

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* Cleanup vega implementation

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* fix minor if statement

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* remove log

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* minor enhancements for robustness

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* Update fixes and more robust

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* fix yarn.lock conflicts

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* Clean up code

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* add additional comments

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* add unit tests

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* add fixes and update tests

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

* fix test name

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

---------

Signed-off-by: Ashish Agrawal <ashisagr@amazon.com>

src/plugins/vis_type_vega/public/components/vega_vis_editor.tsx

@@ -37,7 +37,7 @@ import { i18n } from '@osd/i18n';
 
 import { VisOptionsProps } from 'src/plugins/vis_default_editor/public';
 import { getNotifications } from '../services';
-import { VisParams } from '../vega_fn';
+import { VisParams } from '../expressions/vega_fn';
 import { VegaHelpMenu } from './vega_help_menu';
 import { VegaActionsMenu } from './vega_actions_menu';
 

src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.ts

@@ -0,0 +1,299 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { cloneDeep } from 'lodash';
+import { i18n } from '@osd/i18n';
+import {
+  ExpressionFunctionDefinition,
+  OpenSearchDashboardsDatatable,
+  OpenSearchDashboardsDatatableColumn,
+} from '../../../expressions/public';
+import { VegaVisualizationDependencies } from '../plugin';
+import { VislibDimensions, VisParams } from '../../../visualizations/public';
+
+type Input = OpenSearchDashboardsDatatable;
+type Output = Promise<string>;
+
+interface Arguments {
+  visLayers: string | null;
+  visParams: string;
+  dimensions: string;
+}
+
+export type LineVegaSpecExpressionFunctionDefinition = ExpressionFunctionDefinition<
+  'line_vega_spec',
+  Input,
+  Arguments,
+  Output
+>;
+
+// TODO: move this to the visualization plugin that has VisParams once all of these parameters have been better defined
+interface ValueAxis {
+  id: string;
+  labels: {
+    filter: boolean;
+    rotate: number;
+    show: boolean;
+    truncate: number;
+  };
+  name: string;
+  position: string;
+  scale: {
+    mode: string;
+    type: string;
+  };
+  show: true;
+  style: any;
+  title: {
+    text: string;
+  };
+  type: string;
+}
+
+// Get the first xaxis field as only 1 setup of X Axis will be supported and
+// there won't be support for split series and split chart
+const getXAxisId = (dimensions: any, columns: OpenSearchDashboardsDatatableColumn[]): string => {
+  return columns.filter((column) => column.name === dimensions.x.label)[0].id;
+};
+
+export const cleanString = (rawString: string): string => {
+  return rawString.replaceAll('"', '');
+};
+
+export const formatDataTable = (
+  datatable: OpenSearchDashboardsDatatable
+): OpenSearchDashboardsDatatable => {
+  datatable.columns.forEach((column) => {
+    // clean quotation marks from names in columns
+    column.name = cleanString(column.name);
+  });
+  return datatable;
+};
+
+export const setupConfig = (visParams: VisParams) => {
+  const legendPosition = visParams.legendPosition;
+  return {
+    view: {
+      stroke: null,
+    },
+    concat: {
+      spacing: 0,
+    },
+    legend: {
+      orient: legendPosition,
+    },
+  };
+};
+
+export const buildLayerMark = (seriesParams: {
+  type: string;
+  interpolate: string;
+  lineWidth: number;
+  showCircles: boolean;
+}) => {
+  return {
+    // Possible types are: line, area, histogram. The eligibility checker will
+    // prevent area and histogram (though area works in vega-lite)
+    type: seriesParams.type,
+    // Possible types: linear, cardinal, step-after. All of these types work in vega-lite
+    interpolate: seriesParams.interpolate,
+    // The possible values is any number, which matches what vega-lite supports
+    strokeWidth: seriesParams.lineWidth,
+    // this corresponds to showing the dots in the visbuilder for each data point
+    point: seriesParams.showCircles,
+  };
+};
+
+export const buildXAxis = (
+  xAxisTitle: string,
+  xAxisId: string,
+  startTime: number,
+  endTime: number,
+  visParams: VisParams
+) => {
+  return {
+    axis: {
+      title: xAxisTitle,
+      grid: visParams.grid.categoryLines,
+    },
+    field: xAxisId,
+    // Right now, the line charts can only set the x-axis value to be a date attribute, so
+    // this should always be of type temporal
+    type: 'temporal',
+    scale: {
+      domain: [startTime, endTime],
+    },
+  };
+};
+
+export const buildYAxis = (
+  column: OpenSearchDashboardsDatatableColumn,
+  valueAxis: ValueAxis,
+  visParams: VisParams
+) => {
+  return {
+    axis: {
+      title: cleanString(valueAxis.title.text) || column.name,
+      grid: visParams.grid.valueAxis,
+      orient: valueAxis.position,
+      labels: valueAxis.labels.show,
+      labelAngle: valueAxis.labels.rotate,
+    },
+    field: column.id,
+    type: 'quantitative',
+  };
+};
+
+export const createSpecFromDatatable = (
+  datatable: OpenSearchDashboardsDatatable,
+  visParams: VisParams,
+  dimensions: VislibDimensions
+): object => {
+  // TODO: we can try to use VegaSpec type but it is currently very outdated, where many
+  // of the fields and sub-fields don't have other optional params that we want for customizing.
+  // For now, we make this more loosely-typed by just specifying it as a generic object.
+  const spec = {} as any;
+
+  spec.$schema = 'https://vega.github.io/schema/vega-lite/v5.json';
+  spec.data = {
+    values: datatable.rows,
+  };
+  spec.config = setupConfig(visParams);
+
+  // Get the valueAxes data and generate a map to easily fetch the different valueAxes data
+  const valueAxis = new Map();
+  visParams?.valueAxes?.forEach((yAxis: ValueAxis) => {
+    valueAxis.set(yAxis.id, yAxis);
+  });
+
+  spec.layer = [] as any[];
+
+  if (datatable.rows.length > 0 && dimensions.x !== null) {
+    const xAxisId = getXAxisId(dimensions, datatable.columns);
+    const xAxisTitle = cleanString(dimensions.x.label);
+    // get x-axis bounds for the chart
+    const startTime = new Date(dimensions.x.params.bounds.min).valueOf();
+    const endTime = new Date(dimensions.x.params.bounds.max).valueOf();
+    let skip = 0;
+    datatable.columns.forEach((column, index) => {
+      // Check if it's not xAxis column data
+      if (column.meta?.aggConfigParams?.interval !== undefined) {
+        skip++;
+      } else {
+        const currentSeriesParams = visParams.seriesParams[index - skip];
+        const currentValueAxis = valueAxis.get(currentSeriesParams.valueAxis.toString());
+        let tooltip: Array<{ field: string; type: string; title: string }> = [];
+        if (visParams.addTooltip) {
+          tooltip = [
+            { field: xAxisId, type: 'temporal', title: xAxisTitle },
+            { field: column.id, type: 'quantitative', title: column.name },
+          ];
+        }
+        spec.layer.push({
+          mark: buildLayerMark(currentSeriesParams),
+          encoding: {
+            x: buildXAxis(xAxisTitle, xAxisId, startTime, endTime, visParams),
+            y: buildYAxis(column, currentValueAxis, visParams),
+            tooltip,
+            color: {
+              // This ensures all the different metrics have their own distinct and unique color
+              datum: column.name,
+            },
+          },
+        });
+      }
+    });
+  }
+
+  if (visParams.addTimeMarker) {
+    spec.transform = [
+      {
+        calculate: 'now()',
+        as: 'now_field',
+      },
+    ];
+
+    spec.layer.push({
+      mark: 'rule',
+      encoding: {
+        x: {
+          type: 'temporal',
+          field: 'now_field',
+        },
+        // The time marker on vislib is red, so keeping this consistent
+        color: {
+          value: 'red',
+        },
+        size: {
+          value: 1,
+        },
+      },
+    });
+  }
+
+  if (visParams.thresholdLine.show as boolean) {
+    const layer = {
+      mark: {
+        type: 'rule',
+        color: visParams.thresholdLine.color,
+        strokeDash: [1, 0],
+      },
+      encoding: {
+        y: {
+          datum: visParams.thresholdLine.value,
+        },
+      },
+    };
+
+    // Can only support making a threshold line with full or dashed style, but not dot-dashed
+    // due to vega-lite limitations
+    if (visParams.thresholdLine.style !== 'full') {
+      layer.mark.strokeDash = [8, 8];
+    }
+
+    spec.layer.push(layer);
+  }
+
+  return spec;
+};
+
+export const createLineVegaSpecFn = (
+  dependencies: VegaVisualizationDependencies
+): LineVegaSpecExpressionFunctionDefinition => ({
+  name: 'line_vega_spec',
+  type: 'string',
+  inputTypes: ['opensearch_dashboards_datatable'],
+  help: i18n.translate('visTypeVega.function.help', {
+    defaultMessage: 'Construct line vega spec',
+  }),
+  args: {
+    visLayers: {
+      types: ['string', 'null'],
+      default: '',
+      help: '',
+    },
+    visParams: {
+      types: ['string'],
+      default: '""',
+      help: '',
+    },
+    dimensions: {
+      types: ['string'],
+      default: '""',
+      help: '',
+    },
+  },
+  async fn(input, args, context) {
+    const table = cloneDeep(input);
+
+    // creating initial vega spec from table
+    const spec = createSpecFromDatatable(
+      formatDataTable(table),
+      JSON.parse(args.visParams),
+      JSON.parse(args.dimensions)
+    );
+    return JSON.stringify(spec);
+  },
+});

src/plugins/vis_type_vega/public/vega_fn.ts → src/plugins/vis_type_vega/public/expressions/vega_fn.ts

@@ -35,13 +35,13 @@ import {
   ExpressionFunctionDefinition,
   OpenSearchDashboardsContext,
   Render,
-} from '../../expressions/public';
-import { VegaVisualizationDependencies } from './plugin';
-import { createVegaRequestHandler } from './vega_request_handler';
-import { VegaInspectorAdapters } from './vega_inspector/index';
-import { TimeRange, Query } from '../../data/public';
-import { VisRenderValue } from '../../visualizations/public';
-import { VegaParser } from './data_model/vega_parser';
+} from '../../../expressions/public';
+import { VegaVisualizationDependencies } from '../plugin';
+import { createVegaRequestHandler } from '../vega_request_handler';
+import { VegaInspectorAdapters } from '../vega_inspector';
+import { TimeRange, Query } from '../../../data/public';
+import { VisRenderValue } from '../../../visualizations/public';
+import { VegaParser } from '../data_model/vega_parser';
 
 type Input = OpenSearchDashboardsContext | null;
 type Output = Promise<Render<RenderValue>>;
@@ -52,6 +52,14 @@ interface Arguments {
 
 export type VisParams = Required<Arguments>;
 
+export type VegaExpressionFunctionDefinition = ExpressionFunctionDefinition<
+  'vega',
+  Input,
+  Arguments,
+  Output,
+  ExecutionContext<unknown, VegaInspectorAdapters>
+>;
+
 interface RenderValue extends VisRenderValue {
   visData: VegaParser;
   visType: 'vega';
@@ -60,13 +68,7 @@ interface RenderValue extends VisRenderValue {
 
 export const createVegaFn = (
   dependencies: VegaVisualizationDependencies
-): ExpressionFunctionDefinition<
-  'vega',
-  Input,
-  Arguments,
-  Output,
-  ExecutionContext<unknown, VegaInspectorAdapters>
-> => ({
+): VegaExpressionFunctionDefinition => ({
   name: 'vega',
   type: 'render',
   inputTypes: ['opensearch_dashboards_context', 'null'],

src/plugins/vis_type_vega/public/index.ts

@@ -35,3 +35,6 @@ import { VegaPlugin as Plugin } from './plugin';
 export function plugin(initializerContext: PluginInitializerContext<ConfigSchema>) {
   return new Plugin(initializerContext);
 }
+
+export { VegaExpressionFunctionDefinition } from './expressions/vega_fn';
+export { LineVegaSpecExpressionFunctionDefinition } from './expressions/line_vega_spec_fn';

src/plugins/vis_type_vega/public/plugin.ts

@@ -43,13 +43,14 @@ import {
   setInjectedMetadata,
 } from './services';
 
-import { createVegaFn } from './vega_fn';
+import { createVegaFn } from './expressions/vega_fn';
 import { createVegaTypeDefinition } from './vega_type';
 import { IServiceSettings } from '../../maps_legacy/public';
 import './index.scss';
 import { ConfigSchema } from '../config';
 
 import { getVegaInspectorView } from './vega_inspector';
+import { createLineVegaSpecFn } from './expressions/line_vega_spec_fn';
 
 /** @internal */
 export interface VegaVisualizationDependencies {
@@ -104,6 +105,7 @@ export class VegaPlugin implements Plugin<Promise<void>, void> {
     inspector.registerView(getVegaInspectorView({ uiSettings: core.uiSettings }));
 
     expressions.registerFunction(() => createVegaFn(visualizationDependencies));
+    expressions.registerFunction(() => createLineVegaSpecFn(visualizationDependencies));
 
     visualizations.createBaseVisualization(createVegaTypeDefinition(visualizationDependencies));
   }

src/plugins/vis_type_vega/public/vega_request_handler.ts

@@ -34,7 +34,7 @@ import { SearchAPI } from './data_model/search_api';
 import { TimeCache } from './data_model/time_cache';
 
 import { VegaVisualizationDependencies } from './plugin';
-import { VisParams } from './vega_fn';
+import { VisParams } from './expressions/vega_fn';
 import { getData, getInjectedMetadata } from './services';
 import { VegaInspectorAdapters } from './vega_inspector';