From 96b934f4ae98a951458697ed9ca72be42356285d Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 12:28:45 -0400
Subject: [PATCH 01/13] Generate and save chart annotation and some refactor

---
 .../tera-calibrate-ciemss-drilldown.vue       | 183 +++++++++-------
 .../hmi-client/src/services/chart-settings.ts | 203 ++++++++++++++++++
 .../client/hmi-client/src/services/charts.ts  |   9 +
 .../client/hmi-client/src/temp/Vegalite.vue   |   2 +-
 4 files changed, 316 insertions(+), 81 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index e2f88cdb70..e7b8f42485 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -203,31 +203,35 @@
 					<section ref="outputPanel" v-if="modelConfig && csvAsset">
 						<h5>Parameters</h5>
 						<br />
-						<template v-for="param of selectedParameters" :key="param">
+						<template v-for="setting of selectedParameterSettings" :key="setting.id">
 							<vega-chart
 								expandable
 								:are-embed-actions-visible="true"
-								:visualization-spec="preparedDistributionCharts[param].histogram"
+								:visualization-spec="preparedDistributionCharts[setting.selectedVariables[0]].histogram"
 							>
 								<template v-slot:footer>
 									<table class="distribution-table">
 										<thead>
 											<tr>
 												<th scope="col"></th>
-												<th scope="col">{{ preparedDistributionCharts[param].stat.header[0] }}</th>
-												<th scope="col">{{ preparedDistributionCharts[param].stat.header[1] }}</th>
+												<th scope="col">
+													{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.header[0] }}
+												</th>
+												<th scope="col">
+													{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.header[1] }}
+												</th>
 											</tr>
 										</thead>
 										<tbody>
 											<tr>
 												<th scope="row">Mean</th>
-												<td>{{ preparedDistributionCharts[param].stat.mean[0] }}</td>
-												<td>{{ preparedDistributionCharts[param].stat.mean[1] }}</td>
+												<td>{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.mean[0] }}</td>
+												<td>{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.mean[1] }}</td>
 											</tr>
 											<tr>
 												<th scope="row">Variance</th>
-												<td>{{ preparedDistributionCharts[param].stat.variance[0] }}</td>
-												<td>{{ preparedDistributionCharts[param].stat.variance[1] }}</td>
+												<td>{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.variance[0] }}</td>
+												<td>{{ preparedDistributionCharts[setting.selectedVariables[0]].stat.variance[1] }}</td>
 											</tr>
 										</tbody>
 									</table>
@@ -236,12 +240,16 @@
 						</template>
 						<h5>Variables</h5>
 						<br />
-						<template v-for="variable of selectedVariables" :key="variable">
-							<vega-chart expandable :are-embed-actions-visible="true" :visualization-spec="preparedCharts[variable]" />
+						<template v-for="setting of selectedVariableSettings" :key="setting.id">
+							<vega-chart
+								expandable
+								:are-embed-actions-visible="true"
+								:visualization-spec="preparedCharts[setting.selectedVariables[0]]"
+							/>
 						</template>
 						<h5>Errors</h5>
 						<vega-chart
-							v-if="errorData.length > 0 && selectedErrorVariables.length > 0"
+							v-if="errorData.length > 0 && selectedErrorVariableSettings.length > 0"
 							:expandable="onExpandErrorChart"
 							:are-embed-actions-visible="true"
 							:visualization-spec="errorChart"
@@ -294,7 +302,10 @@
 							@remove="removeChartSetting"
 						/>
 						<tera-chart-control
-							:chart-config="{ selectedRun: 'fixme', selectedVariable: selectedParameters }"
+							:chart-config="{
+								selectedRun: 'fixme',
+								selectedVariable: selectedParameterSettings.map((s) => s.selectedVariables[0])
+							}"
 							:multi-select="true"
 							:show-remove-button="false"
 							:variables="Object.keys(pyciemssMap).filter((c) => modelPartTypesMap[c] === 'parameter')"
@@ -312,7 +323,10 @@
 							@remove="removeChartSetting"
 						/>
 						<tera-chart-control
-							:chart-config="{ selectedRun: 'fixme', selectedVariable: selectedVariables }"
+							:chart-config="{
+								selectedRun: 'fixme',
+								selectedVariable: selectedVariableSettings.map((s) => s.selectedVariables[0])
+							}"
 							:multi-select="true"
 							:show-remove-button="false"
 							:variables="
@@ -332,7 +346,10 @@
 							@remove="removeChartSetting"
 						/>
 						<tera-chart-control
-							:chart-config="{ selectedRun: 'fixme', selectedVariable: selectedErrorVariables }"
+							:chart-config="{
+								selectedRun: 'fixme',
+								selectedVariable: selectedErrorVariableSettings.map((s) => s.selectedVariables[0])
+							}"
 							:multi-select="true"
 							:show-remove-button="false"
 							:variables="Object.keys(pyciemssMap).filter((c) => mapping.find((d) => d.modelVariable === c))"
@@ -357,7 +374,6 @@
 <script setup lang="ts">
 import _ from 'lodash';
 import * as vega from 'vega';
-import { v4 as uuidv4 } from 'uuid';
 import { csvParse, autoType, mean, variance } from 'd3';
 import { computed, onMounted, ref, shallowRef, watch } from 'vue';
 import Button from 'primevue/button';
@@ -366,7 +382,12 @@ import Dropdown from 'primevue/dropdown';
 import Column from 'primevue/column';
 import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
 import { CalibrateMap, setupDatasetInput, setupModelInput } from '@/services/calibrate-workflow';
-import { removeChartSettingById, updateChartSettingsBySelectedVariables } from '@/services/chart-settings';
+import {
+	generateForecastChartAnnotation,
+	removeChartSettingById,
+	saveAnnotation,
+	updateChartSettingsBySelectedVariables
+} from '@/services/chart-settings';
 import TeraDrilldown from '@/components/drilldown/tera-drilldown.vue';
 import TeraDrilldownSection from '@/components/drilldown/tera-drilldown-section.vue';
 import TeraProgressSpinner from '@/components/widgets/tera-progress-spinner.vue';
@@ -402,7 +423,12 @@ import {
 import { getModelConfigurationById } from '@/services/model-configurations';
 
 import { WorkflowNode } from '@/types/workflow';
-import { createForecastChart, createHistogramChart, createErrorChart } from '@/services/charts';
+import {
+	createForecastChart,
+	createHistogramChart,
+	createErrorChart,
+	applyForecastChartAnnotations
+} from '@/services/charts';
 import VegaChart from '@/components/widgets/VegaChart.vue';
 import TeraChartControl from '@/components/workflow/tera-chart-control.vue';
 import TeraInputText from '@/components/widgets/tera-input-text.vue';
@@ -560,40 +586,29 @@ const outputPanel = ref(null);
 const chartSize = computed(() => drilldownChartSize(outputPanel.value));
 
 const chartSettings = computed(() => props.node.state.chartSettings ?? []);
-const selectedParameters = computed(() =>
-	chartSettings.value
-		.filter((setting) => setting.type === ChartSettingType.DISTRIBUTION_COMPARISON)
-		.map((setting) => setting.selectedVariables[0])
+const selectedParameterSettings = computed(() =>
+	chartSettings.value.filter((setting) => setting.type === ChartSettingType.DISTRIBUTION_COMPARISON)
 );
-const selectedVariables = computed(() =>
-	chartSettings.value
-		.filter((setting) => setting.type === ChartSettingType.VARIABLE_COMPARISON)
-		.map((setting) => setting.selectedVariables[0])
+const selectedVariableSettings = computed(() =>
+	chartSettings.value.filter((setting) => setting.type === ChartSettingType.VARIABLE_COMPARISON)
 );
-const selectedErrorVariables = computed(() =>
-	chartSettings.value
-		.filter((setting) => setting.type === ChartSettingType.ERROR_DISTRIBUTION)
-		.map((setting) => setting.selectedVariables[0])
+
+const selectedErrorVariableSettings = computed(() =>
+	chartSettings.value.filter((setting) => setting.type === ChartSettingType.ERROR_DISTRIBUTION)
 );
 
 const chartAnnotations = ref<ChartAnnotation[]>([]);
 const generateAnnotation = async (setting: ChartSetting, query: string) => {
-	// Generate fake annotation. The annotation generation logic for the specific chart setting should go here
-	// Different chart settings type may have different annotation generation logic
-	await new Promise((resolve) => {
-		setTimeout(resolve, 1000);
-	});
-	const annotation: ChartAnnotation = {
-		id: uuidv4(),
-		description: query,
-		nodeId: props.node.id,
-		outputId: '',
-		chartId: setting.id,
-		layerSpec: {},
-		llmGenerated: false,
-		metadata: {}
-	};
-	return annotation;
+	// Note: Currently llm generated chart annotations are supported for the forecast chart only
+	const variable = setting.selectedVariables[0];
+	const annotationLayerSpec = await generateForecastChartAnnotation(
+		query,
+		'timpoint_id',
+		[`${pyciemssMap.value[variable]}:pre`, `${pyciemssMap.value[variable]}`],
+		{ x: modelVarUnits.value._time || 'Time', y: modelVarUnits.value[variable] || '' }
+	);
+	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
+	return saved;
 };
 const addChartAnnotation = (annotation: ChartAnnotation) => {
 	chartAnnotations.value.push(annotation);
@@ -644,40 +659,45 @@ const preparedCharts = computed(() => {
 	const datasetTimeField = state.mapping.find((d) => d.modelVariable === 'timestamp')?.datasetVariable;
 
 	const charts = {};
-	selectedVariables.value.forEach((variable) => {
+	selectedVariableSettings.value.forEach((settings) => {
+		const variable = settings.selectedVariables[0];
+		const annotations = chartAnnotations.value.filter((annotation) => annotation.chartId === settings.id);
 		const datasetVariables: string[] = [];
 		const mapObj = state.mapping.find((d) => d.modelVariable === variable);
 		if (mapObj) {
 			datasetVariables.push(mapObj.datasetVariable);
 		}
-		charts[variable] = createForecastChart(
-			{
-				data: result,
-				variables: [`${pyciemssMap.value[variable]}:pre`, pyciemssMap.value[variable]],
-				timeField: 'timepoint_id',
-				groupField: 'sample_id'
-			},
-			{
-				data: resultSummary,
-				variables: [`${pyciemssMap.value[variable]}_mean:pre`, `${pyciemssMap.value[variable]}_mean`],
-				timeField: 'timepoint_id'
-			},
-			{
-				data: groundTruthData.value,
-				variables: datasetVariables,
-				timeField: datasetTimeField as string,
-				groupField: 'sample_id'
-			},
-			{
-				title: variable,
-				width: chartSize.value.width,
-				height: chartSize.value.height,
-				legend: true,
-				translationMap: reverseMap,
-				xAxisTitle: modelVarUnits.value._time || 'Time',
-				yAxisTitle: modelVarUnits.value[variable] || '',
-				colorscheme: ['#AAB3C6', '#1B8073']
-			}
+		charts[variable] = applyForecastChartAnnotations(
+			createForecastChart(
+				{
+					data: result,
+					variables: [`${pyciemssMap.value[variable]}:pre`, pyciemssMap.value[variable]],
+					timeField: 'timepoint_id',
+					groupField: 'sample_id'
+				},
+				{
+					data: resultSummary,
+					variables: [`${pyciemssMap.value[variable]}:pre`, `${pyciemssMap.value[variable]}_mean`],
+					timeField: 'timepoint_id'
+				},
+				{
+					data: groundTruthData.value,
+					variables: datasetVariables,
+					timeField: datasetTimeField as string,
+					groupField: 'sample_id'
+				},
+				{
+					title: variable,
+					width: chartSize.value.width,
+					height: chartSize.value.height,
+					legend: true,
+					translationMap: reverseMap,
+					xAxisTitle: modelVarUnits.value._time || 'Time',
+					yAxisTitle: modelVarUnits.value[variable] || '',
+					colorscheme: ['#AAB3C6', '#1B8073']
+				}
+			),
+			annotations
 		);
 	});
 	return charts;
@@ -689,7 +709,8 @@ const preparedDistributionCharts = computed(() => {
 	const labelBefore = 'Before calibration';
 	const labelAfter = 'After calibration';
 	const charts = {};
-	selectedParameters.value.forEach((param) => {
+	selectedParameterSettings.value.forEach((setting) => {
+		const param = setting.selectedVariables[0];
 		const fieldName = pyciemssMap.value[param];
 		const beforeFieldName = `${fieldName}:pre`;
 		const histogram = createHistogramChart(result, {
@@ -718,13 +739,15 @@ const preparedDistributionCharts = computed(() => {
 });
 
 const errorChartVariables = computed(() => {
-	if (!selectedErrorVariables.value.length) return [];
+	if (!selectedErrorVariableSettings.value.length) return [];
 	const getDatasetVariable = (modelVariable: string) =>
 		mapping.value.find((d) => d.modelVariable === modelVariable)?.datasetVariable;
-	const variables = selectedErrorVariables.value.map((variable) => ({
-		field: getDatasetVariable(variable) as string,
-		label: variable
-	}));
+	const variables = selectedErrorVariableSettings.value
+		.map((s) => s.selectedVariables[0])
+		.map((variable) => ({
+			field: getDatasetVariable(variable) as string,
+			label: variable
+		}));
 	return variables;
 });
 
diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index a3962827d1..74a73d7148 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -1,5 +1,13 @@
+import API from '@/api/api';
 import type { ChartSetting, ChartSettingType } from '@/types/common';
 import { v4 as uuidv4 } from 'uuid';
+import { b64DecodeUnicode } from '@/utils/binary';
+import { ChartAnnotation } from '@/types/Types';
+
+export interface LLMGeneratedChartAnnotation {
+	request: string;
+	layerSpec: any;
+}
 
 /**
  * Updates the given chart settings based on the selected variables and return it as new settings.
@@ -44,3 +52,198 @@ export function updateChartSettingsBySelectedVariables(
 export function removeChartSettingById(settings: ChartSetting[], id: string) {
 	return settings.filter((setting) => setting.id !== id);
 }
+
+export async function saveAnnotation(
+	annotationLayerSpec: LLMGeneratedChartAnnotation,
+	nodeId: string,
+	chartId: string
+) {
+	const annotation: ChartAnnotation = {
+		id: uuidv4(),
+		description: annotationLayerSpec.request,
+		nodeId,
+		outputId: '',
+		chartId,
+		layerSpec: annotationLayerSpec.layerSpec,
+		llmGenerated: true,
+		metadata: {}
+	};
+	return annotation;
+}
+
+export async function generateForecastChartAnnotation(
+	request: string,
+	timeField: string,
+	variables: string[],
+	axisTitle: { x: string; y: string }
+): Promise<LLMGeneratedChartAnnotation> {
+	const prompt = `
+	  You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
+
+    - The Vega-Lite schema version you must use is https://vega.github.io/schema/vega-lite/v5.json.
+    - Assume that you don’t know the exact data points from the data.
+    - You must give me the single layer object that renders all the necessary drawing objects, including multiple layers within the top layer object if needed.
+    - When adding a label, also draw a constant line perpendicular to the axis to which you are adding the label.
+
+    Assuming you are adding the annotations to the following chart spec,
+    ---- Example Chart Spec Start -----
+    {
+      "data": {"url": "data/samples.csv"},
+      "transform": [
+        {
+          "fold": ["price", "cost", "tax", "profit"],
+          "as": ["variableField", "valueField"]
+        }
+      ],
+      "layer": [
+        {
+          "mark": "line",
+          "encoding": {
+            "x": {"field": "date", "type": "quantitative", "axis": {"title": "Day"}},
+            "y": {"field": "valueField", "type": "quantitative", "axis": {"title": "Dollars"}}
+          }
+        }
+      ]
+    }
+    ---- Example Chart Spec End -----
+    Here are some example requests and the answers:
+
+    Request:
+    At day 200, add a label 'important'
+    Answer:
+    {
+      "description": "At day 200, add a label 'important'",
+      "layer": [
+        {
+          "mark": {
+            "type": "rule",
+            "strokeDash": [4, 4]
+          },
+					"encoding": {
+						"x": { "datum": 200, "axis": { "title": ""} }
+					}
+        },
+        {
+          "mark": {
+            "type": "text",
+            "align": "left",
+            "dx": 5,
+            "dy": -5
+          },
+          "encoding": {
+						"x": { "datum": 200, "axis": { "title": ""} }
+            "text": {"value": "important"}
+          }
+        }
+      ]
+    }
+
+    Request:
+    Add a label 'expensive' at price 20
+    Answer:
+    {
+      "description": "Add a label 'expensive' at price 20",
+      "layer": [
+        {
+          "mark": {
+            "type": "rule",
+            "strokeDash": [4, 4]
+          },
+          "encoding": {
+            "y": { "datum": 20, "axis": { "title": ""} }
+          }
+        },
+        {
+          "mark": {
+            "type": "text",
+            "align": "left",
+            "dx": 5,
+            "dy": -5
+          },
+          "encoding": {
+            "y": { "datum": 20, "axis": { "title": ""} },
+            "text": {"value": "expensive"}
+          }
+        }
+      ]
+    }
+
+    Request:
+    Add a vertical line for the day where the price exceeds 100.
+    Answer:
+    {
+      "description": "Add a vertical line for the day where the price exceeds 100.",
+      "transform": [
+        {"filter": "datum.valueField > 100"},
+        {"aggregate": [{"op": "min", "field": "date", "as": "min_date"}]}
+      ],
+      "layer": [
+        {
+          "mark": {
+            "type": "rule",
+            "strokeDash": [4, 4]
+          },
+          "encoding": {
+            "x": {"field": "min_date", "type": "quantitative", "axis": { "title": ""}}
+          }
+        },
+        {
+          "mark": {
+            "type": "text",
+            "align": "left",
+            "dx": 5,
+            "dy": -10
+          },
+          "encoding": {
+            "x": {"field": "min_date", "type": "quantitative", "axis": { "title": ""}},
+            "text": {"value": "Price > 100"}
+          }
+        }
+      ]
+    }
+
+    Here is the information of the existing target chart spec where you need to add the annotations:
+    - The existing chart follows a similar pattern as the above Example Chart Spec like:
+        {
+          ...
+          "transform": [
+            {
+              "fold": ${JSON.stringify(variables)},
+              "as": ["variableField', "valueField"]
+            }
+          ],
+          "layer": [
+            {
+              ...
+              "encoding": {
+                "x": {"field": "${timeField}", "type": "quantitative", "axis": {"title": "${axisTitle.x}"}},
+                "y": {"field": "valueField", "type": "quantitative", "axis": {"title": "${axisTitle.y}"}}
+              }
+            }
+            ...
+          ]
+        }
+    - Assume all unknown variables except the time field are for the y-axis and are renamed to the valueField.
+
+     Give me the layer object to be added to the existing chart spec based on the following user request.
+		 Please return only a JSON object as a response. Make sure to return plain JSON object that can be parsed as JSON. Do not include code block.
+
+		Request:
+    ${request}
+    Answer
+    {
+	`;
+	// FIXME: Use dedicated endpoint for annotation generation that's configured with JSON response_format instead of using the summary endpoint which is for text output
+	const { data } = await API.post(`/gollm/generate-summary?mode=SYNC`, prompt, {
+		headers: {
+			'Content-Type': 'application/json'
+		}
+	});
+	const str = b64DecodeUnicode(data.output);
+	const result = JSON.parse(str);
+	const layerSpec = JSON.parse(result.response);
+	return {
+		request,
+		layerSpec
+	};
+}
diff --git a/packages/client/hmi-client/src/services/charts.ts b/packages/client/hmi-client/src/services/charts.ts
index ff42b2389e..d47fea2e98 100644
--- a/packages/client/hmi-client/src/services/charts.ts
+++ b/packages/client/hmi-client/src/services/charts.ts
@@ -4,6 +4,7 @@ import { percentile } from '@/utils/math';
 import { isEmpty } from 'lodash';
 import { VisualizationSpec } from 'vega-embed';
 import { v4 as uuidv4 } from 'uuid';
+import { ChartAnnotation } from '@/types/Types';
 
 const VEGALITE_SCHEMA = 'https://vega.github.io/schema/vega-lite/v5.json';
 
@@ -518,6 +519,14 @@ export function createForecastChart(
 	return spec;
 }
 
+export function applyForecastChartAnnotations(chartSpec: any, annotations: ChartAnnotation[]) {
+	const targetLayerIndex = 1; // Assume the target layer is the second layer which is the statistic layer
+	const layerSpecs = annotations.map((a) => a.layerSpec);
+	if (!chartSpec.layer[targetLayerIndex]) return chartSpec;
+	chartSpec.layer[targetLayerIndex].layer.push(...layerSpecs);
+	return chartSpec;
+}
+
 export function createForecastChartAnnotation(axis: 'x' | 'y', datum: number, label: string) {
 	const layerSpec = {
 		description: `At ${axis} ${datum}, add a label '${label}'.`,
diff --git a/packages/client/hmi-client/src/temp/Vegalite.vue b/packages/client/hmi-client/src/temp/Vegalite.vue
index 561f64f59e..5f845c2cb9 100644
--- a/packages/client/hmi-client/src/temp/Vegalite.vue
+++ b/packages/client/hmi-client/src/temp/Vegalite.vue
@@ -46,9 +46,9 @@ import {
 	createForecastChart,
 	createHistogramChart,
 	createErrorChart,
-	generateForecastChartAnnotation,
 	createForecastChartAnnotation
 } from '@/services/charts';
+import { generateForecastChartAnnotation } from '@/services/chart-settings';
 // import { createLLMSummary, getSummaries } from '@/services/summary-service';
 
 const rand = (v: number) => Math.round(Math.random() * v);

From 36a4ca78761fa81de96104be82edcf41979a3da6 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 12:43:50 -0400
Subject: [PATCH 02/13] some clean ups

---
 .../client/hmi-client/src/services/charts.ts  | 196 +-----------------
 .../client/hmi-client/src/temp/Vegalite.vue   |  43 ++--
 2 files changed, 27 insertions(+), 212 deletions(-)

diff --git a/packages/client/hmi-client/src/services/charts.ts b/packages/client/hmi-client/src/services/charts.ts
index d47fea2e98..38a81027ac 100644
--- a/packages/client/hmi-client/src/services/charts.ts
+++ b/packages/client/hmi-client/src/services/charts.ts
@@ -1,5 +1,3 @@
-import API from '@/api/api';
-import { b64DecodeUnicode } from '@/utils/binary';
 import { percentile } from '@/utils/math';
 import { isEmpty } from 'lodash';
 import { VisualizationSpec } from 'vega-embed';
@@ -35,13 +33,6 @@ export interface ForecastChartLayer {
 	groupField?: string;
 }
 
-export interface ForecastChartAnnotation {
-	id: string;
-	layerSpec: any;
-	isLLMGenerated: boolean;
-	metadata: any;
-}
-
 export interface HistogramChartOptions extends BaseChartOptions {
 	maxBins?: number;
 	variables: { field: string; label?: string; width: number; color: string }[];
@@ -553,195 +544,18 @@ export function createForecastChartAnnotation(axis: 'x' | 'y', datum: number, la
 			}
 		]
 	};
-	const annotation: ForecastChartAnnotation = {
+	const annotation: ChartAnnotation = {
 		id: uuidv4(),
+		nodeId: '',
+		outputId: '',
+		chartId: '',
 		layerSpec,
-		isLLMGenerated: false,
+		llmGenerated: false,
 		metadata: { axis, datum, label }
 	};
 	return annotation;
 }
 
-export async function generateForecastChartAnnotation(
-	request: string,
-	timeField: string,
-	variables: string[],
-	axisTitle: { x: string; y: string }
-) {
-	const prompt = `
-	  You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
-
-    - The Vega-Lite schema version you must use is https://vega.github.io/schema/vega-lite/v5.json.
-    - Assume that you don’t know the exact data points from the data.
-    - You must give me the single layer object that renders all the necessary drawing objects, including multiple layers within the top layer object if needed.
-    - When adding a label, also draw a constant line perpendicular to the axis to which you are adding the label.
-
-    Assuming you are adding the annotations to the following chart spec,
-    ---- Example Chart Spec Start -----
-    {
-      "data": {"url": "data/samples.csv"},
-      "transform": [
-        {
-          "fold": ["price", "cost", "tax", "profit"],
-          "as": ["variableField", "valueField"]
-        }
-      ],
-      "layer": [
-        {
-          "mark": "line",
-          "encoding": {
-            "x": {"field": "date", "type": "quantitative", "axis": {"title": "Day"}},
-            "y": {"field": "valueField", "type": "quantitative", "axis": {"title": "Dollars"}}
-          }
-        }
-      ]
-    }
-    ---- Example Chart Spec End -----
-    Here are some example requests and the answers:
-
-    Request:
-    At day 200, add a label 'important'
-    Answer:
-    {
-      "description": "At day 200, add a label 'important'",
-      "layer": [
-        {
-          "mark": {
-            "type": "rule",
-            "strokeDash": [4, 4]
-          },
-					"encoding": {
-						"x": { "datum": 200, "axis": { "title": ""} }
-					}
-        },
-        {
-          "mark": {
-            "type": "text",
-            "align": "left",
-            "dx": 5,
-            "dy": -5
-          },
-          "encoding": {
-						"x": { "datum": 200, "axis": { "title": ""} }
-            "text": {"value": "important"}
-          }
-        }
-      ]
-    }
-
-    Request:
-    Add a label 'expensive' at price 20
-    Answer:
-    {
-      "description": "Add a label 'expensive' at price 20",
-      "layer": [
-        {
-          "mark": {
-            "type": "rule",
-            "strokeDash": [4, 4]
-          },
-          "encoding": {
-            "y": { "datum": 20, "axis": { "title": ""} }
-          }
-        },
-        {
-          "mark": {
-            "type": "text",
-            "align": "left",
-            "dx": 5,
-            "dy": -5
-          },
-          "encoding": {
-            "y": { "datum": 20, "axis": { "title": ""} },
-            "text": {"value": "expensive"}
-          }
-        }
-      ]
-    }
-
-    Request:
-    Add a vertical line for the day where the price exceeds 100.
-    Answer:
-    {
-      "description": "Add a vertical line for the day where the price exceeds 100.",
-      "transform": [
-        {"filter": "datum.valueField > 100"},
-        {"aggregate": [{"op": "min", "field": "date", "as": "min_date"}]}
-      ],
-      "layer": [
-        {
-          "mark": {
-            "type": "rule",
-            "strokeDash": [4, 4]
-          },
-          "encoding": {
-            "x": {"field": "min_date", "type": "quantitative", "axis": { "title": ""}}
-          }
-        },
-        {
-          "mark": {
-            "type": "text",
-            "align": "left",
-            "dx": 5,
-            "dy": -10
-          },
-          "encoding": {
-            "x": {"field": "min_date", "type": "quantitative", "axis": { "title": ""}},
-            "text": {"value": "Price > 100"}
-          }
-        }
-      ]
-    }
-
-    Here is the information of the existing target chart spec where you need to add the annotations:
-    - The existing chart follows a similar pattern as the above Example Chart Spec like:
-        {
-          ...
-          "transform": [
-            {
-              "fold": ${JSON.stringify(variables)},
-              "as": ["variableField', "valueField"]
-            }
-          ],
-          "layer": [
-            {
-              ...
-              "encoding": {
-                "x": {"field": "${timeField}", "type": "quantitative", "axis": {"title": "${axisTitle.x}"}},
-                "y": {"field": "valueField", "type": "quantitative", "axis": {"title": "${axisTitle.y}"}}
-              }
-            }
-            ...
-          ]
-        }
-    - Assume all unknown variables except the time field are for the y-axis and are renamed to the valueField.
-
-     Give me the layer object to be added to the existing chart spec based on the following user request.
-		 Please return only a JSON object as a response. Make sure to return plain JSON object that can be parsed as JSON. Do not include code block.
-
-		Request:
-    ${request}
-    Answer
-    {
-	`;
-	// FIXME: Use dedicated endpoint for annotation generation that's configured with JSON response_format instead of using the summary endpoint which is for text output
-	const { data } = await API.post(`/gollm/generate-summary?mode=SYNC`, prompt, {
-		headers: {
-			'Content-Type': 'application/json'
-		}
-	});
-	const str = b64DecodeUnicode(data.output);
-	const result = JSON.parse(str);
-	const layerSpec = JSON.parse(result.response);
-	const annotation: ForecastChartAnnotation = {
-		id: uuidv4(),
-		layerSpec,
-		isLLMGenerated: true,
-		metadata: { llmRequest: request }
-	};
-	return annotation;
-}
-
 /// /////////////////////////////////////////////////////////////////////////////
 // Optimize charts
 /// /////////////////////////////////////////////////////////////////////////////
diff --git a/packages/client/hmi-client/src/temp/Vegalite.vue b/packages/client/hmi-client/src/temp/Vegalite.vue
index 5f845c2cb9..ef1b406047 100644
--- a/packages/client/hmi-client/src/temp/Vegalite.vue
+++ b/packages/client/hmi-client/src/temp/Vegalite.vue
@@ -36,6 +36,7 @@
 </template>
 
 <script setup lang="ts">
+import { v4 as uuidv4 } from 'uuid';
 import { debounce } from 'lodash';
 import { ref, onMounted, computed } from 'vue';
 import TeraInputText from '@/components/widgets/tera-input-text.vue';
@@ -46,9 +47,11 @@ import {
 	createForecastChart,
 	createHistogramChart,
 	createErrorChart,
-	createForecastChartAnnotation
+	createForecastChartAnnotation,
+	applyForecastChartAnnotations
 } from '@/services/charts';
 import { generateForecastChartAnnotation } from '@/services/chart-settings';
+import { ChartAnnotation } from '@/types/Types';
 // import { createLLMSummary, getSummaries } from '@/services/summary-service';
 
 const rand = (v: number) => Math.round(Math.random() * v);
@@ -160,23 +163,7 @@ const debounceHandleIntervalSelect = debounce(handleIntervalSelect, 200);
 
 // Generate time series data
 const dataNew = generateSimulateData();
-const forecastAnnotations = ref<
-	{
-		id: string;
-		layerSpec: any;
-		isLLMGenerated: boolean;
-		metadata: any;
-	}[]
->([createForecastChartAnnotation('x', 60, 'test label')]);
-
-const applyForecastAnnotations = (chartSpec: any, forecastAnnotations: any[], targetLayerIndex = 1) => {
-	targetLayerIndex = 1; // Assume the target layer is the second layer which is the statistic layer
-	const layerSpecs = forecastAnnotations.map((a) => a.layerSpec);
-	const spec = structuredClone(chartSpec);
-	if (!spec.layer[targetLayerIndex]) return spec;
-	spec.layer[targetLayerIndex].layer.push(...layerSpecs);
-	return spec;
-};
+const forecastAnnotations = ref<ChartAnnotation[]>([createForecastChartAnnotation('x', 60, 'test label')]);
 
 const questionString = ref('');
 const waitingForForecastChartAnnotation = ref(false);
@@ -186,8 +173,22 @@ const generateAndAddAnnotation = async () => {
 	const timeField = 'time';
 	const variables = ['alphaMean', 'betaMean'];
 	const axisTitle = { x: 'Day', y: 'Value' };
-	const annotation = await generateForecastChartAnnotation(questionString.value, timeField, variables, axisTitle);
-	forecastAnnotations.value.push(annotation);
+	const { request, layerSpec } = await generateForecastChartAnnotation(
+		questionString.value,
+		timeField,
+		variables,
+		axisTitle
+	);
+	forecastAnnotations.value.push({
+		id: uuidv4(),
+		description: request,
+		nodeId: '',
+		outputId: '',
+		chartId: 'forecastchart',
+		layerSpec,
+		llmGenerated: true,
+		metadata: {}
+	});
 	waitingForForecastChartAnnotation.value = false;
 };
 const removeAnnotation = async (id: string) => {
@@ -195,7 +196,7 @@ const removeAnnotation = async (id: string) => {
 };
 
 const forecastChartSpec = computed(() =>
-	applyForecastAnnotations(
+	applyForecastChartAnnotations(
 		createForecastChart(
 			{
 				data: dataNew.data,

From b157cf8135e4ffd22ff545636940fbde05f7bc06 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 13:20:56 -0400
Subject: [PATCH 03/13] Added chart annotations crud

---
 .../tera-calibrate-ciemss-drilldown.vue        | 18 ++++++++++--------
 .../hmi-client/src/services/chart-settings.ts  | 13 ++++++++++++-
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index e7b8f42485..fa934d6e93 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -383,6 +383,8 @@ import Column from 'primevue/column';
 import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
 import { CalibrateMap, setupDatasetInput, setupModelInput } from '@/services/calibrate-workflow';
 import {
+	deleteAnnotation,
+	fetchAnnotations,
 	generateForecastChartAnnotation,
 	removeChartSettingById,
 	saveAnnotation,
@@ -597,7 +599,11 @@ const selectedErrorVariableSettings = computed(() =>
 	chartSettings.value.filter((setting) => setting.type === ChartSettingType.ERROR_DISTRIBUTION)
 );
 
+onMounted(() => updateChartAnnotations());
 const chartAnnotations = ref<ChartAnnotation[]>([]);
+const updateChartAnnotations = async () => {
+	chartAnnotations.value = await fetchAnnotations(props.node.id);
+};
 const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	// Note: Currently llm generated chart annotations are supported for the forecast chart only
 	const variable = setting.selectedVariables[0];
@@ -610,14 +616,10 @@ const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
 	return saved;
 };
-const addChartAnnotation = (annotation: ChartAnnotation) => {
-	chartAnnotations.value.push(annotation);
-};
-const removeChartAnnotation = (annotationId: string) => {
-	const index = chartAnnotations.value.findIndex((annotation) => annotation.id === annotationId);
-	if (index !== -1) {
-		chartAnnotations.value.splice(index, 1);
-	}
+const addChartAnnotation = () => updateChartAnnotations();
+const removeChartAnnotation = async (annotationId: string) => {
+	await deleteAnnotation(annotationId);
+	await updateChartAnnotations();
 };
 
 const pyciemssMap = ref<Record<string, string>>({});
diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index 74a73d7148..10818d02f4 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -68,7 +68,18 @@ export async function saveAnnotation(
 		llmGenerated: true,
 		metadata: {}
 	};
-	return annotation;
+	const { data } = await API.post('/chart-annotations', annotation);
+	return data as ChartAnnotation;
+}
+
+export async function deleteAnnotation(annotationId: string) {
+	const { data } = await API.delete(`/chart-annotations/${annotationId}`);
+	return data;
+}
+
+export async function fetchAnnotations(nodeId: string) {
+	const { data } = await API.post(`/chart-annotations/search`, { nodeId });
+	return data as ChartAnnotation[];
 }
 
 export async function generateForecastChartAnnotation(

From 6b6fdce4a7604c0b0cd1c72e0d1b4486353462d4 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 14:08:42 -0400
Subject: [PATCH 04/13] Sync chart annotation crud

---
 .../tera-calibrate-ciemss-drilldown.vue       | 13 ++--
 .../src/composables/useClientEvent.ts         | 10 ++-
 packages/client/hmi-client/src/types/Types.ts |  2 +
 .../ChartAnnotationController.java            | 66 +++++++++++++++++--
 .../hmiserver/models/ClientEventType.java     |  4 +-
 5 files changed, 79 insertions(+), 16 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index fa934d6e93..f362936b8a 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -284,8 +284,7 @@
 						"
 						:active-settings="activeChartSettings"
 						:generate-annotation="generateAnnotation"
-						@create-annotation="addChartAnnotation"
-						@delete-annotation="removeChartAnnotation"
+						@delete-annotation="deleteAnnotation"
 						@close="activeChartSettings = null"
 					/>
 				</template>
@@ -437,6 +436,7 @@ import TeraInputText from '@/components/widgets/tera-input-text.vue';
 import { displayNumber } from '@/utils/number';
 import TeraPyciemssCancelButton from '@/components/pyciemss/tera-pyciemss-cancel-button.vue';
 import TeraSaveAssetModal from '@/components/project/tera-save-asset-modal.vue';
+import { useClientEvent } from '@/composables/useClientEvent';
 import type { CalibrationOperationStateCiemss } from './calibrate-operation';
 import { renameFnGenerator, mergeResults, getErrorData } from './calibrate-utils';
 
@@ -599,11 +599,13 @@ const selectedErrorVariableSettings = computed(() =>
 	chartSettings.value.filter((setting) => setting.type === ChartSettingType.ERROR_DISTRIBUTION)
 );
 
-onMounted(() => updateChartAnnotations());
 const chartAnnotations = ref<ChartAnnotation[]>([]);
 const updateChartAnnotations = async () => {
 	chartAnnotations.value = await fetchAnnotations(props.node.id);
 };
+onMounted(() => updateChartAnnotations());
+useClientEvent([ClientEventType.ChartAnnotationCreate, ClientEventType.ChartAnnotationDelete], updateChartAnnotations);
+
 const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	// Note: Currently llm generated chart annotations are supported for the forecast chart only
 	const variable = setting.selectedVariables[0];
@@ -616,11 +618,6 @@ const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
 	return saved;
 };
-const addChartAnnotation = () => updateChartAnnotations();
-const removeChartAnnotation = async (annotationId: string) => {
-	await deleteAnnotation(annotationId);
-	await updateChartAnnotations();
-};
 
 const pyciemssMap = ref<Record<string, string>>({});
 const preparedChartInputs = computed(() => {
diff --git a/packages/client/hmi-client/src/composables/useClientEvent.ts b/packages/client/hmi-client/src/composables/useClientEvent.ts
index 1b1926a8c7..4c94bc397a 100644
--- a/packages/client/hmi-client/src/composables/useClientEvent.ts
+++ b/packages/client/hmi-client/src/composables/useClientEvent.ts
@@ -2,11 +2,15 @@ import { subscribe, unsubscribe } from '@/services/ClientEventService';
 import { ClientEvent, ClientEventType } from '@/types/Types';
 import { onMounted, onUnmounted } from 'vue';
 
-export function useClientEvent(eventType: ClientEventType, handlerFn: (data: ClientEvent<any>) => void) {
+export function useClientEvent(
+	eventType: ClientEventType | ClientEventType[],
+	handlerFn: (data: ClientEvent<any>) => void
+) {
+	const eventTypes = Array.isArray(eventType) ? eventType : [eventType];
 	onMounted(async () => {
-		await subscribe(eventType, handlerFn);
+		await Promise.all(eventTypes.map((type) => subscribe(type, handlerFn)));
 	});
 	onUnmounted(async () => {
-		await unsubscribe(eventType, handlerFn);
+		await Promise.all(eventTypes.map((type) => unsubscribe(type, handlerFn)));
 	});
 }
diff --git a/packages/client/hmi-client/src/types/Types.ts b/packages/client/hmi-client/src/types/Types.ts
index 9bc636a301..b8248a33fb 100644
--- a/packages/client/hmi-client/src/types/Types.ts
+++ b/packages/client/hmi-client/src/types/Types.ts
@@ -1071,6 +1071,8 @@ export enum ClientEventType {
     TaskEnrichAmr = "TASK_ENRICH_AMR",
     WorkflowUpdate = "WORKFLOW_UPDATE",
     WorkflowDelete = "WORKFLOW_DELETE",
+    ChartAnnotationCreate = "CHART_ANNOTATION_CREATE",
+    ChartAnnotationDelete = "CHART_ANNOTATION_DELETE",
 }
 
 export enum ProgressState {
diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ChartAnnotationController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ChartAnnotationController.java
index cac6c6adb3..6e2117bcab 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ChartAnnotationController.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ChartAnnotationController.java
@@ -22,14 +22,21 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ResponseStatusException;
+import software.uncharted.terarium.hmiserver.models.ClientEvent;
+import software.uncharted.terarium.hmiserver.models.ClientEventType;
 import software.uncharted.terarium.hmiserver.models.dataservice.ChartAnnotation;
 import software.uncharted.terarium.hmiserver.models.dataservice.ResponseDeleted;
+import software.uncharted.terarium.hmiserver.models.dataservice.project.Contributor;
 import software.uncharted.terarium.hmiserver.security.Roles;
+import software.uncharted.terarium.hmiserver.service.ClientEventService;
 import software.uncharted.terarium.hmiserver.service.CurrentUserService;
 import software.uncharted.terarium.hmiserver.service.data.ChartAnnotationService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectAssetService;
+import software.uncharted.terarium.hmiserver.service.data.ProjectPermissionsService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectService;
+import software.uncharted.terarium.hmiserver.utils.rebac.ReBACService;
 import software.uncharted.terarium.hmiserver.utils.rebac.Schema;
+import software.uncharted.terarium.hmiserver.utils.rebac.askem.RebacProject;
 
 @RequestMapping("/chart-annotations")
 @RestController
@@ -46,6 +53,12 @@ public class ChartAnnotationController {
 
 	final CurrentUserService currentUserService;
 
+	final ReBACService reBACService;
+
+	final ProjectPermissionsService projectPermissionsService;
+
+	final ClientEventService clientEventService;
+
 	private static class SearchRequestBody {
 
 		public UUID nodeId;
@@ -117,15 +130,39 @@ public ResponseEntity<ChartAnnotation> createChartAnnotation(
 			currentUserService.get().getId(),
 			projectId
 		);
+
+		final ChartAnnotation chartAnnotation;
 		try {
-			return ResponseEntity.status(HttpStatus.CREATED).body(
-				chartAnnotationService.createAsset(item, projectId, permission)
-			);
+			chartAnnotation = chartAnnotationService.createAsset(item, projectId, permission);
 		} catch (final IOException e) {
 			final String error = "Unable to create chart annotation";
 			log.error(error, e);
 			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
 		}
+
+		final ClientEvent<ChartAnnotation> event = ClientEvent.<ChartAnnotation>builder()
+			.type(ClientEventType.CHART_ANNOTATION_CREATE)
+			.data(chartAnnotation)
+			.build();
+
+		try {
+			final RebacProject rebacProject = new RebacProject(projectId, reBACService);
+			if (rebacProject.isPublic()) {
+				clientEventService.sendToAllUsers(event);
+			} else {
+				final List<String> userIds = projectPermissionsService
+					.getReaders(rebacProject)
+					.stream()
+					.map(Contributor::getUserId)
+					.toList();
+				clientEventService.sendToUsers(event, userIds);
+			}
+		} catch (final Exception e) {
+			log.error("Unable to notify users for created chart annotations ", e);
+			// No response status exception here because the annotation created successfully, and it's just the sending events that's failed.
+		}
+
+		return ResponseEntity.status(HttpStatus.CREATED).body(chartAnnotation);
 	}
 
 	@DeleteMapping("/{id}")
@@ -161,11 +198,32 @@ public ResponseEntity<ResponseDeleted> deleteChartAnnotation(
 
 		try {
 			chartAnnotationService.deleteAsset(id, projectId, permission);
-			return ResponseEntity.ok(new ResponseDeleted("ChartAnnotation", id));
 		} catch (final Exception e) {
 			final String error = String.format("Failed to delete chart annotation %s", id);
 			log.error(error, e);
 			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
 		}
+
+		final ClientEvent<Void> event = ClientEvent.<Void>builder().type(ClientEventType.CHART_ANNOTATION_DELETE).build();
+
+		final RebacProject rebacProject = new RebacProject(projectId, reBACService);
+
+		try {
+			if (rebacProject.isPublic()) {
+				clientEventService.sendToAllUsers(event);
+			} else {
+				final List<String> userIds = projectPermissionsService
+					.getReaders(rebacProject)
+					.stream()
+					.map(Contributor::getUserId)
+					.toList();
+				clientEventService.sendToUsers(event, userIds);
+			}
+		} catch (final Exception e) {
+			log.error("Unable to notify users for deleted chart annotations", e);
+			// No response status exception here because the chart annotation deleted successfully, and it's just the sending events that's failed.
+		}
+
+		return ResponseEntity.ok(new ResponseDeleted("ChartAnnotation", id));
 	}
 }
diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/ClientEventType.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/ClientEventType.java
index 88c9691f59..598b199582 100644
--- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/ClientEventType.java
+++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/ClientEventType.java
@@ -22,5 +22,7 @@ public enum ClientEventType {
 	TASK_MIRA_AMR_TO_MMT,
 	TASK_ENRICH_AMR,
 	WORKFLOW_UPDATE,
-	WORKFLOW_DELETE
+	WORKFLOW_DELETE,
+	CHART_ANNOTATION_CREATE,
+	CHART_ANNOTATION_DELETE
 }

From afd9a9b0e40eb228a81c52ffb636315d369a0e42 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 14:22:33 -0400
Subject: [PATCH 05/13] Apply chart annotations to the node

---
 .../tera-calibrate-ciemss-drilldown.vue       |  2 +
 .../tera-calibrate-node-ciemss.vue            | 90 +++++++++++--------
 2 files changed, 56 insertions(+), 36 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index f362936b8a..8c90b4c648 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -599,6 +599,7 @@ const selectedErrorVariableSettings = computed(() =>
 	chartSettings.value.filter((setting) => setting.type === ChartSettingType.ERROR_DISTRIBUTION)
 );
 
+// --- Handle chart annotations
 const chartAnnotations = ref<ChartAnnotation[]>([]);
 const updateChartAnnotations = async () => {
 	chartAnnotations.value = await fetchAnnotations(props.node.id);
@@ -618,6 +619,7 @@ const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
 	return saved;
 };
+// ---
 
 const pyciemssMap = ref<Record<string, string>>({});
 const preparedChartInputs = computed(() => {
diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-node-ciemss.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-node-ciemss.vue
index 01830cea52..0ca07c1c7f 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-node-ciemss.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-node-ciemss.vue
@@ -1,8 +1,10 @@
 <template>
 	<main>
-		<template v-if="!inProgressCalibrationId && runResult && csvAsset && runResultPre && selectedVariables.length">
+		<template
+			v-if="!inProgressCalibrationId && runResult && csvAsset && runResultPre && selectedVariableSettings.length"
+		>
 			<vega-chart
-				v-for="(_var, index) of selectedVariables"
+				v-for="(_var, index) of selectedVariableSettings"
 				:key="index"
 				:are-embed-actions-visible="false"
 				:visualization-spec="preparedCharts[index]"
@@ -50,15 +52,19 @@ import {
 	Model,
 	ModelConfiguration,
 	SemanticType,
-	InferredParameterSemantic
+	InferredParameterSemantic,
+	ChartAnnotation,
+	ClientEventType
 } from '@/types/Types';
 import { ChartSettingType } from '@/types/common';
 import { createLLMSummary } from '@/services/summary-service';
-import { createForecastChart } from '@/services/charts';
+import { applyForecastChartAnnotations, createForecastChart } from '@/services/charts';
 import VegaChart from '@/components/widgets/VegaChart.vue';
 import * as stats from '@/utils/stats';
 import { createDatasetFromSimulationResult } from '@/services/dataset';
 import { useProjects } from '@/composables/project';
+import { fetchAnnotations } from '@/services/chart-settings';
+import { useClientEvent } from '@/composables/useClientEvent';
 import type { CalibrationOperationStateCiemss } from './calibrate-operation';
 import { CalibrationOperationCiemss } from './calibrate-operation';
 import { renameFnGenerator, mergeResults } from './calibrate-utils';
@@ -87,10 +93,8 @@ const chartSize = { width: 180, height: 120 };
 
 let lossValues: { [key: string]: number }[] = [];
 
-const selectedVariables = computed(() =>
-	(props.node.state.chartSettings ?? [])
-		.filter((setting) => setting.type === ChartSettingType.VARIABLE_COMPARISON)
-		.map((setting) => setting.selectedVariables[0])
+const selectedVariableSettings = computed(() =>
+	(props.node.state.chartSettings ?? []).filter((setting) => setting.type === ChartSettingType.VARIABLE_COMPARISON)
 );
 
 const lossChartSpec = ref();
@@ -161,43 +165,57 @@ const preparedCharts = computed(() => {
 	// Need to get the dataset's time field
 	const datasetTimeField = state.mapping.find((d) => d.modelVariable === 'timestamp')?.datasetVariable;
 
-	return selectedVariables.value.map((variable) => {
+	return selectedVariableSettings.value.map((setting) => {
+		const variable = setting.selectedVariables[0];
 		const datasetVariables: string[] = [];
 		const mapObj = state.mapping.find((d) => d.modelVariable === variable);
 		if (mapObj) {
 			datasetVariables.push(mapObj.datasetVariable);
 		}
-
-		return createForecastChart(
-			{
-				data: result,
-				variables: [`${pyciemssMap[variable]}:pre`, pyciemssMap[variable]],
-				timeField: 'timepoint_id',
-				groupField: 'sample_id'
-			},
-			{
-				data: resultSummary,
-				variables: [`${pyciemssMap[variable]}_mean:pre`, `${pyciemssMap[variable]}_mean`],
-				timeField: 'timepoint_id'
-			},
-			{
-				data: groundTruth,
-				variables: datasetVariables,
-				timeField: datasetTimeField as string
-			},
-			{
-				title: '',
-				legend: true,
-				translationMap: reverseMap,
-				xAxisTitle: modelVarUnits.value._time || 'Time',
-				yAxisTitle: modelVarUnits.value[variable] || '',
-				colorscheme: ['#AAB3C6', '#1B8073'],
-				...chartSize
-			}
+		const annotations = chartAnnotations.value.filter((annotation) => annotation.chartId === setting.id);
+
+		return applyForecastChartAnnotations(
+			createForecastChart(
+				{
+					data: result,
+					variables: [`${pyciemssMap[variable]}:pre`, pyciemssMap[variable]],
+					timeField: 'timepoint_id',
+					groupField: 'sample_id'
+				},
+				{
+					data: resultSummary,
+					variables: [`${pyciemssMap[variable]}_mean:pre`, `${pyciemssMap[variable]}_mean`],
+					timeField: 'timepoint_id'
+				},
+				{
+					data: groundTruth,
+					variables: datasetVariables,
+					timeField: datasetTimeField as string
+				},
+				{
+					title: '',
+					legend: true,
+					translationMap: reverseMap,
+					xAxisTitle: modelVarUnits.value._time || 'Time',
+					yAxisTitle: modelVarUnits.value[variable] || '',
+					colorscheme: ['#AAB3C6', '#1B8073'],
+					...chartSize
+				}
+			),
+			annotations
 		);
 	});
 });
 
+// --- Handle chart annotations
+const chartAnnotations = ref<ChartAnnotation[]>([]);
+const updateChartAnnotations = async () => {
+	chartAnnotations.value = await fetchAnnotations(props.node.id);
+};
+onMounted(() => updateChartAnnotations());
+useClientEvent([ClientEventType.ChartAnnotationCreate, ClientEventType.ChartAnnotationDelete], updateChartAnnotations);
+// ---
+
 const poller = new Poller();
 const pollResult = async (runId: string) => {
 	poller

From 60ba70f4195678ed2e0a7b988ba756cffdddf41b Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 14:37:58 -0400
Subject: [PATCH 06/13] Updated llm endpoint url

---
 packages/client/hmi-client/src/services/chart-settings.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index 10818d02f4..44a8fdc417 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -237,15 +237,13 @@ export async function generateForecastChartAnnotation(
     - Assume all unknown variables except the time field are for the y-axis and are renamed to the valueField.
 
      Give me the layer object to be added to the existing chart spec based on the following user request.
-		 Please return only a JSON object as a response. Make sure to return plain JSON object that can be parsed as JSON. Do not include code block.
 
 		Request:
     ${request}
     Answer
     {
 	`;
-	// FIXME: Use dedicated endpoint for annotation generation that's configured with JSON response_format instead of using the summary endpoint which is for text output
-	const { data } = await API.post(`/gollm/generate-summary?mode=SYNC`, prompt, {
+	const { data } = await API.post(`/gollm/generate-response?mode=SYNC&response-format=json`, prompt, {
 		headers: {
 			'Content-Type': 'application/json'
 		}

From 2e02bf6fe883acb28c4ce602934f97629e3e65a9 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 15:09:41 -0400
Subject: [PATCH 07/13] Fixed variable names

---
 .../ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index 8c90b4c648..70c95f0bf7 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -613,7 +613,7 @@ const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	const annotationLayerSpec = await generateForecastChartAnnotation(
 		query,
 		'timpoint_id',
-		[`${pyciemssMap.value[variable]}:pre`, `${pyciemssMap.value[variable]}`],
+		[`${pyciemssMap.value[variable]}_mean:pre`, `${pyciemssMap.value[variable]}_mean`],
 		{ x: modelVarUnits.value._time || 'Time', y: modelVarUnits.value[variable] || '' }
 	);
 	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
@@ -678,7 +678,7 @@ const preparedCharts = computed(() => {
 				},
 				{
 					data: resultSummary,
-					variables: [`${pyciemssMap.value[variable]}:pre`, `${pyciemssMap.value[variable]}_mean`],
+					variables: [`${pyciemssMap.value[variable]}_mean:pre`, `${pyciemssMap.value[variable]}_mean`],
 					timeField: 'timepoint_id'
 				},
 				{

From e5c5498e855fd0cb0bcf4f04d7cdcbacb7e6b823 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 15:26:41 -0400
Subject: [PATCH 08/13] Pass translate map to the prompt

---
 .../tera-calibrate-ciemss-drilldown.vue                |  8 +++++++-
 .../client/hmi-client/src/services/chart-settings.ts   | 10 +++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index 70c95f0bf7..703a0ad280 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -609,12 +609,18 @@ useClientEvent([ClientEventType.ChartAnnotationCreate, ClientEventType.ChartAnno
 
 const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	// Note: Currently llm generated chart annotations are supported for the forecast chart only
+	if (!preparedChartInputs.value) return {};
+	const { reverseMap } = preparedChartInputs.value;
 	const variable = setting.selectedVariables[0];
 	const annotationLayerSpec = await generateForecastChartAnnotation(
 		query,
 		'timpoint_id',
 		[`${pyciemssMap.value[variable]}_mean:pre`, `${pyciemssMap.value[variable]}_mean`],
-		{ x: modelVarUnits.value._time || 'Time', y: modelVarUnits.value[variable] || '' }
+		{
+			translationMap: reverseMap,
+			xAxisTitle: modelVarUnits.value._time || 'Time',
+			yAxisTitle: modelVarUnits.value[variable] || ''
+		}
 	);
 	const saved = await saveAnnotation(annotationLayerSpec, props.node.id, setting.id);
 	return saved;
diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index 44a8fdc417..e6e47c0a71 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -3,6 +3,7 @@ import type { ChartSetting, ChartSettingType } from '@/types/common';
 import { v4 as uuidv4 } from 'uuid';
 import { b64DecodeUnicode } from '@/utils/binary';
 import { ChartAnnotation } from '@/types/Types';
+import { ForecastChartOptions } from './charts';
 
 export interface LLMGeneratedChartAnnotation {
 	request: string;
@@ -86,8 +87,14 @@ export async function generateForecastChartAnnotation(
 	request: string,
 	timeField: string,
 	variables: string[],
-	axisTitle: { x: string; y: string }
+	options: Partial<ForecastChartOptions>
 ): Promise<LLMGeneratedChartAnnotation> {
+	const axisTitle = {
+		x: options.xAxisTitle ?? '',
+		y: options.yAxisTitle ?? ''
+	};
+	const translateMap = options.translationMap ?? {};
+
 	const prompt = `
 	  You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
 
@@ -235,6 +242,7 @@ export async function generateForecastChartAnnotation(
           ]
         }
     - Assume all unknown variables except the time field are for the y-axis and are renamed to the valueField.
+		- Leverage this variable to human readable name mapping: ${JSON.stringify(translateMap)} if needed.
 
      Give me the layer object to be added to the existing chart spec based on the following user request.
 

From f7d0d5f9124b18d9dc561b3325dc7a0c5f6088ae Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 15:28:36 -0400
Subject: [PATCH 09/13] Fixed axis

---
 packages/client/hmi-client/src/temp/Vegalite.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/hmi-client/src/temp/Vegalite.vue b/packages/client/hmi-client/src/temp/Vegalite.vue
index ef1b406047..e80efb036b 100644
--- a/packages/client/hmi-client/src/temp/Vegalite.vue
+++ b/packages/client/hmi-client/src/temp/Vegalite.vue
@@ -172,7 +172,7 @@ const generateAndAddAnnotation = async () => {
 	waitingForForecastChartAnnotation.value = true;
 	const timeField = 'time';
 	const variables = ['alphaMean', 'betaMean'];
-	const axisTitle = { x: 'Day', y: 'Value' };
+	const axisTitle = { xAxisTitle: 'Day', yAxisTitle: 'Value' };
 	const { request, layerSpec } = await generateForecastChartAnnotation(
 		questionString.value,
 		timeField,

From 89d9e70becce33fc412b4130d94669190b8b69e8 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Fri, 13 Sep 2024 15:33:42 -0400
Subject: [PATCH 10/13] Include only necessary variables

---
 packages/client/hmi-client/src/services/chart-settings.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index e6e47c0a71..96882c53ea 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -1,3 +1,4 @@
+import _ from 'lodash';
 import API from '@/api/api';
 import type { ChartSetting, ChartSettingType } from '@/types/common';
 import { v4 as uuidv4 } from 'uuid';
@@ -93,8 +94,7 @@ export async function generateForecastChartAnnotation(
 		x: options.xAxisTitle ?? '',
 		y: options.yAxisTitle ?? ''
 	};
-	const translateMap = options.translationMap ?? {};
-
+	const translateMap = _.pick(options.translationMap ?? {}, variables);
 	const prompt = `
 	  You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
 

From 339ac71ea4ecee1b935831bfd72dccfbc157f4c2 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Thu, 19 Sep 2024 11:24:19 -0400
Subject: [PATCH 11/13] Fixed indentation

---
 .../hmi-client/src/services/chart-settings.ts      | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index 96882c53ea..98a970cf00 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -137,9 +137,9 @@ export async function generateForecastChartAnnotation(
             "type": "rule",
             "strokeDash": [4, 4]
           },
-					"encoding": {
-						"x": { "datum": 200, "axis": { "title": ""} }
-					}
+          "encoding": {
+            "x": { "datum": 200, "axis": { "title": ""} }
+          }
         },
         {
           "mark": {
@@ -149,7 +149,7 @@ export async function generateForecastChartAnnotation(
             "dy": -5
           },
           "encoding": {
-						"x": { "datum": 200, "axis": { "title": ""} }
+            "x": { "datum": 200, "axis": { "title": ""} }
             "text": {"value": "important"}
           }
         }
@@ -242,11 +242,11 @@ export async function generateForecastChartAnnotation(
           ]
         }
     - Assume all unknown variables except the time field are for the y-axis and are renamed to the valueField.
-		- Leverage this variable to human readable name mapping: ${JSON.stringify(translateMap)} if needed.
+    - Leverage this variable to human readable name mapping: ${JSON.stringify(translateMap)} if needed.
 
-     Give me the layer object to be added to the existing chart spec based on the following user request.
+    Give me the layer object to be added to the existing chart spec based on the following user request.
 
-		Request:
+    Request:
     ${request}
     Answer
     {

From 550d980cdd732ff39103d873929e8ceb122974af Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Thu, 19 Sep 2024 11:25:22 -0400
Subject: [PATCH 12/13] Fixed indentation

---
 packages/client/hmi-client/src/services/chart-settings.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/hmi-client/src/services/chart-settings.ts b/packages/client/hmi-client/src/services/chart-settings.ts
index 98a970cf00..7a6281fb09 100644
--- a/packages/client/hmi-client/src/services/chart-settings.ts
+++ b/packages/client/hmi-client/src/services/chart-settings.ts
@@ -96,7 +96,7 @@ export async function generateForecastChartAnnotation(
 	};
 	const translateMap = _.pick(options.translationMap ?? {}, variables);
 	const prompt = `
-	  You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
+    You are an agent who is an expert in Vega-Lite chart specs. Provide a Vega-Lite layer JSON object for the annotation that can be added to an existing chart spec to satisfy the provided user request.
 
     - The Vega-Lite schema version you must use is https://vega.github.io/schema/vega-lite/v5.json.
     - Assume that you don’t know the exact data points from the data.

From 885929b9a1551ca620b28822c4aec11fff2ca608 Mon Sep 17 00:00:00 2001
From: Jaehwan Ryu <jryu@uncharted.software>
Date: Thu, 19 Sep 2024 11:41:14 -0400
Subject: [PATCH 13/13] Fixed type errors

---
 .../src/components/widgets/tera-chart-settings-panel.vue        | 2 +-
 .../ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue    | 2 +-
 packages/client/hmi-client/src/temp/Vegalite.vue                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/client/hmi-client/src/components/widgets/tera-chart-settings-panel.vue b/packages/client/hmi-client/src/components/widgets/tera-chart-settings-panel.vue
index 9e6e1028ab..9863fab1ff 100644
--- a/packages/client/hmi-client/src/components/widgets/tera-chart-settings-panel.vue
+++ b/packages/client/hmi-client/src/components/widgets/tera-chart-settings-panel.vue
@@ -55,7 +55,7 @@ const props = defineProps<{
 	 * @param setting ChartSetting
 	 * @param query llm query to generate annotation
 	 */
-	generateAnnotation?: (setting: ChartSetting, query: string) => Promise<ChartAnnotation>;
+	generateAnnotation?: (setting: ChartSetting, query: string) => Promise<ChartAnnotation | null>;
 }>();
 
 const emit = defineEmits(['close', 'update:settings', 'delete-annotation', 'create-annotation']);
diff --git a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
index 1a88a6f23f..04ab681fde 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue
@@ -628,7 +628,7 @@ useClientEvent([ClientEventType.ChartAnnotationCreate, ClientEventType.ChartAnno
 
 const generateAnnotation = async (setting: ChartSetting, query: string) => {
 	// Note: Currently llm generated chart annotations are supported for the forecast chart only
-	if (!preparedChartInputs.value) return {};
+	if (!preparedChartInputs.value) return null;
 	const { reverseMap } = preparedChartInputs.value;
 	const variable = setting.selectedVariables[0];
 	const annotationLayerSpec = await generateForecastChartAnnotation(
diff --git a/packages/client/hmi-client/src/temp/Vegalite.vue b/packages/client/hmi-client/src/temp/Vegalite.vue
index e80efb036b..68e568c452 100644
--- a/packages/client/hmi-client/src/temp/Vegalite.vue
+++ b/packages/client/hmi-client/src/temp/Vegalite.vue
@@ -4,7 +4,7 @@
 			<vega-chart expandable :visualization-spec="forecastChartSpec" />
 			<div>
 				<div v-for="a in forecastAnnotations" :key="a.id">
-					{{ a.layerSpec.description }} <i class="pi pi-trash pi-trash" @click="removeAnnotation(a.id)" />
+					{{ a.layerSpec.description }} <i class="pi pi-trash pi-trash" @click="removeAnnotation(a.id ?? '')" />
 				</div>
 			</div>
 			<div style="display: flex; flex-direction: col">