Refactor and clean up of functions related to csv asssets (#5984)

packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-ciemss-drilldown.vue

@@ -490,13 +490,7 @@ import DataTable from 'primevue/datatable';
 import Dropdown from 'primevue/dropdown';
 import Column from 'primevue/column';
 import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
-import {
-	CalibrateMap,
-	setupDatasetInput,
-	setupCsvAsset,
-	setupModelInput,
-	parseCsvAsset
-} from '@/services/calibrate-workflow';
+import { CalibrateMap, setupDatasetInput, setupCsvAsset, setupModelInput } from '@/services/calibrate-workflow';
 import { deleteAnnotation, updateChartSettingsBySelectedVariables } from '@/services/chart-settings';
 import TeraDrilldown from '@/components/drilldown/tera-drilldown.vue';
 import TeraDrilldownSection from '@/components/drilldown/tera-drilldown-section.vue';
@@ -548,6 +542,7 @@ import { getDataset } from '@/services/dataset';
 import { getCalendarSettingsFromModel } from '@/services/model';
 import { useCharts } from '@/composables/useCharts';
 import { useChartSettings } from '@/composables/useChartSettings';
+import { parseCsvAsset } from '@/utils/csv';
 import type { CalibrationOperationStateCiemss } from './calibrate-operation';
 import { renameFnGenerator, getErrorData, usePreparedChartInputs, getSelectedOutputMapping } from './calibrate-utils';
 

packages/client/hmi-client/src/components/workflow/ops/calibrate-ciemss/tera-calibrate-node-ciemss.vue

@@ -47,7 +47,7 @@ import {
 	DataArray
 } from '@/services/models/simulation-service';
 import { getModelConfigurationById, createModelConfiguration } from '@/services/model-configurations';
-import { parseCsvAsset, setupCsvAsset } from '@/services/calibrate-workflow';
+import { setupCsvAsset } from '@/services/calibrate-workflow';
 import { getModelByModelConfigurationId, getUnitsFromModelParts } from '@/services/model';
 import { nodeMetadata, nodeOutputLabel } from '@/components/workflow/util';
 import { logger } from '@/utils/logger';
@@ -75,6 +75,7 @@ import { useChartSettings } from '@/composables/useChartSettings';
 import { useCharts } from '@/composables/useCharts';
 import { filterChartSettingsByVariables } from '@/services/chart-settings';
 import { ChartSettingType } from '@/types/common';
+import { parseCsvAsset } from '@/utils/csv';
 import type { CalibrationOperationStateCiemss } from './calibrate-operation';
 import { CalibrationOperationCiemss } from './calibrate-operation';
 import { renameFnGenerator, usePreparedChartInputs, getSelectedOutputMapping } from './calibrate-utils';

packages/client/hmi-client/src/components/workflow/ops/calibrate-ensemble-ciemss/tera-calibrate-ensemble-ciemss-drilldown.vue

@@ -345,7 +345,7 @@ import TeraInputNumber from '@/components/widgets/tera-input-number.vue';
 import AccordionTab from 'primevue/accordiontab';
 import Accordion from 'primevue/accordion';
 import Dropdown from 'primevue/dropdown';
-import { setupDatasetInput, setupCsvAsset, parseCsvAsset } from '@/services/calibrate-workflow';
+import { setupDatasetInput, setupCsvAsset } from '@/services/calibrate-workflow';
 import TeraDrilldown from '@/components/drilldown/tera-drilldown.vue';
 import TeraDrilldownSection from '@/components/drilldown/tera-drilldown-section.vue';
 import TeraSaveDatasetFromSimulation from '@/components/dataset/tera-save-dataset-from-simulation.vue';
@@ -378,6 +378,7 @@ import { useChartSettings } from '@/composables/useChartSettings';
 import { deleteAnnotation, updateChartSettingsBySelectedVariables } from '@/services/chart-settings';
 import { DataArray } from '@/utils/stats';
 import { GroupedDataArray } from '@/services/charts';
+import { parseCsvAsset } from '@/utils/csv';
 import {
 	CalibrateEnsembleCiemssOperationState,
 	CalibrateEnsembleMappingRow,

packages/client/hmi-client/src/components/workflow/ops/calibrate-ensemble-ciemss/tera-calibrate-ensemble-node-ciemss.vue

@@ -42,7 +42,7 @@ import {
 	getSimulation,
 	DataArray
 } from '@/services/models/simulation-service';
-import { parseCsvAsset, setupCsvAsset } from '@/services/calibrate-workflow';
+import { setupCsvAsset } from '@/services/calibrate-workflow';
 import { nodeMetadata, nodeOutputLabel } from '@/components/workflow/util';
 import { logger } from '@/utils/logger';
 import { Poller, PollerState } from '@/api/api';
@@ -55,6 +55,7 @@ import { useProjects } from '@/composables/project';
 import { useChartSettings } from '@/composables/useChartSettings';
 import { useCharts } from '@/composables/useCharts';
 import { GroupedDataArray } from '@/services/charts';
+import { parseCsvAsset } from '@/utils/csv';
 import {
 	CalibrateEnsembleCiemssOperation,
 	CalibrateEnsembleCiemssOperationState

packages/client/hmi-client/src/components/workflow/ops/compare-datasets/tera-compare-datasets-drilldown.vue

@@ -226,21 +226,6 @@ const activeIndices = ref([0, 1, 2]);
 const isFetchingDatasets = ref(false);
 const isATESelected = ref(false);
 
-const {
-	activeChartSettings,
-	chartSettings,
-	selectedVariableSettings,
-	removeChartSettings,
-	updateChartSettings,
-	updateActiveChartSettings,
-	setActiveChartSettings
-} = useChartSettings(props, emit);
-
-const outputPanel = ref(null);
-const chartSize = useDrilldownChartSize(outputPanel);
-
-const chartData = ref<ChartData | null>(null);
-
 const onRun = () => {
 	console.log('run');
 };
@@ -259,11 +244,6 @@ const knobs = ref<BasicKnobs>({
 	selectedPlotType: PlotValue.PERCENTAGE
 });
 
-const selectedPlotType = computed(() => knobs.value.selectedPlotType);
-const baselineName = computed(
-	() => datasets.value.find((dataset) => dataset.id === knobs.value.selectedDataset)?.name ?? null
-);
-
 const addCriteria = () => {
 	knobs.value.criteriaOfInterestCards.push(blankCriteriaOfInterest);
 };
@@ -276,6 +256,21 @@ const updateCriteria = (card: Partial<CriteriaOfInterestCard>, index: number) =>
 	Object.assign(knobs.value.criteriaOfInterestCards[index], card);
 };
 
+const {
+	activeChartSettings,
+	chartSettings,
+	selectedVariableSettings,
+	removeChartSettings,
+	updateChartSettings,
+	updateActiveChartSettings,
+	setActiveChartSettings
+} = useChartSettings(props, emit);
+
+const outputPanel = ref(null);
+const chartSize = useDrilldownChartSize(outputPanel);
+
+const chartData = ref<ChartData | null>(null);
+
 const { generateAnnotation, getChartAnnotationsByChartId, useCompareDatasetCharts } = useCharts(
 	props.node.id,
 	null,
@@ -285,6 +280,10 @@ const { generateAnnotation, getChartAnnotationsByChartId, useCompareDatasetChart
 	null,
 	null
 );
+const selectedPlotType = computed(() => knobs.value.selectedPlotType);
+const baselineName = computed(
+	() => datasets.value.find((dataset) => dataset.id === knobs.value.selectedDataset)?.name ?? null
+);
 const variableCharts = useCompareDatasetCharts(selectedVariableSettings, selectedPlotType, baselineName);
 
 const initialize = async () => {
@@ -336,6 +335,7 @@ function findDuplicates(strings: string[]): string[] {
 
 async function generateChartData() {
 	if (datasets.value.length <= 1) return;
+	console.log(datasets);
 
 	const rawContents = await Promise.all(
 		datasets.value.map((dataset) => {

packages/client/hmi-client/src/services/calibrate-workflow.ts

@@ -1,7 +1,6 @@
-import { autoType, csvParse } from 'd3';
 import type { Dataset, CsvAsset } from '@/types/Types';
 import { getModelConfigurationById, getObservables } from '@/services/model-configurations';
-import { downloadRawFile } from '@/services/dataset';
+import { getCsvAsset } from '@/services/dataset';
 import { getUnitsFromModelParts, getModelByModelConfigurationId, getTypesFromModelParts } from '@/services/model';
 
 export interface CalibrateMap {
@@ -56,22 +55,8 @@ export const setupCsvAsset = async (dataset: Dataset): Promise<CsvAsset | undefi
 	// FIXME: We are setting the limit to -1 (i.e. no limit) on the number of rows returned.
 	// This is a temporary fix since the datasets could be very large.
 	const limit = -1;
-
-	// We are assuming here there is only a single csv file. This may change in the future as the API allows for it.
-	if (!(dataset.metadata?.format === 'netcdf') || !dataset.esgfId) {
-		const csv = (await downloadRawFile(dataset.id as string, filename, limit)) as CsvAsset;
-		csv.headers = csv.headers.map((header) => header.trim());
-		return csv;
-	}
-	return undefined;
-};
-
-export const parseCsvAsset = (csvAsset: CsvAsset) => {
-	if (!csvAsset) return [];
-	const csv = [csvAsset.headers, ...csvAsset.csv];
-	const csvRaw = csv.map((d) => d.join(',')).join('\n');
-	const parsedCsv = csvParse(csvRaw, autoType);
-	return parsedCsv;
+	const csv = await getCsvAsset(dataset, filename, limit);
+	return csv ?? undefined;
 };
 
 const getFileName = (dataset: Dataset) =>

packages/client/hmi-client/src/services/dataset.ts

@@ -9,6 +9,8 @@ import type { CsvAsset, CsvColumnStats, Dataset, PresignedURL } from '@/types/Ty
 import { Ref } from 'vue';
 import { AxiosResponse } from 'axios';
 import { RunResults } from '@/types/SimulateConfig';
+import { FIFOCache } from '@/utils/FifoCache';
+import { parseCsvAsset } from '@/utils/csv';
 
 /**
  * Get Dataset from the data service
@@ -98,17 +100,28 @@ async function getBulkDatasets(datasetIDs: string[]) {
 	return result;
 }
 
+const rawFileCache = new FIFOCache<Promise<CsvAsset | null>>(100);
 /**
  * Get the raw (CSV) file content for a given dataset
  * @return Array<string>|null - the dataset raw content, or null if none returned by API
  */
 async function downloadRawFile(datasetId: string, filename: string, limit: number = 100): Promise<CsvAsset | null> {
-	const URL = `/datasets/${datasetId}/download-csv?filename=${filename}&limit=${limit}`;
-	console.log('URL', URL);
-	const response = await API.get(URL).catch((error) => {
-		logger.error(`Error: data-service was not able to retrieve the dataset's rawfile ${error}`);
-	});
-	return response?.data ?? null;
+	const cacheKey = `${datasetId}:${filename}:${limit}`;
+	let promise = rawFileCache.get(cacheKey);
+
+	if (!promise) {
+		const URL = `/datasets/${datasetId}/download-csv?filename=${filename}&limit=${limit}`;
+		console.log('URL', URL);
+		promise = API.get(URL)
+			.then((response) => response?.data ?? null)
+			.catch((error) => {
+				logger.error(`Error: data-service was not able to retrieve the dataset's rawfile ${error}`);
+				return null;
+			});
+		rawFileCache.set(cacheKey, promise);
+	}
+
+	return promise;
 }
 
 /**
@@ -368,6 +381,24 @@ async function getRawContent(
 	return null;
 }
 
+async function getCsvAsset(dataset: Dataset, filename: string, limit: number = -1): Promise<CsvAsset | null> {
+	// If it's an ESGF dataset or a NetCDF file, we don't want to download the raw content
+	if (!dataset?.id || dataset.esgfId || dataset.metadata?.format === 'netcdf') return null;
+	const csv = (await downloadRawFile(dataset.id as string, filename, limit)) as CsvAsset;
+	csv.headers = csv.headers.map((header) => header.trim());
+	return csv;
+}
+
+async function getDatasetResultCSV(dataset: Dataset, filename: string, renameFn?: (s: string) => string) {
+	const csvAsset = await getCsvAsset(dataset, filename);
+	if (!csvAsset) return [];
+	if (renameFn) {
+		csvAsset.headers = csvAsset.headers.map(renameFn);
+	}
+	const output = parseCsvAsset(csvAsset);
+	return output;
+}
+
 export {
 	getDataset,
 	getClimateDataset,
@@ -383,5 +414,7 @@ export {
 	saveDataset,
 	createCsvAssetFromRunResults,
 	createDataset,
-	getRawContent
+	getRawContent,
+	getCsvAsset,
+	getDatasetResultCSV
 };

packages/client/hmi-client/src/utils/csv.ts

@@ -0,0 +1,10 @@
+import { autoType, csvParse } from 'd3';
+import type { CsvAsset } from '@/types/Types';
+
+export const parseCsvAsset = (csvAsset: CsvAsset) => {
+	if (!csvAsset) return [];
+	const csv = [csvAsset.headers, ...csvAsset.csv];
+	const csvRaw = csv.map((d) => d.join(',')).join('\n');
+	const parsedCsv = csvParse(csvRaw, autoType);
+	return parsedCsv;
+};