Merge pull request #598 from jvalue/394-feature-save-table-as-csv

Export tables into csv files using fast-csv, the same library that parses the csv in CSVIntperpreter.

closes #394

libs/extensions/tabular/exec/.eslintrc.json

@@ -2,7 +2,11 @@
   "extends": ["../../../../.eslintrc.json"],
   "ignorePatterns": ["!**/*"],
   "parserOptions": {
-    "project": ["libs/extensions/tabular/exec/tsconfig.lib.json", "libs/extensions/tabular/exec/tsconfig.spec.json"]
+    "project": [
+      "libs/extensions/tabular/exec/tsconfig.lib.json",
+      "libs/extensions/tabular/exec/tsconfig.spec.json",
+      "libs/extensions/tabular/exec/tsconfig.mock.json"
+    ]
   },
   "overrides": [
     {

libs/extensions/tabular/exec/__mock__/fs.ts

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { fs } from 'memfs';
+
+export default fs;

libs/extensions/tabular/exec/__mock__/fs/promises.ts

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { fs } from 'memfs';
+
+export default fs.promises;

libs/extensions/tabular/exec/src/extension.ts

@@ -10,6 +10,7 @@ import {
 import { CellRangeSelectorExecutor } from './lib/cell-range-selector-executor';
 import { CellWriterExecutor } from './lib/cell-writer-executor';
 import { ColumnDeleterExecutor } from './lib/column-deleter-executor';
+import { CSVFileLoaderExecutor } from './lib/csv-file-loader-executor';
 import { CSVInterpreterExecutor } from './lib/csv-interpreter-executor';
 import { RowDeleterExecutor } from './lib/row-deleter-executor';
 import { SheetPickerExecutor } from './lib/sheet-picker-executor';
@@ -26,6 +27,7 @@ export class TabularExecExtension extends JayveeExecExtension {
       CellRangeSelectorExecutor,
       TableInterpreterExecutor,
       CSVInterpreterExecutor,
+      CSVFileLoaderExecutor,
       TableTransformerExecutor,
       XLSXInterpreterExecutor,
       SheetPickerExecutor,

libs/extensions/tabular/exec/src/lib/csv-file-loader-executor.spec.ts

@@ -0,0 +1,207 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import * as fsPromise from 'node:fs/promises';
+import path from 'node:path';
+
+import * as R from '@jvalue/jayvee-execution';
+import {
+  constructTable,
+  getTestExecutionContext,
+} from '@jvalue/jayvee-execution/test';
+import {
+  type BlockDefinition,
+  IOType,
+  type JayveeServices,
+  createJayveeServices,
+} from '@jvalue/jayvee-language-server';
+import {
+  type ParseHelperOptions,
+  expectNoParserAndLexerErrors,
+  loadTestExtensions,
+  parseHelper,
+  readJvTestAssetHelper,
+} from '@jvalue/jayvee-language-server/test';
+import {
+  type AstNode,
+  type AstNodeLocator,
+  type LangiumDocument,
+} from 'langium';
+import { NodeFileSystem } from 'langium/node';
+import { vol } from 'memfs';
+import { vi } from 'vitest';
+
+import { CSVFileLoaderExecutor } from './csv-file-loader-executor';
+
+describe('Validation of CSVFileLoaderExecutor', () => {
+  let parse: (
+    input: string,
+    options?: ParseHelperOptions,
+  ) => Promise<LangiumDocument<AstNode>>;
+
+  let locator: AstNodeLocator;
+  let services: JayveeServices;
+
+  const readJvTestAsset = readJvTestAssetHelper(
+    __dirname,
+    '../../test/assets/csv-file-loader-executor/',
+  );
+
+  async function parseAndExecuteExecutor(
+    input: string,
+    IOInput: R.Table,
+  ): Promise<R.Result<R.None>> {
+    const document = await parse(input, { validation: true });
+    expectNoParserAndLexerErrors(document);
+
+    const block = locator.getAstNode<BlockDefinition>(
+      document.parseResult.value,
+      'pipelines@0/blocks@1',
+    ) as BlockDefinition;
+
+    return new CSVFileLoaderExecutor().doExecute(
+      IOInput,
+      getTestExecutionContext(locator, document, services, [block]),
+    );
+  }
+
+  beforeAll(async () => {
+    // Create language services
+    services = createJayveeServices(NodeFileSystem).Jayvee;
+    await loadTestExtensions(services, [
+      path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'),
+    ]);
+    locator = services.workspace.AstNodeLocator;
+    // Parse function for Jayvee (without validation)
+    parse = parseHelper(services);
+  });
+  beforeEach(() => {
+    // NOTE: The virtual filesystem is reset before each test
+    vol.reset();
+  });
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should diagnose no error on valid loader config', async () => {
+    const text = readJvTestAsset('valid-csv-file-loader.jv');
+
+    const inputTable = constructTable(
+      [
+        {
+          columnName: 'Column1',
+          column: {
+            values: ['somestring'],
+            valueType: services.ValueTypeProvider.Primitives.Text,
+          },
+        },
+        {
+          columnName: 'Column2',
+          column: {
+            values: [20.2],
+            valueType: services.ValueTypeProvider.Primitives.Decimal,
+          },
+        },
+      ],
+      1,
+    );
+    const result = await parseAndExecuteExecutor(text, inputTable);
+
+    expect(R.isErr(result)).toEqual(false);
+    if (R.isOk(result)) {
+      expect(result.right.ioType).toEqual(IOType.NONE);
+      const expectedOutput = `Column1,Column2
+somestring,20.2`;
+      const actualOutput = await fsPromise.readFile('test.csv');
+      expect(actualOutput.toString()).toEqual(expectedOutput);
+    }
+  });
+  it('should handle all allowed jayvee representations', async () => {
+    const text = readJvTestAsset('valid-csv-file-loader.jv');
+
+    const inputTable = constructTable(
+      [
+        {
+          columnName: 'Strings',
+          column: {
+            values: ['somestring'],
+            valueType: services.ValueTypeProvider.Primitives.Text,
+          },
+        },
+        {
+          columnName: 'Decimals',
+          column: {
+            values: [20.2],
+            valueType: services.ValueTypeProvider.Primitives.Decimal,
+          },
+        },
+        {
+          columnName: 'Booleans',
+          column: {
+            values: [true],
+            valueType: services.ValueTypeProvider.Primitives.Boolean,
+          },
+        },
+        {
+          columnName: 'Integers',
+          column: {
+            values: [-10],
+            valueType: services.ValueTypeProvider.Primitives.Integer,
+          },
+        },
+      ],
+      1,
+    );
+    const result = await parseAndExecuteExecutor(text, inputTable);
+
+    expect(R.isErr(result)).toEqual(false);
+    if (R.isOk(result)) {
+      expect(result.right.ioType).toEqual(IOType.NONE);
+      const expectedOutput = `Strings,Decimals,Booleans,Integers
+somestring,20.2,true,-10`;
+      const actualOutput = await fsPromise.readFile('test.csv');
+      expect(actualOutput.toString()).toEqual(expectedOutput);
+    }
+  });
+  it('should diagnose no error with user definded properties', async () => {
+    const text = readJvTestAsset('escaping-csv-file-loader.jv');
+
+    const inputTable = constructTable(
+      [
+        {
+          columnName: 'Quoted',
+          column: {
+            values: ['quoted;'],
+            valueType: services.ValueTypeProvider.Primitives.Text,
+          },
+        },
+        {
+          columnName: 'Escaped',
+          column: {
+            values: ['escaped"'],
+            valueType: services.ValueTypeProvider.Primitives.Text,
+          },
+        },
+        {
+          columnName: 'Regular',
+          column: {
+            values: ['regular'],
+            valueType: services.ValueTypeProvider.Primitives.Boolean,
+          },
+        },
+      ],
+      1,
+    );
+    const result = await parseAndExecuteExecutor(text, inputTable);
+
+    expect(R.isErr(result)).toEqual(false);
+    if (R.isOk(result)) {
+      expect(result.right.ioType).toEqual(IOType.NONE);
+      const expectedOutput = `Quoted;Escaped;Regular
+"quoted;";"escaped\\"";regular`;
+      const actualOutput = await fsPromise.readFile('test.csv');
+      expect(actualOutput.toString()).toEqual(expectedOutput);
+    }
+  });
+});

libs/extensions/tabular/exec/src/lib/csv-file-loader-executor.ts

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// eslint-disable-next-line unicorn/prefer-node-protocol
+import assert from 'assert';
+import * as fsPromise from 'node:fs/promises';
+
+import {
+  type FormatterOptionsArgs,
+  type Row,
+  writeToBuffer as writeToCSVBuffer,
+} from '@fast-csv/format';
+import * as R from '@jvalue/jayvee-execution';
+import {
+  AbstractBlockExecutor,
+  type BlockExecutorClass,
+  type ExecutionContext,
+  type Table,
+  implementsStatic,
+} from '@jvalue/jayvee-execution';
+import {
+  IOType,
+  type InternalValueRepresentation,
+} from '@jvalue/jayvee-language-server';
+
+@implementsStatic<BlockExecutorClass>()
+export class CSVFileLoaderExecutor extends AbstractBlockExecutor<
+  IOType.TABLE,
+  IOType.NONE
+> {
+  public static readonly type = 'CSVFileLoader';
+
+  constructor() {
+    super(IOType.TABLE, IOType.NONE);
+  }
+
+  async doExecute(
+    table: Table,
+    context: ExecutionContext,
+  ): Promise<R.Result<R.None>> {
+    const file = context.getPropertyValue(
+      'file',
+      context.valueTypeProvider.Primitives.Text,
+    );
+    const delimiter = context.getPropertyValue(
+      'delimiter',
+      context.valueTypeProvider.Primitives.Text,
+    );
+    const enclosing = context.getPropertyValue(
+      'enclosing',
+      context.valueTypeProvider.Primitives.Text,
+    );
+    const enclosingEscape = context.getPropertyValue(
+      'enclosingEscape',
+      context.valueTypeProvider.Primitives.Text,
+    );
+
+    const formatOptions: FormatterOptionsArgs<Row, Row> = {
+      delimiter,
+      quote: enclosing,
+      escape: enclosingEscape,
+      headers: getHeaders(table),
+    };
+
+    context.logger.logDebug(
+      `Generating csv using delimiter '${formatOptions.delimiter}', enclosing '${formatOptions.quote}' and escape '${formatOptions.escape}'`,
+    );
+    const csv = await writeToCSVBuffer(toRows(table), formatOptions);
+
+    context.logger.logDebug(`Writing csv to file ${file}`);
+    await fsPromise.writeFile(file, csv);
+    context.logger.logDebug(`The data was successfully written to ${file}`);
+
+    return R.ok(R.NONE);
+  }
+}
+
+function getHeaders(table: Table): string[] {
+  return [...table.getColumns().keys()];
+}
+
+function toRows(table: Table): Row[] {
+  const columns: InternalValueRepresentation[][] = [
+    ...table.getColumns().entries(),
+  ].map((column) => column[1].values);
+
+  return transposeArray(columns);
+}
+
+function transposeArray<T>(array: T[][]): T[][] {
+  if (array[0] === undefined) {
+    return [];
+  }
+  return array[0]?.map((_, colIndex) =>
+    array.map((row): T => {
+      const cell = row[colIndex];
+      assert(cell !== undefined);
+      return cell;
+    }),
+  );
+}

libs/extensions/tabular/exec/test/assets/csv-file-loader-executor/escaping-csv-file-loader.jv

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+pipeline TestPipeline {
+
+	block TestExtractor oftype TestFileExtractor { }
+
+	block TestBlock oftype CSVFileLoader {
+		file: "./test.csv";
+		delimiter: ";";
+		enclosing: '"';
+		enclosingEscape: "\\";
+	}
+
+	TestExtractor
+		-> TestBlock;
+}

libs/extensions/tabular/exec/test/assets/csv-file-loader-executor/valid-csv-file-loader.jv

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+pipeline TestPipeline {
+
+	block TestExtractor oftype TestFileExtractor { }
+
+	block TestBlock oftype CSVFileLoader {
+		file: "./test.csv";
+	}
+
+	TestExtractor
+		-> TestBlock;
+}