Skip to content

Commit

Permalink
fix: import arrow dynamically to avoid failed loading
Browse files Browse the repository at this point in the history
error will only be reported when using arrow backend

Fixes #83
  • Loading branch information
dvirtz committed Jul 30, 2023
1 parent 1e5a261 commit 2774fe1
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions src/arrow-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { PassThrough } from 'stream';
import { CancellationToken } from 'vscode';
import { ParquetBackend } from './parquet-backend';
import { jsonSpace } from './settings';
import { readParquet } from 'parquet-reader';
import { Stream } from 'stream';

function bigIntToJson(value: bigint) {
// serialize as a number if it's in bounds, otherwise as a string
Expand All @@ -14,9 +14,23 @@ function bigIntToJson(value: bigint) {
}

export class ArrowBackend extends ParquetBackend {
readParquet_: ((path: string, stream: Stream) => void) | undefined;

private async readParquet(path: string, stream: Stream) {
if (typeof (this.readParquet_) == 'undefined') {
try {
const module = await import("parquet-reader");
this.readParquet_ = module.readParquet;
} catch (_) {
throw new Error('cannot find prebuilt arrow module, either build the module or use another backend');
}
}
this.readParquet_(path, stream);
}

public async * toJsonImpl(parquetPath: string, _token?: CancellationToken): AsyncGenerator<string> {
const stream = new PassThrough;
readParquet(parquetPath, stream);
await this.readParquet(parquetPath, stream);
const batches = await AsyncRecordBatchStreamReader.from(stream);

// read all records from the file and print them
Expand Down

0 comments on commit 2774fe1

Please sign in to comment.