Skip to content

Commit

Permalink
feat: formatting JSON output
Browse files Browse the repository at this point in the history
closes #43
  • Loading branch information
dvirtz committed Jun 17, 2022
1 parent 608da7e commit bf846a5
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 2 deletions.
11 changes: 11 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,17 @@
"trace"
],
"default": "info"
},
"parquet-viewer.jsonSpace": {
"markdownDescription": "JSON indentation space, passed to `JSON.stringify` as is, see [mdn](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#parameters) for details. Only applies when `#parquet-viewer.useParquetTools#` is `false`.",
"type": [
"string",
"number"
],
"default": 0,
"minimum": 0,
"maximum": 10,
"maxLength": 10
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/parquets-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { getLogger } from './logger';
import { ParquetReader } from '@dvirtz/parquets';
import * as os from 'os';
import { ParquetBackend } from './parquet-backend';
import { jsonSpace } from './settings';

export class ParquetsBackend implements ParquetBackend {
public async * toJson(parquetPath: string, token?: vscode.CancellationToken): AsyncGenerator<string> {
Expand All @@ -20,7 +21,7 @@ export class ParquetsBackend implements ParquetBackend {
// read all records from the file and print them
let record = null;
while (!token?.isCancellationRequested && (record = await cursor.next())) {
yield `${JSON.stringify(record)}${os.EOL}`;
yield `${JSON.stringify(record, null, jsonSpace())}${os.EOL}`;
}

await reader.close();
Expand Down
4 changes: 4 additions & 0 deletions src/settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,7 @@ export async function setLogLevel(logLevel: LogLevel | undefined): Promise<void>
export function useParquetTools(): boolean {
return settings().get('useParquetTools', false);
}

export function jsonSpace(): number | string | undefined {
return settings().get('jsonSpace');
}
15 changes: 14 additions & 1 deletion test/unit/parquets-backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { createReadStream } from 'fs';
import * as path from 'path';
import { createInterface } from 'readline';
import { ParquetsBackend } from '../../src/parquets-backend';
import { jsonSpace } from '../../src/settings';


const rootDir = path.join(__dirname, '..', '..');
Expand All @@ -20,13 +21,15 @@ jest.mock('vscode', () => {
};
});

jest.mock('../../src/settings');

describe("ParquetsBackend tests", () => {
const backend = new ParquetsBackend();
const workspace = path.join(rootDir, 'test', 'workspace');

test.each(
["small", "large"]
)('Converts %s parquet to JSON', async function (name) {
const workspace = path.join(rootDir, 'test', 'workspace');
const json = (await toArray(backend.toJson(path.join(workspace, `${name}.parquet`)))).map(line => line.trim());
const expected = await toArray(createInterface({input: createReadStream(path.join(workspace, `${name}.json`))}));

Expand All @@ -38,4 +41,14 @@ describe("ParquetsBackend tests", () => {
'message': expect.stringMatching(/while reading no-such-file: Error: ENOENT: no such file or directory, stat '.*no-such-file'/)
});
});

test.each([0, 2, 10, "\t", "###"])('Test space %s', async function (space) {
jest.mocked(jsonSpace).mockReturnValue(space);

const json = (await toArray(ParquetsBackend.toJson(path.join(workspace, `small.parquet`)))).map(line => line.trim());
const records = (await promises.readFile(path.join(workspace, `small.json`), 'utf-8')).trim().split(os.EOL);
const expected = records.map(record => JSON.stringify(JSON.parse(record), null, space));

expect(json).toEqual(expected);
});
});

0 comments on commit bf846a5

Please sign in to comment.