-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
72 lines (62 loc) · 1.83 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
(async() => {
const fs = require("fs");
const { Transform } = require('stream');
const parquet = require('parquetjs');
const readline = require('readline');
const knex = require('knex')({
client: 'pg',
version: '10.5',
connection: {
host : 'localhost',
user : 'postgre',
password : 'post',
database : 'config',
port : '5433'
}
});
const toFile = fs.createWriteStream('./config.file');
const toJson = new Transform({
objectMode: true,
transform(chunk, encoding, callback) {
this.push(JSON.stringify(chunk))
callback()
}
});
await knex
.select('*')
.from('config')
.stream((stream) => {
stream
.pipe(toJson)
.pipe(toFile);
})
.then(() => {
toFile.end();
})
.catch((e) => { console.error(e); });
const schema = new parquet.ParquetSchema({
id: { type: 'INT64' },
schema_name: { type: 'UTF8' },
table_name: { type: 'UTF8' },
environment: { type: 'UTF8' },
primary_keys: { type: 'UTF8' },
full_load: { type: 'BOOLEAN' },
jndi_connection_id: { type: 'BOOLEAN' },
active: { type: 'BOOLEAN' }
});
let rl = readline.createInterface({
input: fs.createReadStream('./config.file')
});
let line_no = 0;
const writer = await parquet.ParquetWriter.openFile(schema, './config.parquet');
rl.on('line', function(line) {
line_no++;
const contentLine = JSON.parse(line);
writer.appendRow(contentLine);
});
rl.on('close', function(line) {
writer.close();
console.log('Total lines : ' + line_no);
process.exit();
});
})();