-
Notifications
You must be signed in to change notification settings - Fork 3
/
lake-client.ts
76 lines (68 loc) · 2.63 KB
/
lake-client.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { Block } from '@near-lake/primitives';
export default class LakeClient {
constructor (
private readonly network: string = 'mainnet',
private readonly s3Client: S3Client = new S3Client({
maxAttempts: 5,
retryMode: 'adaptive'
})
) {}
// pad with 0s to 12 digits
private normalizeBlockHeight (blockHeight: number): string {
return blockHeight.toString().padStart(12, '0');
}
private fetchShards (blockHeight: number, numberOfShards: number): Array<Promise<any>> {
return ([...Array(numberOfShards).keys()].map(async (shardId) =>
await this.fetchShard(blockHeight, shardId)
));
}
private async fetchShard (blockHeight: number, shardId: number): Promise<any> {
const params = {
Bucket: `near-lake-data-${this.network}`,
Key: `${this.normalizeBlockHeight(blockHeight)}/shard_${shardId}.json`,
};
const response = await this.s3Client.send(new GetObjectCommand(params));
const shardData = await response.Body?.transformToString() ?? '{}';
return JSON.parse(shardData, (_key, value) => this.renameUnderscoreFieldsToCamelCase(value));
}
private async fetchBlockPromise (blockHeight: number): Promise<any> {
const file = 'block.json';
const folder = this.normalizeBlockHeight(blockHeight);
const params = {
Bucket: 'near-lake-data-' + this.network,
Key: `${folder}/${file}`,
};
const response = await this.s3Client.send(new GetObjectCommand(params));
const blockData = await response.Body?.transformToString() ?? '{}';
return JSON.parse(blockData, (_key, value) => this.renameUnderscoreFieldsToCamelCase(value));
}
private renameUnderscoreFieldsToCamelCase (value: Record<string, any>): Record<string, any> {
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
// It's a non-null, non-array object, create a replacement with the keys initially-capped
const newValue: any = {};
for (const key in value) {
const newKey: string = key
.split('_')
.map((word, i) => {
if (i > 0) {
return word.charAt(0).toUpperCase() + word.slice(1);
}
return word;
})
.join('');
newValue[newKey] = value[key];
}
return newValue;
}
return value;
}
async fetchBlock (blockHeight: number): Promise<Block> {
const block = await this.fetchBlockPromise(blockHeight);
const shards = await Promise.all(this.fetchShards(blockHeight, block.chunks.length));
return Block.fromStreamerMessage({
block,
shards,
});
}
}