-
Notifications
You must be signed in to change notification settings - Fork 5
/
log_test.py
46 lines (39 loc) · 1.95 KB
/
log_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from icedb.log import IceLogIO, Schema, FileMarker, LogTombstone, S3Client
from time import time
s3c = S3Client(s3prefix="tenant", s3bucket="testbucket", s3region="us-east-1", s3endpoint="http://localhost:9000", s3accesskey="user", s3secretkey="password")
log = IceLogIO("dan-mbp")
# create a log file
sch = Schema()
logFile = log.append(s3c, 1, sch.accumulate(["a", "b"], ["VARCHAR", "BIGINT"]), [FileMarker(
"tenant/_data/d=2023-08-04/a.parquet", time()-10 * 1000, 123), FileMarker("tenant/_data/d=2023-08-04/b.parquet", time()-11 * 1000, 234), FileMarker("tenant/_data/d=2023-08-04/inevershow.parquet", time()-12 * 1000, 345), FileMarker("tenant/_data/d=2023-08-04/inevershoweither.parquet", time()-12 * 1000, 345)])
print("created log file", logFile)
# pretend inevershow* merged into something
sch = Schema()
logFile = log.append(s3c, 1, sch.accumulate(["a", "b", "c"], ["VARCHAR", "BIGINT", "BIGINT"]), [
FileMarker("tenant/_data/d=2023-08-04/a.parquet", round(time()-10 * 1000), 123),
FileMarker("tenant/_data/d=2023-08-04/b.parquet", round(time()-11 * 1000), 234),
FileMarker("tenant/_data/d=2023-08-05/c.parquet", round(time()-5 * 1000), 123),
FileMarker("tenant/_data/d=2023-08-05/e.parquet", round(time()-4 * 1000), 234),
FileMarker("tenant/_data/d=2023-08-04/inevershow.parquet", round(time()-12 * 1000), 345, 1),
FileMarker("tenant/_data/d=2023-08-04/inevershoweither.parquet", round(time()-12 * 1000), 345, 1)
], [LogTombstone(logFile, round(time()-3 * 1000))])
print("created log file", logFile)
# read in a log file
s1, f1, t1, l1 = log.read_at_max_time(s3c, round(time() * 1000))
print(s1.toJSON())
print(f1)
print(t1)
print(l1)
# s2, f2, t2 = log.readAtMaxTime(s3c, time()-9*1000)
logFiles = s3c.s3.list_objects_v2(
Bucket=s3c.s3bucket,
MaxKeys=1000,
Prefix=s3c.s3prefix
)
for file in logFiles['Contents']:
key = file['Key']
print('deleting', key)
s3c.s3.delete_object(
Bucket=s3c.s3bucket,
Key=key
)