-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
86 lines (76 loc) · 2.41 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from evaluator import Evaluator
from inspect import getmembers, isclass
import algorithms
import json
from tqdm import tqdm
import argparse
import os
import pandas as pd
def parse_args():
parser = argparse.ArgumentParser(description="Argument parser for Hashing Lab")
parser.add_argument(
"-cd",
"--collect-data",
action="store_true",
help="Perform measurements on generated data",
)
parser.add_argument(
"-rd",
"--real-data",
action="store_true",
help="Measure time on real data",
)
return parser.parse_args()
def run(pbar, dtype, additional_save_path="", **kwargs):
measurement_results = {}
for name, hashing_table_cls in pbar:
kwargs["use_k_independent"] = False
pbar.set_description(f"Name: {name}")
if name == "CuckooHashMap":
kwargs["use_k_independent"] = True
evaluator = Evaluator(hashing_table_cls, dtype=dtype, **kwargs)
measurement_results[name] = evaluator()
with open(
f"data/measurement_results_100k_2_power_{dtype}{additional_save_path}.json", "w"
) as fp:
json.dump(measurement_results, fp, indent=4, sort_keys=False)
def main():
args = parse_args()
hashing_classes = reversed(
[o for o in getmembers(algorithms) if isclass(o[1]) if not o[0] == "Item"]
)
if args.collect_data:
pbar = tqdm(hashing_classes)
for dtype in ("int", "tuple", "str"):
run(
pbar,
dtype,
start_value=1000,
max_value=106000,
step=5000,
number_of_bits=63,
)
if args.real_data:
save_path = "data/bbc-text-preprocessed.csv"
if not os.path.exists(save_path):
from utils.process_book import preprocess_text
preprocess_text(save_path)
data = pd.read_csv(save_path)
all_text = []
for _, text in data["text"].items():
all_text.extend(text.split())
all_text = set(all_text)
all_text = [algorithms.Item(word, word) for word in all_text]
pbar = tqdm(hashing_classes)
run(
pbar,
"str",
"_real_data",
external_data=all_text,
start_value=1000,
max_value=25000,
step=1000,
number_of_bits=63,
)
if __name__ == "__main__":
main()