-
Notifications
You must be signed in to change notification settings - Fork 3
/
utils.py
59 lines (53 loc) · 1.85 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import pprint
import codecs
import os
import sys
import time
import pickle as pkl
import numpy as np
from collections import OrderedDict
import io
import collections
def load_embs(path, topk = None, dimension = None):
print(topk)
print("Loading embeddings")
vocab_dict = {}
embeddings = []
with codecs.open(path, encoding = 'utf8', errors = 'replace') as f:
line = f.readline().strip().split()
cntr = 1
if len(line) == 2:
vocab_size = int(line[0])
if not dimension:
dimension = int(line[1])
else:
if not dimension or (dimension and len(line[1:]) == dimension):
vocab_dict[line[0].strip()] = len(vocab_dict)
embeddings.append(np.array(line[1:], dtype=np.float32))
if not dimension:
dimension = len(line) - 1
print("Vector dimensions: " + str(dimension))
while line:
line = f.readline().strip().split()
if (not line):
print("Loaded " + str(cntr) + " vectors.")
break
if line[0].strip() == "":
continue
cntr += 1
if cntr % 20000 == 0:
print(cntr)
if len(line[1:]) == dimension:
if (line[0].strip().lower() not in vocab_dict):
vocab_dict[line[0].strip().lower()] = len(vocab_dict)
embeddings.append(np.array(line[1:], dtype=np.float32))
else:
print("Error in the embeddings file, line " + str(cntr) +
": unexpected vector length (expected " + str(dimension) +
" got " + str(len(np.array(line[1:]))) + " for word '" + line[0] + "'")
if (topk and cntr >= topk):
print("Loaded " + str(cntr) + " vectors.")
break
embeddings = np.array(embeddings, dtype=np.float32)
print(len(vocab_dict), str(embeddings.shape))
return vocab_dict, embeddings