-
Notifications
You must be signed in to change notification settings - Fork 1
/
evaluate.py
125 lines (94 loc) · 4.13 KB
/
evaluate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
# the code is based on https://github.com/pcy1302/DMGI/blob/master/evaluate.py
import torch
import torch.nn as nn
import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import normalized_mutual_info_score, pairwise, f1_score
from models import LogReg
def evaluate(embeds, idx_train, idx_val, idx_test, labels, isTest=True):
nb_classes = labels.shape[1]
train_embs = embeds[idx_train]
xent = nn.CrossEntropyLoss()
val_embs = embeds[idx_val]
test_embs = embeds[idx_test]
train_lbls = torch.argmax(labels[idx_train], dim=1)
val_lbls = torch.argmax(labels[idx_val], dim=1)
test_lbls = torch.argmax(labels[idx_test], dim=1)
accs = []
micro_f1s = []
macro_f1s = []
macro_f1s_val = []
for _ in range(50):
log = LogReg(train_embs.shape[1], nb_classes)
opt = torch.optim.Adam(log.parameters(), lr=0.1)
log.to(train_lbls.device)
val_accs = []; test_accs = []
val_micro_f1s = []; test_micro_f1s = []
val_macro_f1s = []; test_macro_f1s = []
for iter_ in range(100):
# train
log.train()
opt.zero_grad()
logits = log(train_embs)
loss = xent(logits, train_lbls)
loss.backward()
opt.step()
# val
logits = log(val_embs)
preds = torch.argmax(logits, dim=1)
val_acc = torch.sum(preds == val_lbls).float() / val_lbls.shape[0]
val_f1_macro = f1_score(val_lbls.cpu(), preds.cpu(), average='macro')
val_f1_micro = f1_score(val_lbls.cpu(), preds.cpu(), average='micro')
val_accs.append(val_acc.item())
val_macro_f1s.append(val_f1_macro)
val_micro_f1s.append(val_f1_micro)
# test
logits = log(test_embs)
preds = torch.argmax(logits, dim=1)
test_acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0]
test_f1_macro = f1_score(test_lbls.cpu(), preds.cpu(), average='macro')
test_f1_micro = f1_score(test_lbls.cpu(), preds.cpu(), average='micro')
test_accs.append(test_acc.item())
test_macro_f1s.append(test_f1_macro)
test_micro_f1s.append(test_f1_micro)
max_iter = val_accs.index(max(val_accs))
accs.append(test_accs[max_iter])
max_iter = val_macro_f1s.index(max(val_macro_f1s))
macro_f1s.append(test_macro_f1s[max_iter])
macro_f1s_val.append(val_macro_f1s[max_iter])
max_iter = val_micro_f1s.index(max(val_micro_f1s))
micro_f1s.append(test_micro_f1s[max_iter])
if isTest:
print("\t[Classification] Macro-F1: {:.4f} ({:.4f}) | Micro-F1: {:.4f} ({:.4f})".format(
np.mean(macro_f1s), np.std(macro_f1s), np.mean(micro_f1s), np.std(micro_f1s)))
else:
return np.mean(macro_f1s_val), np.mean(macro_f1s)
test_embs = np.array(test_embs.cpu())
test_lbls = np.array(test_lbls.cpu())
k1 = run_kmeans(test_embs, test_lbls, nb_classes)
run_similarity_search(test_embs, test_lbls)
return macro_f1s, micro_f1s, k1
def run_similarity_search(test_embs, test_lbls):
numRows = test_embs.shape[0]
cos_sim_array = pairwise.cosine_similarity(test_embs) - np.eye(numRows)
st = []
for N in [5, 10, 20, 50, 100]:
indices = np.argsort(cos_sim_array, axis=1)[:, -N:]
tmp = np.tile(test_lbls, (numRows, 1))
selected_label = tmp[np.repeat(np.arange(numRows), N), indices.ravel()].reshape(numRows, N)
original_label = np.repeat(test_lbls, N).reshape(numRows,N)
st.append(str(np.round(np.mean(np.sum((selected_label == original_label), 1) / N), 4)))
st = ','.join(st)
print("\t[Similarity] [5,10,20,50,100] : [{}]".format(st))
def run_kmeans(x, y, k):
estimator = KMeans(n_clusters=k)
NMI_list = []
for i in range(10):
estimator.fit(x)
y_pred = estimator.predict(x)
s = normalized_mutual_info_score(y, y_pred, average_method='arithmetic')
NMI_list.append(s)
mean = np.mean(NMI_list)
std = np.std(NMI_list)
print('\t[Clustering] NMI: {:.4f} | {:.4f}'.format(mean, std))
return NMI_list