-
Notifications
You must be signed in to change notification settings - Fork 0
/
logistic_baseline_tensorflow_refresh.py
182 lines (152 loc) · 6.1 KB
/
logistic_baseline_tensorflow_refresh.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
from __future__ import print_function
from project_utils import *
#from diagnostics import *
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import csr
from sklearn.metrics import roc_auc_score
from sys import argv
import os
import pandas as pd
# Getting command args
# -nettype: either 'zero' or 'one', giving the number of hidden layers
# -dataset: either 'toxic' or 'attack', telling which data set to analyze
myargs = getopts(argv)
APPROACH = "ngram"
CLASSIFIER = "logistic"
FLAVOR = "tensorflow-ADAM-refresh"
reg_level = int(myargs['-reglevel'])
if reg_level > 0:
FLAVOR = FLAVOR + '-reglevel' + str(reg_level)
else:
FLAVOR = FLAVOR + '-noReg'
# Parameters
learning_rate = 0.001
training_epochs = 50
beta_reg = 10**(-reg_level) * int(reg_level > 0)
print("beta_reg is ", beta_reg)
batch_size = 100
display_step = 1
lr_decay = 1.0
if myargs['-dataset'] == 'attack':
vecpath = TFIDF_VECTORS_FILE_AGG
if not os.path.isfile(ATTACK_AGGRESSION_FN):
get_and_save_talk_data()
train, dev, test = get_TDT_split(
pd.read_csv(ATTACK_AGGRESSION_FN, index_col=0).fillna(' '))
cnames = train.columns.values[0:2]
aucfn = "auc_scores_attack_baseline_refresh.csv"
elif myargs['-dataset'] == 'toxic':
vecpath = TFIDF_VECTORS_FILE_TOXIC
train, dev, test = get_TDT_split(
pd.read_csv('train.csv').fillna(' '))
cnames = CLASS_NAMES
aucfn = "auc_scores_toxic_baseline_refresh.csv"
# Get data and featurizing
train_vecs, dev_vecs, test_vecs = vectorize_corpus_tf_idf(
train, dev, test, sparse=True, path=vecpath, prot=2
)
n_train = train_vecs.shape[0]
if batch_size is None:
batch_size = train.shape[0]
# tf Graph Input
#x = tf.placeholder(tf.float32, [None, NUM_FEATURES])
x = tf.sparse_placeholder(tf.float32)
y = tf.placeholder(tf.float32, [None, 2])
# Constructing middle layers
if myargs['-nettype'] == 'zero':
beta_reg = 0.0001
training_epochs = 50
W = tf.get_variable("weights",
shape=[NUM_FEATURES, 2],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.zeros([2]))
theta = tf.sparse_tensor_dense_matmul(x, W) + b
elif myargs['-nettype'] == 'one':
beta_reg = 0.0001
FLAVOR = "tensorflow-ADAM-1layer"
training_epochs = 10
W = tf.get_variable("weights",
shape=[NUM_FEATURES, hidden_size],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.zeros([hidden_size]))
z = tf.sparse_tensor_dense_matmul(x, W) + b
h = tf.nn.relu(z)
h = tf.nn.dropout(h, 1 - dropout_rate)
W2 = tf.get_variable("weights2",
shape=[hidden_size, 2],
initializer=tf.contrib.layers.xavier_initializer())
b2 = tf.Variable(tf.zeros([2]))
theta = tf.matmul(h_drop, W2) + b2
# Get prediction (this will only be used for testing)
pred = tf.nn.softmax(theta)
# Get cost directly (without needing prediction above)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=theta, labels=y) + \
tf.nn.l2_loss(W) * beta_reg
)
# Gradient Descent
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Final scoring
def calc_auc_tf(X, Y):
return calc_auc(Y[:, 1], pred.eval({x: X})[:, 1])
# Making weight saving functionality
saver = tf.train.Saver()
# Initialize the variables (i.e. assign their default value)
global_init = tf.global_variables_initializer()
auc_scores = []
pred_mat = np.ndarray(shape = (test_vecs.shape[0],0))
target_mat = np.ndarray(shape = (test_vecs.shape[0],0))
for target_class in range(len(cnames)):
print("doing class " + cnames[target_class])
# Getting labels for training
train_labels = train[cnames[target_class]].values
train_target = get_onehots_from_labels(train_labels)
dev_labels = dev[cnames[target_class]].values
dev_target = get_onehots_from_labels(dev_labels)
test_labels = test[cnames[target_class]].values
test_target = get_onehots_from_labels(test_labels)
# Getting weight saver fn
save_fn = saver_fn(APPROACH, CLASSIFIER, FLAVOR, cnames[target_class])
# Start training
max_auc = 0
with tf.Session() as sess:
# Run initializer
sess.run(global_init)
# Training
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(n_train/batch_size)
# Loop over batches
minibatches = minibatch(train_vecs, train_target, batch_size)
for batch_xs_mat, batch_ys in minibatches:
batch_xs = get_sparse_input(batch_xs_mat)
_, c = sess.run([optimizer, cost], feed_dict={x: batch_xs,
y: batch_ys})
avg_cost += c / total_batch
# Display logs
if (epoch+1) % display_step == 0:
AUC = calc_auc_tf(get_sparse_input(dev_vecs), dev_target)
print("Epoch:", '%04d' % (epoch+1),
"cost=", avg_cost,
"dev.auc=", AUC)
if AUC > max_auc:
print ("New best AUC on dev!")
saver.save(sess, save_fn)
max_auc = AUC
print("Optimization Finished!")
saver.restore(sess, save_fn)
AUC = calc_auc_tf(get_sparse_input(test_vecs), test_target)
print ("Test AUC:", AUC)
auc_scores.append(AUC)
curr_pred = pred.eval({x: get_sparse_input(test_vecs)})
pred_mat = np.column_stack((pred_mat, curr_pred[:,1]))
target_mat = np.column_stack((target_mat, test_target[:,1]))
sess.close()
#pred_mat = np.column_stack((pred_mat, target_mat))
#diagnostics = Diagnostics(build = 'tf',
# model_type = 'logistic-' + myargs['-nettype'],
# preds_targets = pred_mat,
# dataset = myargs['-dataset'])
#diagnostics.do_all_diagnostics()
save_auc_scores(auc_scores, APPROACH, CLASSIFIER, fn=aucfn, cnames=cnames,
flavor=FLAVOR)