-
Notifications
You must be signed in to change notification settings - Fork 1
/
linguistic_analysis.py
288 lines (199 loc) · 9.23 KB
/
linguistic_analysis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
import pickle
import queue
from pathlib import Path
import numpy as np
from scipy.spatial.distance import cosine
from tqdm import tqdm
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from analysis_util import get_numpy_array, BaseFeatureHelper, \
get_sample_feature_value
from structure_temp_analysis import get_post_tweet_deepest_cascade
from temporal_analysis import print_stat_values
from util.constants import REPLY_NODE, POST_NODE
from util.util import tweet_node
def get_deepest_cascade_reply_nodes_avg_sentiment(prop_graph: tweet_node):
deep_cascade, max_height = get_post_tweet_deepest_cascade(prop_graph)
return get_reply_nodes_average_sentiment(deep_cascade)
def get_deepest_cascade_first_level_reply_sentiment(prop_graph: tweet_node):
deep_cascade, max_height = get_post_tweet_deepest_cascade(prop_graph)
return get_first_reply_nodes_average_sentiment(deep_cascade)
def get_first_reply_nodes_average_sentiment(prop_graph: tweet_node):
q = queue.Queue()
q.put(prop_graph)
reply_diff_values = list()
while q.qsize() != 0:
node = q.get()
for child in node.reply_children:
q.put(child)
if child.node_type == REPLY_NODE and node.node_type == POST_NODE:
if child.sentiment:
reply_diff_values.append(child.sentiment)
if len(reply_diff_values) == 0:
return 0
else:
return np.mean(np.array(reply_diff_values))
def get_reply_nodes_average_sentiment(prop_graph: tweet_node):
q = queue.Queue()
q.put(prop_graph)
reply_diff_values = list()
while q.qsize() != 0:
node = q.get()
for child in node.reply_children:
q.put(child)
if node.node_type == REPLY_NODE:
if node.sentiment:
reply_diff_values.append(node.sentiment)
if len(reply_diff_values) == 0:
return 0
else:
return np.mean(np.array(reply_diff_values))
def get_cosine_similarity(reply_node1, reply_node2, reply_id_index_dict, reply_lat_embeddings):
try:
if reply_node1 in reply_id_index_dict and reply_node2 in reply_id_index_dict:
reply1_idx = reply_id_index_dict[reply_node1]
reply2_idx = reply_id_index_dict[reply_node2]
return cosine(reply_lat_embeddings[reply1_idx], reply_lat_embeddings[reply2_idx])
else:
return 0
except:
return 0
def get_supporting_opposing_replies_ratio(prop_graph: tweet_node, news_source, label):
q = queue.Queue()
q.put(prop_graph)
similarity_values = list()
reply_id_index_dict = pickle.load(
open("data/pre_process_data/elmo_features/{}_{}_reply_id_latent_mat_index.pkl".format(news_source, label),
"rb"))
reply_content_latent_embeddings = pickle.load(
open("data/pre_process_data/elmo_features/{}_{}_elmo_lat_embeddings.pkl".format(news_source, label), "rb"))
while q.qsize() != 0:
node = q.get()
for child in node.reply_children:
q.put(child)
if node.node_type == REPLY_NODE and child.node_type == REPLY_NODE:
similarity_values.append(get_cosine_similarity(node.tweet_id, child.tweet_id,
reply_id_index_dict, reply_content_latent_embeddings))
if len(similarity_values) == 0:
return 0
else:
supporting = 1
opposing = 1
for value in similarity_values:
if value > 0.5:
supporting += 1
else:
opposing += 1
return supporting / opposing
def get_reply_nodes_sentiment_ratio(prop_graph: tweet_node):
q = queue.Queue()
q.put(prop_graph)
reply_diff_values = list()
while q.qsize() != 0:
node = q.get()
for child in node.reply_children:
q.put(child)
if node.node_type == REPLY_NODE:
reply_diff_values.append(node.sentiment)
if len(reply_diff_values) == 0:
return 0
else:
positive_sentiment = 1
negative_sentiment = 1
for value in reply_diff_values:
if value > 0.05:
positive_sentiment += 1
elif value < -0.05:
negative_sentiment += 1
return positive_sentiment / negative_sentiment
def get_stats_for_features(news_graps: list, get_feature_fun_ref, print=False, feature_name=None):
result = []
for graph in news_graps:
result.append(get_feature_fun_ref(graph))
if print:
print_stat_values(feature_name, result)
return result
def get_all_linguistic_features(news_graphs, micro_features, macro_features):
all_features = []
if macro_features:
retweet_function_references = []
for function_reference in retweet_function_references:
features_set = get_stats_for_features(news_graphs, function_reference, print=False, feature_name=None)
all_features.append(features_set)
if micro_features:
reply_function_references = [get_reply_nodes_average_sentiment, get_first_reply_nodes_average_sentiment,
get_deepest_cascade_reply_nodes_avg_sentiment,
get_deepest_cascade_first_level_reply_sentiment]
for function_reference in reply_function_references:
features_set = get_stats_for_features(news_graphs, function_reference, print=True, feature_name=None)
all_features.append(features_set)
return np.transpose(get_numpy_array(all_features))
def dump_tweet_reply_sentiment(data_dir, out_dir):
reply_id_content_dict = dict()
reply_id_content_dict.update(pickle.load(
open("{}/{}_{}_reply_id_content_dict.pkl".format(data_dir, "politifact", "fake"), "rb")))
reply_id_content_dict.update(pickle.load(
open("{}/{}_{}_reply_id_content_dict.pkl".format(data_dir, "politifact", "real"), "rb")))
reply_id_content_dict.update(pickle.load(
open("{}/{}_{}_reply_id_content_dict.pkl".format(data_dir, "gossipcop", "fake"), "rb")))
reply_id_content_dict.update(pickle.load(
open("{}/{}_{}_reply_id_content_dict.pkl".format(data_dir, "gossipcop", "real"), "rb")))
print("Total no. of replies : {}".format(len(reply_id_content_dict)))
analyzer = SentimentIntensityAnalyzer()
reply_id_sentiment_output = dict()
for reply_id, content in tqdm(reply_id_content_dict.items()):
sentiment_result = analyzer.polarity_scores(content)
reply_id_sentiment_output[reply_id] = sentiment_result
pickle.dump(reply_id_sentiment_output, open("{}/all_reply_id_sentiment_result.pkl".format(out_dir), "wb"))
class LinguisticFeatureHelper(BaseFeatureHelper):
def get_feature_group_name(self):
return "ling"
def get_micro_feature_method_references(self):
method_refs = [get_reply_nodes_sentiment_ratio,
get_reply_nodes_average_sentiment,
get_first_reply_nodes_average_sentiment,
get_deepest_cascade_reply_nodes_avg_sentiment,
get_deepest_cascade_first_level_reply_sentiment]
return method_refs
def get_micro_feature_method_names(self):
feature_names = ["Sentiment ratio of all replies",
"Average sentiment of all replies",
"Average sentiment of first level replies",
"Average sentiment of replies in deepest cascade",
"Average setiment of first level replies in deepest cascade"]
return feature_names
def get_micro_feature_short_names(self):
feature_names = ["L1", "L2", "L3", "L4", "L5", "L6"]
return feature_names
def get_macro_feature_method_references(self):
method_refs = []
return method_refs
def get_macro_feature_method_names(self):
feature_names = []
return feature_names
feature_names = []
def get_macro_feature_short_names(self):
feature_names = []
return feature_names
def get_features_array(self, prop_graphs, micro_features, macro_features, news_source=None, label=None,
file_dir="/content/FakeNewsPropagation/data/features", use_cache=False):
function_refs = []
file_name = self.get_dump_file_name(news_source, micro_features, macro_features, label, file_dir)
data_file = Path(file_name)
if use_cache and data_file.is_file():
return pickle.load(open(file_name, "rb"))
if micro_features:
function_refs.extend(self.get_micro_feature_method_references())
if len(function_refs) == 0:
return None
all_features = []
for idx in range(len(function_refs)):
features_set = get_sample_feature_value(prop_graphs, function_refs[idx])
all_features.append(features_set)
feature_array = np.transpose(get_numpy_array(all_features))
pickle.dump(feature_array, open(file_name, "wb"))
return feature_array
def get_feature_involving_additional_args(prop_graphs, function_reference, news_source, label):
feature_values = []
for prop_graph in prop_graphs:
feature_values.append(function_reference(prop_graph, news_source, label))
return feature_values