-
Notifications
You must be signed in to change notification settings - Fork 1
/
seed.py
278 lines (237 loc) · 10.2 KB
/
seed.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
""" seed.py downloads a JSON file from SimpleForm and then analyzes it """
import argparse
import os
import sys
import seed_create
import seed_display
import seed_download
import seed_process
import seed_gensim
INDENT = " "
SLASH = "/"
DEFAULT_TOPIC_NUMBER = 3
DEFAULT_PASS_NUMBER = 10
SYMMETRIC = "symmetric"
ASYMMETRIC = "asymmetric"
AUTO = "auto"
DEFAULT_ALPHA = SYMMETRIC
DEFAULT_ETA = SYMMETRIC
SEED_HOME = "SEED_HOME"
SEED_SIMPLEFORM_TOKEN = "SEED_SIMPLEFORM_TOKEN"
def parse_seed_arguments(args):
""" Parses the arguments provided on the command-line """
seed_parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
seed_parser.add_argument(
"--token", help="SimpleForm API token", required=False)
seed_parser.add_argument(
"--analyze-advice",
help="Analyze the 'advice' question",
action="store_true")
seed_parser.add_argument(
"--analyze-challenge",
help="Analyze the 'challenge' question",
action="store_true")
seed_parser.add_argument(
"--analyze-facts",
help="Analyze the 'fact' question",
action="store_true")
seed_parser.add_argument(
"--num-topics",
help="Topics in the LDA model",
type=int,
default=DEFAULT_TOPIC_NUMBER,
required=False)
seed_parser.add_argument(
"--num-passes",
help="Passes when creating the LDA model",
type=int,
default=DEFAULT_PASS_NUMBER,
required=False)
seed_parser.add_argument(
"--alpha",
help="Alpha when creating the LDA model",
choices=[AUTO, ASYMMETRIC, SYMMETRIC],
default=DEFAULT_ALPHA,
required=False)
seed_parser.add_argument(
"--eta",
help="Eta when creating the LDA model",
choices=[AUTO, SYMMETRIC],
default=DEFAULT_ETA,
required=False)
seed_parser.add_argument(
"--create-list", help="Create the mailing list", action="store_true")
seed_parser.add_argument(
"--download-json", help="Download the JSON file", action="store_true")
seed_parser.add_argument(
"--show-respondents",
help="Show the SEED respondents",
action="store_true")
seed_parser.add_argument('--show-respondent', type=str)
seed_parser.add_argument(
"--show-sample", help="Show a SEED sample", action="store_true")
seed_parser.add_argument(
"--verbose", help="Verbose mode", action="store_true")
seed_parser.add_argument(
"--visualize",
help="Create an interactive visualization",
action="store_true")
# verify the arguments, printing help message if they are wrong
seed_arguments = seed_parser.parse_args(args)
return seed_arguments, seed_parser
def get_seed_simpleform_token():
""" Returns the SEED_SIMPLEFORM_TOKEN """
simple_form_token = os.environ.get(SEED_SIMPLEFORM_TOKEN)
return simple_form_token
def get_seed_home():
""" Returns the SEED_HOME """
current_seed_home = os.environ.get(SEED_HOME)
had_to_set = False
# the current SEED_HOME is acceptable, so use it
if verify_seed_home(current_seed_home) is not False:
seed_home = current_seed_home
# the current SEED_HOME is not okay, so guess at one
else:
seed_home = os.getcwd() + SLASH
had_to_set = True
return seed_home, had_to_set
def verify_performing_lda(args):
""" Checks if the command-line arguments ask for LDA """
performing_lda = False
if args.analyze_advice is True or args.analyze_challenge is True or args.analyze_facts is True:
performing_lda = True
return performing_lda
def verify_seed_arguments(args):
""" Checks if the seed_arguments are correct """
verified_arguments = True
if args.download_json is not False and args.token is None:
verified_arguments = False
elif args.create_list is not False and args.token is None:
verified_arguments = False
elif args.num_topics != DEFAULT_TOPIC_NUMBER and verify_performing_lda(
args) is False:
verified_arguments = False
elif args.alpha != DEFAULT_ALPHA and verify_performing_lda(args) is False:
verified_arguments = False
elif args.eta != DEFAULT_ETA and verify_performing_lda(args) is False:
verified_arguments = False
elif args.visualize is not False and verify_performing_lda(args) is False:
verified_arguments = False
return verified_arguments
def verify_seed_home(current_seed_home):
""" Verifies that the SEED_HOME environment variable is set correctly """
verified_seed_home = False
if current_seed_home is not None and current_seed_home.endswith(
SLASH) is True:
verified_seed_home = True
return verified_seed_home
def display_welcome_message():
""" Display a welcome message """
print("SEED: Educational Discussions with Software Engineers")
print("http://www.cs.allegheny.edu/sites/gkapfham/seed/")
print("https://github.com/gkapfham/seed")
print()
def perform_gensim_analysis(seed_arguments, response_list):
""" Use seed_gensim functions to create and analyze a topic model """
if seed_arguments.num_topics is not None:
num_topics_requested = seed_arguments.num_topics
else:
num_topics_requested = DEFAULT_TOPIC_NUMBER
gensim_topic_model, topic_model_corpus, topic_model_dictionary, texts_to_analyze = seed_gensim.create_topic_model(
seed_arguments, response_list)
seed_gensim.show_topic_model_textually(
gensim_topic_model,
topic_model_corpus,
texts_to_analyze,
num_topics=num_topics_requested)
if seed_arguments.visualize is True:
seed_gensim.show_topic_model_visually(
gensim_topic_model, topic_model_corpus, topic_model_dictionary)
if __name__ == '__main__':
display_welcome_message()
current_seed_home, had_to_set = get_seed_home()
if had_to_set is True:
print(INDENT + SEED_HOME,
"is not set or not set with a trailing slash.")
print(INDENT + "Using", current_seed_home, "instead.")
print()
# parse and verify the arguments
seed_arguments, seed_parser = parse_seed_arguments(sys.argv[1:])
did_verify_arguments = verify_seed_arguments(seed_arguments)
if seed_arguments.verbose:
print(seed_arguments)
# arguments were not verified, so print help message
if did_verify_arguments is False:
print("Could not verify the command-line arguments!")
seed_parser.print_help()
# arguments were verified, so perform designated action
else:
# TASK: download the JSON file from SimpleForm
# remove the email addresses
# remove the subscriptions from other forms
# save the final list of dictionaries to a JSON file
if seed_arguments.download_json is True:
seed_json = seed_download.seed_download(seed_arguments.token)
seed_internal_dictionary_list = seed_process.seed_process_create_internal_dictionary(
seed_json)
print(
INDENT,
"Downloaded a total of ",
len(seed_internal_dictionary_list),
" entries",
sep='')
seed_process.seed_process_remove_email_subscriptions(
seed_internal_dictionary_list)
seed_process.seed_process_remove_emails(
seed_internal_dictionary_list)
print(
INDENT,
"Saved a total of ",
len(seed_internal_dictionary_list),
" entries",
sep='')
seed_download.seed_save_json(seed_internal_dictionary_list)
if seed_arguments.verbose:
print(seed_internal_dictionary_list)
# TASK: Create the mailing list for the SEED respondents
elif seed_arguments.create_list is True:
seed_json = seed_download.seed_download(seed_arguments.token)
seed_internal_dictionary_list = seed_process.seed_process_create_internal_dictionary(
seed_json)
list_of_email = seed_create.create_mailing_list(
seed_internal_dictionary_list)
seed_download.seed_save_mailing_list(list_of_email)
# TASK: Show all of the respondents to the SEED survey
elif seed_arguments.show_respondents is True:
seed_dictionary_list = seed_download.seed_load()
seed_display.seed_display_respondents(seed_dictionary_list)
# TASK: Show responses for a respondent to the SEED survey
elif seed_arguments.show_respondent is not None:
seed_dictionary_list = seed_download.seed_load()
seed_respondent_dictionary = seed_process.seed_process_create_respondent_dictionary(
seed_dictionary_list, seed_arguments.show_respondent)
seed_display.seed_display_respondent_markdown(
seed_respondent_dictionary)
# TASK: Show a sample response to the SEED survey
elif seed_arguments.show_sample is True:
seed_dictionary_list = seed_download.seed_load()
seed_display.seed_display_sample(seed_dictionary_list)
# TASK: Analyze the responses to the 'fact' question
elif seed_arguments.analyze_facts is True:
seed_dictionary_list = seed_download.seed_load()
fact_response_list = seed_create.create_fact_answer_list(
seed_dictionary_list)
perform_gensim_analysis(seed_arguments, fact_response_list)
# TASK: Analyze the responses to the 'advice' question
elif seed_arguments.analyze_advice is True:
seed_dictionary_list = seed_download.seed_load()
advice_response_list = seed_create.create_advice_answer_list(
seed_dictionary_list)
perform_gensim_analysis(seed_arguments, advice_response_list)
# TASK: Analyze the responses to the 'advice' question
elif seed_arguments.analyze_challenge is True:
seed_dictionary_list = seed_download.seed_load()
challenge_response_list = seed_create.create_challenge_answer_list(
seed_dictionary_list)
perform_gensim_analysis(seed_arguments, challenge_response_list)