Merge pull request #6 from agoravoting/release-3-4

Release 3 4

agora-results

@@ -76,62 +76,62 @@ def extract_tally(fpath):
     tar.close()
     return extract_dir
 
-def print_csv(data, separator):
+def print_csv(data, separator, output_func=print):
     counts = data['results']['questions']
     for question, i in zip(counts, range(len(counts))):
         if question['tally_type'] not in ["plurality-at-large", "borda", "borda-nauru"] or\
            question.get('no-tally', False):
             continue
 
-        print(separator.join(["Question"]))
-        print(separator.join(["Number", "Title"]))
-        print(separator.join([str(i + 1), question['title']]))
-        print(separator*2)
+        output_func(separator.join(["Question"]))
+        output_func(separator.join(["Number", "Title"]))
+        output_func(separator.join([str(i + 1), question['title']]))
+        output_func(separator*2)
 
-        print(separator.join(["Totals"]))
+        output_func(separator.join(["Totals"]))
 
         question_total = (question['totals']['null_votes']
             + question['totals']['blank_votes']
             + question['totals']['valid_votes'])
 
-        print(separator.join(["Total votes", str(question_total)]))
-        print(separator.join(["Blank votes", str(question['totals']['blank_votes'])]))
-        print(separator.join(["Null votes", str(question['totals']['null_votes'])]))
-        print(separator.join(["Valid votes", str(question['totals']['valid_votes'])]))
-        print(separator*2)
+        output_func(separator.join(["Total votes", str(question_total)]))
+        output_func(separator.join(["Blank votes", str(question['totals']['blank_votes'])]))
+        output_func(separator.join(["Null votes", str(question['totals']['null_votes'])]))
+        output_func(separator.join(["Valid votes", str(question['totals']['valid_votes'])]))
+        output_func(separator*2)
 
-        print(separator.join(["Answers"]))
-        print(separator.join(["Id", "Text", "Category", "Total votes", "Winner position"]))
+        output_func(separator.join(["Answers"]))
+        output_func(separator.join(["Id", "Text", "Category", "Total votes", "Winner position"]))
         for answer in question['answers']:
-            print(separator.join([
+            output_func(separator.join([
                 str(answer['id']),
                 answer['text'],
                 answer['category'],
                 str(answer['total_count']),
                 str(answer['winner_position'])]))
 
 
-def pretty_print(data):
+def pretty_print(data, output_func=print):
     from agora_results.pipes.pretty_print import pretty_print_not_iterative
-    pretty_print_not_iterative([data])
+    pretty_print_not_iterative([data], output_func=output_func)
 
-def print_results(data, output_format):
+def print_results(data, output_format, output_func=print):
   '''
   print results in the specified output format
   '''
   if output_format == "json":
-    print(json.dumps(
+    output_func(json.dumps(
         data['results'],
         indent=4,
         ensure_ascii=False,
         sort_keys=True,
         separators=(',', ': ')))
   elif output_format == "csv":
-      print_csv(data, separator=",")
+      print_csv(data, separator=",", output_func=output_func)
   elif output_format == "tsv":
-      print_csv(data, separator="\t")
+      print_csv(data, separator="\t", output_func=output_func)
   elif output_format == "pretty":
-      pretty_print(data)
+      pretty_print(data, output_func=output_func)
 
 def func_path_sanity_checks(func_path, pipes_whitelist):
     '''
@@ -218,6 +218,8 @@ def main():
     parser.add_argument('-x', '--tar',   nargs='?', help='tar tallies output path')
     parser.add_argument('-p', '--pipes-whitelist', help='path to the file containing the allowed pipes')
     parser.add_argument('-c', '--config', help='config path')
+    parser.add_argument('-eid', '--election-id', help='election id', type=int)
+
     parser.add_argument('-s', '--stdout', help='print output to stdout',
         action='store_true')
     parser.add_argument('-o', '--output-format', help='select the output format',
@@ -276,17 +278,64 @@ def main():
                     print("Extracted tally %s in %s.." % (tally, extract_dir))
                 data_list.append(dict(extract_dir=extract_dir))
 
+        # tar tallies generates a tar file containing everything related to
+        # the results. It also creates a randomly named subdirectory with
+        # all the exportable data.
+
+        priv_results_uuid = None
+        priv_results_dirname = None
+        priv_results_path = None
+        ballots_file = None
+
+        if pargs.tar and len(data_list) > 0:
+            from uuid import uuid4
+            priv_results_uuid = str(uuid4())
+            priv_results_dirname = "results-%s" % priv_results_uuid
+            priv_results_path = os.path.join(pargs.tar, priv_results_dirname)
+
+            # if there's any existing priv_results_path, remove it
+            from glob import glob
+            existing_priv_results = glob(os.path.join(pargs.tar, "results-*"))
+            if len(existing_priv_results) > 0:
+                for existing_priv_result in existing_priv_results:
+                    shutil.rmtree(existing_priv_result)
+
+            # create the priv results
+            os.mkdir(priv_results_path)
+
+            ballots_path = os.path.join(priv_results_path, "ballots.csv")
+            ballots_file = open(ballots_path, "w")
+            # this indicates do_tallies to dump the ballots
+            data_list[0]['ballots_fprint'] = lambda s: ballots_file.write(s + "\n")
+
         execute_pipeline(pipeline_info, data_list,
                          pipes_whitelist=pipes_whitelist)
-        if pargs.stdout and len(data_list) > 0 and 'results' in data_list[0]:
-          print_results(data_list[0], pargs.output_format)
-        data = ""
 
-        # tar tallies
-        if pargs.tar:
+        # tar tallies generates a tar file containing everything related to
+        # the results. It also creates a randomly named subdirectory with
+        # all the exportable data.
+
+        if ballots_file:
+            ballots_file.close()
+        if pargs.tar and len(data_list) > 0 and 'results' in data_list[0]:
+            data_list[0]['results']['results_dirname'] = priv_results_dirname
             from agora_results.utils.tallies import tar_tallies
-            for tally in pargs.tally:
-                tar_tallies(data_list[0], pipeline_info, tally, pargs.tar)
+            tar_tallies(data_list, pipeline_info, pargs.tally, pargs.tar, pargs.election_id)
+
+            # dump the results in CSV,JSON and PRETTY format in the
+            # priv-results dir
+            for res_format in ['csv', 'json', 'pretty']:
+                fpath = os.path.join(
+                    priv_results_path,
+                    "%d.results.%s" % (pargs.election_id, res_format)
+                )
+                with open(fpath, "w") as f:
+                  output_func = lambda s: f.write(s + "\n")
+                  print_results(data_list[0], res_format, output_func)
+
+
+        if pargs.stdout and len(data_list) > 0 and 'results' in data_list[0]:
+          print_results(data_list[0], pargs.output_format)
 
     finally:
         if not pargs.stdout:

agora_results/pipes/duplicate_questions.py

@@ -54,32 +54,50 @@ def write_config(data, config):
         with open(questions_path, 'w', encoding="utf-8") as f:
             f.write(json.dumps(config))
 
-    def do_action(func, dir_path, orig_glob, orig_replace, dest_replace):
-        orig_q_path = os.path.join(dir_path, orig_glob)
+    def do_action(func, orig_path, dest_path, orig_glob, orig_replace, dest_replace):
+        orig_q_path = os.path.join(orig_path, orig_glob)
         orig_q_path = glob.glob(orig_q_path)[0]
         orig_q_id = orig_q_path.split('/')[-1]
         dest_q_id = orig_q_id.replace(orig_replace, dest_replace)
-        dest_q_path = os.path.join(dir_path, dest_q_id)
+        dest_q_path = os.path.join(dest_path, dest_q_id)
         func(orig_q_path, dest_q_path)
 
     for dupl in duplications:
         orig_q = dupl["base_question_index"]
         orig_el = dupl["source_election_index"]
         dest_el = dupl.get("dest_election_index", dupl["source_election_index"])
-        data = data_list[orig_el]
-        data_dest = data_list[dest_el]
-        dir_path = data_dest['extract_dir']
-        qjson = read_config(data)
-        qjson_dest = read_config(data_dest)
+        orig_data = data_list[orig_el]
+        dest_data = data_list[dest_el]
+        dest_path = dest_data['extract_dir']
+        orig_path = orig_data['extract_dir']
+        orig_qjson = read_config(orig_data)
+        qjson_dest = read_config(dest_data)
 
         for dest_q in dupl["duplicated_question_indexes"]:
-            copyq = copy.deepcopy(qjson[orig_q])
+            copyq = copy.deepcopy(orig_qjson[orig_q])
             copyq['source_question_index'] = orig_q
             copyq['source_election_index'] = orig_el
             qjson_dest.insert(dest_q, copyq)
             # +1 to the indexes of the directory of the next questions
             for i in reversed(range(dest_q, len(qjson_dest) - 1)):
-                do_action(os.rename, dir_path, "%d-*" % i, "%d-" % i, "%d-" % (i + 1))
+                # NOTE: we use orig_path twice because we are just renaming, it
+                # is ON PURPOSE
+                do_action(
+                    os.rename,
+                    orig_path,
+                    orig_path,
+                    "%d-*" % i,
+                    "%d-" % i,
+                    "%d-" % (i + 1)
+                )
+
             # duplicate question dir
-            do_action(shutil.copytree, dir_path, "%d-*" % orig_q, "%d-" % orig_q, "%d-" % dest_q)
-        write_config(data_dest, qjson_dest)
+            do_action(
+                shutil.copytree,
+                orig_path,
+                dest_path,
+                "%d-*" % orig_q,
+                "%d-" % orig_q,
+                "%d-" % dest_q
+            )
+        write_config(dest_data, qjson_dest)

agora_results/pipes/multipart.py

@@ -182,7 +182,7 @@ def multipart_tally_plaintexts_append_joiner(data_list, dst_election_id,
 
     dst_plaintexts_path = glob(os.path.join(
           data_list[dst_election_id]['extract_dir'],
-          "%d*" % question_num, "plaintexts_json"))[0]
+          "%d-*" % question_num, "plaintexts_json"))[0]
 
     with codecs.open(dst_plaintexts_path, encoding='utf-8', mode='a') as final_plaintexts:
         # open the plaintexts file from other tallies, and convert each ballot
@@ -193,7 +193,7 @@ def multipart_tally_plaintexts_append_joiner(data_list, dst_election_id,
 
             src_question = data['questions'][question_num]
             plaintexts_path = glob(os.path.join(
-                  data['extract_dir'], "%d*" % question_num, "plaintexts_json"))[0]
+                  data['extract_dir'], "%d-*" % question_num, "plaintexts_json"))[0]
             with codecs.open(plaintexts_path, encoding='utf-8', mode='r') as plaintexts:
                 for line in plaintexts.readlines():
                     final_plaintexts.write(line)
@@ -420,14 +420,14 @@ def translate_ballot(line, dindex, qindex, src_question, dst_question):
         # open last_tally plaintexts file, only used for "append-only", adding
         # more ballots at the end of the file from the other tallies
         last_plaintexts_path = glob(os.path.join(
-              last_data['extract_dir'], "%d*" % qindex, "plaintexts_json"))[0]
+              last_data['extract_dir'], "%d-*" % qindex, "plaintexts_json"))[0]
         with codecs.open(last_plaintexts_path, encoding='utf-8', mode='a') as final_plaintexts:
             # open the plaintexts file from other tallies, and convert each ballot
             # and append it to the last_tally plaintexts file
             for dindex, data in enumerate(data_list[:-1]):
                 src_question = data['questions'][qindex]
                 plaintexts_path = glob(os.path.join(
-                      data['extract_dir'], "%d*" % qindex, "plaintexts_json"))[0]
+                      data['extract_dir'], "%d-*" % qindex, "plaintexts_json"))[0]
                 with codecs.open(plaintexts_path, encoding='utf-8', mode='r') as plaintexts:
                     for line in plaintexts.readlines():
                         try:
@@ -520,7 +520,7 @@ def encode_ballot(ballot, answer_text_to_id, fill_size):
 
     dst_plaintexts_path = glob(os.path.join(
           data_list[dst_election_id]['extract_dir'],
-          "%d*" % question_num, "plaintexts_json"))[0]
+          "%d-*" % question_num, "plaintexts_json"))[0]
 
     with codecs.open(dst_plaintexts_path, encoding='utf-8', mode='a') as final_plaintexts:
           for ballot in parsed_ballots:

agora_results/pipes/pretty_print.py

@@ -18,26 +18,26 @@
 import os
 import subprocess
 
-def pretty_print_stv_winners(data_list):
+def pretty_print_stv_winners(data_list, output_func=print):
     data = data_list[0]
     counts = data['results']['questions']
-    print("Total votes: %d\n", data['results']['total_votes'])
+    output_func("Total votes: %d\n", data['results']['total_votes'])
     for question, i in zip(counts, range(len(counts))):
         if "stv" not in question['tally_type']:
             continue
 
-        print("Q: %s\n" % question['title'])
+        output_func("Q: %s\n" % question['title'])
         winners = [answer for answer in question['answers']
             if answer['winner_position'] != None]
         question['answers'].sort(key=itemgetter('winner_position'))
 
         i = 0
         for answer in question['answers']:
             if answer['winner_position'] != None:
-                print("%d. %s" % (i+1, winner))
+                output_func("%d. %s" % (i+1, winner))
                 i += 1
 
-def __pretty_print_base(data, mark_winners, show_percent, filter_names):
+def __pretty_print_base(data, mark_winners, show_percent, filter_names, output_func=print):
     '''
     percent_base:
       "total" total of the votes, the default
@@ -53,7 +53,7 @@ def get_percentage(num, base):
     for question, i in zip(counts, range(len(counts))):
         if question['tally_type'] not in filter_names or question.get('no-tally', False):
             continue
-        print("\n\nQ: %s\n" % question['title'])
+        output_func("\n\nQ: %s\n" % question['title'])
 
         blank_votes = question['totals']['blank_votes']
         null_votes = question['totals']['null_votes']
@@ -68,32 +68,32 @@ def get_percentage(num, base):
           base_num = question['totals']['valid_votes']
 
 
-        print("Total votes: %d" % total_votes)
-        print("Blank votes: %d (%0.2f%%)" % (
+        output_func("Total votes: %d" % total_votes)
+        output_func("Blank votes: %d (%0.2f%%)" % (
             blank_votes,
             get_percentage(blank_votes, total_votes)))
 
-        print("Null votes: %d (%0.2f%%)" % (
+        output_func("Null votes: %d (%0.2f%%)" % (
             null_votes,
             get_percentage(null_votes, total_votes)))
 
-        print("Total valid votes (votes to options): %d (%0.2f%%)" % (
+        output_func("Total valid votes (votes to options): %d (%0.2f%%)" % (
             valid_votes,
             get_percentage(valid_votes, total_votes)))
-        print("\nOptions (percentages over %s, %d winners):" % (percent_base, question['num_winners']))
+        output_func("\nOptions (percentages over %s, %d winners):" % (percent_base, question['num_winners']))
 
         if mark_winners:
             i = 1
             winners = [answer for answer in question['answers']
                 if answer['winner_position'] != None]
             for answer in winners:
                 if not show_percent:
-                    print("%d. %s (%0.2f votes)" % (
+                    output_func("%d. %s (%0.2f votes)" % (
                         i,
                         answer['text'],
                         answer['total_count']))
                 else:
-                    print("%d. %s (%0.2f votes, %0.2f%%)" % (
+                    output_func("%d. %s (%0.2f votes, %0.2f%%)" % (
                         i,
                         answer['text'],
                         answer['total_count'],
@@ -106,11 +106,11 @@ def get_percentage(num, base):
 
             for loser in losers:
                 if not show_percent:
-                    print("N. %s (%0.2f votes)" % (
+                    output_func("N. %s (%0.2f votes)" % (
                         loser['text'],
                         loser['total_count']))
                 else:
-                    print("N. %s (%0.2f votes, %0.2f%%)" % (
+                    output_func("N. %s (%0.2f votes, %0.2f%%)" % (
                         loser['text'],
                         loser['total_count'],
                         get_percentage(loser['total_count'], base_num)))
@@ -120,17 +120,18 @@ def get_percentage(num, base):
 
             for i, answer in zip(range(len(answers)), answers):
                 if not show_percent:
-                    print("%d. %s (%0.2f votes)" % (
+                    output_func("%d. %s (%0.2f votes)" % (
                         i + 1, answer['text'],
                         answer['total_count']))
                 else:
-                    print("%d. %s (%0.2f votes, %0.2f%%)" % (
+                    output_func("%d. %s (%0.2f votes, %0.2f%%)" % (
                         i + 1, answer['text'],
                         answer['total_count'],
                         get_percentage(answer['total_count'], base_num)))
-    print("")
+    output_func("")
 
-def pretty_print_not_iterative(data_list, mark_winners=True):
+def pretty_print_not_iterative(data_list, mark_winners=True, output_func=print):
     data = data_list[0]
     __pretty_print_base(data, mark_winners, show_percent=True,
-        filter_names=["plurality-at-large", "borda-nauru", "borda", "pairwise-beta", "cup"])
+        filter_names=["plurality-at-large", "borda-nauru", "borda", "pairwise-beta", "cup"],
+        output_func=output_func)