diff --git a/AUTHORS b/AUTHORS index a93beda..efc8313 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,2 @@ -Eduardo Robles Elvira \ No newline at end of file +Eduardo Robles Elvira +Daniel García Moreno \ No newline at end of file diff --git a/agora-results b/agora-results index b64c525..3a27c71 100755 --- a/agora-results +++ b/agora-results @@ -1,6 +1,21 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import signal import argparse @@ -24,6 +39,33 @@ DEFAULT_PIPELINE = [ ] ] +# By default we only allow the most used pipes to reduce default attack surface +# NOTE: keep the list sorted +DEFAULT_PIPES_WHITELIST = [ + #"agora_results.pipes.duplicate_questions.duplicate_questions", + "agora_results.pipes.modifications.apply_modifications", + #"agora_results.pipes.multipart.make_multipart", + #"agora_results.pipes.multipart.election_max_size_corrections", + #"agora_results.pipes.multipart.question_totals_with_corrections", + #"agora_results.pipes.multipart.reduce_answers_with_corrections", + #"agora_results.pipes.multipart.multipart_tally_plaintexts_append_joiner", + #"agora_results.pipes.multipart.data_list_reverse", + #"agora_results.pipes.multipart.multipart_tally_plaintexts_joiner", + #"agora_results.pipes.multipart.append_ballots", + "agora_results.pipes.parity.proportion_rounded", + "agora_results.pipes.parity.parity_zip_non_iterative", + #"agora_results.pipes.parity.reorder_winners", + #"agora_results.pipes.parity.podemos_parity_loreg_zip_non_iterative", + #"agora_results.pipes.podemos.podemos_proportion_rounded_and_duplicates", + #"agora_results.pipes.pretty_print.pretty_print_stv_winners", + "agora_results.pipes.pretty_print.pretty_print_not_iterative", + "agora_results.pipes.results.do_tallies", + #"agora_results.pipes.results.to_files", + #"agora_results.pipes.results.apply_removals", + "agora_results.pipes.sort.sort_non_iterative", + #"agora_results.pipes.stv_tiebreak.stv_first_round_tiebreak" +] + def extract_tally(fpath): ''' extracts the tally and loads the results into a file for convenience @@ -91,10 +133,13 @@ def print_results(data, output_format): elif output_format == "pretty": pretty_print(data) -def func_path_sanity_checks(func_path): +def func_path_sanity_checks(func_path, pipes_whitelist): ''' Check that the func path is valid and reasonably secure ''' + if pipes_whitelist is not None and func_path not in pipes_whitelist: + raise Exception("Pipe not in the whitelist: %s" % func_path) + values = func_path.split(".") if " " in func_path or len(values) == 0 or len(values) > 4 or\ values[0] != "agora_results" or values[1] != "pipes": @@ -104,7 +149,7 @@ def func_path_sanity_checks(func_path): if len(val) == 0 or val.startswith("_"): raise Exception() -def execute_pipeline(pipeline_info, data_list): +def execute_pipeline(pipeline_info, data_list, pipes_whitelist=None): ''' Execute a pipeline of options. pipeline_info must be a list of pairs. Each pair contains (pipe_func_path, params), where pipe_func_path is @@ -121,7 +166,7 @@ def execute_pipeline(pipeline_info, data_list): ''' for func_path, kwargs in pipeline_info: # get access to the function - func_path_sanity_checks(func_path) + func_path_sanity_checks(func_path, pipes_whitelist) func_name = func_path.split(".")[-1] module = __import__( ".".join(func_path.split(".")[:-1]), globals(), locals(), @@ -171,6 +216,7 @@ def main(): parser.add_argument('-t', '--tally', nargs='*', help='tally path', default=[]) parser.add_argument('-e', '--election-config', nargs='*', help='Instead of specifying a tally, you can specify an json election config and an empty ephemeral tally with zero votes will be created. recommended to use together with the multipart.append_ballots pipe.', default=[]) parser.add_argument('-x', '--tar', nargs='?', help='tar tallies output path') + parser.add_argument('-p', '--pipes-whitelist', help='path to the file containing the allowed pipes') parser.add_argument('-c', '--config', help='config path') parser.add_argument('-s', '--stdout', help='print output to stdout', action='store_true') @@ -178,11 +224,20 @@ def main(): default="json", choices=["json", "csv", "tsv", "pretty", "none"]) pargs = parser.parse_args() + # load config if pargs.config is not None: with codecs.open(pargs.config, 'r', encoding="utf-8") as f: pipeline_info = json.loads(f.read()) else: pipeline_info = DEFAULT_PIPELINE + + # load allowed pipes: Format of the file should simply be: one pipe per line + if pargs.pipes_whitelist is not None: + with codecs.open(pargs.pipes_whitelist, 'r', encoding="utf-8") as f: + pipes_whitelist = [l.strip() for l in f.readlines()] + else: + pipes_whitelist = DEFAULT_PIPES_WHITELIST + data_list = [] # remove files on abrupt external exit signal @@ -221,7 +276,8 @@ def main(): print("Extracted tally %s in %s.." % (tally, extract_dir)) data_list.append(dict(extract_dir=extract_dir)) - execute_pipeline(pipeline_info, data_list) + execute_pipeline(pipeline_info, data_list, + pipes_whitelist=pipes_whitelist) if pargs.stdout and len(data_list) > 0 and 'results' in data_list[0]: print_results(data_list[0], pargs.output_format) data = "" diff --git a/agora_results/pipes/bcnencomu.py b/agora_results/pipes/bcnencomu.py index 58ca935..1064b98 100644 --- a/agora_results/pipes/bcnencomu.py +++ b/agora_results/pipes/bcnencomu.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + from collections import defaultdict def team_count_weight_correction(data_list, original_count_weight, team_count_weight, question_indexes, help=""): diff --git a/agora_results/pipes/duplicate_questions.py b/agora_results/pipes/duplicate_questions.py index 1d9f9fc..e559a2f 100644 --- a/agora_results/pipes/duplicate_questions.py +++ b/agora_results/pipes/duplicate_questions.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import json import copy import shutil diff --git a/agora_results/pipes/modifications.py b/agora_results/pipes/modifications.py index d5f2ae2..fc14f98 100644 --- a/agora_results/pipes/modifications.py +++ b/agora_results/pipes/modifications.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import re import os import json diff --git a/agora_results/pipes/multipart.py b/agora_results/pipes/multipart.py index 628c31a..020292b 100644 --- a/agora_results/pipes/multipart.py +++ b/agora_results/pipes/multipart.py @@ -1,12 +1,44 @@ +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import json +import re import codecs +import unicodedata +import string import traceback import copy import types from glob import glob import agora_tally.tally +def curate_text(text): + text = text.replace(""", '"') + text = text.replace("+", '+') + text = text.replace("@", '@') + text = text.replace("'", "'") + text = text.replace("\xa0", ' ') + text = re.sub("[ \n\t]+", " ", text) + text = remove_accents(text) + return text + +def remove_accents(text): + return ''.join(x for x in unicodedata.normalize('NFKD', text) if x in string.ascii_letters).lower() + + def make_multipart(data_list, election_ids, help=""): ''' check that the agora-results is being correctly invoked @@ -113,10 +145,22 @@ def reduce_answers_with_corrections(data_list, mappings, reverse=True, help=""): src_election =[data for data in data_list if data['id'] == src_eid][0] src_q = src_election['results']['questions'][src_qnum] src_answer = [a for a in src_q['answers'] if a['id'] == src_ansid][0] - assert src_answer['text'] == src_anstxt + try: + assert curate_text(src_answer['text']) == curate_text(src_anstxt) + except Exception as e: + print("source_text != expected_source_text, '%s' != '%s'" % + (curate_text(src_answer['text']), + curate_text(src_anstxt))) + raise e dst_answer = [a for a in dst_q['answers'] if a['id'] == dst_ansid][0] - assert dst_answer['text'] == dst_anstxt + try: + assert curate_text(dst_answer['text']) == curate_text(dst_anstxt) + except Exception as e: + print("source_text != expected_source_text, '%s' != '%s'" % + (curate_text(dst_answer['text']), + curate_text(dst_anstxt))) + raise e dst_answer['total_count'] += src_answer['total_count'] diff --git a/agora_results/pipes/parity.py b/agora_results/pipes/parity.py index d8695ce..b359455 100644 --- a/agora_results/pipes/parity.py +++ b/agora_results/pipes/parity.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + from itertools import zip_longest from operator import itemgetter import sys diff --git a/agora_results/pipes/podemos.py b/agora_results/pipes/podemos.py index a88269e..5500a0c 100644 --- a/agora_results/pipes/podemos.py +++ b/agora_results/pipes/podemos.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import json from itertools import groupby, chain import sys diff --git a/agora_results/pipes/pretty_print.py b/agora_results/pipes/pretty_print.py index 899c660..571d3a8 100644 --- a/agora_results/pipes/pretty_print.py +++ b/agora_results/pipes/pretty_print.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import subprocess diff --git a/agora_results/pipes/results.py b/agora_results/pipes/results.py index 3971351..820f22d 100644 --- a/agora_results/pipes/results.py +++ b/agora_results/pipes/results.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import json import agora_tally.tally diff --git a/agora_results/pipes/sort.py b/agora_results/pipes/sort.py index 7c6de96..6b72e5c 100644 --- a/agora_results/pipes/sort.py +++ b/agora_results/pipes/sort.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import json from itertools import groupby, chain from operator import itemgetter diff --git a/agora_results/pipes/stv_tiebreak.py b/agora_results/pipes/stv_tiebreak.py index b501efe..6b50aea 100644 --- a/agora_results/pipes/stv_tiebreak.py +++ b/agora_results/pipes/stv_tiebreak.py @@ -1,5 +1,20 @@ # -*- coding:utf-8 -*- +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import copy import json @@ -7,6 +22,7 @@ import agora_tally.tally from itertools import groupby, chain +# TODO OUTDATED def stv_first_round_tiebreak(data_list): ''' Tie break algorithm for stv sorting of the winners. @@ -31,7 +47,7 @@ def stv_first_round_tiebreak(data_list): continue q_winners = [] - choices = get_choices(data['extract_dir'], tally, question) + choices = __get_choices(data['extract_dir'], tally, question) for iteration, i in zip(log['iterations'], range(len(log['iterations']))): it_winners = [cand for cand in iteration['candidates'] if cand['status'] == 'won'] @@ -42,7 +58,7 @@ def stv_first_round_tiebreak(data_list): # check if there are repeated counts len_set = len(set([i['count'] for i in it_winners])) if len_set != len(it_winners) and i == 0: - it_winners = stv_first_iteration_tie_break( + it_winners = __stv_first_iteration_tie_break( it_winners, iteration, i, data['extract_dir'], question, choices, 1) for winner in it_winners: @@ -50,7 +66,7 @@ def stv_first_round_tiebreak(data_list): question['winners'] = q_winners -def get_choices(extract_dir, tally, question): +def __get_choices(extract_dir, tally, question): question_num = tally.question_num dirs = [os.path.join(extract_dir, d) for d in sorted(os.listdir(extract_dir)) @@ -72,7 +88,7 @@ def get_choices(extract_dir, tally, question): print("invalid vote: %s" % line) return choices -def stv_first_iteration_tie_break( +def __stv_first_iteration_tie_break( it_winners, iteration, question_num, extract_dir, question, choices, break_position=1): ''' @@ -123,7 +139,7 @@ def tie_break(winner): for tie2 in recursive_ties: if len(tie2) == 1: continue - tie2 = stv_first_iteration_tie_break(tie2, iteration, + tie2 = __stv_first_iteration_tie_break(tie2, iteration, question_num, extract_dir, question, choices, break_position + 1) diff --git a/agora_results/utils/deterministic_tar.py b/agora_results/utils/deterministic_tar.py index 2539042..90662e6 100644 --- a/agora_results/utils/deterministic_tar.py +++ b/agora_results/utils/deterministic_tar.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of agora-tools. -# Copyright (C) 2013,2015 Eduardo Robles Elvira +# Copyright (C) 2013-2016 Eduardo Robles Elvira # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by diff --git a/agora_results/utils/join_by_name.py b/agora_results/utils/join_by_name.py index 93108c9..26cb85a 100644 --- a/agora_results/utils/join_by_name.py +++ b/agora_results/utils/join_by_name.py @@ -1,5 +1,20 @@ #!/usr/bin/env python3 +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import argparse import json import codecs diff --git a/agora_results/utils/tallies.py b/agora_results/utils/tallies.py index fe3f330..5122fe6 100644 --- a/agora_results/utils/tallies.py +++ b/agora_results/utils/tallies.py @@ -1,3 +1,18 @@ +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + import os import json import tempfile diff --git a/setup.py b/setup.py index 5c288f9..ba60f75 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,20 @@ #!/usr/bin/env python3 +# This file is part of agora-results. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora-results is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora-results is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora-results. If not, see . + from setuptools import setup from pip.req import parse_requirements @@ -12,7 +27,7 @@ setup( name='Agora Results', - version='1.0.0', + version='3.2.0', author='Agora Voting Team', author_email='agora@agoravoting.com', packages=['agora_results', 'agora_results.pipes'],