diff --git a/collectives/models/event/event_type.py b/collectives/models/event/event_type.py index 571a75497..6f549d90d 100644 --- a/collectives/models/event/event_type.py +++ b/collectives/models/event/event_type.py @@ -1,10 +1,12 @@ """Module to describe the type of event. """ +from typing import Dict, Any from markupsafe import escape - +from flask import current_app from collectives.models.globals import db from collectives.models import Configuration +from collectives.utils.misc import to_ascii class EventType(db.Model): @@ -142,3 +144,40 @@ def get_terms_title(self) -> str: confs = {key: Configuration[key] for key in self.TERMS_CONFIGURATIONS} return self.terms_title.format(**confs) + + @classmethod + def all(cls, include_deprecated=False) -> Dict[int, Dict[str, Any]]: + """Returns type dictionnary as defined by EVENT_TYPES in config. + + :param include_deprecated: Whether to include deprecated activity types + :type include_deprecated: bool + + :type: dict""" + + types = current_app.config["EVENT_TYPES"] + if include_deprecated: + return types + + return { + id: event_type + for id, event_type in types.items() + if not event_type.get("deprecated", False) + } + + @classmethod + def get_type_from_csv_code(cls, csv_code: str) -> int: + """ + :param string short: CSV code of the searched type + :returns: Type id + """ + + # Match without accents, lowercase + csv_code = to_ascii(csv_code.strip().lower()) + for i, event_type in cls.all(include_deprecated=True).items(): + event_type_csv_code = to_ascii( + event_type.get("csv_code", event_type["name"]) + ).lower() + if csv_code == event_type_csv_code: + return i + + return None diff --git a/collectives/models/event_tag.py b/collectives/models/event_tag.py index a10dec4b4..f4084bc4a 100644 --- a/collectives/models/event_tag.py +++ b/collectives/models/event_tag.py @@ -113,6 +113,8 @@ def get_type_from_csv_code(cls, csv_code: str) -> int: :param string short: CSV code of the searched tag :returns: Tag id """ + if csv_code == None: + return None # Match without accents, lowercase csv_code = to_ascii(csv_code.strip().lower()) diff --git a/collectives/routes/activity_supervison.py b/collectives/routes/activity_supervison.py index e0b3f4034..8a12540a1 100644 --- a/collectives/routes/activity_supervison.py +++ b/collectives/routes/activity_supervison.py @@ -238,12 +238,14 @@ def csv_import(): "message", ) + return_code = 200 if len(failed) == 0 else 400 + return render_template( "activity_supervision/import_csv.html", form=form, failed=failed, title="Création d'event par CSV", - ) + ), return_code @blueprint.route("/index", methods=["GET"]) diff --git a/collectives/utils/csv.py b/collectives/utils/csv.py index 5e169a234..388857290 100644 --- a/collectives/utils/csv.py +++ b/collectives/utils/csv.py @@ -5,10 +5,9 @@ from datetime import datetime, timedelta import codecs import csv - +import re from flask import current_app - -from collectives.models import User, Event, EventTag, db +from collectives.models import User, Event, EventTag, EventType, db from collectives.models.user_group import GroupEventCondition, UserGroup from collectives.utils.time import format_date @@ -24,16 +23,17 @@ def fill_from_csv(event, row, template): :type template: string :return: Nothing """ - + # remove all blank spaces in keys + row = {key.replace(" ", ""): value for key, value in row.items()} + event.event_type_id = EventType.get_type_from_csv_code(parse(row, "event_type")) event.title = parse(row, "titre") - # Subscription dates and slots event.start = parse(row, "debut") event.end = parse(row, "fin") event.num_slots = parse(row, "places") parent_event_id = parse(row, "parent") - if parent_event_id != "": + if not parent_event_id in ("", None): if db.session.get(Event, parent_event_id) is None: raise builtins.Exception(f"La collective {parent_event_id} n'existe pas") event.user_group = UserGroup() @@ -72,18 +72,32 @@ def fill_from_csv(event, row, template): minute=0, ) + # Waiting list + if "places_liste_attente" in row and row["places_liste_attente"].strip(): + event.num_waiting_list = parse(row, "places_liste_attente") + if event.num_waiting_list > event.num_slots: + raise builtins.Exception( + "Le nombre de places en liste d'attente doit être inférieur au nombre de places de " + "la collective" + ) + # Description - parse(row, "altitude") - parse(row, "denivele") - parse(row, "distance") - event.description = template.format(**row) + try: + event.description = template.format(**row) + except builtins.Exception as ex: + raise builtins.Exception( + f"La colonne '{ex}' demandée pour la Description de" + "l'événement n'existe pas dans le fichier" + ) event.set_rendered_description(event.description) - # Event tag - tag_id = EventTag.get_type_from_csv_code(parse(row, "tag")) - if tag_id is not None: - tag = EventTag(tag_id=tag_id) - event.tag_refs.append(tag) + # Event tags - takes all column that starts with tag + tags = [[key, value] for key, value in row.items() if key.startswith("tag")] + for [key, value] in tags: + tag_id = EventTag.get_type_from_csv_code(parse(row, key)) + if tag_id is not None: + tag = EventTag(tag_id=tag_id) + event.tag_refs.append(tag) # Leader leader = User.query.filter_by(license=row["id_encadrant"]).first() @@ -92,7 +106,6 @@ def fill_from_csv(event, row, template): f"L'encadrant {row['nom_encadrant']} (numéro de licence {row['id_encadrant']}) n'a " "pas encore créé de compte" ) - # Check if event already exists in same activity if Event.query.filter_by( main_leader_id=leader.id, title=event.title, start=event.start @@ -105,6 +118,29 @@ def fill_from_csv(event, row, template): event.leaders = [leader] event.main_leader_id = leader.id + # Other leaders - takes all column that starts with id_encadrant an try adding them + leaders_table = [ + [key, value] for key, value in row.items() if key.startswith("id_encadrant") + ] + for [key, value] in leaders_table: + leader = User.query.filter_by( + license=value + ).first() # tries to find leader using value as license + if leader is None: + # tries to match: "identifier (license)" or "license (identifier)" + match = re.match(r"(.+)\((.+)\)", value) + first_part, second_part = match.groups() + leader = User.query.filter_by(license=second_part).first() + if leader is None: + leader = User.query.filter_by(license=first_part).first() + if leader is None: + raise builtins.Exception( + f"L'encadrant {value} n'a pas pu etre trouvé. " + "Vérifier que le format et les informations soient correctement reinsegnés." + ) + + event.leaders.append(leader) + def parse(row, column_name): """Parse a column value in csv format to an object depending on column type. @@ -116,12 +152,22 @@ def parse(row, column_name): :return: The parsed value """ csv_columns = current_app.config["CSV_COLUMNS"] + # in case column name is not in standard csv column from app, + # return directly the value + if not column_name in csv_columns: + value_str = row[column_name].strip() + return value_str + column_short_desc = csv_columns[column_name]["short_desc"] - if row[column_name] is None: + # verify if mandatory columns are present or not + if not column_name in row and not csv_columns[column_name].get("optional", 0): raise builtins.Exception( - f"La colonne '{column_short_desc}' n'existe pas dans le fichier" + f"La colonne '{column_short_desc}' est obligatoire et n'existe pas dans le fichier" ) + # if column is not present but a default value can be given, return default + if not column_name in row and "default" in csv_columns[column_name]: + return csv_columns[column_name]["default"] value_str = row[column_name].strip() @@ -149,6 +195,9 @@ def parse(row, column_name): f"La valeur '{value_str}' de la colonne '{column_name}' doit être un " "nombre entier" ) from err + # if value is not present but a default value can be given, return default + if not value_str and "default" in csv_columns[column_name]: + return csv_columns[column_name]["default"] return value_str @@ -158,6 +207,7 @@ def process_stream(base_stream, activity_type, description): Processing will first try to process it as an UTF8 encoded file. If it fails on a decoding error, it will try as Windows encoding (iso-8859-1). + CSV file delimiter will be inferred with csv.Sniffer method. :param base_stream: the csv file as a stream. :type base_stream: :py:class:`io.StringIO` @@ -169,13 +219,24 @@ def process_stream(base_stream, activity_type, description): :return: The number of processed events, and the number of failed attempts :rtype: (int, int) """ + try: + delimiter = ( + csv.Sniffer().sniff(next(codecs.iterdecode(base_stream, "utf8"))).delimiter + ) + base_stream.seek(0) stream = codecs.iterdecode(base_stream, "utf8") - events, processed, failed = csv_to_events(stream, description) + events, processed, failed = csv_to_events(stream, description, delimiter) except UnicodeDecodeError: + base_stream.seek(0) + delimiter = ( + csv.Sniffer() + .sniff(next(codecs.iterdecode(base_stream, "iso-8859-1"))) + .delimiter + ) base_stream.seek(0) stream = codecs.iterdecode(base_stream, "iso-8859-1") - events, processed, failed = csv_to_events(stream, description) + events, processed, failed = csv_to_events(stream, description, delimiter) # Complete event before adding it to db for event in events: @@ -186,7 +247,7 @@ def process_stream(base_stream, activity_type, description): return processed, failed -def csv_to_events(stream, description): +def csv_to_events(stream, description, delimiter): """Decode the csv stream to populate events. :param stream: the csv file as a stream. @@ -194,6 +255,8 @@ def csv_to_events(stream, description): :param description: Description template that will be used to generate new events description. :type description: String + :param delimeter: Delimiter for csv file import. + :type delimiter: String :return: The new events, the number of processed events, and the number of failed attempts :rtype: list(:py:class:`collectives.models.event.Event`), int, int @@ -201,16 +264,7 @@ def csv_to_events(stream, description): events = [] processed = 0 failed = [] - fields = list(current_app.config["CSV_COLUMNS"].keys()) - - reader = csv.DictReader(stream, delimiter=",", fieldnames=fields) - row = next(reader, None) # skip the headers - - if all(row[f] is None for f in fields[1:]): - # Single non-None column, delimiter is likely wrong - # Retry with semi-column - reader = csv.DictReader(stream, delimiter=";", fieldnames=fields) - next(reader, None) # skip the headers + reader = csv.DictReader(stream, delimiter=delimiter, fieldnames=None) for row in reader: processed += 1 diff --git a/config.py b/config.py index d6c8f503d..3979e4328 100644 --- a/config.py +++ b/config.py @@ -491,6 +491,13 @@ "optional": 1, "default": str(DEFAULT_ONLINE_SLOTS), }, + "places_liste_attente": { + "short_desc": "Nombre de places dans la liste d'attente", + "description": "Nombre de places dans la liste d'attente", + "type": "int", + "optional": 1, + "default": 0, + }, "debut_internet": { "short_desc": "Date d'ouverture des inscriptions par internet", "description": "Date d'ouverture des inscriptions par internet de " diff --git a/tests/activity/test_csv_import.py b/tests/activity/test_csv_import.py index 3efe2a3fb..4ac4c9eaf 100644 --- a/tests/activity/test_csv_import.py +++ b/tests/activity/test_csv_import.py @@ -20,11 +20,15 @@ def test_csv_import_form(supervisor_client): def test_csv_import(supervisor_client, user1): """Test upload of a valid csv file.""" csv = ( - ",,,,,,,,,,,,,,,,,\n" - "Jan Johnston,990000000001,26/11/2021 7:00,26/11/2021 7:00,Aiguille des Calvaires," - "Aravis,d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00,,\n" - "Jan Johnston,990000000001,26/11/2021 7:00,26/11/2021 7:00,Mont Sulens,Aravis," - "d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00,,cycle decouverte,," + "nom_encadrant,id_encadrant,debut,fin,titre,secteur,carte_IGN,altitude," + "denivele,cotation,distance,observations,places,places_internet," + "debut_internet,fin_internet,places_liste_attente,parent,tag\n" + "Jan Johnston,990000000001,26/11/2021 7:00,26/11/2021 7:00," + "Aiguille des Calvaires,Aravis,d,2322,1200,F,120,d ,8,4," + "19/11/2021 7:00,25/11/2021 12:00,3,,\n" + "Jan Johnston,990000000001,26/11/2021 7:00,26/11/2021 7:00," + "Mont Sulens,Aravis,d,2322,1200,F,120,d ,8,4," + "19/11/2021 7:00,25/11/2021 12:00,3,,cycle decouverte" ) file = BytesIO(csv.encode("utf8")) activity = supervisor_client.user.get_supervised_activities()[0] @@ -46,6 +50,7 @@ def test_csv_import(supervisor_client, user1): assert event.title == "Aiguille des Calvaires" assert event.num_slots == 8 assert event.num_online_slots == 4 + assert event.num_waiting_list == 3 assert event.registration_open_time == datetime(2021, 11, 19, 7, 0, 0) assert event.registration_close_time == datetime(2021, 11, 25, 12, 0, 0) assert "2322m-1200m-F" in event.rendered_description @@ -56,12 +61,60 @@ def test_csv_import(supervisor_client, user1): assert events[1].tag_refs[0].short == "tag_decouverte" +def test_csv_import_multi_tags_leaders(supervisor_client, user1, user2, user3): + """Test upload of a valid csv file with event_type multiple leaders and multiple tags.""" + csv = ( + "event_type,nom_encadrant,id_encadrant,id_encadrant2,id_encadrant3," + "debut,fin,titre,secteur,carte_IGN,altitude,denivele,cotation,distance," + "observations,places,places_internet,debut_internet,fin_internet," + "places_liste_attente,parent,tag,tag2\n" + "acces libre,Jan Johnston,990000000001,990000000002,990000000003," + "26/11/2021 7:00,26/11/2021 7:00,Baudelaire,,,,,,," + "d ,8,4,19/11/2021 7:00,25/11/2021 12:00,3,,cycle decouverte,rando cool" + ) + file = BytesIO(csv.encode("utf8")) + activity = supervisor_client.user.get_supervised_activities()[0] + + data = { + "csv_file": (file, "import.csv"), + "description": "", + "type": activity.id, + } + + response = supervisor_client.post("/activity_supervision/import", data=data) + assert response.status_code == 200 + + events = Event.query.all() + + assert len(events) == 1 + event = events[0] + assert event.event_type.short == "acces_libre" + assert event.title == "Baudelaire" + assert event.num_slots == 8 + assert event.num_online_slots == 4 + assert event.num_waiting_list == 3 + assert event.registration_open_time == datetime(2021, 11, 19, 7, 0, 0) + assert event.registration_close_time == datetime(2021, 11, 25, 12, 0, 0) + assert event.leaders[0].license == "990000000001" + assert event.leaders[0] == user1 + assert event.leaders[1].license == "990000000002" + assert event.leaders[1] == user2 + assert event.leaders[2].license == "990000000003" + assert event.leaders[2] == user3 + + assert len(events[0].tag_refs) == 2 + assert events[0].tag_refs[0].short == "tag_decouverte" + assert events[0].tag_refs[1].short == "tag_rando_cool" + + def test_csv_import_unknown_leader(supervisor_client, user1): """Test upload of an invalid csv with an unkown leader.""" csv = ( - ",,,,,,,,,,,,,,,,\n" + "nom_encadrant,id_encadrant,debut,fin,titre,secteur," + "carte_IGN,altitude,denivele,cotation,distance,observations," + "places,places_internet,debut_internet,fin_internet,places_liste_attente,parent,tag\n" "Evan Walsh,990000000002,26/11/2021 7:00,26/11/2021 7:00,Aiguille des Calvaires," - "Aravis,d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00,,\n" + "Aravis,d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00,,,\n" ) file = BytesIO(csv.encode("utf8")) activity = supervisor_client.user.get_supervised_activities()[0] diff --git a/tests/unit/utils/test_import_csv.py b/tests/unit/utils/test_import_csv.py index 76be2c956..d80430bbb 100644 --- a/tests/unit/utils/test_import_csv.py +++ b/tests/unit/utils/test_import_csv.py @@ -10,19 +10,26 @@ def test_csv_import(user1): """Test importing an event CSV file""" # pylint: disable=C0301 - csv = f""",,,,,,,,,,,,,,,\nMr TEST,{user1.license},26/11/2021 7:00,26/11/2021 7:00,Aiguille des Calvaires,Aravis,d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00,,rando cool""" - + csv = ( + "event_type,nom_encadrant,id_encadrant,debut,fin,titre,secteur," + "carte_IGN,altitude,denivele,cotation,distance,observations," + "places,places_internet,debut_internet,fin_internet,places_liste_attente,parent,tag" + f"\naccess libre,Mr TEST,{user1.license},26/11/2021 7:00,26/11/2021 7:00," + "Aiguille des Calvaires,Aravis,d,2322,1200,F,120,d ,8,4,19/11/2021 7:00,25/11/2021 12:00," + "2,,rando cool" + ) output = StringIO(csv) events, processed, failed = csv_to_events( - output, "{altitude}m-{denivele}m-{cotation}" + output, "{altitude}m-{denivele}m-{cotation}", "," ) + assert not failed assert len(events) == 1 event = events[0] assert processed == 1 - assert not failed assert event.title == "Aiguille des Calvaires" assert event.num_slots == 8 assert event.num_online_slots == 4 + assert event.num_waiting_list == 2 assert event.registration_open_time == datetime.datetime(2021, 11, 19, 7, 0, 0) assert event.registration_close_time == datetime.datetime(2021, 11, 25, 12, 0, 0) assert "2322m-1200m-F" in event.rendered_description