Merge pull request #58 from PCS-Poli-USP/fix/main

Fix/main

src/blueprints/class_blueprint.py

@@ -29,7 +29,8 @@
 
 class_schema = ClassSchema(unknown=EXCLUDE)
 preferences_schema = PreferencesSchema(unknown=EXCLUDE)
-has_to_be_allocated_schema = HasToBeAllocatedClassesSchema(many=True, unknown=EXCLUDE)
+has_to_be_allocated_schema = HasToBeAllocatedClassesSchema(
+    many=True, unknown=EXCLUDE)
 event_schema = EventSchema()
 user_repository = UserRepository()
 
@@ -65,6 +66,7 @@ def get_all_classes():
                     "week_days": {"$push": "$week_day"},
                     "preferences": {"$first": "$preferences"},
                     "has_to_be_allocated": {"$first": "$has_to_be_allocated"},
+                    "ignore_to_allocate": {"$first": "$ignore_to_allocate"},
                     "subscribers": {"$first": "$subscribers"},
                     "vacancies": {"$first": "$vacancies"},
                     "pendings": {"$first": "$pendings"},
@@ -145,14 +147,14 @@ def update_preferences(subject_code, class_code):
         preferences_schema_load = preferences_schema.load(request.json)
         building_id = preferences_schema_load["building_id"]
         preferences_schema_load["building_id"] = ObjectId(building_id)
-        has_to_be_allocated = request.json["has_to_be_allocated"]
+        ignore_to_allocate = request.json["ignore_to_allocate"]
 
         result = events.update_many(
             query,
             {
                 "$set": {
                     "preferences": preferences_schema_load,
-                    "has_to_be_allocated": has_to_be_allocated,
+                    "ignore_to_allocate": ignore_to_allocate,
                     "updated_at": datetime.now().strftime("%d/%m/%Y %H:%M"),
                 }
             },
@@ -162,6 +164,9 @@ def update_preferences(subject_code, class_code):
 
     except PyMongoError as err:
         return {"message": err._message}
+    except Exception as ex:
+        print(str(ex))
+        return {"message": "Erro ao atualizar prefrerencias", "error": str(ex)}, 500
 
 
 @class_blueprint.route("/<subject_code>/<class_code>", methods=["GET"])
@@ -230,7 +235,8 @@ def edit_class(subject_code, class_code):
 def update_has_to_be_allocated():
     try:
         username = request.user.get("Username")
-        has_to_be_allocated_schema_load = has_to_be_allocated_schema.load(request.json)
+        has_to_be_allocated_schema_load = has_to_be_allocated_schema.load(
+            request.json)
         updated = 0
 
         for cls in has_to_be_allocated_schema_load:
@@ -240,7 +246,8 @@ def update_has_to_be_allocated():
                 "created_by": username,
             }
             result = events.update_many(
-                query, {"$set": {"has_to_be_allocated": cls["has_to_be_allocated"]}}
+                query, {
+                    "$set": {"has_to_be_allocated": cls["has_to_be_allocated"]}}
             )
 
             updated += result.matched_count

src/blueprints/crawler_blueprint.py

@@ -43,29 +43,40 @@ def crawl_subject():
 
     updated = []
     inserted = []
+    failed = []
+    success = []
     for subject_code in subject_codes_list:
-        events = JupiterCrawler.crawl_subject_static(subject_code)
-        for event in events:
-            event["updated_at"] = datetime.now().strftime("%d/%m/%Y %H:%M")
-            event["created_by"] = username
-            event["has_to_be_allocated"] = True
-            event["building"] = building["name"]
+        try:
+            events = JupiterCrawler.crawl_subject_static(subject_code)
+            for event in events:
+                event["updated_at"] = datetime.now().strftime("%d/%m/%Y %H:%M")
+                event["created_by"] = username
+                event["has_to_be_allocated"] = True
+                event["ignore_to_allocate"] = False
+                event["building"] = building["name"]
 
-            event["preferences"] = {
-                "building_id": ObjectId(building_id),
-                "air_conditioning": False,
-                "projector": False,
-                "accessibility": False,
-            }
+                event["preferences"] = {
+                    "building_id": ObjectId(building_id),
+                    "air_conditioning": False,
+                    "projector": False,
+                    "accessibility": False,
+                }
 
-            query = {
-                "class_code": event["class_code"],
-                "subject_code": event["subject_code"],
-                "week_day": event["week_day"],
-            }
-            result = events_tb.update_one(query, {"$set": event}, upsert=True)
-            updated.append(
-                event["subject_code"]
-            ) if result.matched_count else inserted.append(event["subject_code"])
+                query = {
+                    "class_code": event["class_code"],
+                    "subject_code": event["subject_code"],
+                    "week_day": event["week_day"],
+                    "start_time": event["start_time"],
+                }
+                result = events_tb.update_one(
+                    query, {"$set": event}, upsert=True)
+                updated.append(
+                    event["subject_code"]
+                ) if result.matched_count else inserted.append(event["subject_code"])
 
-    return dumps({"updated": updated, "inserted": inserted})
+            success.append(event["subject_code"])
+
+        except Exception as e:
+            failed.append(subject_code)
+
+    return dumps({"updated": updated, "inserted": inserted, "sucess": success, "failed": failed})

src/blueprints/event_blueprint.py

@@ -10,6 +10,7 @@
 from src.common.database import database
 from src.schemas.allocation_schema import AllocatorInputSchema, AllocatorOutputSchema
 from src.common.allocation.new_allocator import allocate_classrooms
+from src.common.utils.prettify_id import recursive_prettify_id
 
 # from src.common.utils.event.events_formatter import clear_event_allocation
 from src.middlewares.auth_middleware import auth_middleware
@@ -39,8 +40,9 @@ def _():
 @swag_from(f"{yaml_files}/get_events.yml")
 def get_events():
     username = request.user.get("Username")
-    result = events.find({"created_by": username}, {"_id": 0})
+    result = events.find({"created_by": username})
     resultList = list(result)
+    recursive_prettify_id(resultList)
 
     return dumps(resultList)
 
@@ -61,7 +63,7 @@ def save_new_allocation():
     try:
         username = request.user.get("Username")
         result = events.update_many(
-            {"created_by": username},
+            {"created_by": username, "ignore_to_allocate" : False},
             {
                 "$unset": {
                     "building": "",
@@ -75,7 +77,8 @@ def save_new_allocation():
                 {"created_by": username, "ignore_to_allocate": False}, {"_id": 0}
             )
         )
-        event_list = list(events.find({"created_by": username}))
+        event_list = list(events.find(
+            {"created_by": username, "ignore_to_allocate": False}))
         result = allocate_classrooms(classroom_list, event_list)
         allocated_events = result[0]
         unallocated_events = result[1]
@@ -104,6 +107,18 @@ def load_allocations():
     try:
         username = request.user.get("Username")
         result = allocations.find_one({"created_by": username})
+
+        if not result:
+            new_allocation = {
+                "allocated_events": [],
+                "unallocated_events": [],
+                "updated_at": datetime.now().strftime(
+                    "%d/%m/%Y %H:%M"),
+                "created_by": username
+            }
+            allocations.insert_one(new_allocation)
+            return dumps(new_allocation)
+
         allocated_events = result["allocated_events"]
         unallocated_events = result["unallocated_events"]
         now = datetime.now().strftime("%d/%m/%Y %H:%M")
@@ -154,6 +169,7 @@ def load_allocations():
         return {"message": err.messages}, 400
 
     except Exception as ex:
+        print(str(ex))
         return {"message": "Erro ao carregar alocação", "error": str(ex)}, 500
 
 

src/common/allocation/new_allocator.py

@@ -4,12 +4,10 @@
 
 events_collection = database["events"]
 
-from src.common.utils.classes.classes_mapper import get_class_schedule
 from src.common.utils.classroom.classroom_mapper import get_classroom_schedule
 from src.common.utils.event.event_mapper import get_events_by_class
-from src.common.utils.classes.classes_sorter import sort_classes_by_vacancies
 from src.common.utils.classroom.classroom_sorter import sort_classrooms_by_capacity
-from src.common.utils.event.event_sorter import sort_events_by_class, sort_events_by_vacancies, sort_events_by_subject_code, sort_events_by_time, sort_events_by_class_and_time
+from src.common.utils.event.event_sorter import sort_events_by_class_and_time
 
 def classroom_capacity_validation(classroom: dict, event: dict) -> bool:
   return classroom["capacity"] >= event["vacancies"]
@@ -19,26 +17,26 @@ def classroom_preferences_validation(classroom: dict, preferences: dict) -> bool
            preferences["air_conditioning"] == classroom["air_conditioning"] and \
            preferences["accessibility"] == classroom["accessibility"]
 
-def classroom_times_validation(classroom: dict, classroom_schedule, event: dict) -> bool:
+def classroom_times_validation(classroom_schedule, event: dict) -> bool:
   start_validation = datetime.strptime(event["start_time"], "%H:%M").time()
   end_validation = datetime.strptime(event["end_time"], "%H:%M").time()
   classroom_times = classroom_schedule[event["week_day"]]
   for classroom_time in classroom_times:
     start = datetime.strptime(classroom_time[0], "%H:%M").time()
     end = datetime.strptime(classroom_time[1], "%H:%M").time()
-    if (start_validation < start and end_validation < end):
+    if (start_validation <= start and end_validation <= end):
       return False
-    if (start_validation > start and end_validation < end):
+    if (start_validation >= start and end_validation <= end):
       return False
-    if (start_validation < end and end_validation > end):
+    if (start_validation <= end and end_validation >= end):
       return False
-    if (start_validation < start and end_validation > end):
+    if (start_validation <= start and end_validation >= end):
       return False
   return True
 
 def classroom_is_allowed_to_allocate(classroom: dict, classroom_schedule, event: dict) -> bool:
   if classroom_capacity_validation(classroom, event):
-    if classroom_times_validation(classroom, classroom_schedule, event):
+    if classroom_times_validation(classroom_schedule, event):
       return True    
   return False
 
@@ -56,15 +54,12 @@ def allocate_classrooms(classroom_list: list, event_list: list):
         for classroom in classroom_list:
             classroom_schedule = get_classroom_schedule(classroom)
             if classroom_is_allowed_to_allocate(classroom, classroom_schedule, event):
-                print("Turma: ", event["subject_code"], event["class_code"], event['week_day'], event['start_time'], "allocada em", classroom["classroom_name"])
                 event["has_to_be_allocated"] = False
                 event["classroom"] = classroom["classroom_name"]
                 event["building"] = classroom["building"]
                 partial_allocated.append(event)
                 break
       if len(partial_allocated) != len(events):
-        difference = len(events) - len(partial_allocated)
-        print("Não foi possível alocar", subject_code, class_code, "tivemos", difference, "horários não alocados")
         unallocated_events.extend(events)
       else:
         for event in partial_allocated:

src/common/cache.py

@@ -3,7 +3,7 @@
 config = {
     "DEBUG": True,
     "CACHE_TYPE": "SimpleCache",
-    "CACHE_DEFAULT_TIMEOUT": 60 * 60 # 1 hour
+    "CACHE_DEFAULT_TIMEOUT": 60 * 5 # 1 hour
 }
 
 cache = Cache(config=config)

src/common/crawler.py

@@ -44,7 +44,8 @@ def __build_soap(self):
         self.soup = BeautifulSoup(page.content, "html.parser")
 
     def __find_classes_divs(self):
-        self.classes_divs = self.soup.find_all("div", attrs=CLASS_DIV_IDENTIFIERS)
+        self.classes_divs = self.soup.find_all(
+            "div", attrs=CLASS_DIV_IDENTIFIERS)
 
     def __extract_classes_info(self):
         for class_div in self.classes_divs:
@@ -54,6 +55,8 @@ def __extract_classes_info(self):
     def __build_events_from_class_div(self, class_div):
         result = []
         info_tables = class_div.find_all("table")
+        if (len(info_tables) != 3):
+            return []
         general_info = self.__get_general_info(info_tables)
         schedule_info_list = self.__get_schedule_info_list(info_tables)
         student_numbers_info = self.__get_student_numbers_info(info_tables)
@@ -67,11 +70,19 @@ def __get_general_info(self, info_tables) -> dict:
         general_info_table = info_tables[0]
         general_info_table_rows = general_info_table.find_all("tr")
 
-        class_code = general_info_table_rows[0].find_all("td")[1].get_text(strip=True)
-        start_period = general_info_table_rows[1].find_all("td")[1].get_text(strip=True)
-        end_period = general_info_table_rows[2].find_all("td")[1].get_text(strip=True)
-        class_type = general_info_table_rows[3].find_all("td")[1].get_text(strip=True)
-        obs = general_info_table_rows[4].find_all("td")[1].get_text(strip=True)
+        class_code = general_info_table_rows[0].find_all(
+            "td")[1].get_text(strip=True)
+        start_period = general_info_table_rows[1].find_all("td")[
+            1].get_text(strip=True)
+        end_period = general_info_table_rows[2].find_all(
+            "td")[1].get_text(strip=True)
+        class_type = general_info_table_rows[3].find_all(
+            "td")[1].get_text(strip=True)
+        try:
+            obs = general_info_table_rows[4].find_all(
+                "td")[1].get_text(strip=True)
+        except IndexError:
+            obs = ""
 
         result["class_code"] = class_code
         result["start_period"] = start_period
@@ -111,14 +122,17 @@ def __get_schedule_info_list(self, info_tables) -> list:
                 and end_time == ""
                 and professor != ""
             ):
-                result[index - 1]["professors"].append(professor)
+                result[len(result) - 1]["professors"].append(professor)
                 continue
 
+            # More than one hour in same day: only week day is empty
+            if (week_day == "" and start_time != "" and end_time != "" and professor != ""):
+                week_day = result[len(result) - 1]["week_day"]
+
             partial_result["week_day"] = week_day
             partial_result["professors"] = [professor]
             partial_result["start_time"] = start_time
             partial_result["end_time"] = end_time
-
             result.append(partial_result)
 
         return result
@@ -138,6 +152,13 @@ def __get_student_numbers_info(self, info_tables) -> dict:
         for row in student_numbers_rows_dropped:
             data = row.find_all("span", attrs=filter)
 
+            try:
+                if len(data) != 5:
+                    raise Exception
+
+            except:
+                continue
+
             vacancies_text = data[1].get_text(strip=True)
             subscribers_text = data[2].get_text(strip=True)
             pendings_text = data[3].get_text(strip=True)
@@ -161,7 +182,8 @@ def __add_subject_info_to_events(self):
             event["subject_code"] = self.subject_code
 
     def __get_subject_name(self):
-        subject = self.soup.find_all("b", text=re.compile("Disciplina:(.*)"))[0]
+        subject = self.soup.find_all(
+            "b", text=re.compile("Disciplina:(.*)"))[0]
         self.subject_name = subject.get_text().replace(
             f"Disciplina: {self.subject_code} - ", ""
         )

src/schemas/event_schema.py

@@ -21,6 +21,7 @@ class EventSchema(Schema):
     pendings = fields.Int()
     preferences = fields.Nested(PreferencesSchema(unknown=EXCLUDE))
     has_to_be_allocated = fields.Bool()
+    ignore_to_allocate = fields.Bool()
     classroom = fields.Str()
     building = fields.Str()
     updated_at = fields.Str()