Merge pull request #19 from xcpcio/bugfix-use-timestamp

fix: use timestamp

tests/spider/csg_cpc/v1/csg_cpc_test.py

@@ -15,7 +15,7 @@ def test_spider_csg_cpc_v1_2023_hncpc(snapshot):
     fetch_uri = os.path.join(current_dir_path, "test_data/2023_hncpc")
 
     csg_cpc = CSG_CPC(c, fetch_uri)
-    csg_cpc.fetch().parse_teams().parse_runs().update_contest()
+    csg_cpc.fetch().update_contest().parse_teams().parse_runs()
 
     assert len(csg_cpc.teams) == 180
     assert len(csg_cpc.runs) == 2336

tests/spider/csg_cpc/v1/snapshots/csg_cpc_test/test_spider_csg_cpc_v1_2023_hncpc/contest

@@ -1 +1 @@
-{"contest_name": "", "start_time": "2023-11-11T15:30:00+08:00", "end_time": "2023-11-11T17:30:00+08:00", "frozen_time": 3600, "penalty": 1200, "problem_quantity": 0, "problem_id": [], "group": {"official": "\u6b63\u5f0f\u961f\u4f0d", "unofficial": "\u6253\u661f\u961f\u4f0d"}, "organization": "School", "status_time_display": {"correct": true, "incorrect": true, "pending": true}, "medal": {}, "options": {}}
+{"contest_name": "", "start_time": 1694912400, "end_time": 1694930400, "frozen_time": 3600, "penalty": 1200, "problem_quantity": 0, "problem_id": [], "group": {"official": "\u6b63\u5f0f\u961f\u4f0d", "unofficial": "\u6253\u661f\u961f\u4f0d"}, "organization": "School", "status_time_display": {"correct": true, "incorrect": true, "pending": true}, "medal": {}, "options": {}}

tests/spider/csg_cpc/v1/test_data/2023_hncpc/contest.json

@@ -1,16 +1,16 @@
 {
-    "contest_id": 1001,
-    "title": "第九届中国大学生程序设计竞赛（深圳）- 热身赛",
-    "start_time": "2023-11-11 15:30:00",
-    "end_time": "2023-11-11 17:30:00",
-    "defunct": "0",
-    "description": "",
+    "contest_id": 1007,
+    "title": "湖南省第十九届大学生计算机程序设计竞赛",
+    "start_time": "2023-09-17 09:00:00",
+    "end_time": "2023-09-17 14:00:00",
+    "defunct": "1",
+    "description": "<p>B题题面有误，实际数据中ai不会出现0，ai为正整数，已更新题面。</p>\n<p>大家可以多看看clarification</p>",
     "private": 2,
     "langmask": 75,
     "password": "",
-    "attach": "2023-11-11_F0EA3F56319521FA",
-    "topteam": 1,
-    "award_ratio": 30020010,
+    "attach": "2023-09-17_49DF64BE8AD940D5",
+    "topteam": 4,
+    "award_ratio": 25020010,
     "frozen_minute": 60,
-    "frozen_after": 0
-}
+    "frozen_after": 150
+}

tests/spider/domjudge/v3/domjudge_test.py

@@ -17,7 +17,7 @@ def test_spider_domjudge_v3_9th_ccpc_guilin(snapshot):
     fetch_uri = os.path.join(current_dir_path, "test_data", test_prefix)
 
     d = DOMjudge(c, fetch_uri)
-    d.fetch().parse_teams().parse_runs().update_contest()
+    d.fetch().update_contest().parse_teams().parse_runs()
 
     assert len(d.teams) == 270
     assert len(d.runs) == 3360

tests/spider/domjudge/v3/snapshots/domjudge_test/test_spider_domjudge_v3_9th_ccpc_guilin/contest

@@ -1 +1 @@
-{"contest_name": "", "start_time": "2023-10-29T09:00:00+08:00", "end_time": "2023-10-29T14:00:00+08:00", "frozen_time": 3600, "penalty": 1200, "problem_quantity": 13, "problem_id": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"], "group": {"official": "\u6b63\u5f0f\u961f\u4f0d", "unofficial": "\u6253\u661f\u961f\u4f0d"}, "organization": "School", "status_time_display": {"correct": true, "incorrect": true, "pending": true}, "medal": {}, "balloon_color": [{"color": "#000", "background_color": "#9d00ff"}, {"color": "#000", "background_color": "#42ff58"}, {"color": "#000", "background_color": "#ffb4f9"}, {"color": "#000", "background_color": "#29ffd1"}, {"color": "#000", "background_color": "#ffffff"}, {"color": "#000", "background_color": "#b5b5b5"}, {"color": "#000", "background_color": "#ff0000"}, {"color": "#000", "background_color": "#007542"}, {"color": "#000", "background_color": "#ffff00"}, {"color": "#000", "background_color": "#ff9d00"}, {"color": "#000", "background_color": "#adfbff"}, {"color": "#000", "background_color": "#8f6900"}, {"color": "#000", "background_color": "#172eff"}], "options": {"submission_timestamp_unit": "millisecond"}}
+{"contest_name": "", "start_time": 1698541200, "end_time": 1698559200, "frozen_time": 3600, "penalty": 1200, "problem_quantity": 13, "problem_id": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"], "group": {"official": "\u6b63\u5f0f\u961f\u4f0d", "unofficial": "\u6253\u661f\u961f\u4f0d"}, "organization": "School", "status_time_display": {"correct": true, "incorrect": true, "pending": true}, "medal": {}, "balloon_color": [{"color": "#000", "background_color": "#9d00ff"}, {"color": "#000", "background_color": "#42ff58"}, {"color": "#000", "background_color": "#ffb4f9"}, {"color": "#000", "background_color": "#29ffd1"}, {"color": "#000", "background_color": "#ffffff"}, {"color": "#000", "background_color": "#b5b5b5"}, {"color": "#000", "background_color": "#ff0000"}, {"color": "#000", "background_color": "#007542"}, {"color": "#000", "background_color": "#ffff00"}, {"color": "#000", "background_color": "#ff9d00"}, {"color": "#000", "background_color": "#adfbff"}, {"color": "#000", "background_color": "#8f6900"}, {"color": "#000", "background_color": "#172eff"}], "options": {"submission_timestamp_unit": "millisecond"}}

xcpcio_board_spider/core/utils.py

@@ -83,5 +83,14 @@ def get_timestamp_second(dt):
     return int(timestamp)
 
 
+def get_timestamp_from_iso8601(dt):
+    from datetime import datetime
+
+    datetime_obj = datetime.fromisoformat(dt)
+    timestamp = datetime_obj.timestamp()
+
+    return int(timestamp)
+
+
 def get_now_timestamp_second():
     return int(time.time())

xcpcio_board_spider/spider/csg_cpc/v1/csg_cpc.py

@@ -66,6 +66,9 @@ def fetch(self):
 
         return self
 
+    def get_timestamp_from_date(self, dt):
+        return utils.get_timestamp_from_iso8601(dt.replace(" ", "T") + "+08:00")
+
     def parse_teams(self):
         teams = Teams()
 
@@ -177,26 +180,6 @@ def parse_result(self, result: int):
 
     def parse_runs(self):
         runs = Submissions()
-
-        """
-
-        [
-            6,
-            1001,
-            1000,
-            "team111",
-            6,
-            "2023-05-13T15:33:27"
-        ],
-
-        solution_id
-        contest_id
-        problem_id
-        user_id
-        result
-        in_date
-        """
-
         for raw_run in self.raw_run_data:
             run = Submission()
 
@@ -205,7 +188,7 @@ def parse_runs(self):
             team_id = str(raw_run["user_id"]).split("_")[-1]
             result = int(raw_run["result"])
             in_date = str(raw_run["in_date"]).replace("T", " ")
-            timestamp = utils.get_timestamp_second(
+            timestamp = self.get_timestamp_from_date(
                 in_date) - self.contest.start_time
 
             run.submission_id = str(submission_id)
@@ -226,6 +209,9 @@ def update_contest(self):
         end_time = self.raw_contest_data["end_time"]
         frozen_minute = self.raw_contest_data["frozen_minute"]
 
-        self.contest.start_time = start_time.replace(" ", "T") + "+08:00"
-        self.contest.end_time = end_time.replace(" ", "T") + "+08:00"
+        self.contest.start_time = self.get_timestamp_from_date(start_time)
+        self.contest.end_time = self.get_timestamp_from_date(end_time)
+
         self.contest.frozen_time = int(frozen_minute) * 60
+
+        return self

xcpcio_board_spider/spider/domjudge/v3/domjudge.py

@@ -1,6 +1,7 @@
 import math
 
 from xcpcio_board_spider.type import Contest, Team, Teams, Submission, Submissions, constants, Color
+from xcpcio_board_spider import utils
 
 from domjudge_utility import Dump, DumpConfig
 
@@ -148,10 +149,12 @@ def update_contest(self):
 
         start_time = self.dump.contest["start_time"]
         end_time = self.dump.contest["end_time"]
-        self.contest.start_time = start_time
-        self.contest.end_time = end_time
+        self.contest.start_time = utils.get_timestamp_from_iso8601(start_time)
+        self.contest.end_time = utils.get_timestamp_from_iso8601(end_time)
 
         if "scoreboard_freeze_duration" in self.dump.contest.keys() and self.dump.contest["scoreboard_freeze_duration"] is not None:
             scoreboard_freeze_duration = self.dump.contest["scoreboard_freeze_duration"]
             self.contest.frozen_time = self.get_submission_timestamp_millisecond(
                 scoreboard_freeze_duration) // 1000
+
+        return self