Merge pull request #818 from openzim/add_notification_timeout

Add notification timeout

dispatcher/backend/maint-scripts/list_running_tasks.py

@@ -0,0 +1,65 @@
+import sys
+
+import requests
+from get_token import get_token, get_token_headers, get_url
+
+running_statuses = [
+    "reserved",
+    "started",
+    "scraper_started",
+    "scraper_completed",
+    "scraper_killed",
+    "cancel_requested",
+]
+
+
+def get_status_query():
+    filters = [f"status={status}" for status in running_statuses]
+    return "&".join(filters)
+
+
+def get_timestamp(task, status):
+    if status in task["timestamp"]:
+        return task["timestamp"][status]
+    else:
+        return "-"
+
+
+def main(username, password):
+    """Print in STDOUT a markdown table of running tasks with various information"""
+    access_token, refresh_token = get_token(username, password)
+    response = requests.get(
+        f"{get_url('/tasks')}?{get_status_query()}",
+        headers=get_token_headers(access_token),
+    )
+    tasks = response.json()["items"]
+    print(
+        "| Task ID | worker | kind | DB Status | last update at | requested"
+        " | reserved | started | scraper_started | scraper_completed |"
+    )
+    print("|--|--|--|--|--|--|--|--|--|--|")
+    for task in sorted(tasks, key=lambda task: task["updated_at"]):
+        response = requests.get(
+            f"{get_url('/tasks')}/{task['_id']}",
+            headers=get_token_headers(access_token),
+        )
+        task_details = response.json()
+
+        print(
+            f"| [{task['_id']}](https://farm.openzim.org/pipeline/{task['_id']}) "
+            f"| {task['worker']} "
+            f"| {task_details['config']['task_name']} "
+            f"| {task['status']} "
+            f"| {task['updated_at'][:19]} "
+            f"| {get_timestamp(task, 'requested')[:19]} "
+            f"| {get_timestamp(task, 'reserved')[:19]} "
+            f"| {get_timestamp(task, 'started')[:19]} "
+            f"| {get_timestamp(task, 'scraper_started')[:19]} "
+            f"| {get_timestamp(task, 'scraper_completed')[:19]} "
+            f"|"
+        )
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    main(*args)

dispatcher/backend/src/common/constants.py

@@ -39,6 +39,8 @@
 
 # NOTIFICATIONS
 
+WEB_NOTIFICATIONS_TIMEOUT = int(os.getenv("WEB_NOTIFICATIONS_TIMEOUT", 5))
+
 # in-notification URLs
 PUBLIC_URL = os.getenv("PUBLIC_URL", "https://farm.openzim.org")
 ZIM_DOWNLOAD_URL = os.getenv(

dispatcher/backend/src/common/emailing.py

@@ -8,7 +8,12 @@
 import requests
 from werkzeug.datastructures import MultiDict
 
-from common.constants import MAILGUN_API_KEY, MAILGUN_API_URL, MAILGUN_FROM
+from common.constants import (
+    MAILGUN_API_KEY,
+    MAILGUN_API_URL,
+    MAILGUN_FROM,
+    WEB_NOTIFICATIONS_TIMEOUT,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +57,7 @@ def send_email_via_mailgun(
             ]
             if attachments
             else [],
+            timeout=WEB_NOTIFICATIONS_TIMEOUT,
         )
         resp.raise_for_status()
     except Exception as exc:

dispatcher/backend/src/common/notifications.py

@@ -19,6 +19,7 @@
     SLACK_ICON,
     SLACK_URL,
     SLACK_USERNAME,
+    WEB_NOTIFICATIONS_TIMEOUT,
     ZIM_DOWNLOAD_URL,
 )
 from common.emailing import send_email_via_mailgun
@@ -95,6 +96,7 @@ def handle_webhook_notification(task, urls):
                 url,
                 data=dumps(task).encode("UTF-8"),
                 headers={"Content-Type": "application/json"},
+                timeout=WEB_NOTIFICATIONS_TIMEOUT,
             )
             resp.raise_for_status()
         except Exception as exc:
@@ -112,6 +114,7 @@ def handle_slack_notification(task, channels):
         try:
             requests.post(
                 SLACK_URL,
+                timeout=WEB_NOTIFICATIONS_TIMEOUT,
                 json={
                     # destination. prefix with # for chans or @ for account
                     "channel": channel,