From 0ad6c9c66cb9dcaf300f3a938b5f243b61ec8eab Mon Sep 17 00:00:00 2001 From: Edward L Platt Date: Mon, 2 Dec 2024 18:20:01 -0500 Subject: [PATCH] Updates email report to query one day at a time. The reporting script running on the analysis machine stopped sending emails. The issue appeared to be mysql queries hanging due to heavy memory usage. In particular, queries of the mod_actions table were not completing. This change makes one query per day rather than making a single query for all days, reducing mysql memory usage. The queries look like they could be optimized further, but it would require a more significant rewrite of the python code. --- utils/email_db_report.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/email_db_report.py b/utils/email_db_report.py index 749f508..b88a403 100644 --- a/utils/email_db_report.py +++ b/utils/email_db_report.py @@ -49,11 +49,16 @@ def str_to_date(date_str, by_day=True): return datetime.datetime.strptime(date_str, date_format) def run_query_for_days(query_str, today, days=7): - today_str = date_to_str(today, by_day=False) - last_week = today - datetime.timedelta(days=days) - last_week_str = date_to_str(last_week, by_day=False) - - result = db_session.execute(query_str, {"from_date": last_week_str, "to_date": today_str}).fetchall() + result = [] + # Some queries (mod_actions) can cause mysql to run out of memory + # Querying a day at a time reduces the memory usage + for day in range(days): + days_ago = days - day + from_dt = today - datetime.timedelta(days=days_ago) + from_str = date_to_str(from_dt, by_day=False) + to_dt = from_dt + datetime.timedelta(days=1) + to_str = date_to_str(to_dt, by_day=False) + result += list(db_session.execute(query_str, {"from_date": from_str, "to_date": to_str}).fetchall()) return result def transform_result_to_dict(result):