diff --git a/server/analytics/base_report/__init__.py b/server/analytics/base_report/__init__.py index 8362fa26e..8c0a40150 100644 --- a/server/analytics/base_report/__init__.py +++ b/server/analytics/base_report/__init__.py @@ -19,7 +19,8 @@ from apps.search import SearchService -from analytics.common import MIME_TYPES, get_elastic_version, get_weekstart_offset_hr, DATE_FILTERS +from analytics.common import MIME_TYPES, get_elastic_version, get_weekstart_offset_hr, DATE_FILTERS, \ + relative_to_absolute_datetime class BaseReportService(SearchService): @@ -99,6 +100,19 @@ def _get_histogram_aggregation(self, interval, args, aggregations): # starting day of the week, based on app.config['START_OF_WEEK'] offset = 0 if interval != 'week' else get_weekstart_offset_hr() + if lt == 'now': + extended_bounds = { + 'max': relative_to_absolute_datetime(lt, '%Y-%m-%dT%H:%M:%S'), + 'min': relative_to_absolute_datetime(gte, '%Y-%m-%dT%H:%M:%S') + } + else: + extended_bounds = { + 'min': gte[:-5], # remove timezone part + 'max': lt[:-5] # remove timezone part + } + + # dates.extended_bounds.min & dates.extended_bounds.max should use dates.format + aggs = { 'dates': { 'date_histogram': { @@ -107,10 +121,7 @@ def _get_histogram_aggregation(self, interval, args, aggregations): 'time_zone': time_zone, 'min_doc_count': 0, 'offset': '{}h'.format(offset), - 'extended_bounds': { - 'min': gte[:-5], # remove timezone part - 'max': lt[:-5] # remove timezone part - }, + 'extended_bounds': extended_bounds, 'format': 'yyyy-MM-dd\'T\'HH:mm:ss' }, 'aggs': aggregations @@ -422,10 +433,10 @@ def _es_get_date_filters(self, params): gte = 'now-1M/M' elif date_filter == DATE_FILTERS.RELATIVE_HOURS: lt = 'now' - gte = 'now-{}h'.format(relative) + gte = 'now-{}h/h'.format(relative) elif date_filter == DATE_FILTERS.RELATIVE_DAYS: lt = 'now' - gte = 'now-{}d'.format(relative) + gte = 'now-{}d/d'.format(relative) elif date_filter == DATE_FILTERS.TODAY: lt = 'now' gte = 'now/d' @@ -437,10 +448,10 @@ def _es_get_date_filters(self, params): gte = 'now/M' elif date_filter == DATE_FILTERS.RELATIVE_WEEKS: lt = 'now' - gte = 'now-{}w/w'.format(relative) + gte = 'now-{}w/d'.format(relative) elif date_filter == DATE_FILTERS.RELATIVE_MONTHS: lt = 'now' - gte = 'now-{}M/M'.format(relative) + gte = 'now-{}M/d'.format(relative) elif date_filter == DATE_FILTERS.LAST_YEAR: lt = 'now/y' gte = 'now-1y/y' diff --git a/server/analytics/base_report/base_report_test.py b/server/analytics/base_report/base_report_test.py index f617e2b9b..b7c8b6ab1 100644 --- a/server/analytics/base_report/base_report_test.py +++ b/server/analytics/base_report/base_report_test.py @@ -229,7 +229,7 @@ def test_generate_elastic_query_from_date_object(self): 'range': { 'versioncreated': { 'lt': 'now', - 'gte': 'now-12h', + 'gte': 'now-12h/h', 'time_zone': '+1000' } } @@ -251,7 +251,7 @@ def test_generate_elastic_query_from_date_object(self): 'range': { 'versioncreated': { 'lt': 'now', - 'gte': 'now-7d', + 'gte': 'now-7d/d', 'time_zone': '+1000' } } diff --git a/server/analytics/common.py b/server/analytics/common.py index 00333b8ae..740b12c51 100644 --- a/server/analytics/common.py +++ b/server/analytics/common.py @@ -9,12 +9,18 @@ # at https://www.sourcefabric.org/superdesk/license from superdesk import get_resource_service +from superdesk.utc import utcnow, utc_to_local from collections import namedtuple from subprocess import check_call, PIPE from flask import current_app as app import pytz -from datetime import datetime +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta from math import floor +import re +import logging + +logger = logging.getLogger(__name__) mime_types = [ @@ -236,3 +242,129 @@ def seconds_to_human_readable(seconds): return '1 second' return '{} seconds'.format(floor(seconds)) + + +def relative_to_absolute_datetime(value, format, now=None, offset=None): + """Converts a relative datetime to an absolute datetime in the format provided + + Formats supported include: + now + now/[g] + now-[o] + now-[o]/[g] + + g = granularity (rounding the value down to the nearest value) + - m = minute + - h = hour + - d = day (first hour/minute/second of the day) + - w = week (first day of the week, using the system configured START_OF_WEEK value) + - M = month (the first day of the month) + - y = year (the first day of the year) + + o = offset from now suffixed by the granularity (see above for supported values) + + examples: + now + now/d + now-1d + now-1d/d + + :param string value: The relative datetime + :param string format: The format used to convert the datetime instance back to a string + :param datetime now: The date and time to use for relative calculations (defaults to now using DEFAULT_TIMEZONE) + :param number offset: The utc offset in minutes (defaults to offset using the DEFAULT_TIMEZONE config) + :return string: The absolute datetime in the format provided + + """ + + try: + values = re.search(r'^now(?P(-\d+[mhdwMy])?)(?P(/[mhdwMy])?)$', value).groupdict() + except AttributeError as e: + logger.exception('Value {} is in incorrect relative format'.format(value)) + raise + + if now is None: + now = utc_to_local(app.config.get('DEFAULT_TIMEZONE'), utcnow()) + + if values.get('offset'): + # Retrieve the offset value and granularity, then shift the datetime + + granularity = values['offset'][-1] + offset = int(values['offset'][1:-1]) + + if granularity == 'm': + now = now - timedelta(minutes=offset) + elif granularity == 'h': + now = now - timedelta(hours=offset) + elif granularity == 'd': + now = now - timedelta(days=offset) + elif granularity == 'w': + now = now - timedelta(weeks=offset) + elif granularity == 'M': + now = now - relativedelta(months=offset) + elif granularity == 'y': + now = now - relativedelta(years=offset) + + if values.get('granularity'): + # Round the value down using the granularity provided + + granularity = values['granularity'][1:] + + parts = None + if granularity == 'm': + parts = {'second': 0} + elif granularity == 'h': + parts = { + 'second': 0, + 'minute': 0 + } + elif granularity == 'd': + parts = { + 'second': 0, + 'minute': 0, + 'hour': 0 + } + elif granularity == 'w': + # Using START_OF_WEEK to calculate the number of days to shift for the beginning of the week + # START_OF_WEEK + # 0: Sunday + # 6: Saturday + + isoweekday = now.isoweekday() + if isoweekday == 7: + isoweekday = 0 + + start_of_week = app.config.get('START_OF_WEEK') or 0 + offset = 7 - start_of_week + isoweekday + + if offset < 7: + now -= timedelta(days=offset) + elif offset > 7: + now -= timedelta(days=offset - 7) + + parts = { + 'second': 0, + 'minute': 0, + 'hour': 0 + } + elif granularity == 'M': + parts = { + 'second': 0, + 'minute': 0, + 'hour': 0, + 'day': 1 + } + elif granularity == 'y': + parts = { + 'second': 0, + 'minute': 0, + 'hour': 0, + 'day': 1, + 'month': 1 + } + + if parts: + # Sets the second, minute, hour, day and/or month of the shifted value provided + now = now.replace(**parts) + + return now.strftime(format) diff --git a/server/analytics/tests/common_test.py b/server/analytics/tests/common_test.py index ab9d6ab3e..5ce863313 100644 --- a/server/analytics/tests/common_test.py +++ b/server/analytics/tests/common_test.py @@ -12,9 +12,9 @@ from superdesk.tests import TestCase from analytics.common import get_weekstart_offset_hr, get_utc_offset_in_minutes,\ - seconds_to_human_readable + seconds_to_human_readable, relative_to_absolute_datetime -from datetime import datetime +from datetime import datetime, timedelta class CommonTestCase(TestCase): @@ -75,3 +75,102 @@ def test_seconds_to_human_readable(self): self.assertEqual(seconds_to_human_readable(129600), '1 day') self.assertEqual(seconds_to_human_readable(172800), '2 days') self.assertEqual(seconds_to_human_readable(216000), '2 days') + + def test_relative_to_absolute_datetime(self): + self.app.config.update({ + 'DEFAULT_TIMEZONE': 'Australia/Sydney', + 'START_OF_WEEK': 0 # Sunday + }) + fm = '%Y-%m-%dT%H:%M:%S' + dt = datetime(2019, 10, 25, 13, 25, 52) + + # Test without any optional parameters + self.assertEqual(relative_to_absolute_datetime('now', fm, dt), '2019-10-25T13:25:52') + + # Test rounding down + self.assertEqual(relative_to_absolute_datetime('now/m', fm, dt), '2019-10-25T13:25:00') + self.assertEqual(relative_to_absolute_datetime('now/h', fm, dt), '2019-10-25T13:00:00') + self.assertEqual(relative_to_absolute_datetime('now/d', fm, dt), '2019-10-25T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now/M', fm, dt), '2019-10-01T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now/y', fm, dt), '2019-01-01T00:00:00') + + # Test subtraction + self.assertEqual(relative_to_absolute_datetime('now-1m', fm, dt), '2019-10-25T13:24:52') + self.assertEqual(relative_to_absolute_datetime('now-1h', fm, dt), '2019-10-25T12:25:52') + self.assertEqual(relative_to_absolute_datetime('now-1d', fm, dt), '2019-10-24T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-1w', fm, dt), '2019-10-18T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-1M', fm, dt), '2019-09-25T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-1y', fm, dt), '2018-10-25T13:25:52') + + self.assertEqual(relative_to_absolute_datetime('now-26m', fm, dt), '2019-10-25T12:59:52') + self.assertEqual(relative_to_absolute_datetime('now-26h', fm, dt), '2019-10-24T11:25:52') + self.assertEqual(relative_to_absolute_datetime('now-26d', fm, dt), '2019-09-29T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-26w', fm, dt), '2019-04-26T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-26M', fm, dt), '2017-08-25T13:25:52') + self.assertEqual(relative_to_absolute_datetime('now-26y', fm, dt), '1993-10-25T13:25:52') + + # Test subtraction and rounding down + self.assertEqual(relative_to_absolute_datetime('now-1m/m', fm, dt), '2019-10-25T13:24:00') + self.assertEqual(relative_to_absolute_datetime('now-1m/h', fm, dt), '2019-10-25T13:00:00') + self.assertEqual(relative_to_absolute_datetime('now-1m/d', fm, dt), '2019-10-25T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-1m/w', fm, dt), '2019-10-20T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-1m/M', fm, dt), '2019-10-01T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-1m/y', fm, dt), '2019-01-01T00:00:00') + + self.assertEqual(relative_to_absolute_datetime('now-26m/m', fm, dt), '2019-10-25T12:59:00') + self.assertEqual(relative_to_absolute_datetime('now-26m/h', fm, dt), '2019-10-25T12:00:00') + self.assertEqual(relative_to_absolute_datetime('now-26m/d', fm, dt), '2019-10-25T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-26m/w', fm, dt), '2019-10-20T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-26m/M', fm, dt), '2019-10-01T00:00:00') + self.assertEqual(relative_to_absolute_datetime('now-26m/y', fm, dt), '2019-01-01T00:00:00') + + def test_relative_to_absolute_datetime_week_granularity(self): + """Starting on Sunday 2019-10-20, test shifting the week using different START_OF_WEEK values""" + + results = { + 'Sunday': { + 'start_of_week': 0, + 'results': [20, 20, 20, 20, 20, 20, 20] + }, + 'Monday': { + 'start_of_week': 1, + 'results': [14, 21, 21, 21, 21, 21, 21] + }, + 'Tuesday': { + 'start_of_week': 2, + 'results': [15, 15, 22, 22, 22, 22, 22] + }, + 'Wednesday': { + 'start_of_week': 3, + 'results': [16, 16, 16, 23, 23, 23, 23] + }, + 'Thursday': { + 'start_of_week': 4, + 'results': [17, 17, 17, 17, 24, 24, 24] + }, + 'Friday': { + 'start_of_week': 5, + 'results': [18, 18, 18, 18, 18, 25, 25] + }, + 'Saturday': { + 'start_of_week': 6, + 'results': [19, 19, 19, 19, 19, 19, 26] + }, + } + + self.app.config['DEFAULT_TIMEZONE'] = 'Australia/Sydney' + fm = '%Y-%m-%dT%H:%M:%S' + + for day, data in results.items(): + self.app.config['START_OF_WEEK'] = data['start_of_week'] + + for i in range(0, 7): + self.assertEqual( + relative_to_absolute_datetime( + 'now/w', + fm, + datetime(2019, 10, 20 + i, 13, 25, 52) + ), + '2019-10-{}T00:00:00'.format(data['results'][i]), + 'Expected: {} {}'.format(day, i) + )