Skip to content

Commit

Permalink
[SDESK-4695] Convert relative dates to absolute for date_histogram bo…
Browse files Browse the repository at this point in the history 
…unds (#112)

* [SDESK-4695] Convert relative dates to absolute for date_histogram bounds

* Remove 'pass' command
  • Loading branch information
@MarkLark86
MarkLark86 authored Dec 20, 2019
1 parent 2e15c27 commit 6065ade
Show file tree
Hide file tree
Showing 4 changed files with 256 additions and 14 deletions.
29 changes: 20 additions & 9 deletions server/analytics/base_report/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

from apps.search import SearchService

from analytics.common import MIME_TYPES, get_elastic_version, get_weekstart_offset_hr, DATE_FILTERS
from analytics.common import MIME_TYPES, get_elastic_version, get_weekstart_offset_hr, DATE_FILTERS, \
relative_to_absolute_datetime


class BaseReportService(SearchService):
Expand Down Expand Up @@ -99,6 +100,19 @@ def _get_histogram_aggregation(self, interval, args, aggregations):
# starting day of the week, based on app.config['START_OF_WEEK']
offset = 0 if interval != 'week' else get_weekstart_offset_hr()

if lt == 'now':
extended_bounds = {
'max': relative_to_absolute_datetime(lt, '%Y-%m-%dT%H:%M:%S'),
'min': relative_to_absolute_datetime(gte, '%Y-%m-%dT%H:%M:%S')
}
else:
extended_bounds = {
'min': gte[:-5], # remove timezone part
'max': lt[:-5] # remove timezone part
}

# dates.extended_bounds.min & dates.extended_bounds.max should use dates.format

aggs = {
'dates': {
'date_histogram': {
Expand All @@ -107,10 +121,7 @@ def _get_histogram_aggregation(self, interval, args, aggregations):
'time_zone': time_zone,
'min_doc_count': 0,
'offset': '{}h'.format(offset),
'extended_bounds': {
'min': gte[:-5], # remove timezone part
'max': lt[:-5] # remove timezone part
},
'extended_bounds': extended_bounds,
'format': 'yyyy-MM-dd\'T\'HH:mm:ss'
},
'aggs': aggregations
Expand Down Expand Up @@ -422,10 +433,10 @@ def _es_get_date_filters(self, params):
gte = 'now-1M/M'
elif date_filter == DATE_FILTERS.RELATIVE_HOURS:
lt = 'now'
gte = 'now-{}h'.format(relative)
gte = 'now-{}h/h'.format(relative)
elif date_filter == DATE_FILTERS.RELATIVE_DAYS:
lt = 'now'
gte = 'now-{}d'.format(relative)
gte = 'now-{}d/d'.format(relative)
elif date_filter == DATE_FILTERS.TODAY:
lt = 'now'
gte = 'now/d'
Expand All @@ -437,10 +448,10 @@ def _es_get_date_filters(self, params):
gte = 'now/M'
elif date_filter == DATE_FILTERS.RELATIVE_WEEKS:
lt = 'now'
gte = 'now-{}w/w'.format(relative)
gte = 'now-{}w/d'.format(relative)
elif date_filter == DATE_FILTERS.RELATIVE_MONTHS:
lt = 'now'
gte = 'now-{}M/M'.format(relative)
gte = 'now-{}M/d'.format(relative)
elif date_filter == DATE_FILTERS.LAST_YEAR:
lt = 'now/y'
gte = 'now-1y/y'
Expand Down
4 changes: 2 additions & 2 deletions server/analytics/base_report/base_report_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def test_generate_elastic_query_from_date_object(self):
'range': {
'versioncreated': {
'lt': 'now',
'gte': 'now-12h',
'gte': 'now-12h/h',
'time_zone': '+1000'
}
}
Expand All @@ -251,7 +251,7 @@ def test_generate_elastic_query_from_date_object(self):
'range': {
'versioncreated': {
'lt': 'now',
'gte': 'now-7d',
'gte': 'now-7d/d',
'time_zone': '+1000'
}
}
Expand Down
134 changes: 133 additions & 1 deletion server/analytics/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,18 @@
# at https://www.sourcefabric.org/superdesk/license

from superdesk import get_resource_service
from superdesk.utc import utcnow, utc_to_local
from collections import namedtuple
from subprocess import check_call, PIPE
from flask import current_app as app
import pytz
from datetime import datetime
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from math import floor
import re
import logging

logger = logging.getLogger(__name__)


mime_types = [
Expand Down Expand Up @@ -236,3 +242,129 @@ def seconds_to_human_readable(seconds):
return '1 second'

return '{} seconds'.format(floor(seconds))


def relative_to_absolute_datetime(value, format, now=None, offset=None):
"""Converts a relative datetime to an absolute datetime in the format provided
Formats supported include:
now
now/[g]
now-[o]
now-[o]/[g]
g = granularity (rounding the value down to the nearest value)
- m = minute
- h = hour
- d = day (first hour/minute/second of the day)
- w = week (first day of the week, using the system configured START_OF_WEEK value)
- M = month (the first day of the month)
- y = year (the first day of the year)
o = offset from now suffixed by the granularity (see above for supported values)
examples:
now
now/d
now-1d
now-1d/d
:param string value: The relative datetime
:param string format: The format used to convert the datetime instance back to a string
:param datetime now: The date and time to use for relative calculations (defaults to now using DEFAULT_TIMEZONE)
:param number offset: The utc offset in minutes (defaults to offset using the DEFAULT_TIMEZONE config)
:return string: The absolute datetime in the format provided
"""

try:
values = re.search(r'^now(?P<offset>(-\d+[mhdwMy])?)(?P<granularity>(/[mhdwMy])?)$', value).groupdict()
except AttributeError as e:
logger.exception('Value {} is in incorrect relative format'.format(value))
raise

if now is None:
now = utc_to_local(app.config.get('DEFAULT_TIMEZONE'), utcnow())

if values.get('offset'):
# Retrieve the offset value and granularity, then shift the datetime

granularity = values['offset'][-1]
offset = int(values['offset'][1:-1])

if granularity == 'm':
now = now - timedelta(minutes=offset)
elif granularity == 'h':
now = now - timedelta(hours=offset)
elif granularity == 'd':
now = now - timedelta(days=offset)
elif granularity == 'w':
now = now - timedelta(weeks=offset)
elif granularity == 'M':
now = now - relativedelta(months=offset)
elif granularity == 'y':
now = now - relativedelta(years=offset)

if values.get('granularity'):
# Round the value down using the granularity provided

granularity = values['granularity'][1:]

parts = None
if granularity == 'm':
parts = {'second': 0}
elif granularity == 'h':
parts = {
'second': 0,
'minute': 0
}
elif granularity == 'd':
parts = {
'second': 0,
'minute': 0,
'hour': 0
}
elif granularity == 'w':
# Using START_OF_WEEK to calculate the number of days to shift for the beginning of the week
# START_OF_WEEK
# 0: Sunday
# 6: Saturday

isoweekday = now.isoweekday()
if isoweekday == 7:
isoweekday = 0

start_of_week = app.config.get('START_OF_WEEK') or 0
offset = 7 - start_of_week + isoweekday

if offset < 7:
now -= timedelta(days=offset)
elif offset > 7:
now -= timedelta(days=offset - 7)

parts = {
'second': 0,
'minute': 0,
'hour': 0
}
elif granularity == 'M':
parts = {
'second': 0,
'minute': 0,
'hour': 0,
'day': 1
}
elif granularity == 'y':
parts = {
'second': 0,
'minute': 0,
'hour': 0,
'day': 1,
'month': 1
}

if parts:
# Sets the second, minute, hour, day and/or month of the shifted value provided
now = now.replace(**parts)

return now.strftime(format)
103 changes: 101 additions & 2 deletions server/analytics/tests/common_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
from superdesk.tests import TestCase

from analytics.common import get_weekstart_offset_hr, get_utc_offset_in_minutes,\
seconds_to_human_readable
seconds_to_human_readable, relative_to_absolute_datetime

from datetime import datetime
from datetime import datetime, timedelta


class CommonTestCase(TestCase):
Expand Down Expand Up @@ -75,3 +75,102 @@ def test_seconds_to_human_readable(self):
self.assertEqual(seconds_to_human_readable(129600), '1 day')
self.assertEqual(seconds_to_human_readable(172800), '2 days')
self.assertEqual(seconds_to_human_readable(216000), '2 days')

def test_relative_to_absolute_datetime(self):
self.app.config.update({
'DEFAULT_TIMEZONE': 'Australia/Sydney',
'START_OF_WEEK': 0 # Sunday
})
fm = '%Y-%m-%dT%H:%M:%S'
dt = datetime(2019, 10, 25, 13, 25, 52)

# Test without any optional parameters
self.assertEqual(relative_to_absolute_datetime('now', fm, dt), '2019-10-25T13:25:52')

# Test rounding down
self.assertEqual(relative_to_absolute_datetime('now/m', fm, dt), '2019-10-25T13:25:00')
self.assertEqual(relative_to_absolute_datetime('now/h', fm, dt), '2019-10-25T13:00:00')
self.assertEqual(relative_to_absolute_datetime('now/d', fm, dt), '2019-10-25T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now/M', fm, dt), '2019-10-01T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now/y', fm, dt), '2019-01-01T00:00:00')

# Test subtraction
self.assertEqual(relative_to_absolute_datetime('now-1m', fm, dt), '2019-10-25T13:24:52')
self.assertEqual(relative_to_absolute_datetime('now-1h', fm, dt), '2019-10-25T12:25:52')
self.assertEqual(relative_to_absolute_datetime('now-1d', fm, dt), '2019-10-24T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-1w', fm, dt), '2019-10-18T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-1M', fm, dt), '2019-09-25T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-1y', fm, dt), '2018-10-25T13:25:52')

self.assertEqual(relative_to_absolute_datetime('now-26m', fm, dt), '2019-10-25T12:59:52')
self.assertEqual(relative_to_absolute_datetime('now-26h', fm, dt), '2019-10-24T11:25:52')
self.assertEqual(relative_to_absolute_datetime('now-26d', fm, dt), '2019-09-29T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-26w', fm, dt), '2019-04-26T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-26M', fm, dt), '2017-08-25T13:25:52')
self.assertEqual(relative_to_absolute_datetime('now-26y', fm, dt), '1993-10-25T13:25:52')

# Test subtraction and rounding down
self.assertEqual(relative_to_absolute_datetime('now-1m/m', fm, dt), '2019-10-25T13:24:00')
self.assertEqual(relative_to_absolute_datetime('now-1m/h', fm, dt), '2019-10-25T13:00:00')
self.assertEqual(relative_to_absolute_datetime('now-1m/d', fm, dt), '2019-10-25T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-1m/w', fm, dt), '2019-10-20T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-1m/M', fm, dt), '2019-10-01T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-1m/y', fm, dt), '2019-01-01T00:00:00')

self.assertEqual(relative_to_absolute_datetime('now-26m/m', fm, dt), '2019-10-25T12:59:00')
self.assertEqual(relative_to_absolute_datetime('now-26m/h', fm, dt), '2019-10-25T12:00:00')
self.assertEqual(relative_to_absolute_datetime('now-26m/d', fm, dt), '2019-10-25T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-26m/w', fm, dt), '2019-10-20T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-26m/M', fm, dt), '2019-10-01T00:00:00')
self.assertEqual(relative_to_absolute_datetime('now-26m/y', fm, dt), '2019-01-01T00:00:00')

def test_relative_to_absolute_datetime_week_granularity(self):
"""Starting on Sunday 2019-10-20, test shifting the week using different START_OF_WEEK values"""

results = {
'Sunday': {
'start_of_week': 0,
'results': [20, 20, 20, 20, 20, 20, 20]
},
'Monday': {
'start_of_week': 1,
'results': [14, 21, 21, 21, 21, 21, 21]
},
'Tuesday': {
'start_of_week': 2,
'results': [15, 15, 22, 22, 22, 22, 22]
},
'Wednesday': {
'start_of_week': 3,
'results': [16, 16, 16, 23, 23, 23, 23]
},
'Thursday': {
'start_of_week': 4,
'results': [17, 17, 17, 17, 24, 24, 24]
},
'Friday': {
'start_of_week': 5,
'results': [18, 18, 18, 18, 18, 25, 25]
},
'Saturday': {
'start_of_week': 6,
'results': [19, 19, 19, 19, 19, 19, 26]
},
}

self.app.config['DEFAULT_TIMEZONE'] = 'Australia/Sydney'
fm = '%Y-%m-%dT%H:%M:%S'

for day, data in results.items():
self.app.config['START_OF_WEEK'] = data['start_of_week']

for i in range(0, 7):
self.assertEqual(
relative_to_absolute_datetime(
'now/w',
fm,
datetime(2019, 10, 20 + i, 13, 25, 52)
),
'2019-10-{}T00:00:00'.format(data['results'][i]),
'Expected: {} {}'.format(day, i)
)

0 comments on commit 6065ade

Please sign in to comment.