diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index b897bca22445a..461341e8ca198 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -1,6 +1,7 @@ #!/usr/bin/env python import csv +from datetime import datetime import gzip import json from subprocess import Popen @@ -8,7 +9,7 @@ from subprocess import Popen from flask.ext.script import Manager from flask.ext.migrate import MigrateCommand from panoramix import db -from sqlalchemy import Column, Integer, String, Table +from sqlalchemy import Column, Integer, String, Table, DateTime from panoramix import app from panoramix import models @@ -59,7 +60,7 @@ def load_examples(sample): Column("year", Integer), Column("name", String(128)), Column("num", Integer), - Column("ds", String(20)), + Column("ds", DateTime), Column("gender", String(10)), Column("sum_boys", Integer), Column("sum_girls", Integer), @@ -78,7 +79,7 @@ def load_examples(sample): continue if num == "NA": num = 0 - ds = str(year) + '-01-01' + ds = datetime(int(year), 1, 1) db.engine.execute( BirthNames.insert(), state=state, diff --git a/panoramix/forms.py b/panoramix/forms.py index b38821e8fd00f..0a0b45d66cda6 100644 --- a/panoramix/forms.py +++ b/panoramix/forms.py @@ -1,6 +1,7 @@ from wtforms import ( Field, Form, SelectMultipleField, SelectField, TextField, TextAreaField, BooleanField, IntegerField, HiddenField) +from wtforms import validators from copy import copy from panoramix import app config = app.config @@ -58,7 +59,7 @@ def __init__(self, viz): description="One or many metrics to display"), 'groupby': SelectMultipleField( 'Group by', - choices=[(s, s) for s in datasource.groupby_column_names], + choices=self.choicify(datasource.groupby_column_names), description="One or many fields to group by"), 'granularity': TextField( 'Time Granularity', default="one day", @@ -75,19 +76,24 @@ def __init__(self, viz): SelectField( 'Row limit', default=config.get("ROW_LIMIT"), - choices=[(s, s) for s in self.row_limits]), + choices=self.choicify(self.row_limits)), 'limit': SelectField( - 'Series limit', choices=[(s, s) for s in self.series_limits], + 'Series limit', + choices=self.choicify(self.series_limits), default=50, description=( "Limits the number of time series that get displayed")), 'rolling_type': SelectField( 'Rolling', + default='mean', choices=[(s, s) for s in ['mean', 'sum', 'std']], description=( "Defines a rolling window function to apply")), - 'rolling_periods': TextField('Periods', description=( + 'rolling_periods': IntegerField( + 'Periods', + validators=[validators.optional()], + description=( "Defines the size of the rolling window function, " "relative to the 'granularity' field")), 'series': SelectField( @@ -118,7 +124,7 @@ def __init__(self, viz): description="Suffix to apply after the percentage display"), 'markup_type': SelectField( "Markup Type", - choices=[(s, s) for s in ['markdown', 'html']], + choices=self.choicify(['markdown', 'html']), default="markdown", description="Pick your favorite markup language"), 'rotation': SelectField( @@ -128,9 +134,9 @@ def __init__(self, viz): description="Rotation to apply to words in the cloud"), 'line_interpolation': SelectField( "Line Interpolation", - choices=[(s, s) for s in [ + choices=self.choicify([ 'linear', 'basis', 'cardinal', 'monotone', - 'step-before', 'step-after']], + 'step-before', 'step-after']), default='linear', description="Line interpolation as defined by d3.js"), 'code': TextAreaField("Code", description="Put your code here"), @@ -168,11 +174,24 @@ def __init__(self, viz): description="Compute the contribution to the total"), 'num_period_compare': IntegerField( "Period Ratio", default=None, + validators=[validators.optional()], description=( - "Number of period to compare against, " + "[integer] Number of period to compare against, " "this is relative to the granularity selected")), + 'time_compare': TextField( + "Time Shift Compare", + default="1 week ago", + description=( + "Overlay a timeseries from a " + "relative time period. Expects relative time delta " + "in natural language (example: 24 hours, 7 days, " + "56 weeks, 365 days")), } + @staticmethod + def choicify(l): + return [("{}".format(obj), "{}".format(obj)) for obj in l] + def get_form(self, previous=False): px_form_fields = self.field_dict viz = self.viz diff --git a/panoramix/models.py b/panoramix/models.py index 91dcd7a0ffccb..3c3a24b4878b5 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -327,6 +327,7 @@ def query( filter=None, is_timeseries=True, timeseries_limit=15, row_limit=None, + inner_from_dttm=None, inner_to_dttm=None, extras=None): qry_start_dttm = datetime.now() @@ -363,10 +364,17 @@ def query( from_clause = table(self.table_name) qry = qry.group_by(*groupby_exprs) - where_clause_and = [ + time_filter = [ timestamp >= from_dttm.isoformat(), - timestamp < to_dttm.isoformat(), + timestamp <= to_dttm.isoformat(), ] + inner_time_filter = copy(time_filter) + if inner_from_dttm: + inner_time_filter[0] = timestamp >= inner_from_dttm.isoformat() + if inner_to_dttm: + inner_time_filter[1] = timestamp <= inner_to_dttm.isoformat() + + where_clause_and = [] for col, op, eq in filter: if op in ('in', 'not in'): values = eq.split(",") @@ -376,14 +384,14 @@ def query( where_clause_and.append(cond) if extras and 'where' in extras: where_clause_and += [text(extras['where'])] - qry = qry.where(and_(*where_clause_and)) + qry = qry.where(and_(*(time_filter + where_clause_and))) qry = qry.order_by(desc(main_metric_expr)) qry = qry.limit(row_limit) if timeseries_limit and groupby: subq = select(inner_groupby_exprs) subq = subq.select_from(table(self.table_name)) - subq = subq.where(and_(*where_clause_and)) + subq = subq.where(and_(*(where_clause_and + inner_time_filter))) subq = subq.group_by(*inner_groupby_exprs) subq = subq.order_by(desc(main_metric_expr)) subq = subq.limit(timeseries_limit) @@ -677,9 +685,13 @@ def query( is_timeseries=True, timeseries_limit=None, row_limit=None, + inner_from_dttm=None, inner_to_dttm=None, extras=None): qry_start_dttm = datetime.now() + inner_from_dttm = inner_from_dttm or from_dttm + inner_to_dttm = inner_to_dttm or to_dttm + # add tzinfo to native datetime with config from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ")) to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ")) @@ -738,6 +750,7 @@ def query( pre_qry['limit_spec'] = { "type": "default", "limit": timeseries_limit, + 'intervals': inner_from_dttm.isoformat() + '/' + inner_to_dttm.isoformat(), "columns": [{ "dimension": metrics[0] if metrics else self.metrics[0], "direction": "descending", diff --git a/panoramix/static/widgets/viz_nvd3.js b/panoramix/static/widgets/viz_nvd3.js index 14dbd9922147f..2d7e4547f5436 100644 --- a/panoramix/static/widgets/viz_nvd3.js +++ b/panoramix/static/widgets/viz_nvd3.js @@ -50,7 +50,9 @@ function viz_nvd3(token_name, json_callback) { chart.yAxis.tickFormat(d3.format('.3s')); if (viz.form_data.contribution || viz.form_data.num_period_compare) { chart.yAxis.tickFormat(d3.format('.3p')); - chart.y2Axis.tickFormat(d3.format('.3p')); + if (chart.y2Axis != undefined) { + chart.y2Axis.tickFormat(d3.format('.3p')); + } } } else if (viz_type === 'dist_bar') { diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index 7beb01c0f9f83..cedb23deaaf48 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -98,6 +98,9 @@