Allowing to define multiple time columns for time series on Sqla

panoramix/bin/panoramix

@@ -116,12 +116,13 @@ def load_examples(sample):
     if not obj:
         obj = TBL(table_name = 'birth_names')
     obj.main_dttm_col = 'ds'
-    obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
+    obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=ds&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
     obj.database = dbobj
     obj.columns = [
         models.TableColumn(column_name="num", sum=True, type="INTEGER"),
         models.TableColumn(column_name="sum_boys", sum=True, type="INTEGER"),
         models.TableColumn(column_name="sum_girls", sum=True, type="INTEGER"),
+        models.TableColumn(column_name="ds", is_dttm=True, type="DATETIME"),
     ]
     models.Table
     session.add(obj)
@@ -141,7 +142,7 @@ def load_examples(sample):
             "flt_col_1": "gender",
             "flt_eq_1": "",
             "flt_op_1": "in",
-            "granularity": "all",
+            "granularity": "ds",
             "groupby": [],
             "metric": 'sum__num',
             "metrics": ["sum__num"],
@@ -194,7 +195,7 @@ def load_examples(sample):
             datasource_type='table',
             table=tbl,
             params=get_slice_json(
-                slice_name, viz_type="big_number", granularity="1 day",
+                slice_name, viz_type="big_number", granularity="ds",
                 compare_lag="5", compare_suffix="over 5Y"))
         session.add(slc)
     slices.append(slc)
@@ -237,7 +238,7 @@ def load_examples(sample):
             table=tbl,
             params=get_slice_json(
                 slice_name, viz_type="line", groupby=['name'],
-                granularity='1 day', rich_tooltip='y', show_legend='y'))
+                granularity='ds', rich_tooltip='y', show_legend='y'))
         session.add(slc)
     slices.append(slc)
 

panoramix/forms.py

@@ -67,6 +67,13 @@ def __init__(self, viz):
                     "The time granularity for the visualization. Note that you "
                     "can type and use simple natural language as in '10 seconds', "
                     "'1 day' or '56 weeks'")),
+            'granularity_sqla': SelectField(
+                'Time Column', default=datasource.main_dttm_col,
+                choices=self.choicify(datasource.dttm_cols),
+                description=(
+                    "The time granularity for the visualization. Note that you "
+                    "can define arbitrary expression that return a DATETIME "
+                    "column in the table editor")),
             'since': TextField(
                 'Since', default="7 days ago", description=(
                     "Timestamp from filter. This supports free form typing and "
@@ -217,7 +224,7 @@ class QueryForm(OmgWtForm):
             standalone = HiddenField()
             async = HiddenField()
             json = HiddenField()
-            previous_viz_type = HiddenField()
+            previous_viz_type = HiddenField(default=viz.viz_type)
 
         filter_cols = datasource.filterable_column_names or ['']
         for i in range(10):
@@ -243,4 +250,11 @@ class QueryForm(OmgWtForm):
         if datasource.__class__.__name__ == 'SqlaTable':
             QueryForm.field_order += ['where']
             setattr(QueryForm, 'where', px_form_fields['where'])
+
+            if 'granularity' in viz.form_fields:
+                setattr(
+                    QueryForm,
+                    'granularity', px_form_fields['granularity_sqla'])
+                field_css_classes['granularity'] = ['form-control', 'select2']
+
         return QueryForm

panoramix/models.py

@@ -164,6 +164,10 @@ class Queryable(object):
     def column_names(self):
         return sorted([c.column_name for c in self.columns])
 
+    @property
+    def main_dttm_col(self):
+        return "timestamp"
+
     @property
     def groupby_column_names(self):
         return sorted([c.column_name for c in self.columns if c.groupby])
@@ -172,6 +176,10 @@ def groupby_column_names(self):
     def filterable_column_names(self):
         return sorted([c.column_name for c in self.columns if c.filterable])
 
+    @property
+    def dttm_cols(self):
+        return []
+
 
 class Database(Model, AuditMixinNullable):
     __tablename__ = 'dbs'
@@ -216,6 +224,13 @@ def perm(self):
             "[{self.database}].[{self.table_name}]"
             "(id:{self.id})").format(self=self)
 
+    @property
+    def dttm_cols(self):
+        l = [c.column_name for c in self.columns if c.is_dttm]
+        if self.main_dttm_col not in l:
+            l.append(self.main_dttm_col)
+        return l
+
     @property
     def name(self):
         return self.table_name
@@ -339,13 +354,20 @@ def query(
             inner_from_dttm=None, inner_to_dttm=None,
             extras=None):
 
+        # For backward compatibility
+        if granularity not in self.dttm_cols:
+            granularity = self.main_dttm_col
+
         cols = {col.column_name: col for col in self.columns}
         qry_start_dttm = datetime.now()
         if not self.main_dttm_col:
             raise Exception(
                 "Datetime column not provided as part table configuration")
-        timestamp = literal_column(
-            self.main_dttm_col).label('timestamp')
+        dttm_expr = cols[granularity].expression
+        if dttm_expr:
+            timestamp = ColumnClause(dttm_expr, is_literal=True).label('timestamp')
+        else:
+            timestamp = literal_column(granularity).label('timestamp')
         metrics_exprs = [
             literal_column(m.expression).label(m.metric_name)
             for m in self.metrics if m.metric_name in metrics]
@@ -381,7 +403,7 @@ def query(
                 inner_groupby_exprs.append(inner)
                 inner_select_exprs.append(inner)
 
-        if granularity != "all":
+        if is_timeseries:
             select_exprs += [timestamp]
             groupby_exprs += [timestamp]
 
@@ -562,7 +584,7 @@ class TableColumn(Model, AuditMixinNullable):
     table = relationship(
         'SqlaTable', backref='columns', foreign_keys=[table_id])
     column_name = Column(String(256))
-    is_dttm = Column(Boolean, default=True)
+    is_dttm = Column(Boolean, default=False)
     is_active = Column(Boolean, default=True)
     type = Column(String(32), default='')
     groupby = Column(Boolean, default=False)
@@ -740,6 +762,9 @@ def query(
             m.metric_name: m.json_obj
             for m in self.metrics if m.metric_name in metrics
         }
+        if granularity != "all":
+            granularity = utils.parse_human_timedelta(
+                granularity).total_seconds() * 1000
         if not isinstance(granularity, basestring):
             granularity = {"type": "duration", "duration": granularity}
 

panoramix/templates/panoramix/datasource.html

@@ -88,7 +88,7 @@ <h4>Filters</h4>
         <input type="hidden" name="datasource_name" value="{{ datasource.name }}">
         <input type="hidden" name="datasource_id" value="{{ datasource.id }}">
         <input type="hidden" name="datasource_type" value="{{ datasource.type }}">
-        {{ form.previous_viz_type() }}
+        <input type="hidden" name="previous_viz_type" value="{{ viz.viz_type or "table" }}">
       </form><br>
     </div>
 

panoramix/views.py

@@ -42,12 +42,18 @@ class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView):
     can_delete = False
     edit_columns = [
         'column_name', 'description', 'groupby', 'filterable', 'table',
-        'count_distinct', 'sum', 'min', 'max', 'expression']
+        'count_distinct', 'sum', 'min', 'max', 'expression', 'is_dttm']
     add_columns = edit_columns
     list_columns = [
         'column_name', 'type', 'groupby', 'filterable', 'count_distinct',
-        'sum', 'min', 'max']
+        'sum', 'min', 'max', 'is_dttm']
     page_size = 100
+    description_columns = {
+        'is_dttm': (
+            "Whether to make this column available as a "
+            "[Time Granularity] option, column has to be DATETIME or "
+            "DATETIME-like"),
+    }
 appbuilder.add_view_no_menu(TableColumnInlineView)
 
 

panoramix/viz.py

@@ -22,9 +22,13 @@ class BaseViz(object):
     viz_type = None
     verbose_name = "Base Viz"
     template = None
+    is_timeseries = False
     form_fields = [
-        'viz_type', 'metrics', 'groupby', 'granularity',
-        ('since', 'until')]
+        'viz_type',
+        'granularity',
+        ('since', 'until'),
+        'metrics', 'groupby',
+    ]
     js_files = []
     css_files = []
 
@@ -37,22 +41,21 @@ def __init__(self, datasource, form_data):
         ff = FormFactory(self)
         form_class = ff.get_form()
         defaults = form_class().data.copy()
+        previous_viz_type = form_data.get('previous_viz_type')
         if isinstance(form_data, ImmutableMultiDict):
             form = form_class(form_data)
         else:
             form = form_class(**form_data)
-
         data = form.data.copy()
+
         if not form.validate():
             for k, v in form.errors.items():
                 if not data.get('json') and not data.get('async'):
                     flash("{}: {}".format(k, " ".join(v)), 'danger')
-        previous_viz_type = form_data.get('previous_viz_type')
-        if previous_viz_type in viz_types and previous_viz_type != self.viz_type:
-            data = {
-                k: form.data[k]
-                for k in form_data.keys()
-                if k in viz_types[previous_viz_type].flat_form_fields() and k in form.data}
+        data = {
+            k: form.data[k]
+            for k in form_data.keys()
+            if k in form.data}
         defaults.update(data)
         self.form_data = defaults
 
@@ -134,10 +137,7 @@ def query_obj(self):
         form_data = self.form_data
         groupby = form_data.get("groupby") or []
         metrics = form_data.get("metrics") or ['count']
-        granularity = form_data.get("granularity", "1 day")
-        if granularity != "all":
-            granularity = utils.parse_human_timedelta(
-                granularity).total_seconds() * 1000
+        granularity = form_data.get("granularity")
         limit = int(form_data.get("limit", 0))
         row_limit = int(
             form_data.get("row_limit", config.get("ROW_LIMIT")))
@@ -160,7 +160,7 @@ def query_obj(self):
             'granularity': granularity,
             'from_dttm': from_dttm,
             'to_dttm': to_dttm,
-            'is_timeseries': True,
+            'is_timeseries': self.is_timeseries,
             'groupby': groupby,
             'metrics': metrics,
             'row_limit': row_limit,
@@ -194,6 +194,7 @@ class TableViz(BaseViz):
     template = 'panoramix/viz_table.html'
     form_fields = BaseViz.form_fields + ['row_limit']
     css_files = ['lib/dataTables/dataTables.bootstrap.css']
+    is_timeseries = False
     js_files = [
         'lib/dataTables/jquery.dataTables.min.js',
         'lib/dataTables/dataTables.bootstrap.js']
@@ -220,6 +221,7 @@ class MarkupViz(BaseViz):
     verbose_name = "Markup Widget"
     template = 'panoramix/viz_markup.html'
     form_fields = ['viz_type', 'markup_type', 'code']
+    is_timeseries = False
 
     def rendered(self):
         markup_type = self.form_data.get("markup_type")
@@ -238,6 +240,7 @@ class WordCloudViz(BaseViz):
     viz_type = "word_cloud"
     verbose_name = "Word Cloud"
     template = 'panoramix/viz_word_cloud.html'
+    is_timeseries = False
     form_fields = [
         'viz_type',
         ('since', 'until'),
@@ -253,7 +256,6 @@ class WordCloudViz(BaseViz):
 
     def query_obj(self):
         d = super(WordCloudViz, self).query_obj()
-        d['granularity'] = 'all'
         metric = self.form_data.get('metric')
         if not metric:
             raise Exception("Pick a metric!")
@@ -271,6 +273,7 @@ class NVD3Viz(BaseViz):
     viz_type = None
     verbose_name = "Base NVD3 Viz"
     template = 'panoramix/viz_nvd3.html'
+    is_timeseries = False
     js_files = [
         'lib/d3.min.js',
         'lib/nvd3/nv.d3.min.js',
@@ -285,6 +288,7 @@ class NVD3Viz(BaseViz):
 class BubbleViz(NVD3Viz):
     viz_type = "bubble"
     verbose_name = "Bubble Chart"
+    is_timeseries = False
     form_fields = [
         'viz_type',
         ('since', 'until'),
@@ -298,7 +302,6 @@ class BubbleViz(NVD3Viz):
     def query_obj(self):
         form_data = self.form_data
         d = super(BubbleViz, self).query_obj()
-        d['granularity'] = 'all'
         d['groupby'] = list({
             form_data.get('series'),
             form_data.get('entity')
@@ -349,6 +352,7 @@ class BigNumberViz(BaseViz):
     viz_type = "big_number"
     verbose_name = "Big Number"
     template = 'panoramix/viz_bignumber.html'
+    is_timeseries = True
     js_files = [
         'lib/d3.min.js',
         'widgets/viz_bignumber.js',
@@ -400,6 +404,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
     viz_type = "line"
     verbose_name = "Time Series - Line Chart"
     sort_series = False
+    is_timeseries = True
     form_fields = [
         'viz_type',
         'granularity', ('since', 'until'),
@@ -553,16 +558,17 @@ class NVD3TimeSeriesStackedViz(NVD3TimeSeriesViz):
 class DistributionPieViz(NVD3Viz):
     viz_type = "pie"
     verbose_name = "Distribution - NVD3 - Pie Chart"
+    is_timeseries = False
     form_fields = [
-        'viz_type', 'metrics', 'groupby',
+        'viz_type',
         ('since', 'until'),
+        'metrics', 'groupby',
         'limit',
         ('donut', 'show_legend'),
     ]
 
     def query_obj(self):
         d = super(DistributionPieViz, self).query_obj()
-        d['granularity'] = "all"
         d['is_timeseries'] = False
         return d
 
@@ -589,6 +595,7 @@ def get_json_data(self):
 class DistributionBarViz(DistributionPieViz):
     viz_type = "dist_bar"
     verbose_name = "Distribution - Bar Chart"
+    is_timeseries = False
     form_fields = [
         'viz_type', 'metrics', 'groupby',
         ('since', 'until'),