diff --git a/MANIFEST.in b/MANIFEST.in index 7a38c62402104..f1b41f659fdc7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,4 @@ recursive-exclude caravel/static/spec * recursive-exclude tests * recursive-include caravel/data * recursive-include caravel/migrations * +include run_prod.sh \ No newline at end of file diff --git a/caravel/assets/javascripts/explore/explore.jsx b/caravel/assets/javascripts/explore/explore.jsx index 79aff8fe3b902..aa72bfdd3aac1 100644 --- a/caravel/assets/javascripts/explore/explore.jsx +++ b/caravel/assets/javascripts/explore/explore.jsx @@ -336,6 +336,51 @@ function initExploreView() { prepSaveDialog(); } +function getAnnotationFilters(annotationSource) { + let sqlaTableId = annotationSource; + if (!sqlaTableId) { + sqlaTableId = px.getParam('annotation_source'); + if (!sqlaTableId || sqlaTableId === 'None') { + return; + } + } + + const annotationFilterSelect = $('#annotation_filter'); + const url = $(location).attr('protocol') + '//' + + $(location).attr('host') + '/caravel/annotations/' + sqlaTableId; + + $.ajax({ + method: 'GET', + url, + dataType: 'json', + contentType: 'application/json; charset=utf-8', + }).done(function (data) { + $(annotationFilterSelect).select2('destroy'); + $(annotationFilterSelect).empty(); + + for (const key in data) { + const annotationText = data[key]; + annotationFilterSelect.append($('') + .attr('value', annotationText) + .html(annotationText)); + } + + $(annotationFilterSelect).select2(); + }).fail(function () { + $(annotationFilterSelect).select2('destroy'); + $(annotationFilterSelect).empty(); + $(annotationFilterSelect).select2(); + }); +} + +function initAnnotationForm() { + const annotationSource = $('#annotation_source'); + annotationSource.change(function () { + getAnnotationFilters(annotationSource.val()); + }); + getAnnotationFilters(null); +} + function initComponents() { const queryAndSaveBtnsEl = document.getElementById('js-query-and-save-btns'); ReactDOM.render( @@ -354,6 +399,8 @@ function initComponents() { />, exploreActionsEl ); + + initAnnotationForm(); } $(document).ready(function () { diff --git a/caravel/assets/package.json b/caravel/assets/package.json index fa678959aa726..a5103b6e0b7a5 100644 --- a/caravel/assets/package.json +++ b/caravel/assets/package.json @@ -47,6 +47,7 @@ "classnames": "^2.2.5", "d3": "^3.5.14", "d3-cloud": "^1.2.1", + "d3-format": "^1.0.2", "d3-sankey": "^0.2.1", "d3-tip": "^0.6.7", "datamaps": "^0.4.4", diff --git a/caravel/assets/visualizations/nvd3_vis.css b/caravel/assets/visualizations/nvd3_vis.css index 20dbd65a74ca6..d11ebf6893cf0 100644 --- a/caravel/assets/visualizations/nvd3_vis.css +++ b/caravel/assets/visualizations/nvd3_vis.css @@ -31,3 +31,28 @@ text.nv-axislabel { .bar svg.nvd3-svg { width: auto; } + +div.annotation_tooltip { + position: absolute; + min-width: 100px; + max-width: 200px; + min-height: 28px; + padding: 7px; + font: 11px sans-serif; + font-weight: bold; + background: white; + border: 1px; + border-color: black; + border-style: solid; + border-radius: 4px; + pointer-events: none; +} + +span.annotation_title { + font: 11px sans-serif; + font-weight: bold; +} + +span.annotation_description { + font: 11px sans-serif; +} diff --git a/caravel/assets/visualizations/nvd3_vis.js b/caravel/assets/visualizations/nvd3_vis.js index 7d8dbfa25a548..f0b2acd6b2c83 100644 --- a/caravel/assets/visualizations/nvd3_vis.js +++ b/caravel/assets/visualizations/nvd3_vis.js @@ -11,8 +11,8 @@ require('./nvd3_vis.css'); const minBarWidth = 15; const animationTime = 1000; -const addTotalBarValues = function (chart, data, stacked) { - const svg = d3.select('svg'); +const addTotalBarValues = function (containerId, chart, data, stacked) { + const svg = d3.select('#' + containerId + ' svg'); const format = d3.format('.3s'); const countSeriesDisplayed = data.length; @@ -53,6 +53,158 @@ const addTotalBarValues = function (chart, data, stacked) { }); }; +function strToRGB(str) { + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = str.charCodeAt(i) + ((hash << 5) - hash); + } + + const c = (hash & 0x00FFFFFF) + .toString(16) + .toUpperCase(); + + return '0000'.substring(0, 6 - c.length) + c; +} + +const addBarAnnotations = function (containerId, chart, data, numberFormat) { + const svg = d3.select('#' + containerId + ' svg'); + const targetAnnotations = svg.select('g.nv-barsWrap').append('g'); + + + const barWidth = parseFloat(d3.select('#' + containerId + ' g.nv-group rect') + .attr('width')); + + const div = d3.select('body') + .append('div') + .attr('class', 'annotation_tooltip') + .style('opacity', 0); + + // Map of "timestamp-value" -> "text" + // to keep track of overlapping annotations + const annotationTitleValues = {}; + + data.forEach( + function (annotation) { + const key = annotation.timestamp + '-' + annotation.value; + if (key in annotationTitleValues) { + annotationTitleValues[key] = annotationTitleValues[key] + + '

' + + '' + + annotation.title + + ''; + } else { + annotationTitleValues[key] = '' + + annotation.title + + ''; + } + + if (annotation.description) { + annotationTitleValues[key] = annotationTitleValues[key] + +'
' + + annotation.description + + ''; + } + + const annotationColor = strToRGB(annotationTitleValues[key]); + const xAxisPosition = chart.xAxis.scale()(annotation.timestamp); + if (isNaN(xAxisPosition)) { + return; + } + + targetAnnotations.append('svg:rect') + .attr('x', xAxisPosition + barWidth * 0.1) + .attr('y', chart.yAxis.scale()( + annotation.value) - 1.5) + .attr('width', barWidth * 0.8) + .attr('height', 3) + .style('fill', annotationColor) + .style('stroke', annotationColor) + .on('mouseover', function () { + d3.event.stopPropagation(); + const rect = d3.select(this); + rect.style('fill', annotationColor); + rect.style('stroke', annotationColor); + rect.attr('opacity', 0.5); + div.transition() + .duration(200) + .style('opacity', 0.8); + div.html('' + + annotationTitleValues[key] + + '

' + + '' + + formatDate(annotation.timestamp) + + '
' + + '' + + 'Value: ' + + d3.format(numberFormat)(annotation.value) + + '') + .style('left', (d3.event.pageX) + 25 + 'px') + .style('top', (d3.event.pageY - 30) + 'px'); + }) + .on('mouseout', function () { + d3.event.stopPropagation(); + const rect = d3.select(this); + rect.style('fill', annotationColor); + rect.style('stroke', annotationColor); + rect.attr('opacity', 1); + div.transition() + .duration(200) + .style('opacity', 0); + }); + } + ); +}; + +// const addVerticalLineAnnotations = function (chart, data) { +// const svg = d3.select('svg'); +// svg.select('g.nv-linesWrap').append('g') +// .attr('class', 'vertical-lines'); +// +// let annotationData = []; +// let numAnnotations = Object.keys(data['annotation_ts']).length; +// for (let i=0; i < numAnnotations; i++) { +// annotationData.push({ +// 'date': data['annotation_ts'][i], +// 'label': data['annotation_val'][i] +// }) +// } +// +// const vertLines = d3.select('.vertical-lines') +// .selectAll('.vertical-line').data(annotationData); +// +// var vertG = vertLines.enter() +// .append('g') +// .attr('class', 'vertical-line'); +// +// vertG.append('svg:line'); +// vertG.append('text'); +// +// vertLines.exit().remove(); +// +// vertLines.selectAll('line') +// .attr('x1', function (d) { +// return chart.xAxis.scale()(d.date); +// }) +// .attr('x2', function (d) { +// return chart.xAxis.scale()(d.date); +// }) +// .attr('y1', chart.yAxis.scale().range()[0] ) +// .attr('y2', chart.yAxis.scale().range()[1] ) +// .style('stroke', 'blue'); +// +// vertLines.selectAll('text') +// .text( function(d) { return d.label }) +// .attr('dy', '1em') +// .attr('transform', function (d) { +// return 'translate(' + +// chart.xAxis.scale()(d.date) + +// ',' + +// chart.yAxis.scale()(2) + +// ') rotate(-90)' +// }) +// .style('font-size','80%') +// }; + function nvd3Vis(slice) { let chart; let colorKey = 'key'; @@ -122,6 +274,12 @@ function nvd3Vis(slice) { chart.xAxis .showMaxMin(fd.x_axis_showminmax) .staggerLabels(false); + + // if (fd.enable_annotations) { + // setTimeout(function () { + // addVerticalLineAnnotations(chart, payload.annotations); + // }, animationTime); + // } break; case 'bar': @@ -140,9 +298,39 @@ function nvd3Vis(slice) { stacked = fd.bar_stacked; chart.stacked(stacked); + if (fd.enable_annotations) { + const chartData = payload.data[0].values; + const latestDataDate = chartData[chartData.length - 1].x; + + const dateValues = {}; + chartData.forEach(function (barData) { + dateValues[barData.x] = true; + }); + + let yMax = 0; + payload.annotations.forEach(function (annotation) { + const annotationTimestamp = annotation.timestamp; + if (!(annotationTimestamp in dateValues)) { + if (annotationTimestamp > latestDataDate) { + chartData.push({ x: annotationTimestamp, y: 0 }); + } + } + + yMax = yMax > annotation.value ? + yMax : annotation.value; + }); + chart.forceY([0, yMax]); + + setTimeout(function () { + addBarAnnotations(slice.containerId, + chart, payload.annotations, fd.y_axis_format); + }, animationTime); + } + if (fd.show_bar_value) { setTimeout(function () { - addTotalBarValues(chart, payload.data, stacked); + addTotalBarValues(slice.containerId, + chart, payload.data, stacked); }, animationTime); } break; diff --git a/caravel/config.py b/caravel/config.py index 267a7777333a9..507236b0b83af 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -8,13 +8,12 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from caravel import app import json import os from dateutil import tz -from flask_appbuilder.security.manager import AUTH_DB +from flask_appbuilder.security.manager import AUTH_LDAP BASE_DIR = os.path.abspath(os.path.dirname(__file__)) DATA_DIR = os.path.join(os.path.expanduser('~'), '.caravel') @@ -43,9 +42,7 @@ SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' # noqa # The SQLAlchemy connection string. -SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db') -# SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp' -# SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp' +SQLALCHEMY_DATABASE_URI = 'mysql://druid_live:jhdftgh8674DDFGFsajdg@druiddata-1-001.renaissance.live.las1.mz-inc.com:3306/caravel?connect_timeout=600' # The limit of queries fetched for query search QUERY_SEARCH_LIMIT = 1000 @@ -86,22 +83,25 @@ # AUTH_DB : Is for database (username/password() # AUTH_LDAP : Is for LDAP # AUTH_REMOTE_USER : Is for using REMOTE_USER from web server -AUTH_TYPE = AUTH_DB +AUTH_TYPE = AUTH_LDAP # Uncomment to setup Full admin role name -# AUTH_ROLE_ADMIN = 'Admin' +AUTH_ROLE_ADMIN = 'Admin' # Uncomment to setup Public role name, no authentication needed # AUTH_ROLE_PUBLIC = 'Public' # Will allow user self registration -# AUTH_USER_REGISTRATION = True +AUTH_USER_REGISTRATION = True # The default user self registration role -# AUTH_USER_REGISTRATION_ROLE = "Public" +AUTH_USER_REGISTRATION_ROLE = "Public" # When using LDAP Auth, setup the ldap server -# AUTH_LDAP_SERVER = "ldap://ldapserver.new" +AUTH_LDAP_SERVER = "ldap://hq-dc1.corpmz.com" +AUTH_LDAP_SEARCH = "dc=corpmz,dc=com" +AUTH_LDAP_UID_FIELD = "userPrincipalName" +AUTH_LDAP_APPEND_DOMAIN = "CORPMZ.com" # Uncomment to setup OpenID providers example for OpenID authentication # OPENID_PROVIDERS = [ @@ -241,4 +241,4 @@ class CeleryConfig(object): pass if not CACHE_DEFAULT_TIMEOUT: - CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT') + CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT') \ No newline at end of file diff --git a/caravel/forms.py b/caravel/forms.py index 638d9f5344176..d53092daf72b7 100755 --- a/caravel/forms.py +++ b/caravel/forms.py @@ -15,7 +15,7 @@ BooleanField, IntegerField, HiddenField, DecimalField) from wtforms import validators, widgets -from caravel import app +from caravel import app, db config = app.config @@ -961,6 +961,24 @@ def __init__(self, viz): ], "description": _("The color for points and clusters in RGB") }), + 'enable_annotations': (BetterBooleanField, { + "label": _("Enable Annotations"), + "default": False, + "description": _("Enable annotations on this graph. Must choose " + "a source for annotation data.") + }), + 'annotation_source': (SelectField, { + "label": _("Annotation Source"), + "choices": self.get_annotation_source_choices(), + "description": _("Source of annotation data"), + }), + 'annotation_filter': (SelectMultipleSortableField, { + "label": _("Annotation Filter"), + "choices": [(text, text) for text + in viz.get_annotation_filter_choices( + viz.orig_form_data.get('annotation_source') + )] + }), } # Override default arguments with form overrides @@ -973,6 +991,17 @@ def __init__(self, viz): for field_name, v in field_data.items() } + @staticmethod + def get_annotation_source_choices(): + from caravel import models + + choices = [('None', '')] + choices.extend([(unicode(table.id), table.full_name) + for table + in db.session.query(models.SqlaTable) + .filter_by(annotation=True)]) + return choices + @staticmethod def choicify(l): return [("{}".format(obj), "{}".format(obj)) for obj in l] diff --git a/caravel/migrations/versions/16f55059a4d6_.py b/caravel/migrations/versions/16f55059a4d6_.py new file mode 100644 index 0000000000000..75c89f5e739a2 --- /dev/null +++ b/caravel/migrations/versions/16f55059a4d6_.py @@ -0,0 +1,22 @@ +"""merge 319 and ef8 + +Revision ID: 16f55059a4d6 +Revises: ('3196bd55582b', 'ef8843b41dac') +Create Date: 2016-10-03 13:08:23.208002 + +""" + +# revision identifiers, used by Alembic. +revision = '16f55059a4d6' +down_revision = ('3196bd55582b', 'ef8843b41dac') + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/caravel/migrations/versions/3196bd55582b_annotations.py b/caravel/migrations/versions/3196bd55582b_annotations.py new file mode 100644 index 0000000000000..90a65f2deee57 --- /dev/null +++ b/caravel/migrations/versions/3196bd55582b_annotations.py @@ -0,0 +1,30 @@ +"""Annotation Support + +Revision ID: 3196bd55582b +Revises: 3b626e2a6783 +Create Date: 2016-09-28 17:08:46.950505 + +""" + +# revision identifiers, used by Alembic. +revision = '3196bd55582b' +down_revision = '3b626e2a6783' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('tables', sa.Column('annotation', sa.Boolean(), server_default='0')) + op.add_column('table_columns', sa.Column('annotation_time', sa.Boolean(), server_default='0')) + op.add_column('table_columns', sa.Column('annotation_value', sa.Boolean(), server_default='0')) + op.add_column('table_columns', sa.Column('annotation_title', sa.Boolean(), server_default='0')) + op.add_column('table_columns', sa.Column('annotation_desc', sa.Boolean(), server_default='0')) + + +def downgrade(): + op.drop_column('tables', 'annotation') + op.drop_column('table_columns', 'annotation_time') + op.drop_column('table_columns', 'annotation_value') + op.drop_column('table_columns', 'annotation_title') + op.drop_column('table_columns', 'annotation_desc') diff --git a/caravel/models.py b/caravel/models.py index ec82efa53144d..fbf18704fb80c 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -677,6 +677,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): default_endpoint = Column(Text) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) is_featured = Column(Boolean, default=False) + annotation = Column(Boolean, default=False) user_id = Column(Integer, ForeignKey('ab_user.id')) owner = relationship('User', backref='tables', foreign_keys=[user_id]) database = relationship( @@ -767,6 +768,44 @@ def get_col(self, col_name): if col_name == col.column_name: return col + def values_for_column(self, + column_name, + from_dttm=None, + to_dttm=None, + limit=500): + """Runs query against sqla to retrieve some sample values for the given column.""" + granularity = self.main_dttm_col + + cols = {col.column_name: col for col in self.columns} + target_col = cols[column_name] + + tbl = table(self.table_name) + qry = select([target_col.sqla_col]) + qry = qry.select_from(tbl) + qry = qry.distinct(column_name) + qry = qry.limit(limit) + + if granularity: + dttm_col = cols[granularity] + timestamp = dttm_col.sqla_col.label('timestamp') + time_filter = [] + if from_dttm: + time_filter.append(timestamp >= text(dttm_col.dttm_sql_literal(from_dttm))) + if to_dttm: + time_filter.append(timestamp <= text(dttm_col.dttm_sql_literal(to_dttm))) + qry = qry.where(and_(*time_filter)) + + engine = self.database.get_sqla_engine() + sql = "{}".format( + qry.compile( + engine, compile_kwargs={"literal_binds": True}, ), + ) + + return pd.read_sql_query( + sql=sql, + con=engine + ) + def query( # sqla self, groupby, metrics, granularity, @@ -949,6 +988,7 @@ def _(element, compiler, **kw): con=engine ) sql = sqlparse.format(sql, reindent=True) + return QueryResult( df=df, duration=datetime.now() - qry_start_dttm, query=sql) @@ -1051,6 +1091,37 @@ def fetch_metadata(self): if not self.main_dttm_col: self.main_dttm_col = any_date_col + def get_annotations(self, from_dttm, to_dttm): + time_column = None + value_column = None + for column in self.table_columns: + if column.annotation_time: + time_column = column + if column.annotation_value: + value_column = column + + if not time_column or not value_column: + return None + + tbl = table(self.table_name) + select_exprs = [ + time_column.sqla_col.label('annotation_ts'), + value_column.sqla_col.label('annotation_val') + ] + qry = select(select_exprs) + qry = qry.select_from(tbl) + + engine = self.database.get_sqla_engine() + sql = "{}".format( + qry.compile( + engine, compile_kwargs={"literal_binds": True}, ), + ) + + return pd.read_sql_query( + sql=sql, + con=engine + ) + class SqlMetric(Model, AuditMixinNullable): @@ -1106,6 +1177,10 @@ class TableColumn(Model, AuditMixinNullable): description = Column(Text, default='') python_date_format = Column(String(255)) database_expression = Column(String(255)) + annotation_time = Column(Boolean, default=False) + annotation_value = Column(Boolean, default=False) + annotation_title = Column(Boolean, default=False) + annotation_desc = Column(Boolean, default=False) num_types = ('DOUBLE', 'FLOAT', 'INT', 'BIGINT', 'LONG') date_types = ('DATE', 'TIME') @@ -1271,7 +1346,8 @@ def perm(self): @property def link(self): name = escape(self.datasource_name) - return Markup('{name}').format(**locals()) + url = self.url + return Markup('{name}').format(url=url, name=name) @property def full_name(self): diff --git a/caravel/utils.py b/caravel/utils.py index 168656986f0aa..b062ef5736562 100644 --- a/caravel/utils.py +++ b/caravel/utils.py @@ -20,6 +20,7 @@ from flask import flash, Markup from flask_appbuilder.security.sqla import models as ab_models from markdown import markdown as md +from semantic.numbers import NumberService from sqlalchemy.types import TypeDecorator, TEXT from pydruid.utils.having import Having @@ -167,6 +168,15 @@ def parse_human_datetime(s): return dttm +def parse_natural_language_number(n): + """ + :param n: natural language number e.g. Two hundred and six + :return: double representation of the words w.g. 206 + """ + service = NumberService() + return service.parse(n) + + def dttm_from_timtuple(d): return datetime( d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec) diff --git a/caravel/views.py b/caravel/views.py index baba51a9388c4..9d3f3866502f0 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -286,11 +286,13 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa edit_columns = [ 'column_name', 'verbose_name', 'description', 'groupby', 'filterable', 'table', 'count_distinct', 'sum', 'min', 'max', 'expression', - 'is_dttm', 'python_date_format', 'database_expression'] + 'is_dttm', 'python_date_format', 'database_expression', + 'annotation_time', 'annotation_value'] add_columns = edit_columns list_columns = [ 'column_name', 'type', 'groupby', 'filterable', 'count_distinct', - 'sum', 'min', 'max', 'is_dttm'] + 'sum', 'min', 'max', 'is_dttm', + 'annotation_time', 'annotation_value', 'annotation_title', 'annotation_desc'] page_size = 500 description_columns = { 'is_dttm': (_( @@ -333,7 +335,11 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa 'expression': _("Expression"), 'is_dttm': _("Is temporal"), 'python_date_format': _("Datetime Format"), - 'database_expression': _("Database Expression") + 'database_expression': _("Database Expression"), + 'annotation_time': _("Annotation Time"), + 'annotation_value': _("Annotation Value"), + 'annotation_title': _("Annotation Title"), + 'annotation_desc': _("Annotation Description"), } appbuilder.add_view_no_menu(TableColumnInlineView) @@ -558,10 +564,10 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'changed_by_', 'changed_on_'] order_columns = [ 'link', 'database', 'is_featured', 'changed_on_'] - add_columns = ['table_name', 'database', 'schema'] + add_columns = ['table_name', 'database', 'schema', 'annotation'] edit_columns = [ - 'table_name', 'sql', 'is_featured', 'database', 'schema', - 'description', 'owner', + 'table_name', 'sql', 'is_featured', 'annotation', + 'database', 'schema','description', 'owner', 'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout'] related_views = [TableColumnInlineView, SqlMetricInlineView] base_order = ('changed_on', 'desc') @@ -587,6 +593,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'database': _("Database"), 'changed_on_': _("Last Changed"), 'is_featured': _("Is Featured"), + 'annotation': _("Annotation"), 'schema': _("Schema"), 'default_endpoint': _("Default Endpoint"), 'offset': _("Offset"), @@ -1942,6 +1949,31 @@ def sqlanvil(self): """SQL Editor""" return self.render_template('caravel/sqllab.html') + @has_access_api + @expose("/annotations/") + def annotation_filter(self, table_id): + sqla_table = db.session.query( + models.SqlaTable).filter_by(id=table_id).first() + + # Find title column + title_column = None + for column in sqla_table.table_columns: + if column.annotation_title: + title_column = column.column_name + + if not title_column: + return Response( + json.dumps({'error': "No annotation text column found."}), + status=403, + mimetype="application/json") + + df = sqla_table.values_for_column(column_name=title_column) + return Response( + df[title_column].to_json(), + status=200, + mimetype="application/json") + + appbuilder.add_view_no_menu(Caravel) if config['DRUID_IS_ACTIVE']: diff --git a/caravel/viz.py b/caravel/viz.py index 524a462acf479..92d09c8c2c1e1 100755 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -11,6 +11,7 @@ import copy import hashlib import logging +import re import uuid import zlib @@ -28,9 +29,9 @@ from werkzeug.urls import Href from dateutil import relativedelta as rdelta -from caravel import app, utils, cache +from caravel import app, db, utils, cache from caravel.forms import FormFactory -from caravel.utils import flasher +from caravel.utils import flasher, parse_natural_language_number config = app.config @@ -288,6 +289,10 @@ def cache_timeout(self): return config.get("CACHE_DEFAULT_TIMEOUT") def get_json(self, force=False): + payload = self.get_payload(force) + return self.json_dumps(payload) + + def get_payload(self, force=False): """Handles caching around the json payload retrieval""" cache_key = self.cache_key payload = None @@ -339,7 +344,7 @@ def get_json(self, force=False): logging.exception(e) cache.delete(cache_key) payload['is_cached'] = is_cached - return self.json_dumps(payload) + return payload def json_dumps(self, obj): """Used by get_json, can be overridden to use specific switches""" @@ -349,7 +354,7 @@ def json_dumps(self, obj): def data(self): """This is the data object serialized to the js layer""" content = { - 'csv_endpoint': self.csv_endpoint, + 'csv_end*point': self.csv_endpoint, 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, 'standalone_endpoint': self.standalone_endpoint, @@ -371,6 +376,9 @@ def get_csv(self): def get_data(self): return [] + def get_annotation_filter_choices(self, annotation_source): + return [] + @property def json_endpoint(self): return self.get_url(json="true") @@ -1142,6 +1150,23 @@ def get_data(self): chart_data = sorted(chart_data, key=lambda x: x['key']) return chart_data + def get_annotations(self): + from caravel import models + datasource = (db.session.query(models.SqlaTable) + .filter_by(id=self.form_data.get('annotation_source')) + .first()) + return datasource.get_annotations(None, None) + + def get_json(self): + payload = super(NVD3TimeSeriesViz, self).get_payload() + + if self.form_data.get('enable_annotations'): + annotations = self.get_annotations() + if annotations is not None: + payload['annotations'] = annotations.to_dict() + + return self.json_dumps(payload) + class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz): @@ -1150,18 +1175,178 @@ class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz): viz_type = "bar" sort_series = True verbose_name = _("Time Series - Bar Chart") - fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [{ - 'label': _('Chart Options'), - 'fields': ( - ('show_brush', 'show_legend', 'show_bar_value'), - ('rich_tooltip', 'y_axis_zero'), - ('y_log_scale', 'contribution'), - ('x_axis_format', 'y_axis_format'), - ('line_interpolation', 'bar_stacked'), - ('x_axis_showminmax', 'bottom_margin'), - ('x_axis_label', 'y_axis_label'), - ('reduce_x_ticks', 'show_controls'), - ), }] + [NVD3TimeSeriesViz.fieldsets[2]] + fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [ + { + 'label': _('Chart Options'), + 'fields': ( + ('show_brush', 'show_legend', 'show_bar_value'), + ('rich_tooltip', 'y_axis_zero'), + ('y_log_scale', 'contribution'), + ('x_axis_showminmax', 'bar_stacked'), + ('reduce_x_ticks', 'show_controls'), + ('x_axis_format', 'y_axis_format'), + ('line_interpolation', 'bottom_margin'), + ('x_axis_label', 'y_axis_label'), + ), + }, { + 'label': _('Annotation Options'), + 'fields': ( + ('enable_annotations'), + ('annotation_source'), + ('annotation_filter') + ), + } + ] + [NVD3TimeSeriesViz.fieldsets[2]] + + TIMEGROUPER_MAPPING = { + 'second': 'S', + 'minute': 'T', + 'hour': 'H', + 'day': 'D', + 'week': 'W', + 'month': 'M', + 'year': 'A', + } + + def _get_timegrouper_freq(self): + from caravel.models import DruidDatasource + if isinstance(self.datasource, DruidDatasource): + gran = self.form_data.get('granularity').strip().lower() + duration_val, duration_unit = gran.rstrip('s').split() + duration_val = parse_natural_language_number(duration_val) + freq = str(int(duration_val)) + self.TIMEGROUPER_MAPPING[duration_unit] + + return freq + + def get_bar_annotations(self): + annotation_source = self.form_data.get('annotation_source') + annotation_filters = self.form_data.get('annotation_filter') + if not annotation_source or str(annotation_source) == 'None': + raise Exception("Annotations are enabled but no " + "annotation source is selected.") + + from caravel.models import SqlaTable + datasource = (db.session.query(SqlaTable) + .filter_by(id=self.form_data.get('annotation_source')) + .first()) + + if not datasource: + raise Exception("Annotation datasource does not exist.") + + time_column, value_column = None, None + title_column, desc_column = None, None + for column in datasource.table_columns: + if column.annotation_time: + time_column = column.column_name + if column.annotation_value: + value_column = column.column_name + if column.annotation_title: + title_column = column.column_name + if column.annotation_desc: + desc_column = column.column_name + + if time_column and value_column and title_column: + break + + if not time_column or not value_column or not title_column: + raise Exception("Time, Text, and Value columns must " + "be selected in annotation table source.") + + query_obj = self.query_obj() + query_obj['granularity'] = time_column + query_obj['groupby'] = None + query_obj['metrics'] = [] + query_obj['filter'] = [] + query_obj['columns'] = [value_column, title_column] + + if desc_column: + query_obj['columns'].append(desc_column) + + if annotation_filters: + cols = {col.column_name: col for col in datasource.columns} + in_clause = cols[title_column].sqla_col.in_(annotation_filters) + engine = datasource.database.get_sqla_engine() + query_obj['extras']['where'] = '{}'.format(in_clause.compile( + engine, compile_kwargs={"literal_binds": True}, )) + + freq = self._get_timegrouper_freq() # Translated Granularity + + annotations = datasource.query(**query_obj).df + if not annotations.empty: + annotations = annotations.set_index('timestamp') + grouby_columns = [pd.TimeGrouper(freq=freq), title_column] + if desc_column: + grouby_columns.append(desc_column) + annotations = annotations.groupby(grouby_columns).sum() + annotations = annotations.reset_index() + if desc_column: + annotations.columns = ['timestamp', 'title', 'description', 'value'] + else: + annotations.columns = ['timestamp', 'title', 'value'] + + return annotations + + def get_json(self): + payload = super(NVD3TimeSeriesBarViz, self).get_payload() + + if self.form_data.get('enable_annotations'): + annotations = self.get_bar_annotations() + payload['annotations'] = annotations.to_dict(orient='records') + + return self.json_dumps(payload) + + def get_annotation_filter_choices(self, annotation_source): + """ + Retrieves values for a column to be used by the filter dropdown. + :param annotation_source: SQLA Table ID + :return: JSON containing the some values for a column + """ + + form_data = self.orig_form_data + + if (not annotation_source + or not form_data.get('enable_annotations') + or form_data.get('enable_annotations') == u'false'): + return [] + + from caravel.models import SqlaTable + datasource = (db.session.query(SqlaTable) + .filter_by(id=annotation_source) + .first()) + + if not datasource: + Exception("Annotations are Enabled. " + "Please select an Annotation Source.") + + title_column = None + for column in datasource.table_columns: + if column.annotation_title: + title_column = column.column_name + + if not title_column: + raise Exception("Please define a title column " + "in the selected annotations source.") + + form_data = self.orig_form_data + + since = form_data.get("since", "1 year ago") + from_dttm = utils.parse_human_datetime(since) + now = datetime.now() + if from_dttm > now: + from_dttm = now - (from_dttm - now) + until = form_data.get("until", "now") + to_dttm = utils.parse_human_datetime(until) + if from_dttm > to_dttm: + flasher("The date range doesn't seem right.", "danger") + from_dttm = to_dttm # Making them identical to not raise + + kwargs = dict( + column_name=title_column, + from_dttm=from_dttm, + to_dttm=to_dttm, + ) + df = datasource.values_for_column(**kwargs) + return df[title_column] class NVD3CompareTimeSeriesViz(NVD3TimeSeriesViz): diff --git a/run_prod.sh b/run_prod.sh new file mode 100644 index 0000000000000..5a48eb2116391 --- /dev/null +++ b/run_prod.sh @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +python setup.py develop + +caravel db upgrade +caravel init + +/root/node_modules/pm2/bin/pm2 start `which caravel` --interpreter python -- runserver \ No newline at end of file diff --git a/setup.py b/setup.py index 0a83a422a6def..2bb5c9924a18d 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ 'PyHive>=0.2.1', 'python-dateutil==2.5.3', 'requests==2.10.0', + 'semantic==1.0.3', 'simplejson==3.8.2', 'six==1.10.0', 'sqlalchemy==1.0.13',