Merge pull request apache#3 in DATA-PLATFORM/caravel from dev to master

* commit '9858b94ed750e79635234673c76c13927a1968af':
  Add prod script and configurations
  Annotation support in timeseries bar chart

MANIFEST.in

@@ -7,3 +7,4 @@ recursive-exclude caravel/static/spec *
 recursive-exclude tests *
 recursive-include caravel/data *
 recursive-include caravel/migrations *
+include run_prod.sh

caravel/assets/javascripts/explore/explore.jsx

@@ -336,6 +336,51 @@ function initExploreView() {
   prepSaveDialog();
 }
 
+function getAnnotationFilters(annotationSource) {
+  let sqlaTableId = annotationSource;
+  if (!sqlaTableId) {
+    sqlaTableId = px.getParam('annotation_source');
+    if (!sqlaTableId || sqlaTableId === 'None') {
+      return;
+    }
+  }
+
+  const annotationFilterSelect = $('#annotation_filter');
+  const url = $(location).attr('protocol') + '//' +
+    $(location).attr('host') + '/caravel/annotations/' + sqlaTableId;
+
+  $.ajax({
+    method: 'GET',
+    url,
+    dataType: 'json',
+    contentType: 'application/json; charset=utf-8',
+  }).done(function (data) {
+    $(annotationFilterSelect).select2('destroy');
+    $(annotationFilterSelect).empty();
+
+    for (const key in data) {
+      const annotationText = data[key];
+      annotationFilterSelect.append($('<option></option>')
+        .attr('value', annotationText)
+        .html(annotationText));
+    }
+
+    $(annotationFilterSelect).select2();
+  }).fail(function () {
+    $(annotationFilterSelect).select2('destroy');
+    $(annotationFilterSelect).empty();
+    $(annotationFilterSelect).select2();
+  });
+}
+
+function initAnnotationForm() {
+  const annotationSource = $('#annotation_source');
+  annotationSource.change(function () {
+    getAnnotationFilters(annotationSource.val());
+  });
+  getAnnotationFilters(null);
+}
+
 function initComponents() {
   const queryAndSaveBtnsEl = document.getElementById('js-query-and-save-btns');
   ReactDOM.render(
@@ -354,6 +399,8 @@ function initComponents() {
     />,
     exploreActionsEl
   );
+
+  initAnnotationForm();
 }
 
 $(document).ready(function () {

caravel/assets/package.json

@@ -47,6 +47,7 @@
     "classnames": "^2.2.5",
     "d3": "^3.5.14",
     "d3-cloud": "^1.2.1",
+    "d3-format": "^1.0.2",
     "d3-sankey": "^0.2.1",
     "d3-tip": "^0.6.7",
     "datamaps": "^0.4.4",

caravel/assets/visualizations/nvd3_vis.css

@@ -31,3 +31,28 @@ text.nv-axislabel {
 .bar svg.nvd3-svg {
   width: auto;
 }
+
+div.annotation_tooltip {
+  position: absolute;
+  min-width: 100px;
+  max-width: 200px;
+  min-height: 28px;
+  padding: 7px;
+  font: 11px sans-serif;
+  font-weight: bold;
+  background: white;
+  border: 1px;
+  border-color: black;
+  border-style: solid;
+  border-radius: 4px;
+  pointer-events: none;
+}
+
+span.annotation_title {
+  font: 11px sans-serif;
+  font-weight: bold;
+}
+
+span.annotation_description {
+  font: 11px sans-serif;
+}

caravel/assets/visualizations/nvd3_vis.js

@@ -11,8 +11,8 @@ require('./nvd3_vis.css');
 const minBarWidth = 15;
 const animationTime = 1000;
 
-const addTotalBarValues = function (chart, data, stacked) {
-  const svg = d3.select('svg');
+const addTotalBarValues = function (containerId, chart, data, stacked) {
+  const svg = d3.select('#' + containerId + ' svg');
   const format = d3.format('.3s');
   const countSeriesDisplayed = data.length;
 
@@ -53,6 +53,158 @@ const addTotalBarValues = function (chart, data, stacked) {
     });
 };
 
+function strToRGB(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = str.charCodeAt(i) + ((hash << 5) - hash);
+  }
+
+  const c = (hash & 0x00FFFFFF)
+    .toString(16)
+    .toUpperCase();
+
+  return '0000'.substring(0, 6 - c.length) + c;
+}
+
+const addBarAnnotations = function (containerId, chart, data, numberFormat) {
+  const svg = d3.select('#' + containerId + ' svg');
+  const targetAnnotations = svg.select('g.nv-barsWrap').append('g');
+
+
+  const barWidth = parseFloat(d3.select('#' + containerId + ' g.nv-group rect')
+    .attr('width'));
+
+  const div = d3.select('body')
+    .append('div')
+    .attr('class', 'annotation_tooltip')
+    .style('opacity', 0);
+
+  // Map of "timestamp-value" -> "text"
+  // to keep track of overlapping annotations
+  const annotationTitleValues = {};
+
+  data.forEach(
+    function (annotation) {
+      const key = annotation.timestamp + '-' + annotation.value;
+      if (key in annotationTitleValues) {
+        annotationTitleValues[key] = annotationTitleValues[key]
+          + '<br/><br/>'
+          + '<span class="annotation_title">'
+          + annotation.title
+          + '</span>';
+      } else {
+        annotationTitleValues[key] = '<span class="annotation_title">'
+          + annotation.title
+          + '</span>';
+      }
+
+      if (annotation.description) {
+        annotationTitleValues[key] = annotationTitleValues[key]
+          +'<br/><span class="annotation_description">'
+          + annotation.description
+          + '</span>';
+      }
+
+      const annotationColor = strToRGB(annotationTitleValues[key]);
+      const xAxisPosition = chart.xAxis.scale()(annotation.timestamp);
+      if (isNaN(xAxisPosition)) {
+        return;
+      }
+
+      targetAnnotations.append('svg:rect')
+        .attr('x', xAxisPosition + barWidth * 0.1)
+        .attr('y', chart.yAxis.scale()(
+          annotation.value) - 1.5)
+        .attr('width', barWidth * 0.8)
+        .attr('height', 3)
+        .style('fill', annotationColor)
+        .style('stroke', annotationColor)
+        .on('mouseover', function () {
+          d3.event.stopPropagation();
+          const rect = d3.select(this);
+          rect.style('fill', annotationColor);
+          rect.style('stroke', annotationColor);
+          rect.attr('opacity', 0.5);
+          div.transition()
+            .duration(200)
+            .style('opacity', 0.8);
+          div.html('<span>' +
+                    annotationTitleValues[key] +
+                  '</span><br/><br/>' +
+                  '<span style="font-weight:normal;">' +
+                      formatDate(annotation.timestamp) +
+                  '</span><br/>' +
+                  '<span>' +
+                    'Value: ' +
+                      d3.format(numberFormat)(annotation.value) +
+                  '</span>')
+            .style('left', (d3.event.pageX) + 25 + 'px')
+            .style('top', (d3.event.pageY - 30) + 'px');
+        })
+        .on('mouseout', function () {
+          d3.event.stopPropagation();
+          const rect = d3.select(this);
+          rect.style('fill', annotationColor);
+          rect.style('stroke', annotationColor);
+          rect.attr('opacity', 1);
+          div.transition()
+            .duration(200)
+            .style('opacity', 0);
+        });
+    }
+  );
+};
+
+// const addVerticalLineAnnotations = function (chart, data) {
+//   const svg = d3.select('svg');
+//   svg.select('g.nv-linesWrap').append('g')
+//     .attr('class', 'vertical-lines');
+//
+//   let annotationData = [];
+//   let numAnnotations = Object.keys(data['annotation_ts']).length;
+//   for (let i=0; i < numAnnotations; i++) {
+//     annotationData.push({
+//       'date': data['annotation_ts'][i],
+//       'label': data['annotation_val'][i]
+//     })
+//   }
+//
+//   const vertLines = d3.select('.vertical-lines')
+//     .selectAll('.vertical-line').data(annotationData);
+//
+//   var vertG = vertLines.enter()
+//         .append('g')
+//         .attr('class', 'vertical-line');
+//
+//   vertG.append('svg:line');
+//   vertG.append('text');
+//
+//   vertLines.exit().remove();
+//
+//   vertLines.selectAll('line')
+//     .attr('x1', function (d) {
+//        return chart.xAxis.scale()(d.date);
+//     })
+//     .attr('x2', function (d) {
+//       return chart.xAxis.scale()(d.date);
+//     })
+//     .attr('y1', chart.yAxis.scale().range()[0] )
+//     .attr('y2', chart.yAxis.scale().range()[1] )
+//     .style('stroke', 'blue');
+//
+//   vertLines.selectAll('text')
+//     .text( function(d) { return d.label })
+//     .attr('dy', '1em')
+//     .attr('transform', function (d) {
+//        return  'translate(' +
+//         chart.xAxis.scale()(d.date) +
+//         ',' +
+//         chart.yAxis.scale()(2) +
+//         ') rotate(-90)'
+//     })
+//     .style('font-size','80%')
+// };
+
 function nvd3Vis(slice) {
   let chart;
   let colorKey = 'key';
@@ -122,6 +274,12 @@ function nvd3Vis(slice) {
             chart.xAxis
             .showMaxMin(fd.x_axis_showminmax)
             .staggerLabels(false);
+
+            // if (fd.enable_annotations) {
+            //   setTimeout(function () {
+            //     addVerticalLineAnnotations(chart, payload.annotations);
+            //   }, animationTime);
+            // }
             break;
 
           case 'bar':
@@ -140,9 +298,39 @@ function nvd3Vis(slice) {
             stacked = fd.bar_stacked;
             chart.stacked(stacked);
 
+            if (fd.enable_annotations) {
+              const chartData = payload.data[0].values;
+              const latestDataDate = chartData[chartData.length - 1].x;
+
+              const dateValues = {};
+              chartData.forEach(function (barData) {
+                dateValues[barData.x] = true;
+              });
+
+              let yMax = 0;
+              payload.annotations.forEach(function (annotation) {
+                const annotationTimestamp = annotation.timestamp;
+                if (!(annotationTimestamp in dateValues)) {
+                  if (annotationTimestamp > latestDataDate) {
+                    chartData.push({ x: annotationTimestamp, y: 0 });
+                  }
+                }
+
+                yMax = yMax > annotation.value ?
+                  yMax : annotation.value;
+              });
+              chart.forceY([0, yMax]);
+
+              setTimeout(function () {
+                addBarAnnotations(slice.containerId,
+                  chart, payload.annotations, fd.y_axis_format);
+              }, animationTime);
+            }
+
             if (fd.show_bar_value) {
               setTimeout(function () {
-                addTotalBarValues(chart, payload.data, stacked);
+                addTotalBarValues(slice.containerId,
+                  chart, payload.data, stacked);
               }, animationTime);
             }
             break;

caravel/config.py

@@ -8,13 +8,12 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-from caravel import app
 
 import json
 import os
 
 from dateutil import tz
-from flask_appbuilder.security.manager import AUTH_DB
+from flask_appbuilder.security.manager import AUTH_LDAP
 
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 DATA_DIR = os.path.join(os.path.expanduser('~'), '.caravel')
@@ -43,9 +42,7 @@
 SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'  # noqa
 
 # The SQLAlchemy connection string.
-SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db')
-# SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp'
-# SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp'
+SQLALCHEMY_DATABASE_URI = 'mysql://druid_live:jhdftgh8674DDFGFsajdg@druiddata-1-001.renaissance.live.las1.mz-inc.com:3306/caravel?connect_timeout=600'
 
 # The limit of queries fetched for query search
 QUERY_SEARCH_LIMIT = 1000
@@ -86,22 +83,25 @@
 # AUTH_DB : Is for database (username/password()
 # AUTH_LDAP : Is for LDAP
 # AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
-AUTH_TYPE = AUTH_DB
+AUTH_TYPE = AUTH_LDAP
 
 # Uncomment to setup Full admin role name
-# AUTH_ROLE_ADMIN = 'Admin'
+AUTH_ROLE_ADMIN = 'Admin'
 
 # Uncomment to setup Public role name, no authentication needed
 # AUTH_ROLE_PUBLIC = 'Public'
 
 # Will allow user self registration
-# AUTH_USER_REGISTRATION = True
+AUTH_USER_REGISTRATION = True
 
 # The default user self registration role
-# AUTH_USER_REGISTRATION_ROLE = "Public"
+AUTH_USER_REGISTRATION_ROLE = "Public"
 
 # When using LDAP Auth, setup the ldap server
-# AUTH_LDAP_SERVER = "ldap://ldapserver.new"
+AUTH_LDAP_SERVER = "ldap://hq-dc1.corpmz.com"
+AUTH_LDAP_SEARCH = "dc=corpmz,dc=com"
+AUTH_LDAP_UID_FIELD = "userPrincipalName"
+AUTH_LDAP_APPEND_DOMAIN = "CORPMZ.com"
 
 # Uncomment to setup OpenID providers example for OpenID authentication
 # OPENID_PROVIDERS = [
@@ -241,4 +241,4 @@ class CeleryConfig(object):
     pass
 
 if not CACHE_DEFAULT_TIMEOUT:
-    CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT')
+    CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT')