Merge branch 'master' of github.com:elastic/kibana into implement/bet…

…terWarningForUrlLength

config/kibana.yml

@@ -61,6 +61,10 @@
 # must be a positive integer.
 # elasticsearch.requestTimeout: 30000
 
+# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
+# headers, set this value to [] (an empty list).
+# elasticsearch.requestHeadersWhitelist: [ authorization ]
+
 # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
 # elasticsearch.shardTimeout: 0
 

docs/advanced-settings.asciidoc

@@ -1,7 +1,7 @@
 [[kibana-settings-reference]]
 
-WARNING: Modifying the following settings can signficantly affect Kibana's performance and cause problems that are 
-difficult to diagnose. Setting a property's value to a blank field will revert to the default behavior, which may not be 
+WARNING: Modifying the following settings can signficantly affect Kibana's performance and cause problems that are
+difficult to diagnose. Setting a property's value to a blank field will revert to the default behavior, which may not be
 compatible with other configuration settings. Deleting a custom setting removes it from Kibana permanently.
 
 .Kibana Settings Reference
@@ -10,38 +10,39 @@ compatible with other configuration settings. Deleting a custom setting removes
 `sort:options`:: Options for the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html[sort] parameter.
 `dateFormat`:: The format to use for displaying pretty-formatted dates.
 `dateFormat:tz`:: The timezone that Kibana uses. The default value of `Browser` uses the timezone detected by the browser.
-`dateFormat:scaled`:: These values define the format used to render ordered time-based data. Formatted timestamps must 
+`dateFormat:scaled`:: These values define the format used to render ordered time-based data. Formatted timestamps must
+`dateFormat:dow`:: This property defines what day weeks should start on.
 adapt to the interval between measurements. Keys are http://en.wikipedia.org/wiki/ISO_8601#Time_intervals[ISO8601 intervals].
 `defaultIndex`:: Default is `null`. This property specifies the default index.
-`metaFields`:: An array of fields outside of `_source`. Kibana merges these fields into the document when displaying the 
+`metaFields`:: An array of fields outside of `_source`. Kibana merges these fields into the document when displaying the
 document.
 `defaultColumns`:: Default is `_source`. Defines the columns that appear by default on the Discover page.
 `discover:sampleSize`:: The number of rows to show in the Discover table.
-`doc_table:highlight`:: Highlight results in Discover and Saved Searches Dashboard. Highlighing makes request slow when 
+`doc_table:highlight`:: Highlight results in Discover and Saved Searches Dashboard. Highlighing makes request slow when
 working on big documents. Set this property to `false` to disable highlighting.
-`courier:maxSegmentCount`:: Kibana splits requests in the Discover app into segments to limit the size of requests sent to 
-the Elasticsearch cluster. This setting constrains the length of the segment list. Long segment lists can significantly 
+`courier:maxSegmentCount`:: Kibana splits requests in the Discover app into segments to limit the size of requests sent to
+the Elasticsearch cluster. This setting constrains the length of the segment list. Long segment lists can significantly
 increase request processing time.
 `fields:popularLimit`:: This setting governs how many of the top most popular fields are shown.
 `histogram:barTarget`:: When date histograms use the `auto` interval, Kibana attempts to generate this number of bars.
-`histogram:maxBars`:: Date histograms are not generated with more bars than the value of this property, scaling values 
+`histogram:maxBars`:: Date histograms are not generated with more bars than the value of this property, scaling values
 when necessary.
-`visualization:tileMap:maxPrecision`:: The maximum geoHash precision displayed on tile maps: 7 is high, 10 is very high, 
+`visualization:tileMap:maxPrecision`:: The maximum geoHash precision displayed on tile maps: 7 is high, 10 is very high,
 12 is the maximum. http://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-geohashgrid-aggregation.html#_cell_dimensions_at_the_equator[Explanation of cell dimensions].
 `visualization:tileMap:WMSdefaults`:: Default properties for the WMS map server support in the tile map.
 `visualization:colorMapping`:: Maps values to specified colors within visualizations.
 `visualization:loadingDelay`:: Time to wait before dimming visualizations during query.
 `csv:separator`:: A string that serves as the separator for exported values.
 `csv:quoteValues`:: Set this property to `true` to quote exported values.
-`history:limit`:: In fields that have history, such as query inputs, the value of this property limits how many recent 
+`history:limit`:: In fields that have history, such as query inputs, the value of this property limits how many recent
 values are shown.
-`shortDots:enable`:: Set this property to `true` to shorten long field names in visualizations. For example, instead of 
+`shortDots:enable`:: Set this property to `true` to shorten long field names in visualizations. For example, instead of
 `foo.bar.baz`, show `f.b.baz`.
-`truncate:maxHeight`:: This property specifies the maximum height that a cell occupies in a table. A value of 0 disables 
+`truncate:maxHeight`:: This property specifies the maximum height that a cell occupies in a table. A value of 0 disables
 truncation.
-`indexPattern:fieldMapping:lookBack`:: The value of this property sets the number of recent matching patterns to query the 
+`indexPattern:fieldMapping:lookBack`:: The value of this property sets the number of recent matching patterns to query the
 field mapping for index patterns with names that contain timestamps.
-`format:defaultTypeMap`:: A map of the default format name for each field type. Field types that are not explicitly 
+`format:defaultTypeMap`:: A map of the default format name for each field type. Field types that are not explicitly
 mentioned use "_default_".
 `format:number:defaultPattern`:: Default numeral format for the "number" format.
 `format:bytes:defaultPattern`:: Default numeral format for the "bytes" format.

docs/kibana-yml.asciidoc

@@ -29,7 +29,9 @@ to `false`.
 wait for Elasticsearch to respond to pings.
 `elasticsearch.requestTimeout:`:: *Default: 300000* Time in milliseconds to wait for responses from the back end or 
 Elasticsearch. This value must be a positive integer.
-`elasticsearch.shardTimeout:`:: *Default: 0* Time in milliseconds for Elasticsearch to wait for responses from shards. Set 
+`elasticsearch.requestHeadersWhitelist:`:: *Default: `[ 'authorization' ]`* List of Kibana client-side headers to send to Elasticsearch.
+To send *no* client-side headers, set this value to [] (an empty list).
+`elasticsearch.shardTimeout:`:: *Default: 0* Time in milliseconds for Elasticsearch to wait for responses from shards. Set
 to 0 to disable.
 `elasticsearch.startupTimeout:`:: *Default: 5000* Time in milliseconds to wait for Elasticsearch at Kibana startup before 
 retrying.

docs/settings.asciidoc

@@ -376,6 +376,10 @@ deprecated[4.2, The names of several Kibana server properties changed in the 4.2
 + 
 *default*: `500000`
 
+`elasticsearch.requestHeadersWhitelist:` added[5.0]:: List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side headers, set this value to [] (an empty list).
++
+*default*: `[ 'authorization' ]`
+
 `elasticsearch.shardTimeout` added[4.2]:: How long Elasticsearch should wait for responses from shards. Set to 0 to disable.
 +
 *alias*: `shard_timeout` deprecated[4.2]

package.json

@@ -67,6 +67,7 @@
   },
   "dependencies": {
     "@bigfunger/decompress-zip": "0.2.0-stripfix2",
+    "@elastic/datemath": "2.2.0",
     "@spalger/angular-bootstrap": "0.12.1",
     "@spalger/filesaver": "1.1.2",
     "@spalger/leaflet-draw": "0.2.3",
@@ -130,6 +131,7 @@
     "raw-loader": "0.5.1",
     "request": "2.61.0",
     "rimraf": "2.4.3",
+    "rison-node": "1.0.0",
     "rjs-repack-loader": "1.0.6",
     "script-loader": "0.6.1",
     "semver": "5.1.0",

src/cli/serve/serve.js

@@ -1,4 +1,5 @@
 import _ from 'lodash';
+import { statSync } from 'fs';
 import { isWorker } from 'cluster';
 import { resolve } from 'path';
 
@@ -117,7 +118,14 @@ module.exports = function (program) {
   command
   .action(async function (opts) {
     if (opts.dev) {
-      opts.config.push(fromRoot('config/kibana.dev.yml'));
+      try {
+        const kbnDevConfig = fromRoot('config/kibana.dev.yml');
+        if (statSync(kbnDevConfig).isFile()) {
+          opts.config.push(kbnDevConfig);
+        }
+      } catch (err) {
+        // ignore, kibana.dev.yml does not exist
+      }
     }
 
     const settings = initServerSettings(opts, this.getUnknownOptions());

src/cli_plugin/install/__tests__/download.js

@@ -5,6 +5,7 @@ import glob from 'glob-all';
 import rimraf from 'rimraf';
 import mkdirp from 'mkdirp';
 import Logger from '../../lib/logger';
+import { UnsupportedProtocolError } from '../../lib/errors';
 import { download, _downloadSingle } from '../download';
 import { join } from 'path';
 
@@ -74,12 +75,12 @@ describe('kibana cli', function () {
           });
         });
 
-        it('should throw an ENOTFOUND error for an invalid url', function () {
+        it('should throw an UnsupportedProtocolError for an invalid url', function () {
           const sourceUrl = 'i am an invalid url';
 
           return _downloadSingle(settings, logger, sourceUrl)
           .then(shouldReject, function (err) {
-            expect(err.message).to.match(/ENOTFOUND/);
+            expect(err).to.be.an(UnsupportedProtocolError);
             expectWorkingPathEmpty();
           });
         });

src/cli_plugin/install/download.js

@@ -1,5 +1,6 @@
 import downloadHttpFile from './downloaders/http';
 import downloadLocalFile from './downloaders/file';
+import { UnsupportedProtocolError } from '../lib/errors';
 import { parse } from 'url';
 
 export function _downloadSingle(settings, logger, sourceUrl) {
@@ -8,8 +9,10 @@ export function _downloadSingle(settings, logger, sourceUrl) {
 
   if (/^file/.test(urlInfo.protocol)) {
     downloadPromise = downloadLocalFile(logger, decodeURI(urlInfo.path), settings.tempArchiveFile);
-  } else {
+  } else if (/^https?/.test(urlInfo.protocol)) {
     downloadPromise = downloadHttpFile(logger, sourceUrl, settings.tempArchiveFile, settings.timeout);
+  } else {
+    downloadPromise = Promise.reject(new UnsupportedProtocolError());
   }
 
   return downloadPromise;
@@ -29,7 +32,9 @@ export function download(settings, logger) {
 
     return _downloadSingle(settings, logger, sourceUrl)
     .catch((err) => {
-      if (err.message === 'ENOTFOUND') {
+      const isUnsupportedProtocol = err instanceof UnsupportedProtocolError;
+      const isDownloadResourceNotFound = err.message === 'ENOTFOUND';
+      if (isUnsupportedProtocol || isDownloadResourceNotFound) {
         return tryNext();
       }
       throw (err);

src/cli_plugin/lib/errors.js

@@ -0,0 +1 @@
+export class UnsupportedProtocolError extends Error {};

src/plugins/elasticsearch/index.js

@@ -5,6 +5,8 @@ import healthCheck from './lib/health_check';
 import exposeClient from './lib/expose_client';
 import createProxy, { createPath } from './lib/create_proxy';
 
+const DEFAULT_REQUEST_HEADERS = [ 'authorization' ];
+
 module.exports = function ({ Plugin }) {
   return new Plugin({
     require: ['kibana'],
@@ -20,6 +22,7 @@ module.exports = function ({ Plugin }) {
         password: string(),
         shardTimeout: number().default(0),
         requestTimeout: number().default(30000),
+        requestHeadersWhitelist: array().items().single().default(DEFAULT_REQUEST_HEADERS),
         pingTimeout: number().default(ref('requestTimeout')),
         startupTimeout: number().default(5000),
         ssl: object({

src/plugins/elasticsearch/lib/__tests__/map_uri.js

@@ -0,0 +1,76 @@
+import expect from 'expect.js';
+import mapUri from '../map_uri';
+import sinon from 'sinon';
+
+describe('plugins/elasticsearch', function () {
+  describe('lib/map_uri', function () {
+
+    let request;
+
+    beforeEach(function () {
+      request = {
+        path: '/elasticsearch/some/path',
+        headers: {
+          cookie: 'some_cookie_string',
+          'accept-encoding': 'gzip, deflate',
+          origin: 'https://localhost:5601',
+          'content-type': 'application/json',
+          'x-my-custom-header': '42',
+          accept: 'application/json, text/plain, */*',
+          authorization: '2343d322eda344390fdw42'
+        }
+      };
+    });
+
+    it('only sends the whitelisted request headers', function () {
+
+      const get = sinon.stub()
+      .withArgs('elasticsearch.url').returns('http://foobar:9200')
+      .withArgs('elasticsearch.requestHeadersWhitelist').returns(['x-my-custom-HEADER', 'Authorization']);
+      const config = function () { return { get: get }; };
+      const server = {
+        config: config
+      };
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(upstreamHeaders).to.have.property('authorization');
+        expect(upstreamHeaders).to.have.property('x-my-custom-header');
+        expect(Object.keys(upstreamHeaders).length).to.be(2);
+      });
+    });
+
+    it('sends no headers if whitelist is set to []', function () {
+
+      const get = sinon.stub()
+      .withArgs('elasticsearch.url').returns('http://foobar:9200')
+      .withArgs('elasticsearch.requestHeadersWhitelist').returns([]);
+      const config = function () { return { get: get }; };
+      const server = {
+        config: config
+      };
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(Object.keys(upstreamHeaders).length).to.be(0);
+      });
+    });
+
+    it('sends no headers if whitelist is set to no value', function () {
+
+      const get = sinon.stub()
+      .withArgs('elasticsearch.url').returns('http://foobar:9200')
+      .withArgs('elasticsearch.requestHeadersWhitelist').returns([ null ]); // This is how Joi returns it
+      const config = function () { return { get: get }; };
+      const server = {
+        config: config
+      };
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(Object.keys(upstreamHeaders).length).to.be(0);
+      });
+    });
+
+  });
+});

src/plugins/elasticsearch/lib/call_with_request.js

@@ -3,12 +3,12 @@ import Promise from 'bluebird';
 import Boom from 'boom';
 import getBasicAuthRealm from './get_basic_auth_realm';
 import toPath from 'lodash/internal/toPath';
+import filterHeaders from './filter_headers';
 
-module.exports = (client) => {
+module.exports = (server, client) => {
   return (req, endpoint, params = {}) => {
-    if (req.headers.authorization) {
-      _.set(params, 'headers.authorization', req.headers.authorization);
-    }
+    const filteredHeaders = filterHeaders(req.headers, server.config().get('elasticsearch.requestHeadersWhitelist'));
+    _.set(params, 'headers', filteredHeaders);
     const path = toPath(endpoint);
     const api = _.get(client, path);
     let apiContext = _.get(client, path.slice(0, -1));

src/plugins/elasticsearch/lib/create_proxy.js

@@ -16,10 +16,12 @@ function createProxy(server, method, route, config) {
     handler: {
       proxy: {
         mapUri: mapUri(server),
-        passThrough: true,
         agent: createAgent(server),
         xforward: true,
-        timeout: server.config().get('elasticsearch.requestTimeout')
+        timeout: server.config().get('elasticsearch.requestTimeout'),
+        onResponse: function (err, responseFromUpstream, request, reply) {
+          reply(err, responseFromUpstream);
+        }
       }
     },
   };

src/plugins/elasticsearch/lib/expose_client.js

@@ -78,8 +78,8 @@ module.exports = function (server) {
   server.expose('ElasticsearchClientLogging', ElasticsearchClientLogging);
   server.expose('client', client);
   server.expose('createClient', createClient);
-  server.expose('callWithRequestFactory', callWithRequest);
-  server.expose('callWithRequest', callWithRequest(noAuthClient));
+  server.expose('callWithRequestFactory', _.partial(callWithRequest, server));
+  server.expose('callWithRequest', callWithRequest(server, noAuthClient));
   server.expose('errors', elasticsearch.errors);
 
   return client;

src/plugins/elasticsearch/lib/filter_headers.js

@@ -0,0 +1,22 @@
+import _ from 'lodash';
+
+export default function (originalHeaders, headersToKeep) {
+
+  const normalizeHeader = function (header) {
+    if (!header) {
+      return '';
+    }
+    header = header.toString();
+    return header.trim().toLowerCase();
+  };
+
+  // Normalize list of headers we want to allow in upstream request
+  const headersToKeepNormalized = headersToKeep.map(normalizeHeader);
+
+  // Normalize original headers in request
+  const originalHeadersNormalized = _.mapKeys(originalHeaders, function (headerValue, headerName) {
+    return normalizeHeader(headerName);
+  });
+
+  return _.pick(originalHeaders, headersToKeepNormalized);
+}

src/plugins/elasticsearch/lib/map_uri.js

@@ -1,6 +1,9 @@
 import querystring from 'querystring';
 import { resolve } from 'url';
+import filterHeaders from './filter_headers';
+
 module.exports = function mapUri(server, prefix) {
+
   const config = server.config();
   return function (request, done) {
     const path = request.path.replace('/elasticsearch', '');
@@ -11,6 +14,7 @@ module.exports = function mapUri(server, prefix) {
     }
     const query = querystring.stringify(request.query);
     if (query) url += '?' + query;
-    done(null, url);
+    const filteredHeaders = filterHeaders(request.headers, server.config().get('elasticsearch.requestHeadersWhitelist'));
+    done(null, url, filteredHeaders);
   };
 };

src/plugins/kibana/public/discover/components/field_chooser/field_chooser.js

@@ -4,7 +4,7 @@ import 'ui/filters/unique';
 import 'plugins/kibana/discover/components/field_chooser/discover_field';
 import _ from 'lodash';
 import $ from 'jquery';
-import rison from 'ui/utils/rison';
+import rison from 'rison-node';
 import fieldCalculator from 'plugins/kibana/discover/components/field_chooser/lib/field_calculator';
 import IndexPatternsFieldListProvider from 'ui/index_patterns/_field_list';
 import uiModules from 'ui/modules';

src/plugins/kibana/public/discover/controllers/discover.js

@@ -2,8 +2,8 @@ import _ from 'lodash';
 import angular from 'angular';
 import moment from 'moment';
 import getSort from 'ui/doc_table/lib/get_sort';
-import rison from 'ui/utils/rison';
-import dateMath from 'ui/utils/date_math';
+import rison from 'rison-node';
+import dateMath from '@elastic/datemath';
 import 'ui/doc_table';
 import 'ui/visualize';
 import 'ui/notify';

src/plugins/kibana/public/settings/sections/objects/_view.js

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import angular from 'angular';
-import rison from 'ui/utils/rison';
+import rison from 'rison-node';
 import registry from 'plugins/kibana/settings/saved_object_registry';
 import objectViewHTML from 'plugins/kibana/settings/sections/objects/_view.html';
 import IndexPatternsCastMappingTypeProvider from 'ui/index_patterns/_cast_mapping_type';