diff --git a/src/core_plugins/kibana/index.js b/src/core_plugins/kibana/index.js index 6b22f5271fb0a..98fab64495695 100644 --- a/src/core_plugins/kibana/index.js +++ b/src/core_plugins/kibana/index.js @@ -1,6 +1,7 @@ import ingest from './server/routes/api/ingest'; import search from './server/routes/api/search'; import settings from './server/routes/api/settings'; +import scripts from './server/routes/api/scripts'; module.exports = function (kibana) { return new kibana.Plugin({ @@ -84,6 +85,7 @@ module.exports = function (kibana) { ingest(server); search(server); settings(server); + scripts(server); } }); diff --git a/src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js b/src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js index e4a00ad8540b6..094d2cb23c633 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js +++ b/src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js @@ -25,6 +25,7 @@ uiModules.get('apps/management') $scope.perPage = 25; $scope.columns = [ { title: 'name' }, + { title: 'lang' }, { title: 'script' }, { title: 'format' }, { title: 'controls', sortable: false } @@ -46,6 +47,7 @@ uiModules.get('apps/management') return [ _.escape(field.name), + _.escape(field.lang), _.escape(field.script), _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']), { diff --git a/src/core_plugins/kibana/server/routes/api/scripts/index.js b/src/core_plugins/kibana/server/routes/api/scripts/index.js new file mode 100644 index 0000000000000..a625649df423e --- /dev/null +++ b/src/core_plugins/kibana/server/routes/api/scripts/index.js @@ -0,0 +1,5 @@ +import { registerLanguages } from './register_languages'; + +export default function (server) { + registerLanguages(server); +} diff --git a/src/core_plugins/kibana/server/routes/api/scripts/register_languages.js b/src/core_plugins/kibana/server/routes/api/scripts/register_languages.js new file mode 100644 index 0000000000000..82bb5cc9ba22c --- /dev/null +++ b/src/core_plugins/kibana/server/routes/api/scripts/register_languages.js @@ -0,0 +1,27 @@ +import _ from 'lodash'; +import handleESError from '../../../lib/handle_es_error'; + +export function registerLanguages(server) { + server.route({ + path: '/api/kibana/scripts/languages', + method: 'GET', + handler: function (request, reply) { + const callWithRequest = server.plugins.elasticsearch.callWithRequest; + + return callWithRequest(request, 'cluster.getSettings', { + include_defaults: true, + filter_path: '**.script.engine.*.inline' + }) + .then((esResponse) => { + const langs = _.get(esResponse, 'defaults.script.engine', {}); + const inlineLangs = _.pick(langs, (lang) => lang.inline === 'true'); + const supportedLangs = _.omit(inlineLangs, 'mustache'); + return _.keys(supportedLangs); + }) + .then(reply) + .catch((error) => { + reply(handleESError(error)); + }); + } + }); +} diff --git a/src/fixtures/logstash_fields.js b/src/fixtures/logstash_fields.js index e4f993da81470..824a66ba25094 100644 --- a/src/fixtures/logstash_fields.js +++ b/src/fixtures/logstash_fields.js @@ -22,6 +22,7 @@ function stubbedLogstashFields() { { name: 'custom_user_field', type: 'conflict', indexed: false, analyzed: false, sortable: false, filterable: true }, { name: 'script string', type: 'string', scripted: true, script: '\'i am a string\'', lang: 'expression' }, { name: 'script number', type: 'number', scripted: true, script: '1234', lang: 'expression' }, + { name: 'script date', type: 'date', scripted: true, script: '1234', lang: 'painless' }, { name: 'script murmur3', type: 'murmur3', scripted: true, script: '1234', lang: 'expression'}, ].map(function (field) { field.count = field.count || 0; diff --git a/src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js b/src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js index 9efc4515038aa..76daadfe70b4b 100644 --- a/src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js +++ b/src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js @@ -35,7 +35,7 @@ describe('AggConfig Filters', function () { interval = interval || 'auto'; if (interval === 'custom') interval = agg.params.customInterval; duration = duration || moment.duration(15, 'minutes'); - field = _.sample(indexPattern.fields.byType.date); + field = _.sample(_.reject(indexPattern.fields.byType.date, 'scripted')); vis = new Vis(indexPattern, { type: 'histogram', aggs: [ diff --git a/src/ui/public/documentation_links/documentation_links.js b/src/ui/public/documentation_links/documentation_links.js index b148e90210f1d..36f83e0e969cd 100644 --- a/src/ui/public/documentation_links/documentation_links.js +++ b/src/ui/public/documentation_links/documentation_links.js @@ -14,5 +14,13 @@ export default { elasticsearchOutputAnchorParameters: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/elasticsearch-output.html#_parameters`, startup: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/_step_5_starting_filebeat.html`, exportedFields: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/exported-fields.html` + }, + scriptedFields: { + scriptFields: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-request-script-fields.html`, + scriptAggs: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-aggregations.html#_values_source`, + painless: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html`, + painlessApi: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html#painless-api`, + painlessSyntax: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless-syntax.html`, + luceneExpressions: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-expression.html` } }; diff --git a/src/ui/public/field_editor/__tests__/field_editor.js b/src/ui/public/field_editor/__tests__/field_editor.js index 43b200fde76ca..366fe48ea2b56 100644 --- a/src/ui/public/field_editor/__tests__/field_editor.js +++ b/src/ui/public/field_editor/__tests__/field_editor.js @@ -4,6 +4,8 @@ import expect from 'expect.js'; import IndexPatternsFieldProvider from 'ui/index_patterns/_field'; import RegistryFieldFormatsProvider from 'ui/registry/field_formats'; import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern'; +import _ from 'lodash'; + describe('FieldEditor directive', function () { let Field; @@ -14,8 +16,15 @@ describe('FieldEditor directive', function () { let $scope; let $el; + let $httpBackend; + beforeEach(ngMock.module('kibana')); beforeEach(ngMock.inject(function ($compile, $injector, Private) { + $httpBackend = $injector.get('$httpBackend'); + $httpBackend + .when('GET', '/api/kibana/scripts/languages') + .respond(['expression', 'painless']); + $rootScope = $injector.get('$rootScope'); Field = Private(IndexPatternsFieldProvider); StringFormat = Private(RegistryFieldFormatsProvider).getType('string'); @@ -127,6 +136,56 @@ describe('FieldEditor directive', function () { }); }); + + describe('scripted fields', function () { + let editor; + let field; + + beforeEach(function () { + $rootScope.field = $rootScope.indexPattern.fields.byName['script string']; + compile(); + editor = $scope.editor; + field = editor.field; + }); + + it('has a scripted flag set to true', function () { + expect(field.scripted).to.be(true); + }); + + it('contains a lang param', function () { + expect(field).to.have.property('lang'); + expect(field.lang).to.be('expression'); + }); + + it('provides lang options based on what is enabled for inline use in ES', function () { + $httpBackend.flush(); + expect(_.isEqual(editor.scriptingLangs, ['expression', 'painless'])).to.be.ok(); + }); + + it('provides curated type options based on language', function () { + $rootScope.$apply(); + expect(editor.fieldTypes).to.have.length(1); + expect(editor.fieldTypes[0]).to.be('number'); + + editor.field.lang = 'painless'; + $rootScope.$apply(); + + expect(editor.fieldTypes).to.have.length(4); + expect(_.isEqual(editor.fieldTypes, ['number', 'string', 'date', 'boolean'])).to.be.ok(); + }); + + it('updates formatter options based on field type', function () { + field.lang = 'painless'; + + $rootScope.$apply(); + expect(editor.field.type).to.be('string'); + const stringFormats = editor.fieldFormatTypes; + + field.type = 'date'; + $rootScope.$apply(); + expect(editor.fieldFormatTypes).to.not.be(stringFormats); + }); + }); }); }); diff --git a/src/ui/public/field_editor/field_editor.html b/src/ui/public/field_editor/field_editor.html index 18e065a6b7b6d..590d15154981a 100644 --- a/src/ui/public/field_editor/field_editor.html +++ b/src/ui/public/field_editor/field_editor.html @@ -1,4 +1,11 @@
+
+

+ + Scripting disabled: + All inline scripting has been disabled in Elasticsearch. You must enable inline scripting for at least one language in order to use scripted fields in Kibana. +

+
+
+ + +
+
+ @@ -86,15 +111,68 @@

- +
-
+
+

+ Proceed with caution +

+ +

+ Please familiarize yourself with script fields and with scripts in aggregations before using scripted fields. +

+ +

+ Scripted fields can be used to display and aggregate calculated values. As such, they can be very slow, and if done incorrectly, can cause Kibana to be unusable. There's no safety net here. If you make a typo, unexpected exceptions will be thrown all over the place! +

+
-
+
+

+ Scripting Help +

+ +

+ By default, Kibana scripted fields use Painless , a simple and secure scripting language designed specifically for use with Elasticsearch. To access values in the document use the following format: +

+ +

doc['some_field'].value

+ +

+ Painless is powerful but easy to use. It provides access to many native Java APIs . Read up on its syntax and you'll be up to speed in no time! +

+ +

+ Coming from an older version of Kibana? The Lucene Expressions you know and love are still available. Lucene expressions are a lot like JavaScript, but limited to basic arithmetic, bitwise and comparison operations. +

+ +

+ There are a few limitations when using Lucene Expressions: +

+
    +
  • Only numeric, boolean, date, and geo_point fields may be accessed
  • +
  • Stored fields are not available
  • +
  • If a field is sparse (only some documents contain a value), documents missing the field will have a value of 0
  • +
+ +

+ Here are all the operations available to lucene expressions: +

+
    +
  • Arithmetic operators: + - * / %
  • +
  • Bitwise operators: | & ^ ~ << >> >>>
  • +
  • Boolean operators (including the ternary operator): && || ! ?:
  • +
  • Comparison operators: < <= == >= >
  • +
  • Common mathematic functions: abs ceil exp floor ln log10 logn max min sqrt pow
  • +
  • Trigonometric library functions: acosh acos asinh asin atanh atan atan2 cosh cos sinh sin tanh tan
  • +
  • Distance functions: haversin
  • +
  • Miscellaneous functions: min, max
  • +
+
diff --git a/src/ui/public/field_editor/field_editor.js b/src/ui/public/field_editor/field_editor.js index 57f9006dfa831..02a257b81644d 100644 --- a/src/ui/public/field_editor/field_editor.js +++ b/src/ui/public/field_editor/field_editor.js @@ -6,15 +6,22 @@ import RegistryFieldFormatsProvider from 'ui/registry/field_formats'; import IndexPatternsFieldProvider from 'ui/index_patterns/_field'; import uiModules from 'ui/modules'; import fieldEditorTemplate from 'ui/field_editor/field_editor.html'; - +import chrome from 'ui/chrome'; +import IndexPatternsCastMappingTypeProvider from 'ui/index_patterns/_cast_mapping_type'; +import { scriptedFields as docLinks } from '../documentation_links/documentation_links'; +import './field_editor.less'; uiModules .get('kibana', ['colorpicker.module']) .directive('fieldEditor', function (Private, $sce) { let fieldFormats = Private(RegistryFieldFormatsProvider); let Field = Private(IndexPatternsFieldProvider); - let scriptingInfo = $sce.trustAsHtml(require('ui/field_editor/scripting_info.html')); - let scriptingWarning = $sce.trustAsHtml(require('ui/field_editor/scripting_warning.html')); + + const fieldTypesByLang = { + painless: ['number', 'string', 'date', 'boolean'], + expression: ['number'], + default: _.keys(Private(IndexPatternsCastMappingTypeProvider).types.byType) + }; return { restrict: 'E', @@ -24,12 +31,17 @@ uiModules getField: '&field' }, controllerAs: 'editor', - controller: function ($scope, Notifier, kbnUrl) { + controller: function ($scope, Notifier, kbnUrl, $http, $q) { let self = this; let notify = new Notifier({ location: 'Field Editor' }); - self.scriptingInfo = scriptingInfo; - self.scriptingWarning = scriptingWarning; + self.docLinks = docLinks; + getScriptingLangs().then((langs) => { + self.scriptingLangs = langs; + if (!_.includes(self.scriptingLangs, self.field.lang)) { + self.field.lang = undefined; + } + }); self.indexPattern = $scope.getIndexPattern(); self.field = shadowCopy($scope.getField()); @@ -39,7 +51,6 @@ uiModules self.creating = !self.indexPattern.fields.byName[self.field.name]; self.selectedFormatId = _.get(self.indexPattern, ['fieldFormatMap', self.field.name, 'type', 'id']); self.defFormatType = initDefaultFormat(); - self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[self.field.type] || []); self.cancel = redirectAway; self.save = function () { @@ -91,6 +102,23 @@ uiModules self.field.format = new FieldFormat(self.formatParams); }, true); + $scope.$watch('editor.field.type', function (newValue) { + self.defFormatType = initDefaultFormat(); + self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[newValue] || []); + + if (_.isUndefined(_.find(self.fieldFormatTypes, {id: self.selectedFormatId}))) { + delete self.selectedFormatId; + } + }); + + $scope.$watch('editor.field.lang', function (newValue) { + self.fieldTypes = _.get(fieldTypesByLang, newValue, fieldTypesByLang.default); + + if (!_.contains(self.fieldTypes, self.field.type)) { + self.field.type = _.first(self.fieldTypes); + } + }); + // copy the defined properties of the field to a plain object // which is mutable, and capture the changed seperately. function shadowCopy(field) { @@ -129,6 +157,14 @@ uiModules else return fieldFormats.getDefaultType(self.field.type); } + function getScriptingLangs() { + return $http.get(chrome.addBasePath('/api/kibana/scripts/languages')) + .then((res) => res.data) + .catch(() => { + return notify.error('Error getting available scripting languages from Elasticsearch'); + }); + } + function initDefaultFormat() { let def = Object.create(fieldFormats.getDefaultType(self.field.type)); diff --git a/src/ui/public/field_editor/field_editor.less b/src/ui/public/field_editor/field_editor.less new file mode 100644 index 0000000000000..5b0ad135175eb --- /dev/null +++ b/src/ui/public/field_editor/field_editor.less @@ -0,0 +1,3 @@ +textarea.field-editor_script-input { + height: 100px; +} diff --git a/src/ui/public/field_editor/scripting_info.html b/src/ui/public/field_editor/scripting_info.html deleted file mode 100644 index 0138c01b2c50f..0000000000000 --- a/src/ui/public/field_editor/scripting_info.html +++ /dev/null @@ -1,32 +0,0 @@ -

- Scripting Help -

- -

- By default, Elasticsearch scripts use Lucene Expressions , which is a lot like JavaScript, but limited to basic arithmetic, bitwise and comparison operations. We'll let you do some reading on Lucene Expressions To access values in the document use the following format: -

- -

doc['some_field'].value

- -

- There are a few limitations when using Lucene Expressions: -

- - -

- Here are all the operations available to scripted fields: -

- diff --git a/src/ui/public/field_editor/scripting_warning.html b/src/ui/public/field_editor/scripting_warning.html deleted file mode 100644 index c525c9ee06dc4..0000000000000 --- a/src/ui/public/field_editor/scripting_warning.html +++ /dev/null @@ -1,11 +0,0 @@ -

- Proceed with caution -

- -

- Please familiarize yourself with script fields and with scripts in aggregations before using scripted fields. -

- -

- Scripted fields can be used to display and aggregate calculated values. As such, they can be very slow, and if done incorrectly, can cause Kibana to be unusable. There's no safety net here. If you make a typo, unexpected exceptions will be thrown all over the place! -

diff --git a/src/ui/public/index_patterns/__tests__/_get_computed_fields.js b/src/ui/public/index_patterns/__tests__/_get_computed_fields.js index ae0da4e11978d..ec7e31ae56c94 100644 --- a/src/ui/public/index_patterns/__tests__/_get_computed_fields.js +++ b/src/ui/public/index_patterns/__tests__/_get_computed_fields.js @@ -32,8 +32,11 @@ describe('get computed fields', function () { it('should request date fields as docvalue_fields', function () { expect(fn().docvalueFields).to.contain('@timestamp'); - expect(fn().docvalueFields).to.not.include.keys('bytes'); + expect(fn().docvalueFields).to.not.contain('bytes'); }); + it('should not request scripted date fields as docvalue_fields', function () { + expect(fn().docvalueFields).to.not.contain('script date'); + }); }); diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index af4471f48feef..986d1fa14ebdd 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -50,7 +50,7 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope // scripted objs obj.fact('scripted', scripted); obj.writ('script', scripted ? spec.script : null); - obj.writ('lang', scripted ? (spec.lang || 'expression') : null); + obj.writ('lang', scripted ? (spec.lang || 'painless') : null); // mapping info obj.fact('indexed', indexed); diff --git a/src/ui/public/index_patterns/_get_computed_fields.js b/src/ui/public/index_patterns/_get_computed_fields.js index 8730183a60066..50f56eda4c47d 100644 --- a/src/ui/public/index_patterns/_get_computed_fields.js +++ b/src/ui/public/index_patterns/_get_computed_fields.js @@ -6,7 +6,7 @@ export default function () { let scriptFields = {}; let docvalueFields = []; - docvalueFields = _.pluck(self.fields.byType.date, 'name'); + docvalueFields = _.map(_.reject(self.fields.byType.date, 'scripted'), 'name'); _.each(self.getScriptedFields(), function (field) { scriptFields[field.name] = { diff --git a/test/intern_api.js b/test/intern_api.js index c30303e96c593..08beab27fb3b4 100644 --- a/test/intern_api.js +++ b/test/intern_api.js @@ -1,7 +1,8 @@ define({ suites: [ 'test/unit/api/ingest/index', - 'test/unit/api/search/index' + 'test/unit/api/search/index', + 'test/unit/api/scripts/index' ], excludeInstrumentation: /(fixtures|node_modules)\//, loaderOptions: { diff --git a/test/unit/api/scripts/_languages.js b/test/unit/api/scripts/_languages.js new file mode 100644 index 0000000000000..9f9afa19926eb --- /dev/null +++ b/test/unit/api/scripts/_languages.js @@ -0,0 +1,27 @@ +define(function (require) { + var expect = require('intern/dojo/node!expect.js'); + + return function (bdd, request) { + bdd.describe('Languages API', function getLanguages() { + + bdd.it('should return 200 with an array of languages', function () { + return request.get('/kibana/scripts/languages') + .expect(200) + .then(function (response) { + expect(response.body).to.be.an('array'); + }); + }); + + bdd.it('should only return langs enabled for inline scripting', function () { + return request.get('/kibana/scripts/languages') + .expect(200) + .then(function (response) { + expect(response.body).to.contain('expression'); + expect(response.body).to.contain('painless'); + + expect(response.body).to.not.contain('groovy'); + }); + }); + }); + }; +}); diff --git a/test/unit/api/scripts/index.js b/test/unit/api/scripts/index.js new file mode 100644 index 0000000000000..7b147eec59bb7 --- /dev/null +++ b/test/unit/api/scripts/index.js @@ -0,0 +1,13 @@ +define(function (require) { + var bdd = require('intern!bdd'); + var serverConfig = require('intern/dojo/node!../../../server_config'); + var request = require('intern/dojo/node!supertest-as-promised'); + var url = require('intern/dojo/node!url'); + var languages = require('./_languages'); + + bdd.describe('scripts API', function () { + request = request(url.format(serverConfig.servers.kibana) + '/api'); + + languages(bdd, request); + }); +});