Merge pull request elastic#7700 from Bargs/painless

Set language for scripted field

src/core_plugins/kibana/index.js

@@ -1,6 +1,7 @@
 import ingest from './server/routes/api/ingest';
 import search from './server/routes/api/search';
 import settings from './server/routes/api/settings';
+import scripts from './server/routes/api/scripts';
 
 module.exports = function (kibana) {
   return new kibana.Plugin({
@@ -84,6 +85,7 @@ module.exports = function (kibana) {
       ingest(server);
       search(server);
       settings(server);
+      scripts(server);
     }
   });
 

src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js

@@ -25,6 +25,7 @@ uiModules.get('apps/management')
       $scope.perPage = 25;
       $scope.columns = [
         { title: 'name' },
+        { title: 'lang' },
         { title: 'script' },
         { title: 'format' },
         { title: 'controls', sortable: false }
@@ -46,6 +47,7 @@ uiModules.get('apps/management')
 
           return [
             _.escape(field.name),
+            _.escape(field.lang),
             _.escape(field.script),
             _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
             {

src/core_plugins/kibana/server/routes/api/scripts/index.js

@@ -0,0 +1,5 @@
+import { registerLanguages } from './register_languages';
+
+export default function (server) {
+  registerLanguages(server);
+}

src/core_plugins/kibana/server/routes/api/scripts/register_languages.js

@@ -0,0 +1,27 @@
+import _ from 'lodash';
+import handleESError from '../../../lib/handle_es_error';
+
+export function registerLanguages(server) {
+  server.route({
+    path: '/api/kibana/scripts/languages',
+    method: 'GET',
+    handler: function (request, reply) {
+      const callWithRequest = server.plugins.elasticsearch.callWithRequest;
+
+      return callWithRequest(request, 'cluster.getSettings', {
+        include_defaults: true,
+        filter_path: '**.script.engine.*.inline'
+      })
+      .then((esResponse) => {
+        const langs = _.get(esResponse, 'defaults.script.engine', {});
+        const inlineLangs = _.pick(langs, (lang) => lang.inline === 'true');
+        const supportedLangs = _.omit(inlineLangs, 'mustache');
+        return _.keys(supportedLangs);
+      })
+      .then(reply)
+      .catch((error) => {
+        reply(handleESError(error));
+      });
+    }
+  });
+}

src/fixtures/logstash_fields.js

@@ -22,6 +22,7 @@ function stubbedLogstashFields() {
     { name: 'custom_user_field',  type: 'conflict',   indexed: false, analyzed: false, sortable: false, filterable: true },
     { name: 'script string',      type: 'string',     scripted: true, script: '\'i am a string\'', lang: 'expression' },
     { name: 'script number',      type: 'number',     scripted: true, script: '1234', lang: 'expression' },
+    { name: 'script date',        type: 'date',       scripted: true, script: '1234', lang: 'painless' },
     { name: 'script murmur3',     type: 'murmur3',    scripted: true, script: '1234', lang: 'expression'},
   ].map(function (field) {
     field.count = field.count || 0;

src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js

@@ -35,7 +35,7 @@ describe('AggConfig Filters', function () {
         interval = interval || 'auto';
         if (interval === 'custom') interval = agg.params.customInterval;
         duration = duration || moment.duration(15, 'minutes');
-        field = _.sample(indexPattern.fields.byType.date);
+        field = _.sample(_.reject(indexPattern.fields.byType.date, 'scripted'));
         vis = new Vis(indexPattern, {
           type: 'histogram',
           aggs: [

src/ui/public/documentation_links/documentation_links.js

@@ -14,5 +14,13 @@ export default {
     elasticsearchOutputAnchorParameters: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/elasticsearch-output.html#_parameters`,
     startup: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/_step_5_starting_filebeat.html`,
     exportedFields: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/exported-fields.html`
+  },
+  scriptedFields: {
+    scriptFields: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-request-script-fields.html`,
+    scriptAggs: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-aggregations.html#_values_source`,
+    painless: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html`,
+    painlessApi: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html#painless-api`,
+    painlessSyntax: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless-syntax.html`,
+    luceneExpressions: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-expression.html`
   }
 };

src/ui/public/field_editor/__tests__/field_editor.js

@@ -4,6 +4,8 @@ import expect from 'expect.js';
 import IndexPatternsFieldProvider from 'ui/index_patterns/_field';
 import RegistryFieldFormatsProvider from 'ui/registry/field_formats';
 import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern';
+import _ from 'lodash';
+
 describe('FieldEditor directive', function () {
 
   let Field;
@@ -14,8 +16,15 @@ describe('FieldEditor directive', function () {
   let $scope;
   let $el;
 
+  let $httpBackend;
+
   beforeEach(ngMock.module('kibana'));
   beforeEach(ngMock.inject(function ($compile, $injector, Private) {
+    $httpBackend = $injector.get('$httpBackend');
+    $httpBackend
+    .when('GET', '/api/kibana/scripts/languages')
+    .respond(['expression', 'painless']);
+
     $rootScope = $injector.get('$rootScope');
     Field = Private(IndexPatternsFieldProvider);
     StringFormat = Private(RegistryFieldFormatsProvider).getType('string');
@@ -127,6 +136,56 @@ describe('FieldEditor directive', function () {
       });
 
     });
+
+    describe('scripted fields', function () {
+      let editor;
+      let field;
+
+      beforeEach(function () {
+        $rootScope.field = $rootScope.indexPattern.fields.byName['script string'];
+        compile();
+        editor = $scope.editor;
+        field = editor.field;
+      });
+
+      it('has a scripted flag set to true', function () {
+        expect(field.scripted).to.be(true);
+      });
+
+      it('contains a lang param', function () {
+        expect(field).to.have.property('lang');
+        expect(field.lang).to.be('expression');
+      });
+
+      it('provides lang options based on what is enabled for inline use in ES', function () {
+        $httpBackend.flush();
+        expect(_.isEqual(editor.scriptingLangs, ['expression', 'painless'])).to.be.ok();
+      });
+
+      it('provides curated type options based on language', function () {
+        $rootScope.$apply();
+        expect(editor.fieldTypes).to.have.length(1);
+        expect(editor.fieldTypes[0]).to.be('number');
+
+        editor.field.lang = 'painless';
+        $rootScope.$apply();
+
+        expect(editor.fieldTypes).to.have.length(4);
+        expect(_.isEqual(editor.fieldTypes, ['number', 'string', 'date', 'boolean'])).to.be.ok();
+      });
+
+      it('updates formatter options based on field type', function () {
+        field.lang = 'painless';
+
+        $rootScope.$apply();
+        expect(editor.field.type).to.be('string');
+        const stringFormats = editor.fieldFormatTypes;
+
+        field.type = 'date';
+        $rootScope.$apply();
+        expect(editor.fieldFormatTypes).to.not.be(stringFormats);
+      });
+    });
   });
 
 });

src/ui/public/field_editor/field_editor.html

@@ -1,4 +1,11 @@
 <form ng-submit="editor.save()" name="form">
+  <div ng-if="editor.scriptingLangs.length === 0" class="hintbox">
+    <p>
+      <i class="fa fa-danger text-danger"></i>
+      <strong>Scripting disabled:</strong>
+      All inline scripting has been disabled in Elasticsearch. You must enable inline scripting for at least one language in order to use scripted fields in Kibana.
+    </p>
+  </div>
   <div ng-if="editor.creating" class="form-group">
     <label>Name</label>
     <input
@@ -17,9 +24,27 @@
     </p>
   </div>
 
+  <div ng-if="editor.field.scripted" class="form-group">
+    <label>Language</label>
+    <select
+      ng-model="editor.field.lang"
+      ng-options="lang as lang for lang in editor.scriptingLangs"
+      required
+      class="form-control">
+      <option value="">-- Select Language --</option>
+    </select>
+  </div>
+
   <div class="form-group">
     <label>Type</label>
+    <select
+      ng-if="editor.field.scripted"
+      ng-model="editor.field.type"
+      ng-options="type as type for type in editor.fieldTypes"
+      class="form-control">
+    </select>
     <input
+      ng-if="!editor.field.scripted"
       ng-model="editor.field.type"
       readonly
       class="form-control">
@@ -86,15 +111,68 @@ <h4 class="hintbox-heading">
   <div ng-if="editor.field.scripted">
     <div class="form-group">
       <label>Script</label>
-      <textarea required class="form-control text-monospace" ng-model="editor.field.script"></textarea>
+      <textarea required class="field-editor_script-input form-control text-monospace" ng-model="editor.field.script"></textarea>
     </div>
 
     <div class="form-group">
-      <div ng-bind-html="editor.scriptingWarning" class="hintbox"></div>
+      <div class="hintbox">
+        <h4>
+          <i class="fa fa-warning text-warning"></i> Proceed with caution
+        </h4>
+
+        <p>
+          Please familiarize yourself with <a target="_window" ng-href="{{ editor.docLinks.scriptFields }}">script fields <i class="fa-link fa"></i></a> and with <a target="_window" ng-href="">scripts in aggregations <i class="fa-link fa"></i></a> before using scripted fields.
+        </p>
+
+        <p>
+          Scripted fields can be used to display and aggregate calculated values. As such, they can be very slow, and if done incorrectly, can cause Kibana to be unusable. There's no safety net here. If you make a typo, unexpected exceptions will be thrown all over the place!
+        </p>
+      </div>
     </div>
 
     <div class="form-group">
-      <div ng-bind-html="editor.scriptingInfo" class="hintbox"></div>
+      <div class="hintbox">
+        <h4>
+          <i class="fa fa-question-circle text-info"></i> Scripting Help
+        </h4>
+
+        <p>
+          By default, Kibana scripted fields use <a target="_window" ng-href="{{editor.docLinks.painless}}">Painless <i class="fa-link fa"></i></a>, a simple and secure scripting language designed specifically for use with Elasticsearch.  To access values in the document use the following format:
+        </p>
+
+        <p><code>doc['some_field'].value</code></p>
+
+        <p>
+          Painless is powerful but easy to use. It provides access to many <a target="_window" ng-href="{{editor.docLinks.painlessApi}}">native Java APIs <i class="fa-link fa"></i></a>. Read up on its <a target="_window" ng-href="{{editor.docLinks.painlessSyntax}}">syntax <i class="fa-link fa"></i></a> and you'll be up to speed in no time!
+        </p>
+
+        <p>
+          Coming from an older version of Kibana? The <a target="_window" ng-href="{{editor.docLinks.luceneExpressions}}">Lucene Expressions <i class="fa-link fa"></i></a> you know and love are still available. Lucene expressions are a lot like JavaScript, but limited to basic arithmetic, bitwise and comparison operations.
+        </p>
+
+        <p>
+          There are a few limitations when using Lucene Expressions:
+        </p>
+        <ul>
+          <li> Only numeric, boolean, date, and geo_point fields may be accessed </li>
+          <li> Stored fields are not available </li>
+          <li> If a field is sparse (only some documents contain a value), documents missing the field will have a value of 0 </li>
+        </ul>
+
+        <p>
+          Here are all the operations available to lucene expressions:
+        </p>
+        <ul>
+          <li> Arithmetic operators: + - * / % </li>
+          <li> Bitwise operators: | & ^ ~ << >> >>> </li>
+          <li> Boolean operators (including the ternary operator): && || ! ?: </li>
+          <li> Comparison operators: < <= == >= > </li>
+          <li> Common mathematic functions: abs ceil exp floor ln log10 logn max min sqrt pow </li>
+          <li> Trigonometric library functions: acosh acos asinh asin atanh atan atan2 cosh cos sinh sin tanh tan </li>
+          <li> Distance functions: haversin </li>
+          <li> Miscellaneous functions: min, max </li>
+        </ul>
+      </div>
     </div>
 
   </div>

src/ui/public/field_editor/field_editor.js

@@ -6,15 +6,22 @@ import RegistryFieldFormatsProvider from 'ui/registry/field_formats';
 import IndexPatternsFieldProvider from 'ui/index_patterns/_field';
 import uiModules from 'ui/modules';
 import fieldEditorTemplate from 'ui/field_editor/field_editor.html';
-
+import chrome from 'ui/chrome';
+import IndexPatternsCastMappingTypeProvider from 'ui/index_patterns/_cast_mapping_type';
+import { scriptedFields as docLinks } from '../documentation_links/documentation_links';
+import './field_editor.less';
 
 uiModules
 .get('kibana', ['colorpicker.module'])
 .directive('fieldEditor', function (Private, $sce) {
   let fieldFormats = Private(RegistryFieldFormatsProvider);
   let Field = Private(IndexPatternsFieldProvider);
-  let scriptingInfo = $sce.trustAsHtml(require('ui/field_editor/scripting_info.html'));
-  let scriptingWarning = $sce.trustAsHtml(require('ui/field_editor/scripting_warning.html'));
+
+  const fieldTypesByLang = {
+    painless: ['number', 'string', 'date', 'boolean'],
+    expression: ['number'],
+    default: _.keys(Private(IndexPatternsCastMappingTypeProvider).types.byType)
+  };
 
   return {
     restrict: 'E',
@@ -24,12 +31,17 @@ uiModules
       getField: '&field'
     },
     controllerAs: 'editor',
-    controller: function ($scope, Notifier, kbnUrl) {
+    controller: function ($scope, Notifier, kbnUrl, $http, $q) {
       let self = this;
       let notify = new Notifier({ location: 'Field Editor' });
 
-      self.scriptingInfo = scriptingInfo;
-      self.scriptingWarning = scriptingWarning;
+      self.docLinks = docLinks;
+      getScriptingLangs().then((langs) => {
+        self.scriptingLangs = langs;
+        if (!_.includes(self.scriptingLangs, self.field.lang)) {
+          self.field.lang = undefined;
+        }
+      });
 
       self.indexPattern = $scope.getIndexPattern();
       self.field = shadowCopy($scope.getField());
@@ -39,7 +51,6 @@ uiModules
       self.creating = !self.indexPattern.fields.byName[self.field.name];
       self.selectedFormatId = _.get(self.indexPattern, ['fieldFormatMap', self.field.name, 'type', 'id']);
       self.defFormatType = initDefaultFormat();
-      self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[self.field.type] || []);
 
       self.cancel = redirectAway;
       self.save = function () {
@@ -91,6 +102,23 @@ uiModules
         self.field.format = new FieldFormat(self.formatParams);
       }, true);
 
+      $scope.$watch('editor.field.type', function (newValue) {
+        self.defFormatType = initDefaultFormat();
+        self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[newValue] || []);
+
+        if (_.isUndefined(_.find(self.fieldFormatTypes, {id: self.selectedFormatId}))) {
+          delete self.selectedFormatId;
+        }
+      });
+
+      $scope.$watch('editor.field.lang', function (newValue) {
+        self.fieldTypes = _.get(fieldTypesByLang, newValue, fieldTypesByLang.default);
+
+        if (!_.contains(self.fieldTypes, self.field.type)) {
+          self.field.type = _.first(self.fieldTypes);
+        }
+      });
+
       // copy the defined properties of the field to a plain object
       // which is mutable, and capture the changed seperately.
       function shadowCopy(field) {
@@ -129,6 +157,14 @@ uiModules
         else return fieldFormats.getDefaultType(self.field.type);
       }
 
+      function getScriptingLangs() {
+        return $http.get(chrome.addBasePath('/api/kibana/scripts/languages'))
+        .then((res) => res.data)
+        .catch(() => {
+          return notify.error('Error getting available scripting languages from Elasticsearch');
+        });
+      }
+
       function initDefaultFormat() {
         let def = Object.create(fieldFormats.getDefaultType(self.field.type));
 

src/ui/public/field_editor/field_editor.less

@@ -0,0 +1,3 @@
+textarea.field-editor_script-input {
+  height: 100px;
+}