Remove support for expression based scripted fields

src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/scripted_fields_table/scripted_fields_table.html

@@ -6,6 +6,48 @@ <h3 class="kuiTextTitle kuiVerticalRhythm">
   These scripted fields are computed on the fly from your data. They can be used in visualizations and displayed in your documents, however they can not be searched. You can manage them here and add new ones as you see fit, but be careful, scripts can be tricky!
 </p>
 
+<div class="kuiInfoPanel kuiInfoPanel--warning kuiVerticalRhythm" ng-if="getDeprecatedLanguagesInUse().length !== 0">
+  <div class="kuiInfoPanelHeader">
+    <span
+      class="kuiInfoPanelHeader__icon kuiIcon kuiIcon--warning fa-bolt"
+      aria-label="Warning"
+      role="img"
+    ></span>
+    <span class="kuiInfoPanelHeader__title">
+      Deprecation Warning
+    </span>
+  </div>
+
+  <div class="kuiInfoPanelBody">
+    <div class="kuiInfoPanelBody__message">
+      We've detected that the following deprecated languages are in use: {{ getDeprecatedLanguagesInUse().join(', ') }}.
+      Support for these languages will be removed in the next major version of Kibana and Elasticsearch.
+      We recommend converting your scripted fields to
+      <a class="kuiLink" ng-href="{{docLinks.painless}}">Painless</a>.
+    </div>
+  </div>
+</div>
+
+<div class="kuiInfoPanel kuiInfoPanel--error kuiVerticalRhythm" ng-if="getUnsupportedLanguagesInUse().length !== 0">
+  <div class="kuiInfoPanelHeader">
+    <span
+      class="kuiInfoPanelHeader__icon kuiIcon kuiIcon--error fa-warning"
+      aria-label="Error"
+      role="img"
+    ></span>
+    <span class="kuiInfoPanelHeader__title">
+      Unsupported Languages
+    </span>
+  </div>
+
+  <div class="kuiInfoPanelBody">
+    <div class="kuiInfoPanelBody__message">
+      We've detected that the following unsupported languages are in use: {{ getUnsupportedLanguagesInUse().join(', ') }}.
+      All scripted fields must be converted to <a class="kuiLink" ng-href="{{docLinks.painless}}">Painless</a>.
+    </div>
+  </div>
+</div>
+
 <a
   data-test-subj="addScriptedFieldLink"
   ng-href="{{ kbnUrl.getRouteHref(indexPattern, 'addField') }}"

src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/scripted_fields_table/scripted_fields_table.js

@@ -4,6 +4,8 @@ import fieldControlsHtml from '../field_controls.html';
 import { dateScripts } from './date_scripts';
 import { uiModules } from 'ui/modules';
 import template from './scripted_fields_table.html';
+import { getSupportedScriptingLanguages, getDeprecatedScriptingLanguages } from 'ui/scripting_languages';
+import { documentationLinks } from 'ui/documentation_links/documentation_links';
 
 uiModules.get('apps/management')
 .directive('scriptedFieldsTable', function (kbnUrl, Notifier, $filter, confirmModal) {
@@ -21,6 +23,7 @@ uiModules.get('apps/management')
       const fieldCreatorPath = '/management/kibana/indices/{{ indexPattern }}/scriptedField';
       const fieldEditorPath = fieldCreatorPath + '/{{ fieldName }}';
 
+      $scope.docLinks = documentationLinks.scriptedFields;
       $scope.perPage = 25;
       $scope.columns = [
         { title: 'name' },
@@ -110,6 +113,19 @@ uiModules.get('apps/management')
         };
         confirmModal(`Are you sure want to delete ${field.name}? This action is irreversible!`, confirmModalOptions);
       };
+
+      function getLanguagesInUse() {
+        const fields = $scope.indexPattern.getScriptedFields();
+        return _.uniq(_.map(fields, 'lang'));
+      }
+
+      $scope.getDeprecatedLanguagesInUse = function () {
+        return _.intersection(getLanguagesInUse(), getDeprecatedScriptingLanguages());
+      };
+
+      $scope.getUnsupportedLanguagesInUse = function () {
+        return _.difference(getLanguagesInUse(), _.union(getSupportedScriptingLanguages(), getDeprecatedScriptingLanguages()));
+      };
     }
   };
 });

src/ui/public/field_editor/__tests__/field_editor.js

@@ -158,7 +158,7 @@ describe('FieldEditor directive', function () {
           .respond(['expression', 'painless', 'groovy']);
 
         $httpBackend.flush();
-        expect(editor.scriptingLangs).to.eql(['expression', 'painless']);
+        expect(editor.scriptingLangs).to.eql(['painless']);
       });
 
       it('provides specific type when language is painless', function () {

src/ui/public/field_editor/field_editor.html

@@ -28,12 +28,33 @@
 
   <div ng-if="editor.field.scripted" class="form-group">
     <label for="scriptedFieldLang">Language</label>
+    <div class="kuiInfoPanel kuiInfoPanel--warning kuiVerticalRhythm" ng-if="editor.field.lang && editor.isDeprecatedLang(editor.field.lang)">
+      <div class="kuiInfoPanelHeader">
+        <span
+          class="kuiInfoPanelHeader__icon kuiIcon kuiIcon--warning fa-bolt"
+          aria-label="Warning"
+          role="img"
+        ></span>
+        <span class="kuiInfoPanelHeader__title">
+          Deprecation Warning
+        </span>
+      </div>
+
+      <div class="kuiInfoPanelBody">
+        <div class="kuiInfoPanelBody__message">
+          <span class="text-capitalize">{{editor.field.lang}}</span> is deprecated and support will be removed in the
+          next major version of Kibana and Elasticsearch. We recommend using
+          <a class="kuiLink" ng-href="{{editor.docLinks.painless}}">Painless</a>
+          for new scripted fields.
+        </div>
+      </div>
+    </div>
     <select
       ng-model="editor.field.lang"
       id="scriptedFieldLang"
       ng-options="lang as lang for lang in editor.scriptingLangs"
       required
-      class="form-control"
+      class="form-control kuiVerticalRhythm"
       data-test-subj="editorFieldLang">
       <option value="">-- Select Language --</option>
     </select>

src/ui/public/field_editor/field_editor.js

@@ -8,7 +8,11 @@ import { uiModules } from 'ui/modules';
 import fieldEditorTemplate from 'ui/field_editor/field_editor.html';
 import '../directives/documentation_href';
 import './field_editor.less';
-import { GetEnabledScriptingLanguagesProvider, getSupportedScriptingLanguages } from '../scripting_languages';
+import {
+  GetEnabledScriptingLanguagesProvider,
+  getSupportedScriptingLanguages,
+  getDeprecatedScriptingLanguages
+} from '../scripting_languages';
 import { getKbnTypeNames } from '../../../utils';
 
 uiModules
@@ -38,7 +42,7 @@ uiModules
       const notify = new Notifier({ location: 'Field Editor' });
 
       getScriptingLangs().then((langs) => {
-        self.scriptingLangs = _.intersection(langs, ['expression', 'painless']);
+        self.scriptingLangs = langs;
         if (!_.includes(self.scriptingLangs, self.field.lang)) {
           self.field.lang = undefined;
         }
@@ -103,6 +107,10 @@ uiModules
         );
       };
 
+      self.isDeprecatedLang = function (lang) {
+        return _.contains(getDeprecatedScriptingLanguages(), lang);
+      };
+
       $scope.$watch('editor.selectedFormatId', function (cur, prev) {
         const format = self.field.format;
         const changedFormat = cur !== prev;
@@ -179,7 +187,7 @@ uiModules
       function getScriptingLangs() {
         return getEnabledScriptingLanguages()
         .then((enabledLanguages) => {
-          return _.intersection(enabledLanguages, getSupportedScriptingLanguages());
+          return _.intersection(enabledLanguages, _.union(getSupportedScriptingLanguages(), getDeprecatedScriptingLanguages()));
         });
       }
 

src/ui/public/scripting_languages/index.js

@@ -4,7 +4,11 @@ import { Notifier } from 'ui/notify/notifier';
 const notify = new Notifier({ location: 'Scripting Language Service' });
 
 export function getSupportedScriptingLanguages() {
-  return ['expression', 'painless'];
+  return ['painless'];
+}
+
+export function getDeprecatedScriptingLanguages() {
+  return [];
 }
 
 export function GetEnabledScriptingLanguagesProvider($http) {

test/functional/apps/management/_scripted_fields.js

@@ -1,9 +1,8 @@
-// Tests for 5 scripted fields;
-// 1. Lucene expression (number type)
-// 2. Painless (number type)
-// 3. Painless (string type)
+// Tests for 4 scripted fields;
+// 1. Painless (number type)
+// 2. Painless (string type)
 // 3. Painless (boolean type)
-// 3. Painless (date type)
+// 4. Painless (date type)
 //
 // Each of these scripted fields has 4 tests (12 tests total);
 // 1. Create scripted field
@@ -38,74 +37,6 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.settings.removeIndexPattern();
     });
 
-    describe('creating and using Lucence expression scripted fields', function describeIndexTests() {
-      const scriptedExpressionFieldName = 'ram_expr1';
-
-      it('should create scripted field', async function () {
-        await PageObjects.settings.navigateTo();
-        await PageObjects.settings.clickKibanaIndices();
-        const startingCount = parseInt(await PageObjects.settings.getScriptedFieldsTabCount());
-        await PageObjects.settings.clickScriptedFieldsTab();
-        await log.debug('add scripted field');
-        await PageObjects.settings
-          .addScriptedField(scriptedExpressionFieldName,
-            'expression', 'number', null, '1', 'doc[\'machine.ram\'].value / (1024 * 1024 * 1024)'
-          );
-        await retry.try(async function () {
-          expect(parseInt(await PageObjects.settings.getScriptedFieldsTabCount())).to.be(startingCount + 1);
-        });
-      });
-
-      it('should see scripted field value in Discover', async function () {
-        const fromTime = '2015-09-17 06:31:44.000';
-        const toTime = '2015-09-18 18:31:44.000';
-        await PageObjects.common.navigateToApp('discover');
-        await log.debug('setAbsoluteRange (' + fromTime + ') to (' + toTime + ')');
-        await PageObjects.header.setAbsoluteRange(fromTime, toTime);
-        await PageObjects.header.waitUntilLoadingHasFinished();
-        await PageObjects.visualize.waitForVisualization();
-        await PageObjects.discover.clickFieldListItem(scriptedExpressionFieldName);
-        await retry.try(async function () {
-          await PageObjects.discover.clickFieldListItemAdd(scriptedExpressionFieldName);
-        });
-        await PageObjects.header.waitUntilLoadingHasFinished();
-        await PageObjects.visualize.waitForVisualization();
-        await retry.try(async function () {
-          const rowData = await PageObjects.discover.getDocTableIndex(1);
-          expect(rowData).to.be('September 18th 2015, 18:20:57.916\n18');
-        });
-      });
-
-      it('should filter by scripted field value in Discover', async function () {
-        await PageObjects.discover.clickFieldListItem(scriptedExpressionFieldName);
-        await log.debug('filter by the first value (14) in the expanded scripted field list');
-        await PageObjects.discover.clickFieldListPlusFilter(scriptedExpressionFieldName, '14');
-        await PageObjects.header.waitUntilLoadingHasFinished();
-        await PageObjects.visualize.waitForVisualization();
-        await retry.try(async function () {
-          expect(await PageObjects.discover.getHitCount()).to.be('31');
-        });
-      });
-
-      it('should visualize scripted field in vertical bar chart', async function () {
-        const expectedChartValues = [ '14', '31', '10', '29', '7', '24', '11', '24', '12', '23',
-          '20', '23', '19', '21', '6', '20', '17', '20', '30', '20', '13', '19', '18', '18', '16', '17', '5', '16',
-          '8', '16', '15', '14', '3', '13', '2', '12', '9', '10', '4', '9'
-        ];
-        await PageObjects.discover.removeAllFilters();
-        await PageObjects.discover.clickFieldListItemVisualize(scriptedExpressionFieldName);
-        await PageObjects.header.waitUntilLoadingHasFinished();
-        await PageObjects.visualize.waitForVisualization();
-        await PageObjects.visualize.toggleSpyPanel();
-        await PageObjects.settings.setPageSize('All');
-        const data = await PageObjects.visualize.getDataTableData();
-        await log.debug('getDataTableData = ' + data.split('\n'));
-        await log.debug('data=' + data);
-        await log.debug('data.length=' + data.length);
-        expect(data.trim().split('\n')).to.eql(expectedChartValues);
-      });
-    });
-
     describe('creating and using Painless numeric scripted fields', function describeIndexTests() {
       const scriptedPainlessFieldName = 'ram_Pain1';
 

test/functional/apps/management/_scripted_fields_filter.js

@@ -5,21 +5,25 @@ export default function ({ getService, getPageObjects }) {
   const retry = getService('retry');
   const log = getService('log');
   const remote = getService('remote');
+  const esArchiver = getService('esArchiver');
   const PageObjects = getPageObjects(['settings']);
 
   describe('filter scripted fields', function describeIndexTests() {
-
-    beforeEach(async function () {
-      await remote.setWindowSize(1200, 800);
+    before(async function () {
       // delete .kibana index and then wait for Kibana to re-create it
+      await remote.setWindowSize(1200, 800);
+      await esArchiver.load('management');
+      await kibanaServer.uiSettings.replace({
+        'dateFormat:tz': 'UTC',
+        'defaultIndex': 'f1e4c910-a2e6-11e7-bb30-233be9be6a15'
+      });
+    });
+
+    after(async function () {
+      await esArchiver.unload('management');
       await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'UTC' });
-      await PageObjects.settings.navigateTo();
-      await PageObjects.settings.clickKibanaIndices();
-      await PageObjects.settings.createIndexPattern();
-      await kibanaServer.uiSettings.update({ 'dateFormat:tz': 'UTC' });
     });
 
-    const scriptedExpressionFieldName = 'ram_expr1';
     const scriptedPainlessFieldName = 'ram_pain1';
 
     it('should filter scripted fields', async function () {
@@ -28,10 +32,9 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.settings.clickScriptedFieldsTab();
       const scriptedFieldLangsBefore = await PageObjects.settings.getScriptedFieldLangs();
       await log.debug('add scripted field');
-      await PageObjects.settings
-      .addScriptedField(scriptedExpressionFieldName,
-        'expression', 'number', null, '1', 'doc[\'machine.ram\'].value / (1024 * 1024 * 1024)'
-      );
+
+      // The expression scripted field has been pre-created in the management esArchiver pack since it is no longer
+      // possible to create an expression script via the UI
       await PageObjects.settings
       .addScriptedField(scriptedPainlessFieldName,
         'painless', 'number', null, '1', 'doc[\'machine.ram\'].value / (1024 * 1024 * 1024)'
@@ -40,7 +43,7 @@ export default function ({ getService, getPageObjects }) {
       // confirm two additional scripted fields were created
       await retry.try(async function () {
         const scriptedFieldLangs = await PageObjects.settings.getScriptedFieldLangs();
-        expect(scriptedFieldLangs.length).to.be(scriptedFieldLangsBefore.length + 2);
+        expect(scriptedFieldLangs.length).to.be(scriptedFieldLangsBefore.length + 1);
       });
 
       await PageObjects.settings.setScriptedFieldLanguageFilter('painless');