Merge branch 'master' into feat/upgrade-ecs-8.0

elastic · Oct 5, 2021 · 1a4306d · 1a4306d
2 parents 54e7a9d + d146b03
commit 1a4306d
Show file tree

Hide file tree

Showing 171 changed files with 3,930 additions and 985 deletions.
diff --git a/.buildkite/scripts/steps/storybooks/build_and_upload.js b/.buildkite/scripts/steps/storybooks/build_and_upload.js
@@ -8,6 +8,7 @@ const STORYBOOKS = [
   'canvas',
   'codeeditor',
   'ci_composite',
+  'custom_integrations',
   'url_template_editor',
   'dashboard',
   'dashboard_enhanced',

diff --git a/.i18nrc.json b/.i18nrc.json
@@ -19,6 +19,7 @@
     "home": "src/plugins/home",
     "flot": "packages/kbn-ui-shared-deps-src/src/flot_charts",
     "charts": "src/plugins/charts",
+    "customIntegrations": "src/plugins/custom_integrations",
     "esUi": "src/plugins/es_ui_shared",
     "devTools": "src/plugins/dev_tools",
     "expressions": "src/plugins/expressions",

diff --git a/docs/dev-tools/grokdebugger/index.asciidoc b/docs/dev-tools/grokdebugger/index.asciidoc
@@ -9,21 +9,22 @@ structure it. Grok is good for parsing syslog, apache, and other
 webserver logs, mysql logs, and in general, any log format that is
 written for human consumption.
 
-Grok patterns are supported in the ingest node
-{ref}/grok-processor.html[grok processor] and the Logstash
-{logstash-ref}/plugins-filters-grok.html[grok filter]. See
-{logstash-ref}/plugins-filters-grok.html#_grok_basics[grok basics]
-for more information on the syntax for a grok pattern.
-
-The Elastic Stack ships
-with more than 120 reusable grok patterns. See
-https://github.com/elastic/elasticsearch/tree/master/libs/grok/src/main/resources/patterns[Ingest node grok patterns] and https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns[Logstash grok patterns]
-for the complete list of patterns.
+Grok patterns are supported in {es} {ref}/runtime.html[runtime fields], the {es}
+{ref}/grok-processor.html[grok ingest processor], and the {ls}
+{logstash-ref}/plugins-filters-grok.html[grok filter]. For syntax, see
+{ref}/grok.html[Grokking grok].
+
+The {stack} ships with more than 120 reusable grok patterns. For a complete
+list of patterns, see
+https://github.com/elastic/elasticsearch/tree/master/libs/grok/src/main/resources/patterns[{es}
+grok patterns] and
+https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns[{ls}
+grok patterns].
 
 Because
-ingest node and Logstash share the same grok implementation and pattern
+{es} and {ls} share the same grok implementation and pattern
 libraries, any grok pattern that you create in the *Grok Debugger* will work
-in ingest node and Logstash.
+in both {es} and {ls}.
 
 [float]
 [[grokdebugger-getting-started]]

diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc
@@ -458,7 +458,7 @@ the infrastructure monitoring use-case within Kibana.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/ingest_pipelines/README.md[ingestPipelines]
-|The ingest_pipelines plugin provides Kibana support for Elasticsearch's ingest nodes. Please refer to the Elasticsearch documentation for more details.
+|The ingest_pipelines plugin provides Kibana support for Elasticsearch's ingest pipelines.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/lens/readme.md[lens]

diff --git a/docs/migration/migrate_8_0.asciidoc b/docs/migration/migrate_8_0.asciidoc
@@ -364,4 +364,34 @@ Configuration management tools and automation will need to be updated to use the
 === `server.xsrf.token` is no longer valid
 *Details:* The deprecated `server.xsrf.token` setting in the `kibana.yml` file has been removed.
 
+[float]
+=== `newsfeed.defaultLanguage` is no longer valid
+*Details:* Specifying a default language to retrieve newsfeed items is no longer supported.
+
+*Impact:* Newsfeed items will be retrieved based on the browser locale and fallback to 'en' if an item does not have a translation for the locale. Configure {kibana-ref}/i18n-settings-kb.html#general-i18n-settings-kb[`i18n.locale`] to override the default behavior. 
+
+[float]
+=== `xpack.banners.placement` has changed value
+*Details:* `xpack.banners.placement: 'header'` setting in `kibana.yml` has changed value.
+
+*Impact:* Use {kibana-ref}/banners-settings-kb.html#banners-settings-kb[`xpack.banners.placement: 'top'`] instead.
+
+[float]
+=== `cpu.cgroup.path.override` is no longer valid
+*Details:* The deprecated `cpu.cgroup.path.override` setting is no longer supported.
+
+*Impact:* Configure {kibana-ref}/settings.html#ops-cGroupOverrides-cpuPath[`ops.cGroupOverrides.cpuPath`] instead.
+
+[float]
+=== `cpuacct.cgroup.path.override` is no longer valid
+*Details:* The deprecated `cpuacct.cgroup.path.override` setting is no longer supported.
+
+*Impact:* Configure {kibana-ref}/settings.html#ops-cGroupOverrides-cpuAcctPath[`ops.cGroupOverrides.cpuAcctPath`] instead.
+
+[float]
+=== `server.xsrf.whitelist` is no longer valid
+*Details:* The deprecated `server.xsrf.whitelist` setting is no longer supported.
+
+*Impact:* Use {kibana-ref}/settings.html#settings-xsrf-allowlist[`server.xsrf.allowlist`] instead.
+
 // end::notable-breaking-changes[]
diff --git a/docs/redirects.asciidoc b/docs/redirects.asciidoc
@@ -293,7 +293,7 @@ This content has moved. Refer to <<dashboard, **Dashboard**>>.
 This content has moved. Refer to <<dashboard, **Dashboard**>>.
 
 [role="exclude",id="ingest-node-pipelines"]
-== Ingest Node Pipelines
+== Ingest Pipelines
 
 This content has moved. Refer to {ref}/ingest.html[Ingest pipelines].
 

diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
@@ -31,11 +31,6 @@ Be sure to back up the encryption key value somewhere safe, as your alerting rul
 [[action-settings]]
 ==== Action settings
 
-`xpack.actions.enabled`::
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported."]
-Feature toggle that enables Actions in {kib}.
-If `false`, all features dependent on Actions are disabled, including the *Observability* and *Security* apps.  Default: `true`.
-
 `xpack.actions.allowedHosts` {ess-icon}::
 A list of hostnames that {kib} is allowed to connect to when built-in actions are triggered. It defaults to `[*]`, allowing any host, but keep in mind the potential for SSRF attacks when hosts are not explicitly added to the allowed hosts. An empty list `[]` can be used to block built-in actions from making any external connections.
 +

diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc
@@ -26,16 +26,6 @@ Toggling this causes the server to regenerate assets on the next startup,
 which may cause a delay before pages start being served.
 Set to `false` to disable Console. *Default: `true`*
 
-| `cpu.cgroup.path.override:`
- | deprecated:[7.10.0,"In 8.0 and later, this setting will no longer be supported."]
- This setting has been renamed to
- <<ops-cGroupOverrides-cpuPath,`ops.cGroupOverrides.cpuPath`>>.
-
-| `cpuacct.cgroup.path.override:`
- | deprecated:[7.10.0,"In 8.0 and later, this setting will no longer be supported."]
- This setting has been renamed to
- <<ops-cGroupOverrides-cpuAcctPath, `ops.cGroupOverrides.cpuAcctPath`>>.
-
 | `csp.rules:`
  | deprecated:[7.14.0,"In 8.0 and later, this setting will no longer be supported."]
 A https://w3c.github.io/webappsec-csp/[Content Security Policy] template

diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc
@@ -17,7 +17,7 @@ Consult your administrator if you do not have the appropriate access.
 [cols="50, 50"]
 |===
 
-| {ref}/ingest.html[Ingest Node Pipelines]
+| {ref}/ingest.html[Ingest Pipelines]
 | Create and manage ingest pipelines that let you perform common transformations
 and enrichments on your data.
 

diff --git a/docs/user/monitoring/monitoring-metricbeat.asciidoc b/docs/user/monitoring/monitoring-metricbeat.asciidoc
@@ -189,8 +189,9 @@ If you configured the monitoring cluster to use encrypted communications, you
 must access it via HTTPS. For example, use a `hosts` setting like
 `https://es-mon-1:9200`.
 
-IMPORTANT: The {es} {monitor-features} use ingest pipelines, therefore the
-cluster that stores the monitoring data must have at least one ingest node.
+IMPORTANT: The {es} {monitor-features} use ingest pipelines. The
+cluster that stores the monitoring data must have at least one node with the
+`ingest` role.
 
 If the {es} {security-features} are enabled on the monitoring cluster, you
 must provide a valid user ID and password so that {metricbeat} can send metrics

diff --git a/packages/kbn-utils/src/path/index.ts b/packages/kbn-utils/src/path/index.ts
@@ -15,14 +15,12 @@ const isString = (v: any): v is string => typeof v === 'string';
 
 const CONFIG_PATHS = [
   process.env.KBN_PATH_CONF && join(process.env.KBN_PATH_CONF, 'kibana.yml'),
-  process.env.KIBANA_PATH_CONF && join(process.env.KIBANA_PATH_CONF, 'kibana.yml'), // deprecated
   join(REPO_ROOT, 'config/kibana.yml'),
   '/etc/kibana/kibana.yml',
 ].filter(isString);
 
 const CONFIG_DIRECTORIES = [
   process.env.KBN_PATH_CONF,
-  process.env.KIBANA_PATH_CONF, // deprecated
   join(REPO_ROOT, 'config'),
   '/etc/kibana',
 ].filter(isString);

diff --git a/src/core/server/config/deprecation/core_deprecations.test.ts b/src/core/server/config/deprecation/core_deprecations.test.ts
@@ -19,55 +19,30 @@ describe('core deprecations', () => {
     process.env = { ...initialEnv };
   });
 
-  describe('kibanaPathConf', () => {
-    it('logs a warning if KIBANA_PATH_CONF environ variable is set', () => {
-      process.env.KIBANA_PATH_CONF = 'somepath';
-      const { messages } = applyCoreDeprecations();
-      expect(messages).toMatchInlineSnapshot(`
-        Array [
-          "Environment variable \\"KIBANA_PATH_CONF\\" is deprecated. It has been replaced with \\"KBN_PATH_CONF\\" pointing to a config folder",
-        ]
-      `);
-    });
-
-    it('does not log a warning if KIBANA_PATH_CONF environ variable is unset', () => {
-      delete process.env.KIBANA_PATH_CONF;
-      const { messages } = applyCoreDeprecations();
-      expect(messages).toHaveLength(0);
-    });
-  });
-
-  describe('xsrfDeprecation', () => {
-    it('logs a warning if server.xsrf.whitelist is set', () => {
-      const { migrated, messages } = applyCoreDeprecations({
-        server: { xsrf: { whitelist: ['/path'] } },
-      });
-      expect(migrated.server.xsrf.allowlist).toEqual(['/path']);
-      expect(messages).toMatchInlineSnapshot(`
-        Array [
-          "Setting \\"server.xsrf.whitelist\\" has been replaced by \\"server.xsrf.allowlist\\"",
-        ]
-      `);
-    });
-  });
-
   describe('server.cors', () => {
     it('renames server.cors to server.cors.enabled', () => {
       const { migrated } = applyCoreDeprecations({
         server: { cors: true },
       });
       expect(migrated.server.cors).toEqual({ enabled: true });
     });
+
     it('logs a warning message about server.cors renaming', () => {
-      const { messages } = applyCoreDeprecations({
+      const { messages, levels } = applyCoreDeprecations({
         server: { cors: true },
       });
       expect(messages).toMatchInlineSnapshot(`
         Array [
           "\\"server.cors\\" is deprecated and has been replaced by \\"server.cors.enabled\\"",
         ]
       `);
+      expect(levels).toMatchInlineSnapshot(`
+        Array [
+          "warning",
+        ]
+      `);
     });
+
     it('does not log deprecation message when server.cors.enabled set', () => {
       const { migrated, messages } = applyCoreDeprecations({
         server: { cors: { enabled: true } },

diff --git a/src/core/server/config/deprecation/core_deprecations.ts b/src/core/server/config/deprecation/core_deprecations.ts
@@ -8,19 +8,6 @@
 
 import { ConfigDeprecationProvider, ConfigDeprecation } from '@kbn/config';
 
-const kibanaPathConf: ConfigDeprecation = (settings, fromPath, addDeprecation) => {
-  if (process.env?.KIBANA_PATH_CONF) {
-    addDeprecation({
-      message: `Environment variable "KIBANA_PATH_CONF" is deprecated. It has been replaced with "KBN_PATH_CONF" pointing to a config folder`,
-      correctiveActions: {
-        manualSteps: [
-          'Use "KBN_PATH_CONF" instead of "KIBANA_PATH_CONF" to point to a config folder.',
-        ],
-      },
-    });
-  }
-};
-
 const rewriteBasePathDeprecation: ConfigDeprecation = (settings, fromPath, addDeprecation) => {
   if (settings.server?.basePath && !settings.server?.rewriteBasePath) {
     addDeprecation({
@@ -44,6 +31,7 @@ const rewriteCorsSettings: ConfigDeprecation = (settings, fromPath, addDeprecati
   if (typeof corsSettings === 'boolean') {
     addDeprecation({
       message: '"server.cors" is deprecated and has been replaced by "server.cors.enabled"',
+      level: 'warning',
       correctiveActions: {
         manualSteps: [
           `Replace "server.cors: ${corsSettings}" with "server.cors.enabled: ${corsSettings}"`,
@@ -114,11 +102,7 @@ const cspRulesDeprecation: ConfigDeprecation = (settings, fromPath, addDeprecati
 };
 
 export const coreDeprecationProvider: ConfigDeprecationProvider = ({ rename, unusedFromRoot }) => [
-  rename('cpu.cgroup.path.override', 'ops.cGroupOverrides.cpuPath'),
-  rename('cpuacct.cgroup.path.override', 'ops.cGroupOverrides.cpuAcctPath'),
-  rename('server.xsrf.whitelist', 'server.xsrf.allowlist'),
   rewriteCorsSettings,
-  kibanaPathConf,
   rewriteBasePathDeprecation,
   cspRulesDeprecation,
 ];
diff --git a/src/core/server/config/test_utils.ts b/src/core/server/config/test_utils.ts
@@ -16,18 +16,22 @@ function collectDeprecations(
 ) {
   const deprecations = provider(configDeprecationFactory);
   const deprecationMessages: string[] = [];
+  const deprecationLevels: string[] = [];
   const { config: migrated } = applyDeprecations(
     settings,
     deprecations.map((deprecation) => ({
       deprecation,
       path,
     })),
     () =>
-      ({ message }) =>
-        deprecationMessages.push(message)
+      ({ message, level }) => {
+        deprecationMessages.push(message);
+        deprecationLevels.push(level ?? '');
+      }
   );
   return {
     messages: deprecationMessages,
+    levels: deprecationLevels,
     migrated,
   };
 }

diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker
@@ -26,8 +26,6 @@ kibana_vars=(
     console.enabled
     console.proxyConfig
     console.proxyFilter
-    cpu.cgroup.path.override
-    cpuacct.cgroup.path.override
     csp.rules
     csp.strict
     csp.warnLegacyBrowsers
@@ -175,7 +173,6 @@ kibana_vars=(
     server.uuid
     server.xsrf.allowlist
     server.xsrf.disableProtection
-    server.xsrf.whitelist
     status.allowAnonymous
     status.v6ApiFormat
     telemetry.allowChangingOptInStatus
@@ -194,7 +191,6 @@ kibana_vars=(
     vis_type_vega.enableExternalUrls
     xpack.actions.allowedHosts
     xpack.actions.customHostSettings
-    xpack.actions.enabled
     xpack.actions.enabledActionTypes
     xpack.actions.maxResponseContentLength
     xpack.actions.preconfigured
@@ -252,7 +248,6 @@ kibana_vars=(
     xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled
     xpack.encryptedSavedObjects.encryptionKey
     xpack.encryptedSavedObjects.keyRotation.decryptionOnlyKeys
-    xpack.event_log.enabled
     xpack.event_log.indexEntries
     xpack.event_log.logEntries
     xpack.fleet.agentPolicies
@@ -384,7 +379,6 @@ kibana_vars=(
     xpack.securitySolution.prebuiltRulesFromSavedObjects
     xpack.spaces.enabled
     xpack.spaces.maxSpaces
-    xpack.task_manager.enabled
     xpack.task_manager.index
     xpack.task_manager.max_attempts
     xpack.task_manager.max_poll_inactivity_cycles
@@ -426,7 +420,7 @@ umask 0002
 # paths. Therefore, Kibana provides a mechanism to override
 # reading the cgroup path from /proc/self/cgroup and instead uses the
 # cgroup path defined the configuration properties
-# cpu.cgroup.path.override and cpuacct.cgroup.path.override.
+# ops.cGroupOverrides.cpuPath and ops.cGroupOverrides.cpuAcctPath.
 # Therefore, we set this value here so that cgroup statistics are
 # available for the container this process will run in.
 

diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts
@@ -12,6 +12,7 @@ export const storybookAliases = {
   canvas: 'x-pack/plugins/canvas/storybook',
   codeeditor: 'src/plugins/kibana_react/public/code_editor/.storybook',
   ci_composite: '.ci/.storybook',
+  custom_integrations: 'src/plugins/custom_integrations/storybook',
   url_template_editor: 'src/plugins/kibana_react/public/url_template_editor/.storybook',
   dashboard: 'src/plugins/dashboard/.storybook',
   dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/.storybook',

diff --git a/src/plugins/custom_integrations/kibana.json b/src/plugins/custom_integrations/kibana.json
@@ -12,5 +12,8 @@
   "extraPublicDirs": [
     "common"
   ],
+  "requiredPlugins": [
+    "presentationUtil"
+  ],
   "optionalPlugins": []
 }
diff --git a/src/plugins/custom_integrations/public/components/index.tsx b/src/plugins/custom_integrations/public/components/index.tsx
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React, { Suspense, ComponentType, ReactElement, Ref } from 'react';
+import { EuiLoadingSpinner, EuiErrorBoundary } from '@elastic/eui';
+
+/**
+ * A HOC which supplies React.Suspense with a fallback component, and a `EuiErrorBoundary` to contain errors.
+ * @param Component A component deferred by `React.lazy`
+ * @param fallback A fallback component to render while things load; default is `EuiLoadingSpinner`
+ */
+export const withSuspense = <P extends {}, R = {}>(
+  Component: ComponentType<P>,
+  fallback: ReactElement | null = <EuiLoadingSpinner />
+) =>
+  React.forwardRef((props: P, ref: Ref<R>) => {
+    return (
+      <EuiErrorBoundary>
+        <Suspense fallback={fallback}>
+          <Component {...props} ref={ref} />
+        </Suspense>
+      </EuiErrorBoundary>
+    );
+  });
+
+export const LazyReplacementCard = React.lazy(() => import('./replacement_card'));