Merge branch '7.17' into bugfix/ua/reindex_loop

docs/user/security/audit-logging.asciidoc

@@ -55,8 +55,9 @@ For information on how to configure `xpack.security.audit.appender`, refer to
 
 Refer to the table of events that can be logged for auditing purposes. 
 
-Each event is broken down into <<field-event-category, category>>, <<field-event-type, type>>, <<field-event-action, action>> and <<field-event-outcome, outcome>> fields
-to make it easy to filter, query and aggregate the resulting logs. 
+Each event is broken down into <<field-event-category, category>>, <<field-event-type, type>>, <<field-event-action, action>> and
+<<field-event-outcome, outcome>> fields to make it easy to filter, query and aggregate the resulting logs. The <<field-trace-id, trace.id>>
+field can be used to correlate multiple events that originate from the same request.
 
 Refer to <<xpack-security-ecs-audit-schema>> for a table of fields that get logged with audit event. 
 
@@ -448,7 +449,57 @@ Example: `https`
 | *Field*
 | *Description*
 
-| `trace.id`
+| [[field-trace-id]] `trace.id`
 | Unique identifier allowing events of the same transaction from {kib} and {es} to be be correlated.
 
 |======
+
+[[xpack-security-ecs-audit-correlation]]
+==== Correlating ECS audit events
+
+Audit events can be correlated in two ways:
+
+1. Multiple {kib} audit events that resulted from the same request can be correlated together.
+2. If {ref}/enable-audit-logging.html[{es} audit logging] is enabled, {kib} audit events from one request can be correlated with backend
+   calls that create {es} audit events.
+
+NOTE: The examples below are simplified, many fields have been omitted and values have been shortened for clarity.
+
+===== Example 1: correlating multiple {kib} audit events
+
+When "thom" creates a new alerting rule, five audit events are written:
+
+[source,json]
+-------------
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/alerting/rule","port":5601,"scheme":"https"},"user":{"name":"thom","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"3dHCZRB..."},"@timestamp":"2022-01-25T13:05:34.449-05:00","message":"User is requesting [/api/alerting/rule] endpoint","trace":{"id":"e300e06..."}}
+{"event":{"action":"space_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"space","id":"default"}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.454-05:00","message":"User has accessed space [id=default]","trace":{"id":"e300e06..."}}
+{"event":{"action":"connector_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"action","id":"5e3b1ae..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.948-05:00","message":"User has accessed connector [id=5e3b1ae...]","trace":{"id":"e300e06..."}}
+{"event":{"action":"connector_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"action","id":"5e3b1ae..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.956-05:00","message":"User has accessed connector [id=5e3b1ae...]","trace":{"id":"e300e06..."}}
+{"event":{"action":"rule_create","category":["database"],"type":["creation"],"outcome":"unknown"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"alert","id":"64517c3..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.956-05:00","message":"User is creating rule [id=64517c3...]","trace":{"id":"e300e06..."}}
+-------------
+
+All of these audit events can be correlated together by the same `trace.id` value `"e300e06..."`. The first event is the HTTP API call, the
+next audit events are checks to validate the space and the connectors, and the last audit event is the actual rule creation.
+
+===== Example 2: correlating a {kib} audit event with {es} audit events
+
+When "thom" logs in, a "user_login" {kib} audit event is written:
+
+[source,json]
+-------------
+{"event":{"action":"user_login","category":["authentication"],"outcome":"success"},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T09:40:39.267-05:00","message":"User [thom] has logged in using basic provider [name=basic]","trace":{"id":"818cbf3..."}}
+-------------
+
+The `trace.id` value `"818cbf3..."` in the {kib} audit event can be correlated with the `opaque_id` value in these six {es} audit events:
+
+[source,json]
+-------------
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,604-0500", "event.action":"access_granted", "user.name":"thom", "user.roles":["superuser"], "request.id":"YCx8wxs...", "action":"cluster:admin/xpack/security/user/authenticate", "request.name":"AuthenticateRequest", "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/index", "request.name":"IndexRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk", "request.name":"BulkRequest", "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk[s]", "request.name":"BulkShardRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/index:op_type/create", "request.name":"BulkItemRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk[s][p]", "request.name":"BulkShardRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+-------------
+
+The {es} audit events show that "thom" authenticated, then subsequently "kibana_system" created a session for that user.

packages/kbn-es/src/cluster.js

@@ -18,7 +18,7 @@ const { createCliError } = require('./errors');
 const { promisify } = require('util');
 const treeKillAsync = promisify(require('tree-kill'));
 const { parseSettings, SettingsFilter } = require('./settings');
-const { CA_CERT_PATH, ES_P12_PATH, ES_P12_PASSWORD, extract } = require('@kbn/dev-utils');
+const { CA_CERT_PATH, ES_NOPASSWORD_P12_PATH, extract } = require('@kbn/dev-utils');
 const readFile = util.promisify(fs.readFile);
 
 // listen to data on stream until map returns anything but undefined
@@ -257,9 +257,10 @@ exports.Cluster = class Cluster {
     // Add to esArgs if ssl is enabled
     if (this._ssl) {
       esArgs.push('xpack.security.http.ssl.enabled=true');
-      esArgs.push(`xpack.security.http.ssl.keystore.path=${ES_P12_PATH}`);
+      esArgs.push(`xpack.security.http.ssl.keystore.path=${ES_NOPASSWORD_P12_PATH}`);
       esArgs.push(`xpack.security.http.ssl.keystore.type=PKCS12`);
-      esArgs.push(`xpack.security.http.ssl.keystore.password=${ES_P12_PASSWORD}`);
+      // We are explicitly using ES_NOPASSWORD_P12_PATH instead of ES_P12_PATH + ES_P12_PASSWORD. The reasoning for this is that setting
+      // the keystore password using environment variables causes Elasticsearch to emit deprecation warnings.
     }
 
     const args = parseSettings(extractConfigFiles(esArgs, installPath, { log: this._log }), {

packages/kbn-es/src/integration_tests/cluster.test.js

@@ -9,8 +9,7 @@
 const {
   ToolingLog,
   ToolingLogCollectingWriter,
-  ES_P12_PATH,
-  ES_P12_PASSWORD,
+  ES_NOPASSWORD_P12_PATH,
   createAnyInstanceSerializer,
   createStripAnsiSerializer,
 } = require('@kbn/dev-utils');
@@ -292,9 +291,8 @@ describe('#start(installPath)', () => {
 
     const config = extractConfigFiles.mock.calls[0][0];
     expect(config).toContain('xpack.security.http.ssl.enabled=true');
-    expect(config).toContain(`xpack.security.http.ssl.keystore.path=${ES_P12_PATH}`);
+    expect(config).toContain(`xpack.security.http.ssl.keystore.path=${ES_NOPASSWORD_P12_PATH}`);
     expect(config).toContain(`xpack.security.http.ssl.keystore.type=PKCS12`);
-    expect(config).toContain(`xpack.security.http.ssl.keystore.password=${ES_P12_PASSWORD}`);
   });
 
   it(`doesn't setup SSL when disabled`, async () => {
@@ -371,9 +369,8 @@ describe('#run()', () => {
 
     const config = extractConfigFiles.mock.calls[0][0];
     expect(config).toContain('xpack.security.http.ssl.enabled=true');
-    expect(config).toContain(`xpack.security.http.ssl.keystore.path=${ES_P12_PATH}`);
+    expect(config).toContain(`xpack.security.http.ssl.keystore.path=${ES_NOPASSWORD_P12_PATH}`);
     expect(config).toContain(`xpack.security.http.ssl.keystore.type=PKCS12`);
-    expect(config).toContain(`xpack.security.http.ssl.keystore.password=${ES_P12_PASSWORD}`);
   });
 
   it(`doesn't setup SSL when disabled`, async () => {

packages/kbn-pm/dist/index.js

@@ -58952,7 +58952,7 @@ async function setupRemoteCache(repoRootPath) {
   try {
     const {
       stdout
-    } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['read', '-field=readonly-key', 'secret/kibana-issues/dev/bazel-remote-cache'], {
+    } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], {
       stdio: 'pipe'
     });
     apiKey = stdout.trim();

packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts

@@ -60,7 +60,7 @@ export async function setupRemoteCache(repoRootPath: string) {
   try {
     const { stdout } = await spawn(
       'vault',
-      ['read', '-field=readonly-key', 'secret/kibana-issues/dev/bazel-remote-cache'],
+      ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'],
       {
         stdio: 'pipe',
       }

src/core/public/application/utils/parse_app_url.test.ts

@@ -39,6 +39,12 @@ describe('parseAppUrl', () => {
       id: 'bar',
       appRoute: '/custom-bar',
     });
+    createApp({
+      id: 're',
+    });
+    createApp({
+      id: 'retail',
+    });
   });
 
   describe('with absolute paths', () => {
@@ -51,6 +57,14 @@ describe('parseAppUrl', () => {
         app: 'bar',
         path: undefined,
       });
+      expect(parseAppUrl('/base-path/app/re', basePath, apps, currentUrl)).toEqual({
+        app: 're',
+        path: undefined,
+      });
+      expect(parseAppUrl('/base-path/app/retail', basePath, apps, currentUrl)).toEqual({
+        app: 'retail',
+        path: undefined,
+      });
     });
     it('parses the path', () => {
       expect(parseAppUrl('/base-path/app/foo/some/path', basePath, apps, currentUrl)).toEqual({
@@ -63,6 +77,14 @@ describe('parseAppUrl', () => {
         app: 'bar',
         path: '/another/path/',
       });
+      expect(parseAppUrl('/base-path/app/re/tail', basePath, apps, currentUrl)).toEqual({
+        app: 're',
+        path: '/tail',
+      });
+      expect(parseAppUrl('/base-path/app/retail/path', basePath, apps, currentUrl)).toEqual({
+        app: 'retail',
+        path: '/path',
+      });
     });
     it('includes query and hash in the path for default app route', () => {
       expect(parseAppUrl('/base-path/app/foo#hash/bang', basePath, apps, currentUrl)).toEqual({
@@ -89,6 +111,18 @@ describe('parseAppUrl', () => {
         app: 'foo',
         path: '/path#hash/bang?hello=dolly',
       });
+      expect(parseAppUrl('/base-path/app/re#hash/bang', basePath, apps, currentUrl)).toEqual({
+        app: 're',
+        path: '#hash/bang',
+      });
+      expect(parseAppUrl('/base-path/app/retail?hello=dolly', basePath, apps, currentUrl)).toEqual({
+        app: 'retail',
+        path: '?hello=dolly',
+      });
+      expect(parseAppUrl('/base-path/app/retail#hash/bang', basePath, apps, currentUrl)).toEqual({
+        app: 'retail',
+        path: '#hash/bang',
+      });
     });
     it('includes query and hash in the path for custom app route', () => {
       expect(parseAppUrl('/base-path/custom-bar#hash/bang', basePath, apps, currentUrl)).toEqual({
@@ -190,7 +224,6 @@ describe('parseAppUrl', () => {
         path: '/path#hash?hello=dolly',
       });
     });
-
     it('returns undefined if the relative path redirect outside of the basePath', () => {
       expect(
         parseAppUrl(
@@ -201,6 +234,19 @@ describe('parseAppUrl', () => {
         )
       ).toBeUndefined();
     });
+    it('works with inclusive app ids', () => {
+      expect(
+        parseAppUrl(
+          './retail/path',
+          basePath,
+          apps,
+          'https://kibana.local:8080/base-path/app/current'
+        )
+      ).toEqual({
+        app: 'retail',
+        path: '/path',
+      });
+    });
   });
 
   describe('with absolute urls', () => {
@@ -217,6 +263,18 @@ describe('parseAppUrl', () => {
         app: 'bar',
         path: undefined,
       });
+      expect(
+        parseAppUrl('https://kibana.local:8080/base-path/app/re', basePath, apps, currentUrl)
+      ).toEqual({
+        app: 're',
+        path: undefined,
+      });
+      expect(
+        parseAppUrl('https://kibana.local:8080/base-path/app/retail', basePath, apps, currentUrl)
+      ).toEqual({
+        app: 'retail',
+        path: undefined,
+      });
     });
     it('parses the path', () => {
       expect(
@@ -241,6 +299,29 @@ describe('parseAppUrl', () => {
         app: 'bar',
         path: '/another/path/',
       });
+
+      expect(
+        parseAppUrl(
+          'https://kibana.local:8080/base-path/app/re/some/path',
+          basePath,
+          apps,
+          currentUrl
+        )
+      ).toEqual({
+        app: 're',
+        path: '/some/path',
+      });
+      expect(
+        parseAppUrl(
+          'https://kibana.local:8080/base-path/app/retail/another/path/',
+          basePath,
+          apps,
+          currentUrl
+        )
+      ).toEqual({
+        app: 'retail',
+        path: '/another/path/',
+      });
     });
     it('includes query and hash in the path for default app routes', () => {
       expect(
@@ -298,6 +379,28 @@ describe('parseAppUrl', () => {
         app: 'foo',
         path: '/path#hash/bang?hello=dolly',
       });
+      expect(
+        parseAppUrl(
+          'https://kibana.local:8080/base-path/app/re#hash/bang',
+          basePath,
+          apps,
+          currentUrl
+        )
+      ).toEqual({
+        app: 're',
+        path: '#hash/bang',
+      });
+      expect(
+        parseAppUrl(
+          'https://kibana.local:8080/base-path/app/re?hello=dolly',
+          basePath,
+          apps,
+          currentUrl
+        )
+      ).toEqual({
+        app: 're',
+        path: '?hello=dolly',
+      });
     });
     it('includes query and hash in the path for custom app route', () => {
       expect(

src/core/public/application/utils/parse_app_url.ts

@@ -61,7 +61,7 @@ export const parseAppUrl = (
   for (const app of apps.values()) {
     const appPath = app.appRoute || `/app/${app.id}`;
 
-    if (url.startsWith(appPath)) {
+    if (urlInApp(url, appPath)) {
       const path = url.substr(appPath.length);
       return {
         app: app.id,
@@ -70,3 +70,16 @@ export const parseAppUrl = (
     }
   }
 };
+
+const separators = ['/', '?', '#'];
+
+const urlInApp = (url: string, appPath: string) => {
+  if (url === appPath) {
+    return true;
+  }
+  if (url.startsWith(appPath)) {
+    const nextChar = url.substring(appPath.length, appPath.length + 1);
+    return separators.includes(nextChar);
+  }
+  return false;
+};

src/core/server/elasticsearch/client/log_query_and_deprecation.test.ts

@@ -532,13 +532,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
       });
       client.emit('response', new errors.ResponseError(response), response);
 
-      // One debug log entry from 'elasticsearch.query' context
-      expect(loggingSystemMock.collect(logger).debug.length).toEqual(1);
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch(
+      // Test debug[1] since theree is one log entry from 'elasticsearch.query' context
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch(
         'Elasticsearch deprecation: 299 Elasticsearch-8.1.0 "GET /_path is deprecated"'
       );
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch('Origin:kibana');
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch(
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch('Origin:kibana');
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch(
         /Query:\n.*400\n.*GET \/_path\?hello\=dolly \[illegal_argument_exception\]: request \[\/_path\] contains unrecognized parameter: \[name\]/
       );
     });
@@ -564,7 +563,6 @@ describe('instrumentQueryAndDeprecationLogger', () => {
       });
       client.emit('response', null, response);
 
-      expect(loggingSystemMock.collect(logger).info).toEqual([]);
       // Test debug[1] since theree is one log entry from 'elasticsearch.query' context
       expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch(
         'Elasticsearch deprecation: 299 Elasticsearch-8.1.0 "GET /_path is deprecated"'
@@ -598,13 +596,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
       });
       client.emit('response', null, response);
 
-      // One debug log entry from 'elasticsearch.query' context
-      expect(loggingSystemMock.collect(logger).debug.length).toEqual(1);
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch(
+      // Test debug[1] since theree is one log entry from 'elasticsearch.query' context
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch(
         'Elasticsearch deprecation: 299 Elasticsearch-8.1.0 "GET /_path is deprecated"'
       );
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch('Origin:kibana');
-      expect(loggingSystemMock.collect(logger).info[0][0]).toMatch(
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch('Origin:kibana');
+      expect(loggingSystemMock.collect(logger).debug[1][0]).toMatch(
         /Query:\n.*200\n.*GET \/_path\?hello\=dolly/
       );
     });

src/core/server/elasticsearch/client/log_query_and_deprecation.ts

@@ -136,12 +136,9 @@ export const instrumentEsQueryAndDeprecationLogger = ({
             ? 'kibana'
             : 'user';
 
-        const deprecationMsg = `Elasticsearch deprecation: ${event.warnings}\nOrigin:${requestOrigin}\nQuery:\n${queryMsg}`;
-        if (requestOrigin === 'kibana') {
-          deprecationLogger.info(deprecationMsg);
-        } else {
-          deprecationLogger.debug(deprecationMsg);
-        }
+        deprecationLogger.debug(
+          `Elasticsearch deprecation: ${event.warnings}\nOrigin:${requestOrigin}\nQuery:\n${queryMsg}`
+        );
       }
     }
   });