Merge branch '7.x' into issue-xxx-session-deprecations

src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts

@@ -16,6 +16,7 @@ import { sortOrderSchema } from './common_schemas';
  * - filter
  * - histogram
  * - nested
+ * - reverse_nested
  * - terms
  *
  * Not implemented:
@@ -37,7 +38,6 @@ import { sortOrderSchema } from './common_schemas';
  * - parent
  * - range
  * - rare_terms
- * - reverse_nested
  * - sampler
  * - significant_terms
  * - significant_text
@@ -76,6 +76,9 @@ export const bucketAggsSchemas: Record<string, ObjectType> = {
   nested: s.object({
     path: s.string(),
   }),
+  reverse_nested: s.object({
+    path: s.maybe(s.string()),
+  }),
   terms: s.object({
     field: s.maybe(s.string()),
     collect_mode: s.maybe(s.string()),

x-pack/plugins/security/server/config_deprecations.ts

@@ -33,22 +33,62 @@ export const securityConfigDeprecationProvider: ConfigDeprecationProvider = ({
   (settings, fromPath, addDeprecation, { branch }) => {
     const auditLoggingEnabled = settings?.xpack?.security?.audit?.enabled ?? false;
     const legacyAuditLoggerEnabled = !settings?.xpack?.security?.audit?.appender;
-    if (auditLoggingEnabled && legacyAuditLoggerEnabled) {
+
+    // Gross, but the cloud plugin depends on the security plugin already,
+    // so we can't add a dependency in the other direction to check this in a more conventional manner.
+    const isCloudInstance = typeof settings?.xpack?.cloud?.id === 'string';
+
+    const isUsingLegacyAuditLogger = auditLoggingEnabled && legacyAuditLoggerEnabled;
+
+    if (!isUsingLegacyAuditLogger) {
+      return;
+    }
+
+    const title = i18n.translate('xpack.security.deprecations.auditLoggerTitle', {
+      defaultMessage: 'The legacy audit logger is deprecated',
+    });
+
+    const message = i18n.translate('xpack.security.deprecations.auditLoggerMessage', {
+      defaultMessage: 'Use the new ECS-compliant audit logger.',
+    });
+
+    const documentationUrl = `https://www.elastic.co/guide/en/kibana/${branch}/security-settings-kb.html#audit-logging-settings`;
+
+    const configPath = 'xpack.security.audit.appender';
+    if (isCloudInstance) {
       addDeprecation({
-        configPath: 'xpack.security.audit.appender',
-        title: i18n.translate('xpack.security.deprecations.auditLoggerTitle', {
-          defaultMessage: 'The legacy audit logger is deprecated',
-        }),
-        message: i18n.translate('xpack.security.deprecations.auditLoggerMessage', {
-          defaultMessage:
-            'The legacy audit logger is deprecated in favor of the new ECS-compliant audit logger.',
-        }),
-        documentationUrl: `https://www.elastic.co/guide/en/kibana/${branch}/security-settings-kb.html#audit-logging-settings`,
+        title,
+        message,
+        configPath,
+        level: 'warning',
+        documentationUrl,
+        correctiveActions: {
+          manualSteps: [
+            i18n.translate('xpack.security.deprecations.auditLogger.manualStepOneMessageCloud', {
+              defaultMessage:
+                'To enable the ECS audit logger now, add the "xpack.security.audit.appender.type: rolling-file" setting.',
+            }),
+            i18n.translate('xpack.security.deprecations.auditLogger.manualStepTwoMessageCloud', {
+              defaultMessage: `If you don't make any changes, the ECS audit logger will be enabled when you upgrade to 8.0.`,
+            }),
+          ],
+        },
+      });
+    } else {
+      addDeprecation({
+        title,
+        message,
+        configPath,
+        level: 'warning',
+        documentationUrl,
         correctiveActions: {
           manualSteps: [
             i18n.translate('xpack.security.deprecations.auditLogger.manualStepOneMessage', {
               defaultMessage:
-                'Declare an audit logger "appender" via "xpack.security.audit.appender" to enable the ECS audit logger.',
+                'To enable the ECS audit logger now, configure an appender with "xpack.security.audit.appender".',
+            }),
+            i18n.translate('xpack.security.deprecations.auditLogger.manualStepTwoMessage', {
+              defaultMessage: `If you don't make any changes, the ECS audit logger will be enabled when you upgrade to 8.0.`,
             }),
           ],
         },

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -479,7 +479,6 @@ export const getRuleExecutionStatuses = (): Array<
     type: 'my-type',
     id: 'e0b86950-4e9f-11ea-bdbd-07b56aa159b3',
     attributes: {
-      alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bc',
       statusDate: '2020-02-18T15:26:49.783Z',
       status: RuleExecutionStatus.succeeded,
       lastFailureAt: undefined,
@@ -492,15 +491,20 @@ export const getRuleExecutionStatuses = (): Array<
       bulkCreateTimeDurations: ['800.43'],
     },
     score: 1,
-    references: [],
+    references: [
+      {
+        id: '04128c15-0d1b-4716-a4c5-46997ac7f3bc',
+        type: 'alert',
+        name: 'alert_0',
+      },
+    ],
     updated_at: '2020-02-18T15:26:51.333Z',
     version: 'WzQ2LDFd',
   },
   {
     type: 'my-type',
     id: '91246bd0-5261-11ea-9650-33b954270f67',
     attributes: {
-      alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
       statusDate: '2020-02-18T15:15:58.806Z',
       status: RuleExecutionStatus.failed,
       lastFailureAt: '2020-02-18T15:15:58.806Z',
@@ -514,7 +518,13 @@ export const getRuleExecutionStatuses = (): Array<
       bulkCreateTimeDurations: ['800.43'],
     },
     score: 1,
-    references: [],
+    references: [
+      {
+        id: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
+        type: 'alert',
+        name: 'alert_0',
+      },
+    ],
     updated_at: '2020-02-18T15:15:58.860Z',
     version: 'WzMyLDFd',
   },
@@ -523,7 +533,6 @@ export const getRuleExecutionStatuses = (): Array<
 export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({
   '04128c15-0d1b-4716-a4c5-46997ac7f3bd': [
     {
-      alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
       statusDate: '2020-02-18T15:26:49.783Z',
       status: RuleExecutionStatus.succeeded,
       lastFailureAt: undefined,
@@ -538,7 +547,6 @@ export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({
   ],
   '1ea5a820-4da1-4e82-92a1-2b43a7bece08': [
     {
-      alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
       statusDate: '2020-02-18T15:15:58.806Z',
       status: RuleExecutionStatus.failed,
       lastFailureAt: '2020-02-18T15:15:58.806Z',

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts

@@ -31,7 +31,7 @@ import { updatePrepackagedRules } from '../../rules/update_prepacked_rules';
 import { getRulesToInstall } from '../../rules/get_rules_to_install';
 import { getRulesToUpdate } from '../../rules/get_rules_to_update';
 import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules';
-import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client';
+import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset/rule_asset_saved_objects_client';
 
 import { buildSiemResponse } from '../utils';
 import { RulesClient } from '../../../../../../alerting/server';

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts

@@ -20,7 +20,7 @@ import { getRulesToUpdate } from '../../rules/get_rules_to_update';
 import { findRules } from '../../rules/find_rules';
 import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules';
 import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules';
-import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client';
+import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset/rule_asset_saved_objects_client';
 import { buildFrameworkRequest } from '../../../timeline/utils/common';
 import { ConfigType } from '../../../../config';
 import { SetupPlugins } from '../../../../plugin';

x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts

@@ -136,16 +136,16 @@ describe.each([
 
   describe('mergeStatuses', () => {
     it('merges statuses and converts from camelCase saved object to snake_case HTTP response', () => {
+      //
       const statusOne = exampleRuleStatus();
       statusOne.attributes.status = RuleExecutionStatus.failed;
       const statusTwo = exampleRuleStatus();
       statusTwo.attributes.status = RuleExecutionStatus.failed;
       const currentStatus = exampleRuleStatus();
       const foundRules = [currentStatus.attributes, statusOne.attributes, statusTwo.attributes];
-      const res = mergeStatuses(currentStatus.attributes.alertId, foundRules, {
+      const res = mergeStatuses(currentStatus.references[0].id, foundRules, {
         'myfakealertid-8cfac': {
           current_status: {
-            alert_id: 'myfakealertid-8cfac',
             status_date: '2020-03-27T22:55:59.517Z',
             status: RuleExecutionStatus.succeeded,
             last_failure_at: null,
@@ -163,7 +163,6 @@ describe.each([
       expect(res).toEqual({
         'myfakealertid-8cfac': {
           current_status: {
-            alert_id: 'myfakealertid-8cfac',
             status_date: '2020-03-27T22:55:59.517Z',
             status: 'succeeded',
             last_failure_at: null,
@@ -179,7 +178,6 @@ describe.each([
         },
         'f4b8e31d-cf93-4bde-a265-298bde885cd7': {
           current_status: {
-            alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7',
             status_date: '2020-03-27T22:55:59.517Z',
             status: 'succeeded',
             last_failure_at: null,
@@ -193,7 +191,6 @@ describe.each([
           },
           failures: [
             {
-              alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7',
               status_date: '2020-03-27T22:55:59.517Z',
               status: 'failed',
               last_failure_at: null,
@@ -206,7 +203,6 @@ describe.each([
               last_look_back_date: null, // NOTE: This is no longer used on the UI, but left here in case users are using it within the API
             },
             {
-              alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7',
               status_date: '2020-03-27T22:55:59.517Z',
               status: 'failed',
               last_failure_at: null,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts

@@ -42,7 +42,7 @@ export class EventLogAdapter implements IRuleExecutionLogClient {
   }
 
   public async update(args: UpdateExecutionLogArgs) {
-    const { attributes, spaceId, ruleName, ruleType } = args;
+    const { attributes, spaceId, ruleId, ruleName, ruleType } = args;
 
     await this.savedObjectsAdapter.update(args);
 
@@ -51,7 +51,7 @@ export class EventLogAdapter implements IRuleExecutionLogClient {
       this.eventLogClient.logStatusChange({
         ruleName,
         ruleType,
-        ruleId: attributes.alertId,
+        ruleId,
         newStatus: attributes.status,
         spaceId,
       });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts

@@ -5,27 +5,33 @@
  * 2.0.
  */
 
-import { get } from 'lodash';
 import {
-  SavedObjectsClientContract,
   SavedObject,
-  SavedObjectsUpdateResponse,
+  SavedObjectsClientContract,
+  SavedObjectsCreateOptions,
   SavedObjectsFindOptions,
+  SavedObjectsFindOptionsReference,
   SavedObjectsFindResult,
-} from '../../../../../../../../src/core/server';
-import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
+  SavedObjectsUpdateResponse,
+} from 'kibana/server';
+import { get } from 'lodash';
+// eslint-disable-next-line no-restricted-imports
+import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings';
 import { IRuleStatusSOAttributes } from '../../rules/types';
-import { buildChunkedOrFilter } from '../../signals/utils';
 
 export interface RuleStatusSavedObjectsClient {
   find: (
     options?: Omit<SavedObjectsFindOptions, 'type'>
   ) => Promise<Array<SavedObjectsFindResult<IRuleStatusSOAttributes>>>;
   findBulk: (ids: string[], statusesPerId: number) => Promise<FindBulkResponse>;
-  create: (attributes: IRuleStatusSOAttributes) => Promise<SavedObject<IRuleStatusSOAttributes>>;
+  create: (
+    attributes: IRuleStatusSOAttributes,
+    options: SavedObjectsCreateOptions
+  ) => Promise<SavedObject<IRuleStatusSOAttributes>>;
   update: (
     id: string,
-    attributes: Partial<IRuleStatusSOAttributes>
+    attributes: Partial<IRuleStatusSOAttributes>,
+    options: SavedObjectsCreateOptions
   ) => Promise<SavedObjectsUpdateResponse<IRuleStatusSOAttributes>>;
   delete: (id: string) => Promise<{}>;
 }
@@ -35,63 +41,80 @@ export interface FindBulkResponse {
 }
 
 /**
- * @pdeprecated Use RuleExecutionLogClient instead
+ * @deprecated Use RuleExecutionLogClient instead
  */
 export const ruleStatusSavedObjectsClientFactory = (
   savedObjectsClient: SavedObjectsClientContract
 ): RuleStatusSavedObjectsClient => ({
   find: async (options) => {
     const result = await savedObjectsClient.find<IRuleStatusSOAttributes>({
       ...options,
-      type: ruleStatusSavedObjectType,
+      type: legacyRuleStatusSavedObjectType,
     });
     return result.saved_objects;
   },
   findBulk: async (ids, statusesPerId) => {
     if (ids.length === 0) {
       return {};
     }
-    const filter = buildChunkedOrFilter(`${ruleStatusSavedObjectType}.attributes.alertId`, ids);
+    const references = ids.map<SavedObjectsFindOptionsReference>((alertId) => ({
+      id: alertId,
+      type: 'alert',
+    }));
     const order: 'desc' = 'desc';
     const aggs = {
-      alertIds: {
-        terms: {
-          field: `${ruleStatusSavedObjectType}.attributes.alertId`,
-          size: ids.length,
+      references: {
+        nested: {
+          path: `${legacyRuleStatusSavedObjectType}.references`,
         },
         aggs: {
-          most_recent_statuses: {
-            top_hits: {
-              sort: [
-                {
-                  [`${ruleStatusSavedObjectType}.statusDate`]: {
-                    order,
+          alertIds: {
+            terms: {
+              field: `${legacyRuleStatusSavedObjectType}.references.id`,
+              size: ids.length,
+            },
+            aggs: {
+              rule_status: {
+                reverse_nested: {},
+                aggs: {
+                  most_recent_statuses: {
+                    top_hits: {
+                      sort: [
+                        {
+                          [`${legacyRuleStatusSavedObjectType}.statusDate`]: {
+                            order,
+                          },
+                        },
+                      ],
+                      size: statusesPerId,
+                    },
                   },
                 },
-              ],
-              size: statusesPerId,
+              },
             },
           },
         },
       },
     };
     const results = await savedObjectsClient.find({
-      filter,
+      hasReference: references,
       aggs,
-      type: ruleStatusSavedObjectType,
+      type: legacyRuleStatusSavedObjectType,
       perPage: 0,
     });
-    const buckets = get(results, 'aggregations.alertIds.buckets');
+    const buckets = get(results, 'aggregations.references.alertIds.buckets');
     return buckets.reduce((acc: Record<string, unknown>, bucket: unknown) => {
       const key = get(bucket, 'key');
-      const hits = get(bucket, 'most_recent_statuses.hits.hits');
+      const hits = get(bucket, 'rule_status.most_recent_statuses.hits.hits');
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const statuses = hits.map((hit: any) => hit._source['siem-detection-engine-rule-status']);
-      acc[key] = statuses;
+      acc[key] = hits.map((hit: any) => hit._source[legacyRuleStatusSavedObjectType]);
       return acc;
     }, {});
   },
-  create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes),
-  update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes),
-  delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id),
+  create: (attributes, options) => {
+    return savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options);
+  },
+  update: (id, attributes, options) =>
+    savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes, options),
+  delete: (id) => savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id),
 });