Merge branch 'master' into 106510

# Conflicts:
#	x-pack/plugins/lens/public/lens_attribute_service.ts
#	x-pack/plugins/lens/public/plugin.ts

docs/development/core/server/kibana-plugin-core-server.deprecationsservicesetup.md

@@ -31,10 +31,10 @@ async function getDeprecations({ esClient, savedObjectsClient }: GetDeprecations
     // Example of a manual correctiveAction
     deprecations.push({
       title: i18n.translate('xpack.timelion.deprecations.worksheetsTitle', {
-        defaultMessage: 'Found Timelion worksheets.'
+        defaultMessage: 'Timelion worksheets are deprecated'
       }),
       message: i18n.translate('xpack.timelion.deprecations.worksheetsMessage', {
-        defaultMessage: 'You have {count} Timelion worksheets. The Timelion app will be removed in 8.0. To continue using your Timelion worksheets, migrate them to a dashboard.',
+        defaultMessage: 'You have {count} Timelion worksheets. Migrate your Timelion worksheets to a dashboard to continue using them.',
         values: { count },
       }),
       documentationUrl:

docs/management/connectors/action-types/email.asciidoc

@@ -51,7 +51,7 @@ Use the <<action-settings, Action configuration settings>> to customize connecto
 
 Config defines information for the connector type.
 
-`service`:: The name of a https://nodemailer.com/smtp/well-known/[well-known email service provider]. If `service` is provided, `host`, `port`, and `secure` properties are ignored. For more information on the `gmail` service value, see the https://nodemailer.com/usage/using-gmail/[Nodemailer Gmail documentation].
+`service`:: The name of the email service. If `service` is `elastic_cloud` (for Elastic Cloud notifications) or one of Nodemailer's https://nodemailer.com/smtp/well-known/[well-known email service providers], `host`, `port`, and `secure` properties are ignored. If `service` is `other`, `host` and `port` properties must be defined. For more information on the `gmail` service value, see the https://nodemailer.com/usage/using-gmail/[Nodemailer Gmail documentation].
 `from`:: An email address that corresponds to *Sender*.
 `host`:: A string that corresponds to *Host*.
 `port`:: A number that corresponds to *Port*.

docs/user/alerting/troubleshooting/alerting-common-issues.asciidoc

@@ -68,7 +68,7 @@ Rules  are taking a long time to execute and are impacting the overall health of
 
 [IMPORTANT]
 ==============================================
-By default, only users with a `superuser` role can query the {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index.
+By default, only users with a `superuser` role can query the experimental[] {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index.
 ==============================================
 
 *Solution*

docs/user/alerting/troubleshooting/event-log-index.asciidoc

@@ -2,6 +2,8 @@
 [[event-log-index]]
 === Event log index
 
+experimental[]
+
 Use the event log index to determine:
 
 * Whether a rule successfully ran but its associated actions did not

docs/user/production-considerations/alerting-production-considerations.asciidoc

@@ -54,6 +54,8 @@ Predicting the buffer required to account for actions depends heavily on the rul
 [[event-log-ilm]]
 === Event log index lifecycle managment
 
+experimental[]
+
 Alerts and actions log activity in a set of "event log" indices.  These indices are configured with an index lifecycle management (ILM) policy, which you can customize.  The default policy rolls over the index when it reaches 50GB, or after 30 days.  Indices over 90 days old are deleted.
 
 The name of the index policy is `kibana-event-log-policy`.  {kib} creates the index policy on startup, if it doesn't already exist.  The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it.

packages/kbn-test/src/kbn_client/kbn_client_plugins.ts

@@ -8,26 +8,14 @@
 
 import { KbnClientStatus } from './kbn_client_status';
 
-const PLUGIN_STATUS_ID = /^plugin:(.+?)@/;
-
 export class KbnClientPlugins {
   constructor(private readonly status: KbnClientStatus) {}
   /**
    * Get a list of plugin ids that are enabled on the server
    */
   public async getEnabledIds() {
-    const pluginIds: string[] = [];
     const apiResp = await this.status.get();
 
-    for (const status of apiResp.status.statuses) {
-      if (status.id) {
-        const match = status.id.match(PLUGIN_STATUS_ID);
-        if (match) {
-          pluginIds.push(match[1]);
-        }
-      }
-    }
-
-    return pluginIds;
+    return Object.keys(apiResp.status.plugins);
   }
 }

packages/kbn-test/src/kbn_client/kbn_client_status.ts

@@ -9,13 +9,11 @@
 import { KbnClientRequester } from './kbn_client_requester';
 
 interface Status {
-  state: 'green' | 'red' | 'yellow';
-  title?: string;
-  id?: string;
-  icon: string;
-  message: string;
-  uiColor: string;
-  since: string;
+  level: 'available' | 'degraded' | 'unavailable' | 'critical';
+  summary: string;
+  detail?: string;
+  documentationUrl?: string;
+  meta?: Record<string, unknown>;
 }
 
 interface ApiResponseStatus {
@@ -29,7 +27,8 @@ interface ApiResponseStatus {
   };
   status: {
     overall: Status;
-    statuses: Status[];
+    core: Record<string, Status>;
+    plugins: Record<string, Status>;
   };
   metrics: unknown;
 }
@@ -55,6 +54,6 @@ export class KbnClientStatus {
    */
   public async getOverallState() {
     const status = await this.get();
-    return status.status.overall.state;
+    return status.status.overall.level;
   }
 }

src/core/public/core_app/status/lib/load_status.ts

@@ -145,7 +145,7 @@ export async function loadStatus({
   let response: StatusResponse;
 
   try {
-    response = await http.get('/api/status', { query: { v8format: true } });
+    response = await http.get('/api/status');
   } catch (e) {
     // API returns a 503 response if not all services are available.
     // In this case, we want to treat this as a successful API call, so that we can

src/core/server/core_usage_data/core_usage_data_service.mock.ts

@@ -10,12 +10,14 @@ import { PublicMethodsOf } from '@kbn/utility-types';
 import { BehaviorSubject } from 'rxjs';
 import { CoreUsageDataService } from './core_usage_data_service';
 import { coreUsageStatsClientMock } from './core_usage_stats_client.mock';
-import { CoreUsageData, CoreUsageDataSetup, CoreUsageDataStart } from './types';
+import { CoreUsageData, InternalCoreUsageDataSetup, CoreUsageDataStart } from './types';
 
 const createSetupContractMock = (usageStatsClient = coreUsageStatsClientMock.create()) => {
-  const setupContract: jest.Mocked<CoreUsageDataSetup> = {
+  const setupContract: jest.Mocked<InternalCoreUsageDataSetup> = {
     registerType: jest.fn(),
     getClient: jest.fn().mockReturnValue(usageStatsClient),
+    registerUsageCounter: jest.fn(),
+    incrementUsageCounter: jest.fn(),
   };
   return setupContract;
 };

src/core/server/core_usage_data/core_usage_data_service.test.ts

@@ -150,6 +150,50 @@ describe('CoreUsageDataService', () => {
         expect(usageStatsClient).toBeInstanceOf(CoreUsageStatsClient);
       });
     });
+
+    describe('Usage Counter', () => {
+      it('registers a usage counter and uses it to increment the counters', async () => {
+        const http = httpServiceMock.createInternalSetupContract();
+        const metrics = metricsServiceMock.createInternalSetupContract();
+        const savedObjectsStartPromise = Promise.resolve(
+          savedObjectsServiceMock.createStartContract()
+        );
+        const changedDeprecatedConfigPath$ = configServiceMock.create().getDeprecatedConfigPath$();
+        const coreUsageData = service.setup({
+          http,
+          metrics,
+          savedObjectsStartPromise,
+          changedDeprecatedConfigPath$,
+        });
+        const myUsageCounter = { incrementCounter: jest.fn() };
+        coreUsageData.registerUsageCounter(myUsageCounter);
+        coreUsageData.incrementUsageCounter({ counterName: 'test' });
+        expect(myUsageCounter.incrementCounter).toHaveBeenCalledWith({ counterName: 'test' });
+      });
+
+      it('swallows errors when provided increment counter fails', async () => {
+        const http = httpServiceMock.createInternalSetupContract();
+        const metrics = metricsServiceMock.createInternalSetupContract();
+        const savedObjectsStartPromise = Promise.resolve(
+          savedObjectsServiceMock.createStartContract()
+        );
+        const changedDeprecatedConfigPath$ = configServiceMock.create().getDeprecatedConfigPath$();
+        const coreUsageData = service.setup({
+          http,
+          metrics,
+          savedObjectsStartPromise,
+          changedDeprecatedConfigPath$,
+        });
+        const myUsageCounter = {
+          incrementCounter: jest.fn(() => {
+            throw new Error('Something is really wrong');
+          }),
+        };
+        coreUsageData.registerUsageCounter(myUsageCounter);
+        expect(() => coreUsageData.incrementUsageCounter({ counterName: 'test' })).not.toThrow();
+        expect(myUsageCounter.incrementCounter).toHaveBeenCalledWith({ counterName: 'test' });
+      });
+    });
   });
 
   describe('start', () => {

src/core/server/core_usage_data/core_usage_data_service.ts

@@ -27,7 +27,7 @@ import type {
   CoreServicesUsageData,
   CoreUsageData,
   CoreUsageDataStart,
-  CoreUsageDataSetup,
+  InternalCoreUsageDataSetup,
   ConfigUsageData,
   CoreConfigUsageData,
 } from './types';
@@ -39,6 +39,7 @@ import { LEGACY_URL_ALIAS_TYPE } from '../saved_objects/object_types';
 import { CORE_USAGE_STATS_TYPE } from './constants';
 import { CoreUsageStatsClient } from './core_usage_stats_client';
 import { MetricsServiceSetup, OpsMetrics } from '..';
+import { CoreIncrementUsageCounter } from './types';
 
 export type ExposedConfigsToUsage = Map<string, Record<string, boolean>>;
 
@@ -86,7 +87,8 @@ const isCustomIndex = (index: string) => {
   return index !== '.kibana';
 };
 
-export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, CoreUsageDataStart> {
+export class CoreUsageDataService
+  implements CoreService<InternalCoreUsageDataSetup, CoreUsageDataStart> {
   private logger: Logger;
   private elasticsearchConfig?: ElasticsearchConfigType;
   private configService: CoreContext['configService'];
@@ -98,6 +100,7 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
   private kibanaConfig?: KibanaConfigType;
   private coreUsageStatsClient?: CoreUsageStatsClient;
   private deprecatedConfigPaths: ChangedDeprecatedPaths = { set: [], unset: [] };
+  private incrementUsageCounter: CoreIncrementUsageCounter = () => {}; // Initially set to noop
 
   constructor(core: CoreContext) {
     this.logger = core.logger.get('core-usage-stats-service');
@@ -495,7 +498,24 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
 
     this.coreUsageStatsClient = getClient();
 
-    return { registerType, getClient } as CoreUsageDataSetup;
+    const contract: InternalCoreUsageDataSetup = {
+      registerType,
+      getClient,
+      registerUsageCounter: (usageCounter) => {
+        this.incrementUsageCounter = (params) => usageCounter.incrementCounter(params);
+      },
+      incrementUsageCounter: (params) => {
+        try {
+          this.incrementUsageCounter(params);
+        } catch (e) {
+          // Self-defense mechanism since the handler is externally registered
+          this.logger.debug('Failed to increase the usage counter');
+          this.logger.debug(e);
+        }
+      },
+    };
+
+    return contract;
   }
 
   start({ savedObjects, elasticsearch, exposedConfigsToUsage }: StartDeps) {

src/core/server/core_usage_data/index.ts

@@ -7,7 +7,15 @@
  */
 
 export { CORE_USAGE_STATS_TYPE, CORE_USAGE_STATS_ID } from './constants';
-export type { CoreUsageDataSetup, ConfigUsageData, CoreUsageDataStart } from './types';
+export type {
+  InternalCoreUsageDataSetup,
+  ConfigUsageData,
+  CoreUsageDataStart,
+  CoreUsageDataSetup,
+  CoreUsageCounter,
+  CoreIncrementUsageCounter,
+  CoreIncrementCounterParams,
+} from './types';
 export { CoreUsageDataService } from './core_usage_data_service';
 export { CoreUsageStatsClient, REPOSITORY_RESOLVE_OUTCOME_STATS } from './core_usage_stats_client';
 

src/core/server/core_usage_data/types.ts

@@ -280,12 +280,59 @@ export interface CoreConfigUsageData {
   };
 }
 
+/**
+ * @internal Details about the counter to be incremented
+ */
+export interface CoreIncrementCounterParams {
+  /** The name of the counter **/
+  counterName: string;
+  /** The counter type ("count" by default) **/
+  counterType?: string;
+  /** Increment the counter by this number (1 if not specified) **/
+  incrementBy?: number;
+}
+
+/**
+ * @internal
+ * Method to call whenever an event occurs, so the counter can be increased.
+ */
+export type CoreIncrementUsageCounter = (params: CoreIncrementCounterParams) => void;
+
+/**
+ * @internal
+ * API to track whenever an event occurs, so the core can report them.
+ */
+export interface CoreUsageCounter {
+  /** @internal {@link CoreIncrementUsageCounter} **/
+  incrementCounter: CoreIncrementUsageCounter;
+}
+
 /** @internal */
-export interface CoreUsageDataSetup {
+export interface InternalCoreUsageDataSetup extends CoreUsageDataSetup {
   registerType(
     typeRegistry: ISavedObjectTypeRegistry & Pick<SavedObjectTypeRegistry, 'registerType'>
   ): void;
   getClient(): CoreUsageStatsClient;
+
+  /** @internal {@link CoreIncrementUsageCounter} **/
+  incrementUsageCounter: CoreIncrementUsageCounter;
+}
+
+/**
+ * Internal API for registering the Usage Tracker used for Core's usage data payload.
+ *
+ * @note This API should never be used to drive application logic and is only
+ * intended for telemetry purposes.
+ *
+ * @internal
+ */
+export interface CoreUsageDataSetup {
+  /**
+   * @internal
+   * API for a usage tracker plugin to inject the {@link CoreUsageCounter} to use
+   * when tracking events.
+   */
+  registerUsageCounter: (usageCounter: CoreUsageCounter) => void;
 }
 
 /**

src/core/server/elasticsearch/integration_tests/client.test.ts

@@ -96,8 +96,8 @@ describe('fake elasticsearch', () => {
 
   test('should return unknown product when it cannot perform the Product check (503 response)', async () => {
     const resp = await supertest(kibanaHttpServer).get('/api/status').expect(503);
-    expect(resp.body.status.overall.state).toBe('red');
-    expect(resp.body.status.statuses[0].message).toBe(
+    expect(resp.body.status.overall.level).toBe('critical');
+    expect(resp.body.status.core.elasticsearch.summary).toBe(
       'Unable to retrieve version information from Elasticsearch nodes. The client noticed that the server is not Elasticsearch and we do not support this unknown product.'
     );
   });

src/core/server/index.ts

@@ -55,7 +55,7 @@ import { CapabilitiesSetup, CapabilitiesStart } from './capabilities';
 import { MetricsServiceSetup, MetricsServiceStart } from './metrics';
 import { StatusServiceSetup } from './status';
 import { AppenderConfigType, appendersSchema, LoggingServiceSetup } from './logging';
-import { CoreUsageDataStart } from './core_usage_data';
+import { CoreUsageDataStart, CoreUsageDataSetup } from './core_usage_data';
 import { I18nServiceSetup } from './i18n';
 import { DeprecationsServiceSetup, DeprecationsClient } from './deprecations';
 // Because of #79265 we need to explicitly import, then export these types for
@@ -410,7 +410,13 @@ export type {
 export { ServiceStatusLevels } from './status';
 export type { CoreStatus, ServiceStatus, ServiceStatusLevel, StatusServiceSetup } from './status';
 
-export type { CoreUsageDataStart } from './core_usage_data';
+export type {
+  CoreUsageDataSetup,
+  CoreUsageDataStart,
+  CoreUsageCounter,
+  CoreIncrementUsageCounter,
+  CoreIncrementCounterParams,
+} from './core_usage_data';
 
 /**
  * Plugin specific context passed to a route handler.
@@ -500,6 +506,8 @@ export interface CoreSetup<TPluginsStart extends object = object, TStart = unkno
   deprecations: DeprecationsServiceSetup;
   /** {@link StartServicesAccessor} */
   getStartServices: StartServicesAccessor<TPluginsStart, TStart>;
+  /** @internal {@link CoreUsageDataSetup} */
+  coreUsageData: CoreUsageDataSetup;
 }
 
 /**

src/core/server/internal_types.ts

@@ -36,7 +36,7 @@ import { InternalRenderingServiceSetup } from './rendering';
 import { InternalHttpResourcesPreboot, InternalHttpResourcesSetup } from './http_resources';
 import { InternalStatusServiceSetup } from './status';
 import { InternalLoggingServicePreboot, InternalLoggingServiceSetup } from './logging';
-import { CoreUsageDataStart } from './core_usage_data';
+import { CoreUsageDataStart, InternalCoreUsageDataSetup } from './core_usage_data';
 import { I18nServiceSetup } from './i18n';
 import { InternalDeprecationsServiceSetup, InternalDeprecationsServiceStart } from './deprecations';
 import type {
@@ -73,6 +73,7 @@ export interface InternalCoreSetup {
   logging: InternalLoggingServiceSetup;
   metrics: InternalMetricsServiceSetup;
   deprecations: InternalDeprecationsServiceSetup;
+  coreUsageData: InternalCoreUsageDataSetup;
 }
 
 /**