Merge branch 'main' into task-manager/mget-poll-interval

.buildkite/ftr_base_serverless_configs.yml

@@ -1,6 +1,8 @@
 disabled:
   # Base config files, only necessary to inform config finding script
 
+  # Serverless deployment-agnostic default config for api-integration tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts
   # Serverless base config files
   - x-pack/test_serverless/api_integration/config.base.ts
   - x-pack/test_serverless/functional/config.base.ts

.buildkite/ftr_oblt_serverless_configs.yml

@@ -26,3 +26,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -1,4 +1,6 @@
 disabled:
+  # Stateful base config for deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts
   # Base config files, only necessary to inform config finding script
   - test/functional/config.base.js
   - test/functional/firefox/config.base.ts
@@ -155,7 +157,6 @@ enabled:
   - x-pack/test/api_integration/apis/monitoring/config.ts
   - x-pack/test/api_integration/apis/monitoring_collection/config.ts
   - x-pack/test/api_integration/apis/osquery/config.ts
-  - x-pack/test/api_integration/apis/painless_lab/config.ts
   - x-pack/test/api_integration/apis/search/config.ts
   - x-pack/test/api_integration/apis/searchprofiler/config.ts
   - x-pack/test/api_integration/apis/security/config.ts
@@ -359,3 +360,5 @@ enabled:
   - x-pack/performance/journeys_e2e/apm_service_inventory.ts
   - x-pack/performance/journeys_e2e/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
+  # stateful config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/stateful.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -16,3 +16,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -97,3 +97,5 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts

.eslintrc.js

@@ -618,7 +618,7 @@ module.exports = {
         'test/*/*.config.ts',
         'test/*/{tests,test_suites,apis,apps}/**/*',
         'test/server_integration/**/*.ts',
-        'x-pack/test/*/{tests,test_suites,apis,apps}/**/*',
+        'x-pack/test/*/{tests,test_suites,apis,apps,deployment_agnostic}/**/*',
         'x-pack/test/*/*config.*ts',
         'x-pack/test/saved_object_api_integration/*/apis/**/*',
         'x-pack/test/ui_capabilities/*/tests/**/*',

.github/CODEOWNERS

@@ -1274,6 +1274,7 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /x-pack/test_serverless/functional/test_suites/security/ftr/ @elastic/appex-qa
 /x-pack/test_serverless/functional/test_suites/common/home_page/  @elastic/appex-qa
 /x-pack/test_serverless/**/services/ @elastic/appex-qa
+/packages/kbn-es/src/stateful_resources/roles.yml @elastic/appex-qa
 
 # Core
 /config/ @elastic/kibana-core

packages/kbn-es/index.ts

@@ -21,4 +21,4 @@ export {
   readRolesDescriptorsFromResource,
 } from './src/utils';
 export type { ArtifactLicense } from './src/artifact';
-export { SERVERLESS_ROLES_ROOT_PATH } from './src/paths';
+export { SERVERLESS_ROLES_ROOT_PATH, STATEFUL_ROLES_ROOT_PATH } from './src/paths';

packages/kbn-es/src/paths.ts

@@ -25,6 +25,8 @@ export const ES_CONFIG = 'config/elasticsearch.yml';
 
 export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore');
 
+export const STATEFUL_ROLES_ROOT_PATH = resolve(__dirname, './stateful_resources');
+
 export const SERVERLESS_OPERATOR_USERS_PATH = resolve(
   __dirname,
   './serverless_resources/operator_users.yml'

packages/kbn-es/src/stateful_resources/roles.yml

@@ -0,0 +1,130 @@
+# -----
+# This file is for information purpose only. 'viewer' and 'editor' roles are defined in stateful Elasticsearch by default
+# Source: https://github.com/elastic/elasticsearch/blob/4272164530807787d4d8b991e3095a6e79176dbf/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java#L861-L952
+# Note: inconsistency between these roles definition and the same roles of serverless project may break FTR deployment-agnostic tests
+# -----
+viewer:
+  cluster: []
+  indices:
+    - names:
+        - '.alerts*'
+        - '.preview.alerts*'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+    - names:
+        - '.items-*'
+        - '.lists-*'
+        - '.siem-signals*'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+    - names:
+        - '/~(([.]|ilm-history-).*)/'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+    - names:
+        - '.profiling-*'
+        - 'profiling-*'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+  applications:
+    - application: 'kibana-.kibana'
+      privileges:
+        - 'read'
+      resources:
+        - '*'
+  run_as: []
+
+editor:
+  cluster: []
+  indices:
+    - names:
+        - 'observability-annotations'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+        - 'write'
+      allow_restricted_indices: false
+    - names:
+        - '.items-*'
+        - '.lists-*'
+        - '.siem-signals*'
+      privileges:
+        - 'maintenance'
+        - 'read'
+        - 'view_index_metadata'
+        - 'write'
+      allow_restricted_indices: false
+    - names:
+        - '/~(([.]|ilm-history-).*)/'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+    - names:
+        - '.profiling-*'
+        - 'profiling-*'
+      privileges:
+        - 'read'
+        - 'view_index_metadata'
+      allow_restricted_indices: false
+    - names:
+        - '.alerts*'
+        - '.internal.alerts*'
+        - '.internal.preview.alerts*'
+        - '.preview.alerts*'
+      privileges:
+        - 'maintenance'
+        - 'read'
+        - 'view_index_metadata'
+        - 'write'
+      allow_restricted_indices: false
+  applications:
+    - application: 'kibana-.kibana'
+      privileges:
+        - 'all'
+      resources:
+        - '*'
+  run_as: []
+
+# Admin role without 'remote_indices' access definition
+# There is no such built-in role in stateful, and it's a role "similar" to the built-in 'admin' role in serverless
+admin:
+  # TODO: 'all' should be replaced with explicit list both here and serverless for deployment-agnostic tests with 'admin' role to be compatible
+  cluster: ['all']
+  indices:
+    - names: ['*']
+      privileges: ['all']
+      allow_restricted_indices: false
+    - names: ['*']
+      privileges:
+        - 'monitor'
+        - 'read'
+        - 'read_cross_cluster'
+        - 'view_index_metadata'
+      allow_restricted_indices: true
+  applications:
+    - application: '*'
+      privileges: ['*']
+      resources: ['*']
+  run_as: ['*']
+
+# temporarily added for testing purpose
+system_indices_superuser:
+  cluster: ['all']
+  indices:
+    - names: ['*']
+      privileges: ['all']
+      allow_restricted_indices: true
+  applications:
+    - application: '*'
+      privileges: ['*']
+      resources: ['*']
+  run_as: ['*']

packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts

@@ -14,6 +14,9 @@ const visibleIndices = indexes
   .map(({ name, suggestedAs }) => suggestedAs || name)
   .sort();
 
+const addTrailingSpace = (strings: string[], predicate: (s: string) => boolean = (_s) => true) =>
+  strings.map((string) => (predicate(string) ? `${string} ` : string));
+
 const metadataFields = [...METADATA_FIELDS].sort();
 
 describe('autocomplete.suggest', () => {
@@ -33,17 +36,17 @@ describe('autocomplete.suggest', () => {
       test('suggests visible indices on space', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from /', visibleIndices);
-        await assertSuggestions('FROM /', visibleIndices);
-        await assertSuggestions('from /index', visibleIndices);
+        await assertSuggestions('from /', addTrailingSpace(visibleIndices));
+        await assertSuggestions('FROM /', addTrailingSpace(visibleIndices));
+        await assertSuggestions('from /index', addTrailingSpace(visibleIndices));
       });
 
       test('suggests visible indices on comma', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('FROM a,/', visibleIndices);
-        await assertSuggestions('FROM a, /', visibleIndices);
-        await assertSuggestions('from *,/', visibleIndices);
+        await assertSuggestions('FROM a,/', addTrailingSpace(visibleIndices));
+        await assertSuggestions('FROM a, /', addTrailingSpace(visibleIndices));
+        await assertSuggestions('from *,/', addTrailingSpace(visibleIndices));
       });
 
       test('can suggest integration data sources', async () => {
@@ -52,17 +55,21 @@ describe('autocomplete.suggest', () => {
           .filter(({ hidden }) => !hidden)
           .map(({ name, suggestedAs }) => suggestedAs || name)
           .sort();
+        const expectedSuggestions = addTrailingSpace(
+          visibleDataSources,
+          (s) => !integrations.find(({ name }) => name === s)
+        );
         const { assertSuggestions, callbacks } = await setup();
         const cb = {
           ...callbacks,
           getSources: jest.fn().mockResolvedValue(dataSources),
         };
 
-        assertSuggestions('from /', visibleDataSources, { callbacks: cb });
-        assertSuggestions('FROM /', visibleDataSources, { callbacks: cb });
-        assertSuggestions('FROM a,/', visibleDataSources, { callbacks: cb });
-        assertSuggestions('from a, /', visibleDataSources, { callbacks: cb });
-        assertSuggestions('from *,/', visibleDataSources, { callbacks: cb });
+        await assertSuggestions('from /', expectedSuggestions, { callbacks: cb });
+        await assertSuggestions('FROM /', expectedSuggestions, { callbacks: cb });
+        await assertSuggestions('FROM a,/', expectedSuggestions, { callbacks: cb });
+        await assertSuggestions('from a, /', expectedSuggestions, { callbacks: cb });
+        await assertSuggestions('from *,/', expectedSuggestions, { callbacks: cb });
       });
     });
 
@@ -71,7 +78,7 @@ describe('autocomplete.suggest', () => {
 
       test('on <kbd>SPACE</kbd> without comma ",", suggests adding metadata', async () => {
         const { assertSuggestions } = await setup();
-        const expected = ['METADATA $0', ',', '|'].sort();
+        const expected = ['METADATA $0', ',', '| '].sort();
 
         await assertSuggestions('from a, b /', expected);
       });
@@ -86,10 +93,10 @@ describe('autocomplete.suggest', () => {
       test('on <kbd>SPACE</kbd> after "METADATA" column suggests command and pipe operators', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from a, b [metadata _index  /]', [',', '|']);
-        await assertSuggestions('from a, b metadata _index /', [',', '|']);
-        await assertSuggestions('from a, b metadata _index, _source /', [',', '|']);
-        await assertSuggestions(`from a, b metadata ${METADATA_FIELDS.join(', ')} /`, ['|']);
+        await assertSuggestions('from a, b [metadata _index  /]', [',', '| ']);
+        await assertSuggestions('from a, b metadata _index /', [',', '| ']);
+        await assertSuggestions('from a, b metadata _index, _source /', [',', '| ']);
+        await assertSuggestions(`from a, b metadata ${METADATA_FIELDS.join(', ')} /`, ['| ']);
       });
 
       test('filters out already used metadata fields', async () => {