From 5e2e8db255b0a73e8c0ac2294165b8d9c0f5f0c4 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Fri, 4 Oct 2024 15:43:02 +0200 Subject: [PATCH 01/11] Initial commit for cypress tests --- .../e2e/integrations_automatic_import.cy.ts | 37 +++++++ ...ileges_integrations_automatic_import.cy.ts | 101 ++++++++++++++++++ .../screens/integrations_automatic_import.ts | 15 +++ .../plugins/fleet/cypress/tasks/privileges.ts | 89 +++++++++++++++ .../missing_privileges_description.tsx | 2 +- .../create_integration_landing.tsx | 5 +- .../integration_assistant_card.tsx | 5 +- 7 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts create mode 100644 x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts create mode 100644 x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts diff --git a/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts new file mode 100644 index 0000000000000..9a3475adc45db --- /dev/null +++ b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { deleteIntegrations } from '../tasks/integrations'; +import { + UPLOAD_PACKAGE_LINK, + ASSISTANT_BUTTON, + TECH_PREVIEW_BADGE, + CREATE_INTEGRATION_LANDING_PAGE, +} from '../screens/integrations_automatic_import'; +import { cleanupAgentPolicies } from '../tasks/cleanup'; +import { login } from '../tasks/login'; + +describe('Add Integration - Automatic Import', () => { + beforeEach(() => { + login(); + + cleanupAgentPolicies(); + deleteIntegrations(); + }); + + afterEach(() => {}); + + it('should provide list of connectors to choose from', () => { + cy.visit(CREATE_INTEGRATION_LANDING_PAGE); + + cy.getBySel(ASSISTANT_BUTTON).should('exist'); + cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); + cy.getBySel(TECH_PREVIEW_BADGE).should('exist'); + + cy.getBySel(ASSISTANT_BUTTON).click(); + }); +}); diff --git a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts new file mode 100644 index 0000000000000..4bcae07bab82a --- /dev/null +++ b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + deleteUsersAndRoles, + AutomaticImportIntegrRole, + AutomaticImportIntegrUser, + createUsersAndRoles, + AutomaticImportConnectorNoneUser, + AutomaticImportConnectorNoneRole, + AutomaticImportConnectorAllUser, + AutomaticImportConnectorAllRole, +} from '../tasks/privileges'; +import { login, loginWithUserAndWaitForPage, logout } from '../tasks/login'; +import { + ASSISTANT_BUTTON, + CREATE_INTEGRATION_ASSISTANT, + CREATE_INTEGRATION_UPLOAD, + MISSING_PRIVILEGES, + UPLOAD_PACKAGE_LINK, +} from '../screens/integrations_automatic_import'; + +describe('When the user has Read previleges for Integrations', () => { + before(() => { + createUsersAndRoles([AutomaticImportIntegrUser], [AutomaticImportIntegrRole]); + }); + + beforeEach(() => { + login(); + }); + + afterEach(() => { + logout(); + }); + + after(() => { + deleteUsersAndRoles([AutomaticImportIntegrUser], [AutomaticImportIntegrRole]); + }); + + it('Create Assistant is not accessible if user has read role in integrations', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportIntegrUser); + cy.getBySel(MISSING_PRIVILEGES).should('exist'); + }); + + it('Create upload is not accessible if user has read role in integrations', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_UPLOAD, AutomaticImportIntegrUser); + cy.getBySel(MISSING_PRIVILEGES).should('exist'); + }); +}); + +describe('When the user has All previleges for Integrations and No permissions for Connectors', () => { + before(() => { + createUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); + }); + + beforeEach(() => { + login(); + }); + + afterEach(() => { + logout(); + }); + + after(() => { + deleteUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); + }); + + it('Create Assistant is not accessible but upload is accessible', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorNoneUser); + cy.getBySel(ASSISTANT_BUTTON).should('not.exist'); + cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); + }); +}); + +describe('When the user has All previleges for Integrations and All permissions for Connectors', () => { + before(() => { + createUsersAndRoles([AutomaticImportConnectorAllUser], [AutomaticImportConnectorAllRole]); + }); + + beforeEach(() => { + login(); + }); + + afterEach(() => { + logout(); + }); + + after(() => { + deleteUsersAndRoles([AutomaticImportConnectorAllUser], [AutomaticImportConnectorAllRole]); + }); + + it('Create Assistant is not accessible but upload is accessible', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorAllUser); + cy.getBySel(ASSISTANT_BUTTON).should('not.exist'); + cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); + }); +}); diff --git a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts new file mode 100644 index 0000000000000..c0046e6884b86 --- /dev/null +++ b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const UPLOAD_PACKAGE_LINK = 'uploadPackageLink'; +export const ASSISTANT_BUTTON = 'assistantButton'; +export const TECH_PREVIEW_BADGE = 'techPreviewBadge'; +export const MISSING_PRIVILEGES = 'missingPrivilegesCallOut'; + +export const CREATE_INTEGRATION_LANDING_PAGE = '/app/integrations/create'; +export const CREATE_INTEGRATION_ASSISTANT = '/app/integrations/create/assistant'; +export const CREATE_INTEGRATION_UPLOAD = '/app/integrations/create/upload'; diff --git a/x-pack/plugins/fleet/cypress/tasks/privileges.ts b/x-pack/plugins/fleet/cypress/tasks/privileges.ts index 214bd0f14e6e6..65e8fe360a6c0 100644 --- a/x-pack/plugins/fleet/cypress/tasks/privileges.ts +++ b/x-pack/plugins/fleet/cypress/tasks/privileges.ts @@ -193,6 +193,95 @@ export const FleetNoneIntegrAllUser: User = { roles: [FleetNoneIntegrAllRole.name], }; +export const AutomaticImportIntegrRole: Role = { + name: 'automatic_import_integrations_read_role', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + cluster: ['manage_service_account'], + }, + kibana: [ + { + feature: { + fleetv2: ['read'], + fleet: ['read'], + }, + spaces: ['*'], + }, + ], + }, +}; +export const AutomaticImportIntegrUser: User = { + username: 'automatic_import_integrations_read_user', + password: 'password', + roles: [AutomaticImportIntegrRole.name], +}; + +export const AutomaticImportConnectorNoneRole: Role = { + name: 'automatic_import_connectors_none_role', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + cluster: ['manage_service_account'], + }, + kibana: [ + { + feature: { + fleetv2: ['all'], + fleet: ['all'], + connectors: ['none'], + }, + spaces: ['*'], + }, + ], + }, +}; +export const AutomaticImportConnectorNoneUser: User = { + username: 'automatic_import_connectors_none_user', + password: 'password', + roles: [AutomaticImportConnectorNoneRole.name], +}; + +export const AutomaticImportConnectorAllRole: Role = { + name: 'automatic_import_connectors_all_role', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + cluster: ['manage_service_account'], + }, + kibana: [ + { + feature: { + fleetv2: ['all'], + fleet: ['all'], + connectors: ['all'], + }, + spaces: ['*'], + }, + ], + }, +}; +export const AutomaticImportConnectorAllUser: User = { + username: 'automatic_import_connectors_all_user', + password: 'password', + roles: [AutomaticImportConnectorAllRole.name], +}; + export const BuiltInEditorUser: User = { username: 'editor_user', password: 'password', diff --git a/x-pack/plugins/integration_assistant/public/common/components/authorization/missing_privileges_description.tsx b/x-pack/plugins/integration_assistant/public/common/components/authorization/missing_privileges_description.tsx index 15365aeb3a08e..ccc65a2e49f0e 100644 --- a/x-pack/plugins/integration_assistant/public/common/components/authorization/missing_privileges_description.tsx +++ b/x-pack/plugins/integration_assistant/public/common/components/authorization/missing_privileges_description.tsx @@ -13,7 +13,7 @@ type MissingPrivilegesDescriptionProps = Partial; export const MissingPrivilegesDescription = React.memo( ({ canCreateIntegrations, canCreateConnectors, canExecuteConnectors }) => { return ( - + {i18n.PRIVILEGES_REQUIRED_TITLE} diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_landing/create_integration_landing.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_landing/create_integration_landing.tsx index 39cbd2cea1026..71706625f636f 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_landing/create_integration_landing.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_landing/create_integration_landing.tsx @@ -54,7 +54,10 @@ export const CreateIntegrationLanding = React.memo(() => { defaultMessage="If you have an existing integration package, {link}" values={{ link: ( - navigate(Page.upload)}> + navigate(Page.upload)} + data-test-subj="uploadPackageLink" + > { tooltipContent={i18n.TECH_PREVIEW_TOOLTIP} size="s" color="hollow" + data-test-subj="techPreviewBadge" /> @@ -64,7 +65,9 @@ export const IntegrationAssistantCard = React.memo(() => { {canExecuteConnectors ? ( - navigate(Page.assistant)}>{i18n.ASSISTANT_BUTTON} + navigate(Page.assistant)} data-test-subj="assistantButton"> + {i18n.ASSISTANT_BUTTON} + ) : ( {i18n.ASSISTANT_BUTTON} From 18890e01732c2eb083d98d2bcc4c1b8b36eb957a Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 7 Oct 2024 13:56:19 +0200 Subject: [PATCH 02/11] add RBAC tests --- ...ileges_integrations_automatic_import.cy.ts | 97 +++++++++++++++---- .../screens/integrations_automatic_import.ts | 2 + .../plugins/fleet/cypress/tasks/privileges.ts | 50 +++++++--- .../steps/connector_step/connector_setup.tsx | 5 +- 4 files changed, 118 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts index 4bcae07bab82a..e0ca3488b8f08 100644 --- a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts @@ -5,28 +5,81 @@ * 2.0. */ +import type { User } from '../tasks/privileges'; import { deleteUsersAndRoles, - AutomaticImportIntegrRole, - AutomaticImportIntegrUser, + getIntegrationsAutoImportRole, createUsersAndRoles, AutomaticImportConnectorNoneUser, AutomaticImportConnectorNoneRole, AutomaticImportConnectorAllUser, AutomaticImportConnectorAllRole, + AutomaticImportConnectorReadUser, + AutomaticImportConnectorReadRole, } from '../tasks/privileges'; import { login, loginWithUserAndWaitForPage, logout } from '../tasks/login'; import { ASSISTANT_BUTTON, + CONNECTOR_ACTION_ID, + CONNECTOR_ACTION_NAME, CREATE_INTEGRATION_ASSISTANT, + CREATE_INTEGRATION_LANDING_PAGE, CREATE_INTEGRATION_UPLOAD, MISSING_PRIVILEGES, UPLOAD_PACKAGE_LINK, } from '../screens/integrations_automatic_import'; -describe('When the user has Read previleges for Integrations', () => { +describe('When the user does not have enough previleges for Integrations', () => { + const runs = [ + { fleetRole: 'read', integrationsRole: 'read' }, + { fleetRole: 'read', integrationsRole: 'all' }, + { fleetRole: 'all', integrationsRole: 'read' }, + ]; + + runs.forEach(function (run) { + describe(`When the user has '${run.fleetRole}' role for fleet and '${run.integrationsRole}' role for Integrations`, () => { + const automaticImportIntegrRole = getIntegrationsAutoImportRole({ + fleetv2: [run.fleetRole], // fleet + fleet: [run.integrationsRole], // integrations + }); + const AutomaticImportIntegrUser: User = { + username: 'automatic_import_integrations_read_user', + password: 'password', + roles: [automaticImportIntegrRole.name], + }; + + before(() => { + createUsersAndRoles([AutomaticImportIntegrUser], [automaticImportIntegrRole]); + }); + + beforeEach(() => { + login(); + }); + + afterEach(() => { + logout(); + }); + + after(() => { + deleteUsersAndRoles([AutomaticImportIntegrUser], [automaticImportIntegrRole]); + }); + + it('Create Assistant is not accessible if user has read role in integrations', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportIntegrUser); + cy.getBySel(MISSING_PRIVILEGES).should('exist'); + }); + + it('Create upload is not accessible if user has read role in integrations', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_UPLOAD, AutomaticImportIntegrUser); + cy.getBySel(MISSING_PRIVILEGES).should('exist'); + }); + }); + }); +}); + +describe('When the user has All permissions for Integrations and No permissions for actions', () => { before(() => { - createUsersAndRoles([AutomaticImportIntegrUser], [AutomaticImportIntegrRole]); + createUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); }); beforeEach(() => { @@ -38,23 +91,19 @@ describe('When the user has Read previleges for Integrations', () => { }); after(() => { - deleteUsersAndRoles([AutomaticImportIntegrUser], [AutomaticImportIntegrRole]); - }); - - it('Create Assistant is not accessible if user has read role in integrations', () => { - loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportIntegrUser); - cy.getBySel(MISSING_PRIVILEGES).should('exist'); + deleteUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); }); - it('Create upload is not accessible if user has read role in integrations', () => { - loginWithUserAndWaitForPage(CREATE_INTEGRATION_UPLOAD, AutomaticImportIntegrUser); - cy.getBySel(MISSING_PRIVILEGES).should('exist'); + it('Create Assistant is not accessible but upload is accessible', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_LANDING_PAGE, AutomaticImportConnectorNoneUser); + cy.getBySel(ASSISTANT_BUTTON).should('not.exist'); + cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); }); }); -describe('When the user has All previleges for Integrations and No permissions for Connectors', () => { +describe('When the user has All permissions for Integrations and read permissions for actions', () => { before(() => { - createUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); + createUsersAndRoles([AutomaticImportConnectorReadUser], [AutomaticImportConnectorReadRole]); }); beforeEach(() => { @@ -66,17 +115,23 @@ describe('When the user has All previleges for Integrations and No permissions f }); after(() => { - deleteUsersAndRoles([AutomaticImportConnectorNoneUser], [AutomaticImportConnectorNoneRole]); + deleteUsersAndRoles([AutomaticImportConnectorReadUser], [AutomaticImportConnectorReadRole]); }); it('Create Assistant is not accessible but upload is accessible', () => { - loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorNoneUser); - cy.getBySel(ASSISTANT_BUTTON).should('not.exist'); + loginWithUserAndWaitForPage(CREATE_INTEGRATION_LANDING_PAGE, AutomaticImportConnectorReadUser); + cy.getBySel(ASSISTANT_BUTTON).should('exist'); cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); }); + + it('Create Assistant is not accessible but upload is accessible', () => { + loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorReadUser); + cy.getBySel(CONNECTOR_ACTION_ID).should('not.exist'); + cy.getBySel(CONNECTOR_ACTION_NAME).should('not.exist'); + }); }); -describe('When the user has All previleges for Integrations and All permissions for Connectors', () => { +describe('When the user has All permissions for Integrations and All permissions for actions', () => { before(() => { createUsersAndRoles([AutomaticImportConnectorAllUser], [AutomaticImportConnectorAllRole]); }); @@ -95,7 +150,7 @@ describe('When the user has All previleges for Integrations and All permissions it('Create Assistant is not accessible but upload is accessible', () => { loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorAllUser); - cy.getBySel(ASSISTANT_BUTTON).should('not.exist'); - cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); + cy.getBySel(CONNECTOR_ACTION_ID).should('exist'); + cy.getBySel(CONNECTOR_ACTION_NAME).should('exist'); }); }); diff --git a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts index c0046e6884b86..63c93c8cf5c65 100644 --- a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts +++ b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts @@ -9,6 +9,8 @@ export const UPLOAD_PACKAGE_LINK = 'uploadPackageLink'; export const ASSISTANT_BUTTON = 'assistantButton'; export const TECH_PREVIEW_BADGE = 'techPreviewBadge'; export const MISSING_PRIVILEGES = 'missingPrivilegesCallOut'; +export const CONNECTOR_ACTION_ID = 'connectorActionId'; +export const CONNECTOR_ACTION_NAME = 'connectorActionName'; export const CREATE_INTEGRATION_LANDING_PAGE = '/app/integrations/create'; export const CREATE_INTEGRATION_ASSISTANT = '/app/integrations/create/assistant'; diff --git a/x-pack/plugins/fleet/cypress/tasks/privileges.ts b/x-pack/plugins/fleet/cypress/tasks/privileges.ts index 65e8fe360a6c0..876b88ac9d5b5 100644 --- a/x-pack/plugins/fleet/cypress/tasks/privileges.ts +++ b/x-pack/plugins/fleet/cypress/tasks/privileges.ts @@ -8,7 +8,7 @@ import { request } from './common'; import { constructUrlWithUser, getEnvAuth } from './login'; -interface User { +export interface User { username: string; password: string; description?: string; @@ -193,7 +193,7 @@ export const FleetNoneIntegrAllUser: User = { roles: [FleetNoneIntegrAllRole.name], }; -export const AutomaticImportIntegrRole: Role = { +export const getIntegrationsAutoImportRole = (feature: FeaturesPrivileges): Role => ({ name: 'automatic_import_integrations_read_role', privileges: { elasticsearch: { @@ -207,20 +207,12 @@ export const AutomaticImportIntegrRole: Role = { }, kibana: [ { - feature: { - fleetv2: ['read'], - fleet: ['read'], - }, + feature, spaces: ['*'], }, ], }, -}; -export const AutomaticImportIntegrUser: User = { - username: 'automatic_import_integrations_read_user', - password: 'password', - roles: [AutomaticImportIntegrRole.name], -}; +}); export const AutomaticImportConnectorNoneRole: Role = { name: 'automatic_import_connectors_none_role', @@ -239,7 +231,7 @@ export const AutomaticImportConnectorNoneRole: Role = { feature: { fleetv2: ['all'], fleet: ['all'], - connectors: ['none'], + actions: ['none'], }, spaces: ['*'], }, @@ -252,6 +244,36 @@ export const AutomaticImportConnectorNoneUser: User = { roles: [AutomaticImportConnectorNoneRole.name], }; +export const AutomaticImportConnectorReadRole: Role = { + name: 'automatic_import_connectors_read_role', + privileges: { + elasticsearch: { + indices: [ + { + names: ['*'], + privileges: ['all'], + }, + ], + cluster: ['manage_service_account'], + }, + kibana: [ + { + feature: { + fleetv2: ['all'], + fleet: ['all'], + actions: ['read'], + }, + spaces: ['*'], + }, + ], + }, +}; +export const AutomaticImportConnectorReadUser: User = { + username: 'automatic_import_connectors_read_user', + password: 'password', + roles: [AutomaticImportConnectorReadRole.name], +}; + export const AutomaticImportConnectorAllRole: Role = { name: 'automatic_import_connectors_all_role', privileges: { @@ -269,7 +291,7 @@ export const AutomaticImportConnectorAllRole: Role = { feature: { fleetv2: ['all'], fleet: ['all'], - connectors: ['all'], + actions: ['all'], }, spaces: ['*'], }, diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/connector_step/connector_setup.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/connector_step/connector_setup.tsx index 8715f42eb8f58..e85481378f4dd 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/connector_step/connector_setup.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/connector_step/connector_setup.tsx @@ -104,10 +104,13 @@ export const ConnectorSetup = React.memo( size="xl" color="text" type={actionTypeRegistry.get(actionType.id).iconClass} + data-test-subj="connectorActionId" /> - {actionType.name} + + {actionType.name} + From 9414ec108f489b6e5066817dbe1bc6755e7261fe Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 7 Oct 2024 17:04:06 +0200 Subject: [PATCH 03/11] Add e2e integration test flow --- .../e2e/integrations_automatic_import.cy.ts | 53 +++++++++++- ...ileges_integrations_automatic_import.cy.ts | 17 ++-- .../fleet/cypress/fixtures/teleport.ndjson | 1 + .../screens/integrations_automatic_import.ts | 16 +++- .../cypress/tasks/api_calls/connectors.ts | 86 +++++++++++++++++++ .../cypress/tasks/api_calls/graph_calls.ts | 28 ++++++ 6 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/fleet/cypress/fixtures/teleport.ndjson create mode 100644 x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts create mode 100644 x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts diff --git a/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts index 9a3475adc45db..f0abc801c1054 100644 --- a/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts @@ -11,9 +11,19 @@ import { ASSISTANT_BUTTON, TECH_PREVIEW_BADGE, CREATE_INTEGRATION_LANDING_PAGE, + BUTTON_FOOTER_NEXT, + INTEGRATION_TITLE_INPUT, + INTEGRATION_DESCRIPTION_INPUT, + DATASTREAM_TITLE_INPUT, + DATASTREAM_DESCRIPTION_INPUT, + DATASTREAM_NAME_INPUT, + DATA_COLLECTION_METHOD_INPUT, + LOGS_SAMPLE_FILE_PICKER, } from '../screens/integrations_automatic_import'; import { cleanupAgentPolicies } from '../tasks/cleanup'; import { login } from '../tasks/login'; +import { createBedrockConnector, deleteConnectors } from '../tasks/api_calls/connectors'; +import { results } from '../tasks/api_calls/graph_calls'; describe('Add Integration - Automatic Import', () => { beforeEach(() => { @@ -21,17 +31,58 @@ describe('Add Integration - Automatic Import', () => { cleanupAgentPolicies(); deleteIntegrations(); + + // Create a mock connector + deleteConnectors(); + createBedrockConnector(); + // Mock API Responses + cy.intercept('POST', '/api/integration_assistant/ecs', { + statusCode: 200, + body: { + results, + }, + }); + cy.intercept('POST', '/api/integration_assistant/categorization', { + statusCode: 200, + body: { + results, + }, + }); + cy.intercept('POST', '/api/integration_assistant/related', { + statusCode: 200, + body: { + results, + }, + }); }); afterEach(() => {}); - it('should provide list of connectors to choose from', () => { + it('should create an integration', () => { cy.visit(CREATE_INTEGRATION_LANDING_PAGE); cy.getBySel(ASSISTANT_BUTTON).should('exist'); cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); cy.getBySel(TECH_PREVIEW_BADGE).should('exist'); + // Create Integration Assistant Page cy.getBySel(ASSISTANT_BUTTON).click(); + cy.getBySel(BUTTON_FOOTER_NEXT).click(); + + // Integration details Page + cy.getBySel(INTEGRATION_TITLE_INPUT).type('Test Integration'); + cy.getBySel(INTEGRATION_DESCRIPTION_INPUT).type('Test Integration Description'); + cy.getBySel(BUTTON_FOOTER_NEXT).click(); + + // Datastream details page + cy.getBySel(DATASTREAM_TITLE_INPUT).type('Audit'); + cy.getBySel(DATASTREAM_DESCRIPTION_INPUT).type('Test Datastream Description'); + cy.getBySel(DATASTREAM_NAME_INPUT).type('audit'); + cy.getBySel(DATA_COLLECTION_METHOD_INPUT).type('file stream').trigger('mousemove').click(); + + // Select sample logs file and Analyze logs + cy.fixture('teleport.ndjson', null).as('myFixture'); + cy.getBySel(LOGS_SAMPLE_FILE_PICKER).selectFile('@myFixture'); + cy.getBySel(BUTTON_FOOTER_NEXT).click(); }); }); diff --git a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts index e0ca3488b8f08..29eaab7eaca0a 100644 --- a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts @@ -20,8 +20,9 @@ import { import { login, loginWithUserAndWaitForPage, logout } from '../tasks/login'; import { ASSISTANT_BUTTON, - CONNECTOR_ACTION_ID, - CONNECTOR_ACTION_NAME, + CONNECTOR_BEDROCK, + CONNECTOR_GEMINI, + CONNECTOR_OPENAI, CREATE_INTEGRATION_ASSISTANT, CREATE_INTEGRATION_LANDING_PAGE, CREATE_INTEGRATION_UPLOAD, @@ -124,10 +125,11 @@ describe('When the user has All permissions for Integrations and read permission cy.getBySel(UPLOAD_PACKAGE_LINK).should('exist'); }); - it('Create Assistant is not accessible but upload is accessible', () => { + it('Create Assistant is accessible but execute connector is not accessible', () => { loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorReadUser); - cy.getBySel(CONNECTOR_ACTION_ID).should('not.exist'); - cy.getBySel(CONNECTOR_ACTION_NAME).should('not.exist'); + cy.getBySel(CONNECTOR_BEDROCK).should('not.exist'); + cy.getBySel(CONNECTOR_OPENAI).should('not.exist'); + cy.getBySel(CONNECTOR_GEMINI).should('not.exist'); }); }); @@ -150,7 +152,8 @@ describe('When the user has All permissions for Integrations and All permissions it('Create Assistant is not accessible but upload is accessible', () => { loginWithUserAndWaitForPage(CREATE_INTEGRATION_ASSISTANT, AutomaticImportConnectorAllUser); - cy.getBySel(CONNECTOR_ACTION_ID).should('exist'); - cy.getBySel(CONNECTOR_ACTION_NAME).should('exist'); + cy.getBySel(CONNECTOR_BEDROCK).should('exist'); + cy.getBySel(CONNECTOR_OPENAI).should('exist'); + cy.getBySel(CONNECTOR_GEMINI).should('exist'); }); }); diff --git a/x-pack/plugins/fleet/cypress/fixtures/teleport.ndjson b/x-pack/plugins/fleet/cypress/fixtures/teleport.ndjson new file mode 100644 index 0000000000000..82774ac2297d6 --- /dev/null +++ b/x-pack/plugins/fleet/cypress/fixtures/teleport.ndjson @@ -0,0 +1 @@ +{"ei":0,"event":"cert.create","uid":"efd326fc-dd13-4df8-acef-3102c2d717d3","code":"TC000I","time":"2024-02-24T06:56:50.648137154Z"} \ No newline at end of file diff --git a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts index 63c93c8cf5c65..098387337099e 100644 --- a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts +++ b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts @@ -9,8 +9,20 @@ export const UPLOAD_PACKAGE_LINK = 'uploadPackageLink'; export const ASSISTANT_BUTTON = 'assistantButton'; export const TECH_PREVIEW_BADGE = 'techPreviewBadge'; export const MISSING_PRIVILEGES = 'missingPrivilegesCallOut'; -export const CONNECTOR_ACTION_ID = 'connectorActionId'; -export const CONNECTOR_ACTION_NAME = 'connectorActionName'; + +export const CONNECTOR_BEDROCK = 'actionType-.bedrock'; +export const CONNECTOR_OPENAI = 'actionType-.gen-ai'; +export const CONNECTOR_GEMINI = 'actionType-.gemini'; + +export const BUTTON_FOOTER_NEXT = 'buttonsFooter-nextButton'; + +export const INTEGRATION_TITLE_INPUT = 'integrationTitleInput'; +export const INTEGRATION_DESCRIPTION_INPUT = 'integrationDescriptionInput'; +export const DATASTREAM_TITLE_INPUT = 'dataStreamTitleInput'; +export const DATASTREAM_DESCRIPTION_INPUT = 'dataStreamDescriptionInput'; +export const DATASTREAM_NAME_INPUT = 'dataStreamNameInput'; +export const DATA_COLLECTION_METHOD_INPUT = 'dataCollectionMethodInput'; +export const LOGS_SAMPLE_FILE_PICKER = 'logsSampleFilePicker'; export const CREATE_INTEGRATION_LANDING_PAGE = '/app/integrations/create'; export const CREATE_INTEGRATION_ASSISTANT = '/app/integrations/create/assistant'; diff --git a/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts b/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts new file mode 100644 index 0000000000000..12d3ff768a6fb --- /dev/null +++ b/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AllConnectorsResponse } from '@kbn/actions-plugin/common/routes/connector/response'; + +import { API_AUTH, COMMON_API_HEADERS } from '../common'; + +export const bedrockId = '23933aeb-a3cb-476f-8465-3235085a9dd8'; +export const azureId = 'e68f5cbe-3604-468f-a5e1-809b80b614aa'; + +// Replaces request - adds baseline authentication + global headers +export const request = ({ + headers, + ...options +}: Partial): Cypress.Chainable> => { + return cy.request({ + auth: API_AUTH, + headers: { ...COMMON_API_HEADERS, ...headers }, + ...options, + }); +}; +export const INTERNAL_CLOUD_CONNECTORS = ['Elastic-Cloud-SMTP']; + +export const getConnectors = () => + request({ + method: 'GET', + url: 'api/actions/connectors', + }); + +export const createConnector = (connector: Record, id: string) => + cy.request({ + method: 'POST', + url: `/api/actions/connector/${id}`, + body: connector, + headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + }); + +export const deleteConnectors = () => { + getConnectors().then(($response) => { + if ($response.body.length > 0) { + const ids = $response.body.map((connector) => { + return connector.id; + }); + ids.forEach((id) => { + if (!INTERNAL_CLOUD_CONNECTORS.includes(id)) { + request({ + method: 'DELETE', + url: `api/actions/connector/${id}`, + }); + } + }); + } + }); +}; + +export const azureConnectorAPIPayload = { + connector_type_id: '.gen-ai', + secrets: { + apiKey: '123', + }, + config: { + apiUrl: + 'https://goodurl.com/openai/deployments/good-gpt4o/chat/completions?api-version=2024-02-15-preview', + apiProvider: 'Azure OpenAI', + }, + name: 'Azure OpenAI cypress test e2e connector', +}; + +export const bedrockConnectorAPIPayload = { + connector_type_id: '.bedrock', + secrets: { + accessKey: '123', + secret: '123', + }, + config: { + apiUrl: 'https://bedrock.com', + }, + name: 'Bedrock cypress test e2e connector', +}; + +export const createAzureConnector = () => createConnector(azureConnectorAPIPayload, azureId); +export const createBedrockConnector = () => createConnector(bedrockConnectorAPIPayload, bedrockId); diff --git a/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts b/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts new file mode 100644 index 0000000000000..4f6a44905a295 --- /dev/null +++ b/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const results = { + mapping: { + properties: { + '@timestamp': { + type: 'date', + }, + message: { + type: 'text', + }, + }, + }, + pipeline: { + processors: [ + { + script: { + source: 'ctx._source.message = "hello world"', + }, + }, + ], + }, +}; From 5c5e55c15668994c295a7298e2a5b1047fcb24eb Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Tue, 8 Oct 2024 10:39:40 +0200 Subject: [PATCH 04/11] Complete integration creation test --- .../e2e/integrations_automatic_import.cy.ts | 41 +- .../screens/integrations_automatic_import.ts | 6 + .../cypress/tasks/api_calls/connectors.ts | 6 +- .../cypress/tasks/api_calls/graph_calls.ts | 28 - .../cypress/tasks/api_calls/graph_results.ts | 531 ++++++++++++++++++ .../success_section/success_section.tsx | 14 +- 6 files changed, 587 insertions(+), 39 deletions(-) delete mode 100644 x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts create mode 100644 x-pack/plugins/fleet/cypress/tasks/api_calls/graph_results.ts diff --git a/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts index f0abc801c1054..e2454cb1dcf77 100644 --- a/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/integrations_automatic_import.cy.ts @@ -19,11 +19,20 @@ import { DATASTREAM_NAME_INPUT, DATA_COLLECTION_METHOD_INPUT, LOGS_SAMPLE_FILE_PICKER, + EDIT_PIPELINE_BUTTON, + SAVE_PIPELINE_BUTTON, + VIEW_INTEGRATION_BUTTON, + INTEGRATION_SUCCESS_SECTION, + SAVE_ZIP_BUTTON, } from '../screens/integrations_automatic_import'; import { cleanupAgentPolicies } from '../tasks/cleanup'; -import { login } from '../tasks/login'; +import { login, logout } from '../tasks/login'; import { createBedrockConnector, deleteConnectors } from '../tasks/api_calls/connectors'; -import { results } from '../tasks/api_calls/graph_calls'; +import { + ecsResultsForJson, + categorizationResultsForJson, + relatedResultsForJson, +} from '../tasks/api_calls/graph_results'; describe('Add Integration - Automatic Import', () => { beforeEach(() => { @@ -39,24 +48,29 @@ describe('Add Integration - Automatic Import', () => { cy.intercept('POST', '/api/integration_assistant/ecs', { statusCode: 200, body: { - results, + results: ecsResultsForJson, }, }); cy.intercept('POST', '/api/integration_assistant/categorization', { statusCode: 200, body: { - results, + results: categorizationResultsForJson, }, }); cy.intercept('POST', '/api/integration_assistant/related', { statusCode: 200, body: { - results, + results: relatedResultsForJson, }, }); }); - afterEach(() => {}); + afterEach(() => { + deleteConnectors(); + cleanupAgentPolicies(); + deleteIntegrations(); + logout(); + }); it('should create an integration', () => { cy.visit(CREATE_INTEGRATION_LANDING_PAGE); @@ -78,11 +92,24 @@ describe('Add Integration - Automatic Import', () => { cy.getBySel(DATASTREAM_TITLE_INPUT).type('Audit'); cy.getBySel(DATASTREAM_DESCRIPTION_INPUT).type('Test Datastream Description'); cy.getBySel(DATASTREAM_NAME_INPUT).type('audit'); - cy.getBySel(DATA_COLLECTION_METHOD_INPUT).type('file stream').trigger('mousemove').click(); + cy.getBySel(DATA_COLLECTION_METHOD_INPUT).type('file stream'); + cy.get('body').click(0, 0); // Select sample logs file and Analyze logs cy.fixture('teleport.ndjson', null).as('myFixture'); cy.getBySel(LOGS_SAMPLE_FILE_PICKER).selectFile('@myFixture'); cy.getBySel(BUTTON_FOOTER_NEXT).click(); + + // Edit Pipeline + cy.getBySel(EDIT_PIPELINE_BUTTON).click(); + cy.getBySel(SAVE_PIPELINE_BUTTON).click(); + + // Deploy + cy.getBySel(BUTTON_FOOTER_NEXT).click(); + cy.getBySel(INTEGRATION_SUCCESS_SECTION).should('exist'); + cy.getBySel(SAVE_ZIP_BUTTON).should('exist'); + + // View Integration + cy.getBySel(VIEW_INTEGRATION_BUTTON).click(); }); }); diff --git a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts index 098387337099e..e549f88294a3b 100644 --- a/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts +++ b/x-pack/plugins/fleet/cypress/screens/integrations_automatic_import.ts @@ -24,6 +24,12 @@ export const DATASTREAM_NAME_INPUT = 'dataStreamNameInput'; export const DATA_COLLECTION_METHOD_INPUT = 'dataCollectionMethodInput'; export const LOGS_SAMPLE_FILE_PICKER = 'logsSampleFilePicker'; +export const EDIT_PIPELINE_BUTTON = 'editPipelineButton'; +export const SAVE_PIPELINE_BUTTON = 'savePipelineButton'; +export const VIEW_INTEGRATION_BUTTON = 'viewIntegrationButton'; +export const INTEGRATION_SUCCESS_SECTION = 'integrationSuccessSection'; +export const SAVE_ZIP_BUTTON = 'saveZipButton'; + export const CREATE_INTEGRATION_LANDING_PAGE = '/app/integrations/create'; export const CREATE_INTEGRATION_ASSISTANT = '/app/integrations/create/assistant'; export const CREATE_INTEGRATION_UPLOAD = '/app/integrations/create/upload'; diff --git a/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts b/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts index 12d3ff768a6fb..230fdcd124562 100644 --- a/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts +++ b/x-pack/plugins/fleet/cypress/tasks/api_calls/connectors.ts @@ -7,10 +7,12 @@ import type { AllConnectorsResponse } from '@kbn/actions-plugin/common/routes/connector/response'; +import { v4 as uuidv4 } from 'uuid'; + import { API_AUTH, COMMON_API_HEADERS } from '../common'; -export const bedrockId = '23933aeb-a3cb-476f-8465-3235085a9dd8'; -export const azureId = 'e68f5cbe-3604-468f-a5e1-809b80b614aa'; +export const bedrockId = uuidv4(); +export const azureId = uuidv4(); // Replaces request - adds baseline authentication + global headers export const request = ({ diff --git a/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts b/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts deleted file mode 100644 index 4f6a44905a295..0000000000000 --- a/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_calls.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const results = { - mapping: { - properties: { - '@timestamp': { - type: 'date', - }, - message: { - type: 'text', - }, - }, - }, - pipeline: { - processors: [ - { - script: { - source: 'ctx._source.message = "hello world"', - }, - }, - ], - }, -}; diff --git a/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_results.ts b/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_results.ts new file mode 100644 index 0000000000000..3276b6ecf055f --- /dev/null +++ b/x-pack/plugins/fleet/cypress/tasks/api_calls/graph_results.ts @@ -0,0 +1,531 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ecsResultsForJson = { + mapping: { + teleport2: { + audit: { + ei: null, + event: { + target: 'event.action', + confidence: 0.9, + type: 'string', + date_formats: [], + }, + uid: { + target: 'event.id', + confidence: 0.95, + type: 'string', + date_formats: [], + }, + code: { + target: 'event.code', + confidence: 0.9, + type: 'string', + date_formats: [], + }, + }, + }, + }, + pipeline: { + description: 'Pipeline to process teleport2 audit logs', + processors: [ + { + set: { + field: 'ecs.version', + tag: 'set_ecs_version', + value: '8.11.0', + }, + }, + { + remove: { + field: 'message', + ignore_missing: true, + tag: 'remove_message', + }, + }, + { + json: { + field: 'event.original', + tag: 'json_original', + target_field: 'teleport2.audit', + }, + }, + { + rename: { + field: 'teleport2.audit.event', + target_field: 'event.action', + ignore_missing: true, + }, + }, + { + script: { + description: 'Ensures the date processor does not receive an array value.', + tag: 'script_convert_array_to_string', + lang: 'painless', + source: + 'if (ctx.teleport2?.audit?.time != null &&\n ctx.teleport2.audit.time instanceof ArrayList){\n ctx.teleport2.audit.time = ctx.teleport2.audit.time[0];\n}\n', + }, + }, + { + date: { + field: 'teleport2.audit.time', + target_field: 'event.start', + formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'], + tag: 'date_processor_teleport2.audit.time', + if: 'ctx.teleport2?.audit?.time != null', + }, + }, + { + set: { + field: 'event.kind', + value: 'pipeline_error', + }, + }, + ], + }, +}; + +export const categorizationResultsForJson = { + docs: [ + { + ecs: { + version: '8.11.0', + }, + teleport2: { + audit: { + cert_type: 'user', + time: '2024-02-24T06:56:50.648137154Z', + ei: 0, + identity: { + expires: '2024-02-24T06:56:50.648137154Z', + traits: { + logins: ['root', 'ubuntu', 'ec2-user'], + }, + private_key_policy: 'none', + teleport_cluster: 'teleport.com', + prev_identity_expires: '0001-01-01T00:00:00Z', + route_to_cluster: 'teleport.com', + logins: ['root', 'ubuntu', 'ec2-user', '-teleport-internal-join'], + }, + }, + }, + organization: { + name: 'teleport.com', + }, + source: { + ip: '1.2.3.4', + }, + event: { + code: 'TC000I', + start: '2024-02-24T06:56:50.648Z', + action: 'cert.create', + end: '0001-01-01T00:00:00.000Z', + id: 'efd326fc-dd13-4df8-acef-3102c2d717d3', + category: ['iam', 'authentication'], + type: ['creation', 'start'], + }, + user: { + name: 'teleport-admin', + changes: { + name: '2024-02-24T06:56:50.648Z', + }, + roles: ['access', 'editor'], + }, + tags: [ + '_geoip_database_unavailable_GeoLite2-City.mmdb', + '_geoip_database_unavailable_GeoLite2-ASN.mmdb', + '_geoip_database_unavailable_GeoLite2-City.mmdb', + '_geoip_database_unavailable_GeoLite2-ASN.mmdb', + ], + }, + ], + pipeline: { + description: 'Pipeline to process teleport2 audit logs', + processors: [ + { + set: { + field: 'ecs.version', + tag: 'set_ecs_version', + value: '8.11.0', + }, + }, + { + remove: { + field: 'message', + ignore_missing: true, + tag: 'remove_message', + }, + }, + { + json: { + field: 'event.original', + tag: 'json_original', + target_field: 'teleport2.audit', + }, + }, + { + rename: { + field: 'teleport2.audit.event', + target_field: 'event.action', + ignore_missing: true, + }, + }, + { + script: { + description: 'Ensures the date processor does not receive an array value.', + tag: 'script_convert_array_to_string', + lang: 'painless', + source: + 'if (ctx.teleport2?.audit?.time != null &&\n ctx.teleport2.audit.time instanceof ArrayList){\n ctx.teleport2.audit.time = ctx.teleport2.audit.time[0];\n}\n', + }, + }, + { + date: { + field: 'teleport2.audit.time', + target_field: 'event.start', + formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'], + tag: 'date_processor_teleport2.audit.time', + if: 'ctx.teleport2?.audit?.time != null', + }, + }, + { + set: { + field: 'event.kind', + value: 'pipeline_error', + }, + }, + ], + }, +}; + +export const relatedResultsForJson = { + docs: [ + { + ecs: { + version: '8.11.0', + }, + related: { + user: ['teleport-admin'], + ip: ['1.2.3.4'], + }, + teleport2: { + audit: { + cert_type: 'user', + time: '2024-02-24T06:56:50.648137154Z', + ei: 0, + identity: { + expires: '2024-02-24T06:56:50.648137154Z', + traits: { + logins: ['root', 'ubuntu', 'ec2-user'], + }, + private_key_policy: 'none', + teleport_cluster: 'teleport.com', + prev_identity_expires: '0001-01-01T00:00:00Z', + route_to_cluster: 'teleport.com', + logins: ['root', 'ubuntu', 'ec2-user', '-teleport-internal-join'], + }, + }, + }, + organization: { + name: 'teleport.com', + }, + source: { + ip: '1.2.3.4', + }, + event: { + code: 'TC000I', + start: '2024-02-24T06:56:50.648Z', + action: 'cert.create', + end: '0001-01-01T00:00:00.000Z', + id: 'efd326fc-dd13-4df8-acef-3102c2d717d3', + category: ['iam', 'authentication'], + type: ['creation', 'start'], + }, + user: { + name: 'teleport-admin', + changes: { + name: '2024-02-24T06:56:50.648Z', + }, + roles: ['access', 'editor'], + }, + tags: [ + '_geoip_database_unavailable_GeoLite2-City.mmdb', + '_geoip_database_unavailable_GeoLite2-ASN.mmdb', + '_geoip_database_unavailable_GeoLite2-City.mmdb', + '_geoip_database_unavailable_GeoLite2-ASN.mmdb', + ], + }, + ], + pipeline: { + description: 'Pipeline to process teleport2 audit logs', + processors: [ + { + set: { + tag: 'set_ecs_version', + field: 'ecs.version', + value: '8.11.0', + }, + }, + { + set: { + tag: 'copy_original_message', + field: 'originalMessage', + copy_from: 'message', + }, + }, + { + rename: { + ignore_missing: true, + if: 'ctx.event?.original == null', + tag: 'rename_message', + field: 'originalMessage', + target_field: 'event.original', + }, + }, + { + rename: { + ignore_missing: true, + field: 'teleport2.audit.user', + target_field: 'user.name', + }, + }, + { + rename: { + ignore_missing: true, + field: 'teleport2.audit.login', + target_field: 'user.id', + }, + }, + { + rename: { + ignore_missing: true, + field: 'teleport2.audit.server_hostname', + target_field: 'destination.domain', + }, + }, + { + rename: { + ignore_missing: true, + field: 'teleport2.audit.addr.remote', + target_field: 'source.address', + }, + }, + { + rename: { + ignore_missing: true, + field: 'teleport2.audit.proto', + target_field: 'network.protocol', + }, + }, + { + script: { + tag: 'script_drop_null_empty_values', + description: 'Drops null/empty values recursively.', + lang: 'painless', + source: + 'boolean dropEmptyFields(Object object) {\n if (object == null || object == "") {\n return true;\n } else if (object instanceof Map) {\n ((Map) object).values().removeIf(value -> dropEmptyFields(value));\n return (((Map) object).size() == 0);\n } else if (object instanceof List) {\n ((List) object).removeIf(value -> dropEmptyFields(value));\n return (((List) object).length == 0);\n }\n return false;\n}\ndropEmptyFields(ctx);\n', + }, + }, + { + geoip: { + ignore_missing: true, + tag: 'geoip_source_ip', + field: 'source.ip', + target_field: 'source.geo', + }, + }, + { + geoip: { + ignore_missing: true, + tag: 'geoip_source_asn', + database_file: 'GeoLite2-ASN.mmdb', + field: 'source.ip', + target_field: 'source.as', + properties: ['asn', 'organization_name'], + }, + }, + { + rename: { + ignore_missing: true, + tag: 'rename_source_as_asn', + field: 'source.as.asn', + target_field: 'source.as.number', + }, + }, + { + rename: { + ignore_missing: true, + tag: 'rename_source_as_organization_name', + field: 'source.as.organization_name', + target_field: 'source.as.organization.name', + }, + }, + { + geoip: { + ignore_missing: true, + tag: 'geoip_destination_ip', + field: 'destination.ip', + target_field: 'destination.geo', + }, + }, + { + geoip: { + ignore_missing: true, + tag: 'geoip_destination_asn', + database_file: 'GeoLite2-ASN.mmdb', + field: 'destination.ip', + target_field: 'destination.as', + properties: ['asn', 'organization_name'], + }, + }, + { + rename: { + ignore_missing: true, + tag: 'rename_destination_as_asn', + field: 'destination.as.asn', + target_field: 'destination.as.number', + }, + }, + { + rename: { + ignore_missing: true, + tag: 'rename_destination_as_organization_name', + field: 'destination.as.organization_name', + target_field: 'destination.as.organization.name', + }, + }, + { + append: { + if: "ctx.event?.action == 'cert.create'", + field: 'event.category', + value: ['iam'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.event?.action == 'cert.create'", + field: 'event.type', + value: ['creation'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.event?.action == 'cert.create'", + field: 'event.category', + value: ['authentication'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.event?.action == 'cert.create'", + field: 'event.type', + value: ['start'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.event?.action == 'session.start'", + field: 'event.category', + value: ['session'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.event?.action == 'session.start'", + field: 'event.type', + value: ['start'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.network?.protocol == 'ssh'", + field: 'event.category', + value: ['network'], + allow_duplicates: false, + }, + }, + { + append: { + if: "ctx.network?.protocol == 'ssh'", + field: 'event.type', + value: ['connection', 'start'], + allow_duplicates: false, + }, + }, + { + append: { + field: 'related.ip', + value: '{{{source.ip}}}', + if: 'ctx.source?.ip != null', + allow_duplicates: false, + }, + }, + { + append: { + field: 'related.user', + value: '{{{user.name}}}', + if: 'ctx.user?.name != null', + allow_duplicates: false, + }, + }, + { + append: { + field: 'related.hosts', + value: '{{{destination.domain}}}', + if: 'ctx.destination?.domain != null', + allow_duplicates: false, + }, + }, + { + append: { + field: 'related.user', + value: '{{{user.id}}}', + if: 'ctx.user?.id != null', + allow_duplicates: false, + }, + }, + { + remove: { + ignore_missing: true, + tag: 'remove_fields', + field: ['teleport2.audit.identity.client_ip'], + }, + }, + { + remove: { + ignore_failure: true, + ignore_missing: true, + if: 'ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))', + tag: 'remove_original_event', + field: 'event.original', + }, + }, + ], + on_failure: [ + { + append: { + field: 'error.message', + value: + 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}', + }, + }, + { + set: { + field: 'event.kind', + value: 'pipeline_error', + }, + }, + ], + }, +}; diff --git a/x-pack/plugins/integration_assistant/public/common/components/success_section/success_section.tsx b/x-pack/plugins/integration_assistant/public/common/components/success_section/success_section.tsx index 62df4a8f98660..08da1329770cd 100644 --- a/x-pack/plugins/integration_assistant/public/common/components/success_section/success_section.tsx +++ b/x-pack/plugins/integration_assistant/public/common/components/success_section/success_section.tsx @@ -35,7 +35,13 @@ export const SuccessSection = React.memo(({ integrationName return ( - + (({ integrationName icon={} title={i18n.VIEW_INTEGRATION_TITLE} description={i18n.VIEW_INTEGRATION_DESCRIPTION} - footer={{i18n.VIEW_INTEGRATION_BUTTON}} + footer={ + + {i18n.VIEW_INTEGRATION_BUTTON} + + } /> From fb579283076937667ff677a8345fd56b0faf1531 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 8 Oct 2024 08:50:40 +0000 Subject: [PATCH 05/11] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- x-pack/plugins/fleet/cypress/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/cypress/tsconfig.json b/x-pack/plugins/fleet/cypress/tsconfig.json index ee3dd7cd1e246..6d1433482b1c2 100644 --- a/x-pack/plugins/fleet/cypress/tsconfig.json +++ b/x-pack/plugins/fleet/cypress/tsconfig.json @@ -29,5 +29,6 @@ "force": true }, "@kbn/rison", + "@kbn/actions-plugin", ] } From 3035c0855b8fedf6850a04565c279715fe3556b4 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Tue, 8 Oct 2024 14:59:35 +0200 Subject: [PATCH 06/11] try catch buildkite error --- .../server/integration_builder/readme_files.ts | 13 ++++++++++--- .../{package_readme.md.njk => package_rdm.md.njk} | 0 2 files changed, 10 insertions(+), 3 deletions(-) rename x-pack/plugins/integration_assistant/server/templates/{package_readme.md.njk => package_rdm.md.njk} (100%) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index 163b2b04b52f9..f8d24573581c9 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -11,13 +11,20 @@ import { join as joinPath } from 'path'; import { createSync, ensureDirSync } from '../util'; export function createReadme(packageDir: string, integrationName: string, fields: object[]) { - createPackageReadme(packageDir, integrationName, fields); - createBuildReadme(packageDir, integrationName, fields); + try { + createPackageReadme(packageDir, integrationName, fields); + createBuildReadme(packageDir, integrationName, fields); + } catch (e) { + // eslint-disable-next-line no-console + console.log('********************************'); + // eslint-disable-next-line no-console + console.log(e); + } } function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) { const dirPath = joinPath(packageDir, 'docs/'); - createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, fields); + createReadmeFile(dirPath, 'package_rdm.md.njk', integrationName, fields); } function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) { diff --git a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk similarity index 100% rename from x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk rename to x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk From 8fc88d71444840f45863a592c8afc10589e72269 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Wed, 9 Oct 2024 09:24:07 +0200 Subject: [PATCH 07/11] Modify template paths for readme --- .../server/integration_builder/readme_files.ts | 13 ++++++++++--- .../server/templates/package_rdm.md.njk | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index f8d24573581c9..c9d021071c6dc 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -5,7 +5,7 @@ * 2.0. */ -import nunjucks from 'nunjucks'; +import { Environment, FileSystemLoader } from 'nunjucks'; import { join as joinPath } from 'path'; import { createSync, ensureDirSync } from '../util'; @@ -40,10 +40,17 @@ function createReadmeFile( ) { ensureDirSync(targetDir); - const template = nunjucks.render(templateName, { + const templatesPath = joinPath(__dirname, '../templates'); + const env = new Environment(new FileSystemLoader(templatesPath), { + autoescape: false, + }); + + const template = env.getTemplate(templateName); + + const renderedTemplate = template.render({ package_name: integrationName, fields, }); - createSync(joinPath(targetDir, 'README.md'), template); + createSync(joinPath(targetDir, 'README.md'), renderedTemplate); } diff --git a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk index b47e3491b5bc2..917abce65de8a 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk @@ -1,4 +1,4 @@ -{% include "readme.njk" %} +{% include "./readme.njk" %} {% for data_stream in fields %} ### {{ data_stream.datastream }} From e2b31ebf73b420436b2c4e5d8590adbd50e9e6c4 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Wed, 9 Oct 2024 10:50:47 +0200 Subject: [PATCH 08/11] modify readme naing --- .../server/integration_builder/readme_files.ts | 2 +- .../server/templates/{build_readme.md.njk => build_rdm.md.njk} | 2 +- .../server/templates/{readme.njk => description_rdm.njk} | 0 .../integration_assistant/server/templates/package_rdm.md.njk | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename x-pack/plugins/integration_assistant/server/templates/{build_readme.md.njk => build_rdm.md.njk} (84%) rename x-pack/plugins/integration_assistant/server/templates/{readme.njk => description_rdm.njk} (100%) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index c9d021071c6dc..5dcee02b98330 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -29,7 +29,7 @@ function createPackageReadme(packageDir: string, integrationName: string, fields function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) { const dirPath = joinPath(packageDir, '_dev/build/docs/'); - createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, fields); + createReadmeFile(dirPath, 'build_rdm.md.njk', integrationName, fields); } function createReadmeFile( diff --git a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk similarity index 84% rename from x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk rename to x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk index e23fa4af9efe8..8c490fefe28da 100644 --- a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk @@ -1,4 +1,4 @@ -{% include "readme.njk" %} +{% include "./description_rdm.njk" %} {% for data_stream in fields %} ### {{ data_stream.datastream }} diff --git a/x-pack/plugins/integration_assistant/server/templates/readme.njk b/x-pack/plugins/integration_assistant/server/templates/description_rdm.njk similarity index 100% rename from x-pack/plugins/integration_assistant/server/templates/readme.njk rename to x-pack/plugins/integration_assistant/server/templates/description_rdm.njk diff --git a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk index 917abce65de8a..035f342d90d89 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk @@ -1,4 +1,4 @@ -{% include "./readme.njk" %} +{% include "./description_rdm.njk" %} {% for data_stream in fields %} ### {{ data_stream.datastream }} From 8acfee83611d012f7fbdfc24d483596224a9bb34 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Wed, 9 Oct 2024 13:08:03 +0200 Subject: [PATCH 09/11] remove try catch --- .../server/integration_builder/readme_files.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index 5dcee02b98330..143f056c1a9f2 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -11,15 +11,8 @@ import { join as joinPath } from 'path'; import { createSync, ensureDirSync } from '../util'; export function createReadme(packageDir: string, integrationName: string, fields: object[]) { - try { - createPackageReadme(packageDir, integrationName, fields); - createBuildReadme(packageDir, integrationName, fields); - } catch (e) { - // eslint-disable-next-line no-console - console.log('********************************'); - // eslint-disable-next-line no-console - console.log(e); - } + createPackageReadme(packageDir, integrationName, fields); + createBuildReadme(packageDir, integrationName, fields); } function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) { From 757fb9b2e6493a16036ae512bc84298fc8b8b400 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Wed, 9 Oct 2024 15:38:53 +0200 Subject: [PATCH 10/11] rename file --- ...mport.cy.ts => privileges_integrations_automatic_import.cy.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugins/fleet/cypress/e2e/{previleges_integrations_automatic_import.cy.ts => privileges_integrations_automatic_import.cy.ts} (100%) diff --git a/x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts b/x-pack/plugins/fleet/cypress/e2e/privileges_integrations_automatic_import.cy.ts similarity index 100% rename from x-pack/plugins/fleet/cypress/e2e/previleges_integrations_automatic_import.cy.ts rename to x-pack/plugins/fleet/cypress/e2e/privileges_integrations_automatic_import.cy.ts From b47add51a3ac81ee864df5bc4662f5dc9926802f Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 10 Oct 2024 08:01:02 +0200 Subject: [PATCH 11/11] provide readme clarification --- .../server/integration_builder/readme_files.ts | 6 ++++-- .../templates/{build_rdm.md.njk => build_readme.md.njk} | 2 +- .../{description_rdm.njk => description_readme.njk} | 0 .../templates/{package_rdm.md.njk => package_readme.md.njk} | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) rename x-pack/plugins/integration_assistant/server/templates/{build_rdm.md.njk => build_readme.md.njk} (83%) rename x-pack/plugins/integration_assistant/server/templates/{description_rdm.njk => description_readme.njk} (100%) rename x-pack/plugins/integration_assistant/server/templates/{package_rdm.md.njk => package_readme.md.njk} (92%) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index 143f056c1a9f2..5467a1549cea2 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -17,12 +17,14 @@ export function createReadme(packageDir: string, integrationName: string, fields function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) { const dirPath = joinPath(packageDir, 'docs/'); - createReadmeFile(dirPath, 'package_rdm.md.njk', integrationName, fields); + // The readme nunjucks template files should be named in the format `somename_readme.md.njk` and not just `readme.md.njk` + // since any file with `readme.*` pattern is skipped in build process in buildkite. + createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, fields); } function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) { const dirPath = joinPath(packageDir, '_dev/build/docs/'); - createReadmeFile(dirPath, 'build_rdm.md.njk', integrationName, fields); + createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, fields); } function createReadmeFile( diff --git a/x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk similarity index 83% rename from x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk rename to x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk index 8c490fefe28da..1b58e55aebd37 100644 --- a/x-pack/plugins/integration_assistant/server/templates/build_rdm.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk @@ -1,4 +1,4 @@ -{% include "./description_rdm.njk" %} +{% include "./description_readme.njk" %} {% for data_stream in fields %} ### {{ data_stream.datastream }} diff --git a/x-pack/plugins/integration_assistant/server/templates/description_rdm.njk b/x-pack/plugins/integration_assistant/server/templates/description_readme.njk similarity index 100% rename from x-pack/plugins/integration_assistant/server/templates/description_rdm.njk rename to x-pack/plugins/integration_assistant/server/templates/description_readme.njk diff --git a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk similarity index 92% rename from x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk rename to x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk index 035f342d90d89..bd56aba5ac1e5 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_rdm.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk @@ -1,4 +1,4 @@ -{% include "./description_rdm.njk" %} +{% include "./description_readme.njk" %} {% for data_stream in fields %} ### {{ data_stream.datastream }}