Merge branch 'main' into oas-schema-rules-api

.buildkite/ftr_configs.yml

@@ -11,12 +11,13 @@ disabled:
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.trial.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/config.base.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.base.ts
-  - x-pack/test/security_solution_endpoint/config.base.ts
-  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/security_solution_endpoint/configs/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -408,6 +409,10 @@ enabled:
   - x-pack/test/security_functional/insecure_cluster_warning.config.ts
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
+  - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -578,9 +583,17 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.trial.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/integrations.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml

@@ -1,18 +1,143 @@
 steps:
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run
-    label: "Cypress MKI - Defend Workflows "
-    key: test_defend_workflows
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      enableNestedVirtualization: true
-      localSsds: 1
-      localSsdInterface: nvme
-      machineType: n2-standard-4
-    timeout_in_minutes: 300
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: "*"
-          limit: 1
+  - group: "Cypress MKI - Defend Workflows"
+    key: cypress_test_defend_workflows
+    steps:
+      - label: "Running cypress:dw:qa:serverless:run"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run
+        key: test_defend_workflows
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 300
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1
+
+  - group: "API MKI - Defend Workflows"
+    key: api_test_defend_workflows
+    steps:
+#      - label: "Running edr_workflows:artifacts:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:artifacts:qa:serverless
+#        key: edr_workflows:artifacts:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:authentication:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:authentication:qa:serverless
+#        key: edr_workflows:authentication:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:metadata:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:metadata:qa:serverless
+#        key: edr_workflows:metadata:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:package:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:package:qa:serverless
+#        key: edr_workflows:package:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+
+      - label: "Running edr_workflows:policy_response:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:policy_response:qa:serverless
+        key: edr_workflows:policy_response:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1
+
+      - label: "Running edr_workflows:resolver:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:resolver:qa:serverless
+        key: edr_workflows:resolver:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1
+
+      - label: "Running edr_workflows:response_actions:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:response_actions:qa:serverless
+        key: edr_workflows:response_actions:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1

.buildkite/scripts/common/env.sh

@@ -47,7 +47,6 @@ export MERGE_QUEUE_TARGET_BRANCH
 BUILDKITE_BRANCH_MERGE_QUEUE="${MERGE_QUEUE_TARGET_BRANCH:-${BUILDKITE_BRANCH:-}}"
 export BUILDKITE_BRANCH_MERGE_QUEUE
 
-
 BUILDKITE_AGENT_GCP_REGION=""
 if [[ "$(curl -is metadata.google.internal || true)" ]]; then
   # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
@@ -62,7 +61,6 @@ fi
 
 export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
 export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
-export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
 export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX/cypress"
 
 export NODE_OPTIONS="--max-old-space-size=4096"
@@ -133,15 +131,19 @@ export TEST_GROUP_TYPE_FUNCTIONAL="Functional Tests"
 export GH_REPO=github.com/elastic/kibana
 
 FTR_ENABLE_FIPS_AGENT=false
-# used by FIPS agents to link FIPS OpenSSL modules
 if [[ "${KBN_ENABLE_FIPS:-}" == "true" ]] || is_pr_with_label "ci:enable-fips-agent"; then
   FTR_ENABLE_FIPS_AGENT=true
+  # used by FIPS agents to link FIPS OpenSSL modules
   export OPENSSL_MODULES=$HOME/openssl/lib/ossl-modules
 
   if [[ -f "$KIBANA_DIR/config/node.options" ]]; then
     echo -e '\n--enable-fips' >>"$KIBANA_DIR/config/node.options"
     echo "--openssl-config=$HOME/nodejs.cnf" >>"$KIBANA_DIR/config/node.options"
   fi
+
+  if [[ -f "$KIBANA_DIR/config/kibana.yml" ]]; then
+    echo -e '\nxpack.security.experimental.fipsMode.enabled: true' >>"$KIBANA_DIR/config/kibana.yml"
+  fi
 fi
 
 export FTR_ENABLE_FIPS_AGENT

.buildkite/scripts/common/util.sh

@@ -33,7 +33,7 @@ check_for_changed_files() {
 
   SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
   CUSTOM_FIX_MESSAGE="${3:-}"
-  GIT_CHANGES="$(git status --porcelain -- . ':!:.bazelrc' ':!:config/node.options')"
+  GIT_CHANGES="$(git status --porcelain -- . ':!:.bazelrc' ':!:config/node.options' ':!config/kibana.yml')"
 
   if [ "$GIT_CHANGES" ]; then
     if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
@@ -56,7 +56,7 @@ check_for_changed_files() {
       git config --global user.name kibanamachine
       git config --global user.email '42973632+kibanamachine@users.noreply.github.com'
       gh pr checkout "${BUILDKITE_PULL_REQUEST}"
-      git add -A -- . ':!.bazelrc' ':!config/node.options'
+      git add -A -- . ':!.bazelrc' ':!config/node.options' ':!config/kibana.yml'
 
       git commit -m "$NEW_COMMIT_MESSAGE"
       git push

.buildkite/scripts/steps/artifacts/publish.sh

@@ -54,39 +54,51 @@ echo "--- Pull latest Release Manager CLI"
 docker pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
-if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
+if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then
   export VAULT_ROLE_ID="$(get_vault_role_id)"
   export VAULT_SECRET_ID="$(get_vault_secret_id)"
   export VAULT_ADDR="https://secrets.elastic.co:8200"
 
   download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json)
 
-  PUBLISH_CMD=$(cat << EOF
-  docker run --rm \
-    --name release-manager \
-    -e VAULT_ADDR \
-    -e VAULT_ROLE_ID \
-    -e VAULT_SECRET_ID \
-    --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
-    docker.elastic.co/infra/release-manager:latest \
-      cli collect \
-        --project kibana \
-        --branch "$KIBANA_BASE_BRANCH" \
-        --commit "$GIT_COMMIT" \
-        --workflow "$WORKFLOW" \
-        --version "$BASE_VERSION" \
-        --qualifier "$VERSION_QUALIFIER" \
-        --dependency "beats:$BEATS_MANIFEST_URL" \
-        --artifact-set main
-EOF
-)
-  if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then
-    PUBLISH_CMD+=(" --dry-run")
+  if [[ "$DRY_RUN" =~ ^(1|true)$ ]]; then
+      docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR \
+        -e VAULT_ROLE_ID \
+        -e VAULT_SECRET_ID \
+        --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+        docker.elastic.co/infra/release-manager:latest \
+          cli collect \
+            --project kibana \
+            --branch "$KIBANA_BASE_BRANCH" \
+            --commit "$GIT_COMMIT" \
+            --workflow "$WORKFLOW" \
+            --version "$BASE_VERSION" \
+            --qualifier "$VERSION_QUALIFIER" \
+            --dependency "beats:$BEATS_MANIFEST_URL" \
+            --artifact-set main \
+            --dry-run
+  else
+      docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR \
+        -e VAULT_ROLE_ID \
+        -e VAULT_SECRET_ID \
+        --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+        docker.elastic.co/infra/release-manager:latest \
+          cli collect \
+            --project kibana \
+            --branch "$KIBANA_BASE_BRANCH" \
+            --commit "$GIT_COMMIT" \
+            --workflow "$WORKFLOW" \
+            --version "$BASE_VERSION" \
+            --qualifier "$VERSION_QUALIFIER" \
+            --dependency "beats:$BEATS_MANIFEST_URL" \
+            --artifact-set main
   fi
 
-  "${PUBLISH_CMD[@]}"
-
   KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url')
 
   cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary

.buildkite/scripts/steps/code_generation/security_solution_codegen.sh

@@ -6,17 +6,22 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Security Solution OpenAPI Code Generation
 
-echo OpenAPI Common Package
+echo -e "\n[Security Solution OpenAPI Code Generation] OpenAPI Common Package"
 
 (cd packages/kbn-openapi-common && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true
 
-echo Lists API Common Package
+echo -e "\n[Security Solution OpenAPI Code Generation] Lists Common Package\n"
 
 (cd packages/kbn-securitysolution-lists-common && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true
 
-echo Security Solution Plugin
+echo -e "\n[Security Solution OpenAPI Code Generation] Exceptions Common Package"
+
+(cd packages/kbn-securitysolution-exceptions-common && yarn openapi:generate)
+check_for_changed_files "yarn openapi:generate" true
+
+echo -e "\n[Security Solution OpenAPI Code Generation] Security Solution Plugin"
 
 (cd x-pack/plugins/security_solution && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true