From bc5a97fa159208311cef5b484b038d6dec90dccb Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:12:22 -0400 Subject: [PATCH 01/31] [updatecli] update elastic stack version for testing 8.14.0-14244116 (#38122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine Co-authored-by: Denis --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 859e94b0672..7f78746abe0 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-b9699c81-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-faa2d2d3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-b9699c81-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-faa2d2d3-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-b9699c81-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-faa2d2d3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From fb2594a36c7c0e7aa973f7f44e990938506ee801 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:17:28 +1030 Subject: [PATCH 02/31] x-pack/filebeat/input/http_endpoint: generalise event path using CEL (#38193) Allow users to handle events where multiple events are provided via single JSON objects in an array field. This is enabled by providing an optional CEL program with only minimal support for CEL evaluations; no extensions from the mito lib extensions are made available. --- CHANGELOG.next.asciidoc | 2 + .../docs/inputs/input-http-endpoint.asciidoc | 5 + x-pack/filebeat/input/http_endpoint/config.go | 1 + .../filebeat/input/http_endpoint/handler.go | 113 +++++++++++++++++- .../input/http_endpoint/handler_test.go | 38 +++++- x-pack/filebeat/input/http_endpoint/input.go | 15 ++- 6 files changed, 163 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 24ccc48ffdc..7fa92411e0f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -201,6 +201,8 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add logging for cache processor file reads and writes. {pull}38052[38052] - Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] - Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] +- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] +- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc b/x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc index b7a7ee06f70..7f3050d1f68 100644 --- a/x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc @@ -227,6 +227,11 @@ The prefix for the signature. Certain webhooks prefix the HMAC signature with a By default the input expects the incoming POST to include a Content-Type of `application/json` to try to enforce the incoming data to be valid JSON. In certain scenarios when the source of the request is not able to do that, it can be overwritten with another value or set to null. +[float] +==== `program` + +The normal operation of the input treats the body either as a single event when the body is an object, or as a set of events when the body is an array. If the body should be treated handled differently, for example a set of events in an array field of an object to be handled as a set of events, then a https://opensource.google.com/projects/cel[Common Expression Language (CEL)] program can be provided through this configuration field. No CEL extensions are provided beyond the function in the CEL https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard[standard library]. CEL https://pkg.go.dev/github.com/google/cel-go/cel#OptionalTypes[optional types] are supported. + [float] ==== `response_code` diff --git a/x-pack/filebeat/input/http_endpoint/config.go b/x-pack/filebeat/input/http_endpoint/config.go index 3b0c97741de..1618dc90758 100644 --- a/x-pack/filebeat/input/http_endpoint/config.go +++ b/x-pack/filebeat/input/http_endpoint/config.go @@ -37,6 +37,7 @@ type config struct { URL string `config:"url" validate:"required"` Prefix string `config:"prefix"` ContentType string `config:"content_type"` + Program string `config:"program"` SecretHeader string `config:"secret.header"` SecretValue string `config:"secret.value"` HMACHeader string `config:"hmac.header"` diff --git a/x-pack/filebeat/input/http_endpoint/handler.go b/x-pack/filebeat/input/http_endpoint/handler.go index 0e2620b5b65..3d0948489ac 100644 --- a/x-pack/filebeat/input/http_endpoint/handler.go +++ b/x-pack/filebeat/input/http_endpoint/handler.go @@ -12,10 +12,16 @@ import ( "io" "net" "net/http" + "reflect" "time" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/checker/decls" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "google.golang.org/protobuf/types/known/structpb" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/beat" @@ -24,6 +30,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/mito/lib" ) const headerContentEncoding = "Content-Encoding" @@ -43,6 +50,7 @@ type handler struct { reqLogger *zap.Logger host, scheme string + program *program messageField string responseCode int responseBody string @@ -80,7 +88,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Body = io.NopCloser(&buf) } - objs, _, status, err := httpReadJSON(body) + objs, _, status, err := httpReadJSON(body, h.program) if err != nil { h.sendAPIErrorResponse(w, r, h.log, status, err) h.metrics.apiErrors.Add(1) @@ -218,22 +226,22 @@ func (h *handler) publishEvent(obj, headers mapstr.M) error { return nil } -func httpReadJSON(body io.Reader) (objs []mapstr.M, rawMessages []json.RawMessage, status int, err error) { +func httpReadJSON(body io.Reader, prg *program) (objs []mapstr.M, rawMessages []json.RawMessage, status int, err error) { if body == http.NoBody { return nil, nil, http.StatusNotAcceptable, errBodyEmpty } - obj, rawMessage, err := decodeJSON(body) + obj, rawMessage, err := decodeJSON(body, prg) if err != nil { return nil, nil, http.StatusBadRequest, err } return obj, rawMessage, http.StatusOK, err } -func decodeJSON(body io.Reader) (objs []mapstr.M, rawMessages []json.RawMessage, err error) { +func decodeJSON(body io.Reader, prg *program) (objs []mapstr.M, rawMessages []json.RawMessage, err error) { decoder := json.NewDecoder(body) for decoder.More() { var raw json.RawMessage - if err := decoder.Decode(&raw); err != nil { + if err = decoder.Decode(&raw); err != nil { if err == io.EOF { //nolint:errorlint // This will never be a wrapped error. break } @@ -241,9 +249,22 @@ func decodeJSON(body io.Reader) (objs []mapstr.M, rawMessages []json.RawMessage, } var obj interface{} - if err := newJSONDecoder(bytes.NewReader(raw)).Decode(&obj); err != nil { + if err = newJSONDecoder(bytes.NewReader(raw)).Decode(&obj); err != nil { return nil, nil, fmt.Errorf("malformed JSON object at stream position %d: %w", decoder.InputOffset(), err) } + + if prg != nil { + obj, err = prg.eval(obj) + if err != nil { + return nil, nil, err + } + // Re-marshal to ensure the raw bytes agree with the constructed object. + raw, err = json.Marshal(obj) + if err != nil { + return nil, nil, fmt.Errorf("failed to remarshal object: %w", err) + } + } + switch v := obj.(type) { case map[string]interface{}: objs = append(objs, v) @@ -265,6 +286,86 @@ func decodeJSON(body io.Reader) (objs []mapstr.M, rawMessages []json.RawMessage, return objs, rawMessages, nil } +type program struct { + prg cel.Program + ast *cel.Ast +} + +func newProgram(src string) (*program, error) { + if src == "" { + return nil, nil + } + + registry, err := types.NewRegistry() + if err != nil { + return nil, fmt.Errorf("failed to create env: %w", err) + } + env, err := cel.NewEnv( + cel.Declarations(decls.NewVar("obj", decls.Dyn)), + cel.OptionalTypes(cel.OptionalTypesVersion(lib.OptionalTypesVersion)), + cel.CustomTypeAdapter(&numberAdapter{registry}), + cel.CustomTypeProvider(registry), + ) + if err != nil { + return nil, fmt.Errorf("failed to create env: %w", err) + } + + ast, iss := env.Compile(src) + if iss.Err() != nil { + return nil, fmt.Errorf("failed compilation: %w", iss.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, fmt.Errorf("failed program instantiation: %w", err) + } + return &program{prg: prg, ast: ast}, nil +} + +var _ types.Adapter = (*numberAdapter)(nil) + +type numberAdapter struct { + fallback types.Adapter +} + +func (a *numberAdapter) NativeToValue(value any) ref.Val { + if n, ok := value.(json.Number); ok { + var errs []error + i, err := n.Int64() + if err == nil { + return types.Int(i) + } + errs = append(errs, err) + f, err := n.Float64() + if err == nil { + return types.Double(f) + } + errs = append(errs, err) + return types.NewErr("%v", errors.Join(errs...)) + } + return a.fallback.NativeToValue(value) +} + +func (p *program) eval(obj interface{}) (interface{}, error) { + out, _, err := p.prg.Eval(map[string]interface{}{"obj": obj}) + if err != nil { + err = lib.DecoratedError{AST: p.ast, Err: err} + return nil, fmt.Errorf("failed eval: %w", err) + } + + v, err := out.ConvertToNative(reflect.TypeOf((*structpb.Value)(nil))) + if err != nil { + return nil, fmt.Errorf("failed proto conversion: %w", err) + } + switch v := v.(type) { + case *structpb.Value: + return v.AsInterface(), nil + default: + // This should never happen. + return nil, fmt.Errorf("unexpected native conversion type: %T", v) + } +} + func decodeJSONArray(raw *bytes.Reader) (objs []mapstr.M, rawMessages []json.RawMessage, err error) { dec := newJSONDecoder(raw) token, err := dec.Token() diff --git a/x-pack/filebeat/input/http_endpoint/handler_test.go b/x-pack/filebeat/input/http_endpoint/handler_test.go index 6660508b15b..cb911f8ab18 100644 --- a/x-pack/filebeat/input/http_endpoint/handler_test.go +++ b/x-pack/filebeat/input/http_endpoint/handler_test.go @@ -38,6 +38,7 @@ func Test_httpReadJSON(t *testing.T) { tests := []struct { name string body string + program string wantObjs []mapstr.M wantStatus int wantErr bool @@ -135,10 +136,43 @@ func Test_httpReadJSON(t *testing.T) { }, wantStatus: http.StatusOK, }, + { + name: "kinesis", + body: `{ + "requestId": "ed4acda5-034f-9f42-bba1-f29aea6d7d8f", + "timestamp": 1578090901599, + "records": [ + { + "data": "aGVsbG8=" + }, + { + "data": "aGVsbG8gd29ybGQ=" + } + ] +}`, + program: `obj.records.map(r, { + "requestId": obj.requestId, + "timestamp": string(obj.timestamp), // leave timestamp in unix milli for ingest to handle. + "event": r, + })`, + wantRawMessage: []json.RawMessage{ + []byte(`{"event":{"data":"aGVsbG8="},"requestId":"ed4acda5-034f-9f42-bba1-f29aea6d7d8f","timestamp":"1578090901599"}`), + []byte(`{"event":{"data":"aGVsbG8gd29ybGQ="},"requestId":"ed4acda5-034f-9f42-bba1-f29aea6d7d8f","timestamp":"1578090901599"}`), + }, + wantObjs: []mapstr.M{ + {"event": map[string]any{"data": "aGVsbG8="}, "requestId": "ed4acda5-034f-9f42-bba1-f29aea6d7d8f", "timestamp": "1578090901599"}, + {"event": map[string]any{"data": "aGVsbG8gd29ybGQ="}, "requestId": "ed4acda5-034f-9f42-bba1-f29aea6d7d8f", "timestamp": "1578090901599"}, + }, + wantStatus: http.StatusOK, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotObjs, rawMessages, gotStatus, err := httpReadJSON(strings.NewReader(tt.body)) + prg, err := newProgram(tt.program) + if err != nil { + t.Fatalf("failed to compile program: %v", err) + } + gotObjs, rawMessages, gotStatus, err := httpReadJSON(strings.NewReader(tt.body), prg) if (err != nil) != tt.wantErr { t.Errorf("httpReadJSON() error = %v, wantErr %v", err, tt.wantErr) return @@ -344,7 +378,7 @@ func Test_apiResponse(t *testing.T) { pub := new(publisher) metrics := newInputMetrics("") defer metrics.Close() - apiHandler := newHandler(ctx, tracerConfig(tc.name, tc.conf, *withTraces), pub, logp.NewLogger("http_endpoint.test"), metrics) + apiHandler := newHandler(ctx, tracerConfig(tc.name, tc.conf, *withTraces), nil, pub, logp.NewLogger("http_endpoint.test"), metrics) // Execute handler. respRec := httptest.NewRecorder() diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index ca648b69747..7d5055ebe65 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -131,6 +131,14 @@ func (p *pool) serve(ctx v2.Context, e *httpEndpoint, pub stateless.Publisher, m metrics.route.Set(u.Path) metrics.isTLS.Set(e.tlsConfig != nil) + var prg *program + if e.config.Program != "" { + prg, err = newProgram(e.config.Program) + if err != nil { + return err + } + } + p.mu.Lock() s, ok := p.servers[e.addr] if ok { @@ -149,7 +157,7 @@ func (p *pool) serve(ctx v2.Context, e *httpEndpoint, pub stateless.Publisher, m return err } log.Infof("Adding %s end point to server on %s", pattern, e.addr) - s.mux.Handle(pattern, newHandler(s.ctx, e.config, pub, log, metrics)) + s.mux.Handle(pattern, newHandler(s.ctx, e.config, prg, pub, log, metrics)) s.idOf[pattern] = ctx.ID p.mu.Unlock() <-s.ctx.Done() @@ -165,7 +173,7 @@ func (p *pool) serve(ctx v2.Context, e *httpEndpoint, pub stateless.Publisher, m srv: srv, } s.ctx, s.cancel = ctxtool.WithFunc(ctx.Cancelation, func() { srv.Close() }) - mux.Handle(pattern, newHandler(s.ctx, e.config, pub, log, metrics)) + mux.Handle(pattern, newHandler(s.ctx, e.config, prg, pub, log, metrics)) p.servers[e.addr] = s p.mu.Unlock() @@ -287,7 +295,7 @@ func (s *server) getErr() error { return s.err } -func newHandler(ctx context.Context, c config, pub stateless.Publisher, log *logp.Logger, metrics *inputMetrics) http.Handler { +func newHandler(ctx context.Context, c config, prg *program, pub stateless.Publisher, log *logp.Logger, metrics *inputMetrics) http.Handler { h := &handler{ log: log, publisher: pub, @@ -305,6 +313,7 @@ func newHandler(ctx context.Context, c config, pub stateless.Publisher, log *log hmacType: c.HMACType, hmacPrefix: c.HMACPrefix, }, + program: prg, messageField: c.Prefix, responseCode: c.ResponseCode, responseBody: c.ResponseBody, From e0d5fe5fefaf3ad12cf68b7c89f3da4b4f435cae Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:05:05 +0200 Subject: [PATCH 03/31] Migrate xpack packetbeat (#38135) * migrate xpack-metricbeat --------- Co-authored-by: Victor Martinez Co-authored-by: Julien Lind --- .buildkite/hooks/pre-command | 10 +- .buildkite/scripts/common.sh | 5 + .../generate_xpack_metricbeat_pipeline.sh | 11 +- .../generate_xpack_packetbeat_pipeline.sh | 195 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 2 +- .buildkite/scripts/win_unit_tests.ps1 | 42 +++- .../x-pack/pipeline.xpack.packetbeat.yml | 60 +++++- x-pack/packetbeat/magefile.go | 10 +- 8 files changed, 318 insertions(+), 17 deletions(-) create mode 100644 .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 282fb5a0085..c23f5c3af64 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,6 +3,7 @@ set -euo pipefail AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" retry() { local retries=$1 @@ -33,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" @@ -47,3 +48,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_ACCESS_KEY fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + export PRIVATE_CI_GCS_CREDENTIALS_SECRET + fi +fi diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ebb15c937dd..d6a91a48243 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -22,6 +22,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" +[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" +[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" metricbeat_changeset=( "^metricbeat/.*" @@ -105,6 +107,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; + "beats-xpack-packetbeat") + BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} + ;; *) echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c9c65a5e757..af116b17209 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -31,8 +31,6 @@ steps: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -42,8 +40,6 @@ steps: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":windows: Windows Unit Tests - {{matrix.image}}" @@ -106,8 +102,7 @@ else fi #TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then -# if [[ are_conditions_met_macos_tests ]]; then +if are_conditions_met_aws_tests || are_conditions_met_macos_tests ; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -140,8 +135,6 @@ if are_conditions_met_aws_tests; then provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML @@ -166,6 +159,8 @@ if are_conditions_met_packaging; then provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh new file mode 100644 index 00000000000..4eb2a1c3e04 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-packetbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":linux: Ubuntu System Tests" + key: "mandatory-linux-system-test" + command: "cd $BEATS_PROJECT_NAME && mage systemTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + # - label: ":windows: Windows 2022 System Tests" + # key: "mandatory-win-2022-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2022}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + # - label: ":windows: Windows 10 System Tests" + # key: "extended-win-10-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 974886061d4..29a8a05446e 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,9 +11,9 @@ SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" -ASDF_TERRAFORM_VERSION="1.0.2" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +ASDF_TERRAFORM_VERSION="1.0.2" AWS_REGION="eu-central-1" export SETUP_GVM_VERSION diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index b3c5c58fac0..b61e4107c3c 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -1,3 +1,7 @@ +param( + [string]$testType = "unittest" +) + $ErrorActionPreference = "Stop" # set -e $WorkFolder = $env:BEATS_PROJECT_NAME $WORKSPACE = Get-Location @@ -120,6 +124,23 @@ function withNmap($version) { } Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait } +function google_cloud_auth { + $tempFileName = "google-cloud-credentials.json" + $secretFileLocation = Join-Path $env:TEMP $tempFileName + $null = New-Item -ItemType File -Path $secretFileLocation + Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET + gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1 + $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation +} + +function google_cloud_auth_cleanup { + if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { + Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force + Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS + } else { + Write-Host "No GCP credentials were added" + } +} fixCRLF @@ -142,10 +163,23 @@ $env:MAGEFILE_CACHE = $magefile New-Item -ItemType Directory -Force -Path "build" -if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { - mage -w reader/etw build goUnitTest -} else { - mage build unitTest +if ($testType -eq "unittest") { + if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { + mage -w reader/etw build goUnitTest + } else { + mage build unitTest + } +} +elseif ($testType -eq "systemtest") { + try { + google_cloud_auth + mage systemTest + } finally { + google_cloud_auth_cleanup + } +} +else { + Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." } $EXITCODE=$LASTEXITCODE diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 34321b61161..750b59e716d 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -1,5 +1,61 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-packetbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/packetbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Packetbeat - run_xpack_packetbeat" + key: "run_xpack_packetbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_packetbeat_macos_tests" + key: "run_xpack_packetbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_packetbeat_arm_tests" + key: "run_xpack_packetbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 03104ab9157..357e5e23585 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -172,6 +172,13 @@ func SystemTest(ctx context.Context) error { return devtools.GoTest(ctx, args) } +func getBucketName() string { + if os.Getenv("BUILDKITE") == "true" { + return "ingest-buildkite-ci" + } + return "obs-ci-cache" +} + // getNpcapInstaller gets the installer from the Google Cloud Storage service. // // On Windows platforms, if getNpcapInstaller is invoked with the environment variables @@ -198,7 +205,8 @@ func getNpcapInstaller() error { return err } } + ciBucketName := getBucketName() fmt.Printf("getting %s from private cache\n", installer) - return sh.RunV("gsutil", "cp", "gs://obs-ci-cache/private/"+installer, dstPath) + return sh.RunV("gsutil", "cp", "gs://"+ciBucketName+"/private/"+installer, dstPath) } From 1e05407b636800139bf59889c88cd6831ab620c2 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 14 Mar 2024 10:07:17 +0200 Subject: [PATCH 04/31] Use native support BK-pr-bot support for merge commits (#38305) When implementing the migration from Jenkins to Buildkite, we implemented the default behavior of Jenkins GitHub Pull Request plugin i.e. to use merge commits, rather than HEAD, via a post-checkout script. Since then, native support has been added[^1], so there's no need for this script anymore. This commit forces the BK PR Bot to use merge commits. [^1]: https://github.com/elastic/buildkite-pr-bot/pull/7 --- .buildkite/hooks/post-checkout | 53 -------------------------------- .buildkite/pull-requests.json | 56 +++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 67 deletions(-) delete mode 100644 .buildkite/hooks/post-checkout diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout deleted file mode 100644 index b6cc7ad60bd..00000000000 --- a/.buildkite/hooks/post-checkout +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -checkout_merge() { - local target_branch=$1 - local pr_commit=$2 - local merge_branch=$3 - - if [[ -z "${target_branch}" ]]; then - echo "No pull request target branch" - exit 1 - fi - - git fetch -v origin "${target_branch}" - git checkout FETCH_HEAD - echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" - - # create temporal branch to merge the PR with the target branch - git checkout -b ${merge_branch} - echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" - - # set author identity so it can be run git merge - git config user.name "github-merged-pr-post-checkout" - git config user.email "auto-merge@buildkite" - - git merge --no-edit "${BUILDKITE_COMMIT}" || { - local merge_result=$? - echo "Merge failed: ${merge_result}" - git merge --abort - exit ${merge_result} - } -} - -pull_request="${BUILDKITE_PULL_REQUEST:-false}" - -if [[ "${pull_request}" == "false" ]]; then - echo "Not a pull request, skipping" - exit 0 -fi - -TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" -PR_COMMIT="${BUILDKITE_COMMIT}" -PR_ID=${BUILDKITE_PULL_REQUEST} -MERGE_BRANCH="pr_merge_${PR_ID}" - -checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" - -echo "Commit information" -git --no-pager log --format=%B -n 1 - -# Ensure buildkite groups are rendered -echo "" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8018411a743..26f785b7522 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -14,7 +14,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ ] + "always_require_ci_on_changed": [ ], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -30,7 +32,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ] + "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -46,7 +50,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -62,7 +68,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -78,7 +86,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -94,7 +104,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"] + "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -110,7 +122,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -126,7 +140,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -142,7 +158,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], - "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -158,7 +176,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -174,7 +194,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/winlogbeat/.*", ".buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "always_require_ci_on_changed": ["^x-pack/winlogbeat/.*", ".buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -190,7 +212,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -206,7 +230,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true }, { "enabled": true, @@ -222,7 +248,9 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], + "use_merge_commit": true, + "fail_on_not_mergeable": true } ] } From ffae5035204b2e0518c1b130c8e595752e56466c Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 14 Mar 2024 11:02:34 +0200 Subject: [PATCH 05/31] Revert "Use native support BK-pr-bot support for merge commits (#38305)" (#38317) After merging #38305 we've hit an issue with the commit that Buildkite PR Bot uses. e.g. using #38316 we see it's still using: ``` git fetch -v --prune -- origin refs/pull/38316/head ``` rather than the merge commit. Reverting for now, until a fix[^1] in the Buildkite PR bot has been merged. [^1]: https://github.com/elastic/buildkite-pr-bot/pull/12 --- .buildkite/hooks/post-checkout | 53 ++++++++++++++++++++++++++++++++ .buildkite/pull-requests.json | 56 +++++++++------------------------- 2 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 .buildkite/hooks/post-checkout diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 00000000000..b6cc7ad60bd --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git --no-pager log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 26f785b7522..8018411a743 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -14,9 +14,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ ], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": [ ] }, { "enabled": true, @@ -32,9 +30,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ] }, { "enabled": true, @@ -50,9 +46,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -68,9 +62,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -86,9 +78,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -104,9 +94,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"] }, { "enabled": true, @@ -122,9 +110,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -140,9 +126,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -158,9 +142,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], - "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -176,9 +158,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, @@ -194,9 +174,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/winlogbeat/.*", ".buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^x-pack/winlogbeat/.*", ".buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, @@ -212,9 +190,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, @@ -230,9 +206,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, @@ -248,9 +222,7 @@ "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"], - "use_merge_commit": true, - "fail_on_not_mergeable": true + "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } From 5bd33b7f3f93c7d09d31e93df6482bdd8ab3ad5c Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:54:48 +0200 Subject: [PATCH 06/31] xpack pipelines initialization (#38319) * xpack pipelines initialization * update yaml-language-server --- .buildkite/pull-requests.json | 80 +++++ .../x-pack/pipeline.xpack.auditbeat.yml | 6 + .../x-pack/pipeline.xpack.dockerlogbeat.yml | 6 + .buildkite/x-pack/pipeline.xpack.filebeat.yml | 6 + .../x-pack/pipeline.xpack.heartbeat.yml | 6 + .../x-pack/pipeline.xpack.osquerybeat.yml | 6 + catalog-info.yaml | 294 ++++++++++++++++-- 7 files changed, 372 insertions(+), 32 deletions(-) create mode 100644 .buildkite/x-pack/pipeline.xpack.auditbeat.yml create mode 100644 .buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml create mode 100644 .buildkite/x-pack/pipeline.xpack.filebeat.yml create mode 100644 .buildkite/x-pack/pipeline.xpack.heartbeat.yml create mode 100644 .buildkite/x-pack/pipeline.xpack.osquerybeat.yml diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8018411a743..669b70d6570 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -223,6 +223,86 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-auditbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/auditbeat$", + "always_trigger_comment_regex": "^/test x-pack/auditbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-dockerlogbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/dockerlogbeat$", + "always_trigger_comment_regex": "^/test x-pack/dockerlogbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/dockerlogbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-filebeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/filebeat$", + "always_trigger_comment_regex": "^/test x-pack/filebeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-heartbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/heartbeat$", + "always_trigger_comment_regex": "^/test x-pack/heartbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-osquerybeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/osquerybeat$", + "always_trigger_comment_regex": "^/test x-pack/osquerybeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml new file mode 100644 index 00000000000..2343eb6a4dd --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-auditbeat" + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml new file mode 100644 index 00000000000..c4a0805615b --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-dockerlogbeat" + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml new file mode 100644 index 00000000000..6d3d7d9daee --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-filebeat" + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml new file mode 100644 index 00000000000..65175d3b029 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-heartbeat" + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml new file mode 100644 index 00000000000..22297e33ab5 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-osquerybeat" + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/catalog-info.yaml b/catalog-info.yaml index f52f80df3c7..7f99ab9ce83 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -109,7 +109,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -155,7 +155,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -201,7 +201,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -247,7 +247,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -293,7 +293,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -339,7 +339,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -385,7 +385,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -430,7 +430,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -476,7 +476,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -497,9 +497,9 @@ spec: name: beats-xpack-winlogbeat description: "Beats x-pack winlogbeat pipeline" spec: - # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.winlogbeat.yml" - # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + maximum_timeout_in_minutes: 120 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false @@ -513,8 +513,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -522,7 +522,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -543,9 +543,9 @@ spec: name: beats-xpack-packetbeat description: "Beats x-pack packetbeat pipeline" spec: - # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.packetbeat.yml" - # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + maximum_timeout_in_minutes: 120 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false @@ -559,8 +559,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -568,7 +568,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -589,9 +589,9 @@ spec: name: beats-xpack-libbeat description: "Beats x-pack libbeat pipeline" spec: - # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.libbeat.yml" - # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + maximum_timeout_in_minutes: 120 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false @@ -605,8 +605,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -614,7 +614,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -635,9 +635,9 @@ spec: name: beats-xpack-metricbeat description: "Beats x-pack metricbeat pipeline" spec: - # branch_configuration: "7.17" #TODO: uncomment after tests + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" - maximum_timeout_in_minutes: 480 + maximum_timeout_in_minutes: 120 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false @@ -648,11 +648,11 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!7.17" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!7.17" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -660,7 +660,7 @@ spec: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -694,7 +694,7 @@ spec: everyone: access_level: READ_ONLY --- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: @@ -728,3 +728,233 @@ spec: access_level: MANAGE_BUILD_AND_READ everyone: access_level: BUILD_AND_READ + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-auditbeat + description: "Beats x-pack auditbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-auditbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-auditbeat + description: "Beats x-pack auditbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.auditbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-dockerlogbeat + description: "Beats x-pack dockerlogbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-dockerlogbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-dockerlogbeat + description: "Beats x-pack dockerlogbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-filebeat + description: "Beats x-pack filebeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-filebeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-filebeat + description: "Beats x-pack filebeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.filebeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-heartbeat + description: "Beats x-pack heartbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-heartbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-heartbeat + description: "Beats x-pack heartbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.heartbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-osquerybeat + description: "Beats x-pack osquerybeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-osquerybeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-osquerybeat + description: "Beats x-pack osquerybeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.osquerybeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY From e0675778b8eed65c6fff6a1d1911833c21563755 Mon Sep 17 00:00:00 2001 From: Yoel Spotts Date: Thu, 14 Mar 2024 10:25:16 -0400 Subject: [PATCH 07/31] support for Timestamp in file outputter path (#38029) * support for TIME_NOW var * fix import ordering * add documentation * add unit tests * fix linting error * path is a date time formatter * fmt * remove unneeded * update docs * add UTC * improve message * change link * create new Unpacker * fmt * description of the use case * unit tests --- libbeat/outputs/fileout/config.go | 28 +++-- libbeat/outputs/fileout/config_test.go | 100 ++++++++++++++++++ libbeat/outputs/fileout/docs/fileout.asciidoc | 8 ++ libbeat/outputs/fileout/file.go | 17 +-- libbeat/outputs/fileout/pathformatstring.go | 66 ++++++++++++ .../outputs/fileout/pathformatstring_test.go | 87 +++++++++++++++ 6 files changed, 290 insertions(+), 16 deletions(-) create mode 100644 libbeat/outputs/fileout/config_test.go create mode 100644 libbeat/outputs/fileout/pathformatstring.go create mode 100644 libbeat/outputs/fileout/pathformatstring_test.go diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index e72a9f87d6f..69af40e4289 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -26,14 +26,14 @@ import ( ) type fileOutConfig struct { - Path string `config:"path"` - Filename string `config:"filename"` - RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` - NumberOfFiles uint `config:"number_of_files"` - Codec codec.Config `config:"codec"` - Permissions uint32 `config:"permissions"` - RotateOnStartup bool `config:"rotate_on_startup"` - Queue config.Namespace `config:"queue"` + Path *PathFormatString `config:"path"` + Filename string `config:"filename"` + RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` + NumberOfFiles uint `config:"number_of_files"` + Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` + RotateOnStartup bool `config:"rotate_on_startup"` + Queue config.Namespace `config:"queue"` } func defaultConfig() fileOutConfig { @@ -45,6 +45,18 @@ func defaultConfig() fileOutConfig { } } +func readConfig(cfg *config.C) (*fileOutConfig, error) { + foConfig := defaultConfig() + if err := cfg.Unpack(&foConfig); err != nil { + return nil, err + } + + // disable bulk support in publisher pipeline + _ = cfg.SetInt("bulk_max_size", -1, -1) + + return &foConfig, nil +} + func (c *fileOutConfig) Validate() error { if c.NumberOfFiles < 2 || c.NumberOfFiles > file.MaxBackupsLimit { return fmt.Errorf("the number_of_files to keep should be between 2 and %v", diff --git a/libbeat/outputs/fileout/config_test.go b/libbeat/outputs/fileout/config_test.go new file mode 100644 index 00000000000..7e149173f6d --- /dev/null +++ b/libbeat/outputs/fileout/config_test.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fileout + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestConfig(t *testing.T) { + for name, test := range map[string]struct { + config *config.C + useWindowsPath bool + assertion func(t *testing.T, config *fileOutConfig, err error) + }{ + "default config": { + config: config.MustNewConfigFrom([]byte(`{ }`)), + assertion: func(t *testing.T, actual *fileOutConfig, err error) { + expectedConfig := &fileOutConfig{ + NumberOfFiles: 7, + RotateEveryKb: 10 * 1024, + Permissions: 0600, + RotateOnStartup: true, + } + + assert.Equal(t, expectedConfig, actual) + assert.Nil(t, err) + }, + }, + "config given with posix path": { + config: config.MustNewConfigFrom(mapstr.M{ + "number_of_files": 10, + "rotate_every_kb": 5 * 1024, + "path": "/tmp/packetbeat/%{+yyyy-MM-dd-mm-ss-SSSSSS}", + "filename": "pb", + }), + assertion: func(t *testing.T, actual *fileOutConfig, err error) { + assert.Equal(t, uint(10), actual.NumberOfFiles) + assert.Equal(t, uint(5*1024), actual.RotateEveryKb) + assert.Equal(t, true, actual.RotateOnStartup) + assert.Equal(t, uint32(0600), actual.Permissions) + assert.Equal(t, "pb", actual.Filename) + + path, runErr := actual.Path.Run(time.Date(2024, 1, 2, 3, 4, 5, 67890, time.UTC)) + assert.Nil(t, runErr) + + assert.Equal(t, "/tmp/packetbeat/2024-01-02-04-05-000067", path) + assert.Nil(t, err) + }, + }, + "config given with windows path": { + useWindowsPath: true, + config: config.MustNewConfigFrom(mapstr.M{ + "number_of_files": 10, + "rotate_every_kb": 5 * 1024, + "path": "c:\\tmp\\packetbeat\\%{+yyyy-MM-dd-mm-ss-SSSSSS}", + "filename": "pb", + }), + assertion: func(t *testing.T, actual *fileOutConfig, err error) { + assert.Equal(t, uint(10), actual.NumberOfFiles) + assert.Equal(t, uint(5*1024), actual.RotateEveryKb) + assert.Equal(t, true, actual.RotateOnStartup) + assert.Equal(t, uint32(0600), actual.Permissions) + assert.Equal(t, "pb", actual.Filename) + + path, runErr := actual.Path.Run(time.Date(2024, 1, 2, 3, 4, 5, 67890, time.UTC)) + assert.Nil(t, runErr) + + assert.Equal(t, "c:\\tmp\\packetbeat\\2024-01-02-04-05-000067", path) + assert.Nil(t, err) + }, + }, + } { + t.Run(name, func(t *testing.T) { + isWindowsPath = test.useWindowsPath + cfg, err := readConfig(test.config) + test.assertion(t, cfg, err) + }) + } +} diff --git a/libbeat/outputs/fileout/docs/fileout.asciidoc b/libbeat/outputs/fileout/docs/fileout.asciidoc index 54dfdd0772a..bb2a953ec75 100644 --- a/libbeat/outputs/fileout/docs/fileout.asciidoc +++ b/libbeat/outputs/fileout/docs/fileout.asciidoc @@ -49,6 +49,14 @@ The default value is `true`. The path to the directory where the generated files will be saved. This option is mandatory. +The path may include the timestamp when the file output is initialized using the `+FORMAT` syntax where `FORMAT` is a +valid https://github.com/elastic/beats/blob/{doc-branch}/libbeat/common/dtfmt/doc.go[time format], +and enclosed with expansion braces: `%{+FORMAT}`. For example: + +``` +path: 'fileoutput-%{+yyyy.MM.dd}' +``` + ===== `filename` The name of the generated files. The default is set to the Beat name. For example, the files diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 4ddc5955d6e..34c57f29791 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -52,20 +52,17 @@ func makeFileout( observer outputs.Observer, cfg *c.C, ) (outputs.Group, error) { - foConfig := defaultConfig() - if err := cfg.Unpack(&foConfig); err != nil { + foConfig, err := readConfig(cfg) + if err != nil { return outputs.Fail(err) } - // disable bulk support in publisher pipeline - _ = cfg.SetInt("bulk_max_size", -1, -1) - fo := &fileOutput{ log: logp.NewLogger("file"), beat: beat, observer: observer, } - if err := fo.init(beat, foConfig); err != nil { + if err = fo.init(beat, *foConfig); err != nil { return outputs.Fail(err) } @@ -74,10 +71,14 @@ func makeFileout( func (out *fileOutput) init(beat beat.Info, c fileOutConfig) error { var path string + configPath, runErr := c.Path.Run(time.Now().UTC()) + if runErr != nil { + return runErr + } if c.Filename != "" { - path = filepath.Join(c.Path, c.Filename) + path = filepath.Join(configPath, c.Filename) } else { - path = filepath.Join(c.Path, out.beat.Beat) + path = filepath.Join(configPath, out.beat.Beat) } out.filePath = path diff --git a/libbeat/outputs/fileout/pathformatstring.go b/libbeat/outputs/fileout/pathformatstring.go new file mode 100644 index 00000000000..acd2a7605fe --- /dev/null +++ b/libbeat/outputs/fileout/pathformatstring.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fileout + +import ( + "os" + "strings" + "time" + + "github.com/elastic/beats/v7/libbeat/common/fmtstr" + + "github.com/elastic/beats/v7/libbeat/beat" +) + +var isWindowsPath = os.PathSeparator == '\\' + +// PathFormatString is a wrapper around EventFormatString for the +// handling paths with a format expression that has access to the timestamp format. +// It has special handling for paths, specifically for windows path separator +// which would be interpreted as an escape character. This formatter double escapes +// the path separator so it is properly interpreted by the fmtstr processor +type PathFormatString struct { + efs *fmtstr.EventFormatString +} + +// Run executes the format string returning a new expanded string or an error +// if execution or event field expansion fails. +func (fs *PathFormatString) Run(timestamp time.Time) (string, error) { + placeholderEvent := &beat.Event{ + Timestamp: timestamp, + } + return fs.efs.Run(placeholderEvent) +} + +// Unpack tries to initialize the PathFormatString from provided value +// (which must be a string). Unpack method satisfies go-ucfg.Unpacker interface +// required by config.C, in order to use PathFormatString with +// `common.(*Config).Unpack()`. +func (fs *PathFormatString) Unpack(v interface{}) error { + path, ok := v.(string) + if !ok { + return nil + } + + if isWindowsPath { + path = strings.ReplaceAll(path, "\\", "\\\\") + } + + fs.efs = &fmtstr.EventFormatString{} + return fs.efs.Unpack(path) +} diff --git a/libbeat/outputs/fileout/pathformatstring_test.go b/libbeat/outputs/fileout/pathformatstring_test.go new file mode 100644 index 00000000000..b8eee4e44ea --- /dev/null +++ b/libbeat/outputs/fileout/pathformatstring_test.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fileout + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPathFormatString(t *testing.T) { + tests := []struct { + title string + useWindowsPath bool + format string + timestamp time.Time + expected string + }{ + { + "empty string", + false, + "", + time.Time{}, + "", + }, + { + "no fields configured", + false, + "format string", + time.Time{}, + "format string", + }, + { + "test timestamp formatter", + false, + "timestamp: %{+YYYY.MM.dd}", + time.Date(2015, 5, 1, 20, 12, 34, 0, time.UTC), + "timestamp: 2015.05.01", + }, + { + "test timestamp formatter with posix path", + false, + "/tmp/%{+YYYY.MM.dd}", + time.Date(2015, 5, 1, 20, 12, 34, 0, time.UTC), + "/tmp/2015.05.01", + }, + { + "test timestamp formatter with windows path", + true, + "C:\\tmp\\%{+YYYY.MM.dd}", + time.Date(2015, 5, 1, 20, 12, 34, 0, time.UTC), + "C:\\tmp\\2015.05.01", + }, + } + + for i, test := range tests { + t.Logf("test(%v): %v", i, test.title) + isWindowsPath = test.useWindowsPath + pfs := &PathFormatString{} + err := pfs.Unpack(test.format) + if err != nil { + t.Error(err) + continue + } + + actual, err := pfs.Run(test.timestamp) + + assert.NoError(t, err) + assert.Equal(t, test.expected, actual) + } +} From 361a62fc17bd077ec6115696d4f1bd03f1d12f4c Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:27:34 -0400 Subject: [PATCH 08/31] chore: Update snapshot.yml (#38330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 7f78746abe0..3e0cd8a6d90 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-faa2d2d3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-bd8c2be5-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-faa2d2d3-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-bd8c2be5-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-faa2d2d3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-bd8c2be5-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 9c9ae3568309bb4c715fe1c25834eafd06f0f82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Thu, 14 Mar 2024 20:21:44 +0100 Subject: [PATCH 09/31] [Heartbeat] Add prctl dumpable flag reset after cap drop (#38269) Enforce dumpable attribute on heartbeat process for /proc/io to be readable by elastic-agent. --------- Co-authored-by: Vignesh Shanmugam --- CHANGELOG.next.asciidoc | 1 + heartbeat/security/security.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7fa92411e0f..be217566c4c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,6 +107,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix setuid root when running under cgroups v2. {pull}37794[37794] - Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] +- Reset prctl dumpable flag after cap drop. {pull}38269[38269] *Metricbeat* diff --git a/heartbeat/security/security.go b/heartbeat/security/security.go index 8e15102f7b8..597e3a5bda9 100644 --- a/heartbeat/security/security.go +++ b/heartbeat/security/security.go @@ -26,6 +26,7 @@ import ( "strconv" "syscall" + "golang.org/x/sys/unix" "kernel.org/pub/linux/libs/security/libcap/cap" ) @@ -46,6 +47,9 @@ func init() { // The beat should use `getcap` at a later point to examine available capabilities // rather than relying on errors from `setcap` _ = setCapabilities() + + // Make heartbeat dumpable so elastic-agent can access process metrics. + _ = setDumpable() } func setNodeProcAttr(localUserName string) error { @@ -99,3 +103,13 @@ func setCapabilities() error { return nil } + +// Enforce PR_SET_DUMPABLE=true to allow user-level access to /proc//io. +func setDumpable() error { + _, err := cap.Prctl(unix.PR_SET_DUMPABLE, 1) + if err != nil { + return fmt.Errorf("error setting dumpable flag via prctl: %w", err) + } + + return nil +} From 9172e9c8254b20e2ca4c9e349ebca0b17c58d919 Mon Sep 17 00:00:00 2001 From: Gabriel Pop <94497545+gpop63@users.noreply.github.com> Date: Fri, 15 Mar 2024 21:54:26 +0200 Subject: [PATCH 10/31] [metricbeat][mysql] Add SSL support (#37997) * add config * add ssl support for metricsets * update docs * add changelog entry * make update * fix config * add NewDB argument * fix imports * Update mysql.go * Update config.go * Update CHANGELOG.next.asciidoc Co-authored-by: Aman <38116245+devamanv@users.noreply.github.com> * add integration test with tls * add TLS MinVersion --------- Co-authored-by: Aman <38116245+devamanv@users.noreply.github.com> Co-authored-by: Pierre HILBERT --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/mysql.asciidoc | 13 ++++- metricbeat/metricbeat.reference.yml | 11 ++++ metricbeat/module/mysql/_meta/Dockerfile | 4 ++ .../module/mysql/_meta/certs/client-cert.pem | 19 +++++++ .../module/mysql/_meta/certs/client-key.pem | 28 ++++++++++ .../module/mysql/_meta/certs/client-req.pem | 16 ++++++ .../module/mysql/_meta/certs/root-ca-key.pem | 28 ++++++++++ .../module/mysql/_meta/certs/root-ca.pem | 21 ++++++++ .../module/mysql/_meta/certs/server-cert.pem | 19 +++++++ .../module/mysql/_meta/certs/server-key.pem | 28 ++++++++++ .../module/mysql/_meta/certs/server-req.pem | 16 ++++++ .../module/mysql/_meta/config.reference.yml | 12 +++++ metricbeat/module/mysql/_meta/config.yml | 12 +++++ metricbeat/module/mysql/_meta/test.cnf | 4 ++ metricbeat/module/mysql/config.go | 32 +++++++++++ metricbeat/module/mysql/docker-compose.yml | 4 +- .../module/mysql/galera_status/status.go | 12 +++-- metricbeat/module/mysql/mysql.go | 53 +++++++++++++++--- .../module/mysql/mysql_integration_test.go | 54 ++++++++++++++++++- metricbeat/module/mysql/query/query.go | 27 ++++++++-- metricbeat/module/mysql/status/status.go | 11 ++-- metricbeat/modules.d/mysql.yml.disabled | 12 +++++ x-pack/metricbeat/metricbeat.reference.yml | 11 ++++ 24 files changed, 428 insertions(+), 20 deletions(-) create mode 100755 metricbeat/module/mysql/_meta/certs/client-cert.pem create mode 100755 metricbeat/module/mysql/_meta/certs/client-key.pem create mode 100755 metricbeat/module/mysql/_meta/certs/client-req.pem create mode 100755 metricbeat/module/mysql/_meta/certs/root-ca-key.pem create mode 100755 metricbeat/module/mysql/_meta/certs/root-ca.pem create mode 100755 metricbeat/module/mysql/_meta/certs/server-cert.pem create mode 100755 metricbeat/module/mysql/_meta/certs/server-key.pem create mode 100755 metricbeat/module/mysql/_meta/certs/server-req.pem create mode 100644 metricbeat/module/mysql/config.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be217566c4c..d06ffb5f705 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -230,6 +230,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] - Add linux IO metrics to system/process {pull}37213[37213] - Add new memory/cgroup metrics to Kibana module {pull}37232[37232] +- Add SSL support to mysql module {pull}37997[37997] *Metricbeat* diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 0c5a793a29a..5433fe94224 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -89,7 +89,18 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false ----- + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key"---- [float] === Metricsets diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 6659ca29276..59a8d5d6c38 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -764,6 +764,17 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: diff --git a/metricbeat/module/mysql/_meta/Dockerfile b/metricbeat/module/mysql/_meta/Dockerfile index 2051c726595..b701ad617ea 100644 --- a/metricbeat/module/mysql/_meta/Dockerfile +++ b/metricbeat/module/mysql/_meta/Dockerfile @@ -5,4 +5,8 @@ ENV MYSQL_ROOT_PASSWORD test HEALTHCHECK --interval=1s --retries=90 CMD mysql -u root -p$MYSQL_ROOT_PASSWORD -h$HOSTNAME -P 3306 -e "SHOW STATUS" > /dev/null +COPY /certs/root-ca.pem /etc/certs/root-ca.pem +COPY /certs/server-cert.pem /etc/certs/server-cert.pem +COPY /certs/server-key.pem /etc/certs/server-key.pem + COPY test.cnf /etc/mysql/conf.d/test.cnf diff --git a/metricbeat/module/mysql/_meta/certs/client-cert.pem b/metricbeat/module/mysql/_meta/certs/client-cert.pem new file mode 100755 index 00000000000..df9c76e0862 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDDCCAfQCAQEwDQYJKoZIhvcNAQELBQAwSjELMAkGA1UEBhMCVVMxEzARBgNV +BAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdm +YWtlLUNBMB4XDTI0MDIxNTIzNTA0MloXDTMzMTIyNDIzNTA0MlowTjELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJh +MRQwEgYDVQQDDAtmYWtlLWNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAIqHZbSUB1x/iW6DxaRlkFWjPuZ+F1wYTGvfpqnxZgZY1k5vSJTy3ETe +y3TelpEPBWEmsgHDx4bjuqeG+3my9dDEKEIYgXkfkfHREndVxPDfnRdfXPfp3qbm +wV2bdJnpSQzCg+lv8e8U+kMv0WcmwTuwlpVG0Rnb6vFdOs67/IIlBvI9sP5BKDYL +YFRxaoc8fLb8UMkfQ0BSmT4Rvmq5MSETh4re7OecV6pN0naEWhZf72mr/HiTAhb6 +xZJNSvNAzvdkQnhwt9aHemGQLRZD+4dduZYn27cwK4ySTZdyMoKn66HqMIfXPvr8 +LlICP4Gb8Df/JuUZVRbI13P+Xqujd8kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +gwA1+nZISC6QF9JtkOGrPpBZk6v1iy4iLsZSNaoinkB/FgesIpNrTFG0k6exSBV1 +pwQSmMVNSEsUOOjEq/Vk98014Kf8QVqfkdcujaBNPtxMqsocOO9Od78UuX5QdZXi +ayhkzrcPX4HTwjTqKFlJxb92rHrBx/GIWa68TeAjwbRiZmDASpVCEI2HnkBkFWTs +5Ux4wlC3JrnY3Jxb7QfDK94g9r5s1ljHeVki83cUYaI5TdY7F0uP+O6TvlhCPrjd +5708kRZJHnKThu3aE8HJYIbYhHocm9DszbnObd4SqECjfd6YNbREBhyaHJdCY/j2 +hm1zhBiW24dazs108uhFsQ== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/client-key.pem b/metricbeat/module/mysql/_meta/certs/client-key.pem new file mode 100755 index 00000000000..33430372fd2 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCKh2W0lAdcf4lu +g8WkZZBVoz7mfhdcGExr36ap8WYGWNZOb0iU8txE3st03paRDwVhJrIBw8eG47qn +hvt5svXQxChCGIF5H5Hx0RJ3VcTw350XX1z36d6m5sFdm3SZ6UkMwoPpb/HvFPpD +L9FnJsE7sJaVRtEZ2+rxXTrOu/yCJQbyPbD+QSg2C2BUcWqHPHy2/FDJH0NAUpk+ +Eb5quTEhE4eK3uznnFeqTdJ2hFoWX+9pq/x4kwIW+sWSTUrzQM73ZEJ4cLfWh3ph +kC0WQ/uHXbmWJ9u3MCuMkk2XcjKCp+uh6jCH1z76/C5SAj+Bm/A3/yblGVUWyNdz +/l6ro3fJAgMBAAECggEAEPRCAHQrA/k4c9oFBQoonHCMrNdDCuKO7NdsHYm1ucJi +5SnVxWQFTRkC59hrr1B6MTIUEGb6iyHhOOpqafI7B0xQnIlFBFLWsPSseMY6opvN +jTwew9k/xqfAg/E4F7OvXPRMAnSQ1LjZqcInE+Owe9qQjW/DvPFXS2fEgCOOA4vw +M6w6USf8UTsXBzMvRnDHMTQM0vfKNNSdopYDPeQc4YQ1A2AjkpYXZVWXFcFsE9zw +xFVZ9k6tP+gzk6shJjsbBoQ7qWwhdq1Q5tJ28FTaCVXDAp8l6yIFuZuI7r23O7+0 +ngxSejABJ3m9NmG0J7DPGU6zXhJW5nylWcSk5vwMkQKBgQDCWIRe4iSW0eGYBSe5 +hBoQgLe7aMAbsaCrHjTYQkKvI25YlfJ08OVU7oB/Bng/9AlpJlouGz67/W0PiRaz +jlP370p92IiwehUl9PkuVDpex4l2rDLCM1iVrPbxhbm/7+2nro2M/0/4iUyIK+Gr +Rpcqj2dQ3qarD+UmLXYPOoyRuQKBgQC2ec0sWyU67QuFaTemvTH8YFu68BfQqg6t +YQMc4+wj30ww0TZHFYVwyvR4agTOdFjwIUTERRN3EcFmlV5x+fGz/LfUdVYJj8B0 +lXakqeATsGJHngrdlyM+m+g+6JI1SUTshMa/xXVAUx8NZESOVE5JeZH6TD4/9Q3y +ijtithtekQKBgQCPeso/QrXAozLqCORLEjwr8tuygKNTzs/PhX1+K20P4BiXThyy +OScWjP5QyXX9wS0xdB8f6v1lzLO3xH3+EhXr9b4JKtO/dmImo7VTftuZHbde5cKT +nVTJK+kkZpW8HmZWZYgbkGJ6GuNlpP/2cycnRLgB/F8P66xBg06l75PYAQKBgGap +GhR1ZvnC+TNiocuuL5wkfhcrEsrzkfRbWwv68xSvgUcJvTa61etCU84XH4MjlBHt +NaoSjsPzelKDgLIxA5nWeXoPVYtlk8pDeI9lf0q0dmaCdOx8JnkH797Mq81M3nkO +rl6f8bpxyUuYeLV2muDdg5JFKNSEwwcMXCLJ/5XxAoGAKIkS02jWudDoBzubdFe/ +c5jSYufTZOmErnjnSKGkj9oZGVP6RYDhkHMPOxadO/4OLOKo6Phkg9yRmPG2tyKA ++ddgYP7zXEnsLxrjumoYTvcWgs1AHUUH4kA5SdImzYbSSfPW5h0KkvB+gYaukBGa +XHILry/59LkxU+nP1ZCVvt8= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/client-req.pem b/metricbeat/module/mysql/_meta/certs/client-req.pem new file mode 100755 index 00000000000..3295c803f8d --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-req.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkzCCAXsCAQAwTjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FDASBgNVBAcMC1NhbnRhIENsYXJhMRQwEgYDVQQDDAtmYWtlLWNsaWVudDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIqHZbSUB1x/iW6DxaRlkFWjPuZ+ +F1wYTGvfpqnxZgZY1k5vSJTy3ETey3TelpEPBWEmsgHDx4bjuqeG+3my9dDEKEIY +gXkfkfHREndVxPDfnRdfXPfp3qbmwV2bdJnpSQzCg+lv8e8U+kMv0WcmwTuwlpVG +0Rnb6vFdOs67/IIlBvI9sP5BKDYLYFRxaoc8fLb8UMkfQ0BSmT4Rvmq5MSETh4re +7OecV6pN0naEWhZf72mr/HiTAhb6xZJNSvNAzvdkQnhwt9aHemGQLRZD+4dduZYn +27cwK4ySTZdyMoKn66HqMIfXPvr8LlICP4Gb8Df/JuUZVRbI13P+Xqujd8kCAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQBr6+WE3t0KdMpEBBC81IUHkXNB9Mf5EYKG +d1ev6jq1bi2jw6WqAGbqYp1W0awEjZJZcS2skXoy8QIFDNjznHPgKEXB9b98nj34 +TLpszCrlcQteWmzRCspwkhdrXNGE4Z4UMgN+xoh2P/dujK4kGH6HFcF1Fo4ajDUX +HT5vybjQuQlPDgt6Ufs+Pjotr5uCzLbIsFN1QG6gKVY90WAzPsa0XYN1ehMpkLsM +8vbVP0uRT6/VXTenbTtqqQ5Y70gmeiF/EssnQ9rM3vkGUW1A/9j23agLmlOVaCWw +HSN5HqrFUIlsLFIDDTgi7icW4Uk+7qdMSF7ooMOJIm27PGc49u4U +-----END CERTIFICATE REQUEST----- diff --git a/metricbeat/module/mysql/_meta/certs/root-ca-key.pem b/metricbeat/module/mysql/_meta/certs/root-ca-key.pem new file mode 100755 index 00000000000..2343e39b149 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/root-ca-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSrYQWHfzCy/+N +Nzbg03pW/xnJ4wsEOeSuaYJfrkzU87pwlOFY57bhxe+3rektDOOziwjCxo4a8rKu +YmHRYKx4XdBtTjPnimRiwymSnemZdABWLNuJyvWen6iNJQqrcSesvobAtaQ265A9 +faRPn/Hjx5CH5x52hLhcpo6Yg4Ae6K2dnGbahFb1DI7Btfcf+PYiUau5DRiJiIpU +9K9hBbPmPuo0hsGiAYCJkTspdDMrFsBA6hNadamzsXy6AzB82Pu19nckR20kJVlG +Ioebg6mlHlcTV1qCsiWZBR/ghGGNHBp15EIXvIDpEJ4rcuy4AER4lXIdpG2RPD7Y ++Y7EGi0zAgMBAAECggEAU/SCuR+dHPGXdqqEq4aK78U7/SiFuNrrRxfTtRZsFsbD +yt6BiODaD9HFrCBZVjMXQHLM/HWMpq+Fxhl9mqcYQ+U6zHxIEeKkC3lzhTJ5p0XD +ZpP8rsYbKGm+jPSwck6m/V91qrEX7izkb6S0iGiYR+m8rnPLP3a3U3CqTZvFwErG +n7jk7caLZcT9+p7/TLlDIyx4ha4+7RRaL9OC1dNH8ADOkSHk/vaE6aU8J8PJ4YZg +QvNfsuo7FtDMq3OIkMAsHseuX90X8c3ZS7lNdCTRU7YuC1+8+l6xGs1Arjv1jqnd +9gIo6kh88Ng8zi4TkGLVAnfc55eXmB+f7PPN93fMeQKBgQD0uqDSsvPNnaY6DJIF +Gyz4qExyYH/h2QFT5M4bb0hQNIkP187JhBZw8Et2AvBtSBhs8dXfBxu736KRs8XG +b60iw2qXqo1XUEUO7R0VMO6NcA8Hk206X+p7ukn5RExzv2MurD+3f8QM8CypFA57 +UnSWdDCrOAh6WU5zfcz9woOM2QKBgQDcYWvqbV8XoyhJBG5PkG3UzYtOi/Es/fGH +qt03ZyyfYdCbhAknqftuj6ZzlMfVV3xOSXX+sdr0rLzammnRdlPJtJfjk8lUYa/i +0hy4eTHm7o1iZJfMS9rCMH9uTwyNGnb67u8kW16BuzaLbJMtd7IKtEG69U63abZX +t+zqmxGy6wKBgQCD43w+cNCxdA+cYx/ifpXK4DBqx5TDq0Zq5vkokd1/1AA1uJEp +yvSpIucYD1dxHZSESgR/sH4Czu/249JnMdI11OjCGdkYQBsngyPUQs2dDdIbvBj2 +h7B/w5KQMn2dN3yFL7Ea/FE0w87dxABV98b7OlzsOUNgZHbCCP8LluN8aQKBgGS3 +RTly2JWV5DBSjRNhn0A026h+/i6gs8RbyxOp3FPOwSaBlimBXr4telWyNg2DGPUy +T3Gh2L4fP4PsM9YdbLdvCEdiYA1nQ5m2ipeoE61Fcmn4LQOZ2xUKUwKXr9XAtYWC +stn7w9ooNApOCYkq/bw0myGVQG9EKag3D1g8nD8XAoGAZLJlDhlfFaWa7jy1VF/g +JWcsN/+BfTjBY6t3npxzg4pdi7lHhuAZ45PLnQMTIdWCkqgigt224kcbUy3b351u +lzoSiLatNXj5Q3on85ZNRaOMLqp0ueIzOLWvC+CRp46wXlwxTrPxghXatUBPsG47 +mO/mtw9gmaJ8UBW/SuxS24g= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/root-ca.pem b/metricbeat/module/mysql/_meta/certs/root-ca.pem new file mode 100755 index 00000000000..9b3e4f60fe8 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/root-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgIUUp8x6W/bui3FjHLnJfIb7AsKBIwwDQYJKoZIhvcNAQEL +BQAwSjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcM +C1NhbnRhIENsYXJhMRAwDgYDVQQDDAdmYWtlLUNBMB4XDTI0MDIxNTIzNTAzNVoX +DTMzMTIyNDIzNTAzNVowSjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3Ju +aWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdmYWtlLUNBMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0q2EFh38wsv/jTc24NN6Vv8ZyeML +BDnkrmmCX65M1PO6cJThWOe24cXvt63pLQzjs4sIwsaOGvKyrmJh0WCseF3QbU4z +54pkYsMpkp3pmXQAVizbicr1np+ojSUKq3EnrL6GwLWkNuuQPX2kT5/x48eQh+ce +doS4XKaOmIOAHuitnZxm2oRW9QyOwbX3H/j2IlGruQ0YiYiKVPSvYQWz5j7qNIbB +ogGAiZE7KXQzKxbAQOoTWnWps7F8ugMwfNj7tfZ3JEdtJCVZRiKHm4OppR5XE1da +grIlmQUf4IRhjRwadeRCF7yA6RCeK3LsuABEeJVyHaRtkTw+2PmOxBotMwIDAQAB +o1MwUTAdBgNVHQ4EFgQURA7Q9JPfB4mveB0vzmoqNJ2HSZUwHwYDVR0jBBgwFoAU +RA7Q9JPfB4mveB0vzmoqNJ2HSZUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAQEAB4NGJFZpzltHLqvInSU/EQxdIHgifihOFzsXTEXkdrmkfEw5puVL +fzg6qnLOunh3GAwLCnM0aIzDLS8WAS509Jwwidn7OtBpYV+jIzJrrTycWjAdvcHC +WToPTueXxwaAD3pCrus0w9H8egoQ1haNVmQm0OWcv3My82cNbZwViuQSCrky1srL +N5l7UM0gbXKeZjTGHIoTIjQJDgJT8PydsxpOZq7CcKRDBdF5nYMcUq8wltneb0Nh +7DuLLdxEM11XzIRT4GLRxT2xqwW7UpLfWpuo+niCvmNFY6SzyHFR1vFI3Kw1rYXh +3cbEtHtRvcNQg6Jp/zoHDcXMS3hDMeN2vQ== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/server-cert.pem b/metricbeat/module/mysql/_meta/certs/server-cert.pem new file mode 100755 index 00000000000..1ca56e3f44f --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDDCCAfQCAQEwDQYJKoZIhvcNAQELBQAwSjELMAkGA1UEBhMCVVMxEzARBgNV +BAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdm +YWtlLUNBMB4XDTI0MDIxNTIzNTAzOFoXDTMzMTIyNDIzNTAzOFowTjELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJh +MRQwEgYDVQQDDAtmYWtlLXNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAMuPqkUt/Ax9s/h5LPxXU0m6OAEp1InLbR6x//hGVgmIiQu5/Fg1VfmZ +YbwraXxs4JDfMUyK6bd/bk2o71I1pnLmoFmQvawDRxOqkA1NLpF2FJtk0eevkF1D +crC9T1SfrzlwrucqqUXowdprVXFFVbFQTXsSyD8Nv/MGzDgmDtmMXQ8sLVqjGIEM +akuPMbNCVNTVnd/53WMaDzopnam/NCJNDGp2RVhf+KuOWLTURXFYN6j1z+f/1BNa +4QW+WtofzYkAWEcvCc8zeXUhwL6xE5gDyq1NkQ/ejqQq+iIJLd1FUFOH1jPSgmW5 +3CiWih2Is6VA0hCzDirdFtAHTui/OekCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +vdGGVxaeSEfOkx+D7uYCx0blnobJoclggQP3fOIpyrU/LCeka+F8dvFvuGJLvn3A +JOMZZHCVnK6jjJYHFeolRCxd9iULYHD+dkWDr6uhvNMfwIt7UzUmtbznHAaD+ays +X0H70Z9+jmr3uFkevRbFkvDZqzdRYi/12oPM+0Skra3ouYen6zAtPU0Hruc0jyBP +W7V6mMSmCUPKTOJRZgDEIEBvu43rwEbQUG0ayqF1sLv+D6hjFrFJ2gCxgVH/+C9E +h0NF2Kdpb+jECCu3yhQA536Ugi9k96zJqJonu9jP4ODXMTG2qmsdFFW1zyFb9DbV +bjUsiDE7bEumHY2NEfzr3A== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/server-key.pem b/metricbeat/module/mysql/_meta/certs/server-key.pem new file mode 100755 index 00000000000..d1a7d286a1c --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLj6pFLfwMfbP4 +eSz8V1NJujgBKdSJy20esf/4RlYJiIkLufxYNVX5mWG8K2l8bOCQ3zFMium3f25N +qO9SNaZy5qBZkL2sA0cTqpANTS6RdhSbZNHnr5BdQ3KwvU9Un685cK7nKqlF6MHa +a1VxRVWxUE17Esg/Db/zBsw4Jg7ZjF0PLC1aoxiBDGpLjzGzQlTU1Z3f+d1jGg86 +KZ2pvzQiTQxqdkVYX/irjli01EVxWDeo9c/n/9QTWuEFvlraH82JAFhHLwnPM3l1 +IcC+sROYA8qtTZEP3o6kKvoiCS3dRVBTh9Yz0oJludwoloodiLOlQNIQsw4q3RbQ +B07ovznpAgMBAAECggEADLAux9Me89ReBG3hLPVwfpb56LCny9L/QTuNHfecY0m8 +aRu1q/XfHwi9e9Ik6BmNQdp3ozLBcKujv3l5OWGYt27CrfKEsBUgOAyYoAugjHaU +wD7fipZ55CZRHs0eBcNSU70/Wa9iD7Z7Ztbr43yT49KCkdpQ2wVLYqWY0yMkJ9Eo +ZUJ8fL+yDMeJxnhQSIejK62TQI3FdMz+aNXA6AO0YiSfqagTS8GVNZQvZzvyxYS0 +DpiydzKSbS2RXkf3waClU5hDGwqhNxXa9bya/KrLvm4ag/VaV0O1M9jwFOKwfUGY +0SDELz/mxsOmGntTUbtuH7VSvnqkJHfACUcNkkIjAQKBgQD5pwIzrPnGrljDcFqu +OCRxhiRjgCNth4ObBbmj2n0BV5Uw33o1VlN/+GCfKcIQ1+tHOUrEtkwP5mMatUbf +4G4K/+bO3eWAf+ia5hkSVASbU0ui36iSkPWLYJr0oDx0N6Vw+ZK7oxqLGqW2dm4Y +Q1TFaIDd2wUGPYAuDaqPDHecCQKBgQDQvKXy9Ueh4iTbz3sH6Kp4wGN2BsjWWOVn +Hi4QoqnDoLrguhCe5vvNyxfayziu9hUKzP8kBHQOY/2xpKv+epPuw6hgaD0Mnh/w +UcWEqZs102y0zZcQISfG8TUoLHW31T87veB3YEVIB+8uZg1CWJ7aDKe8UmugVGV2 +k2sMG7fm4QKBgHq0z6w+lPZGs3I8QxXmmmMCH9iYHtGzDcigY8JZnZ+PQNEoxpR4 +vcnkdvlEORK2TfpP+qP9Rh16i7OQ7ikT0oKtjPCYuDkUpWudNS2BBlKh+kcvz1da +0JWVAhTCvXQR9cs1oB2B6YX9rv2j8DEUxxHQb6acBDgw+lOoe/CbnB6hAoGBAKxg +bcbjCcHFCF1BzT8tw8GuVzS7y5U/mkp64N26BunXzRwSa/FdnOpI4q07j9bkv2HJ +ApZS2yibKIFQFP01av8NMvpSer/1wThrvuqcSeG8dJQnB645QykGPrirZpdmki6a +0kijBvPCIaI2gpKcrqoxMz/Q7LJdn+C5Qvif11HhAoGAfai8GYFiXuShvf+8gjOt +qIsBMV3YexcX11qPD5vVJMCW1xLbpb9f3sPf8P31TB8tz5JA3aG24k8yURtgTA4Z +2I6Jo9vwMjAdOxHTalqMllDvBj5S5+cX38kGdcxdcbAiUHwIoXy6cjcGbeO/SesR +L1bbyZA45gpsWFxFr5V67G0= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/server-req.pem b/metricbeat/module/mysql/_meta/certs/server-req.pem new file mode 100755 index 00000000000..035ab7e2faf --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-req.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkzCCAXsCAQAwTjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FDASBgNVBAcMC1NhbnRhIENsYXJhMRQwEgYDVQQDDAtmYWtlLXNlcnZlcjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMuPqkUt/Ax9s/h5LPxXU0m6OAEp +1InLbR6x//hGVgmIiQu5/Fg1VfmZYbwraXxs4JDfMUyK6bd/bk2o71I1pnLmoFmQ +vawDRxOqkA1NLpF2FJtk0eevkF1DcrC9T1SfrzlwrucqqUXowdprVXFFVbFQTXsS +yD8Nv/MGzDgmDtmMXQ8sLVqjGIEMakuPMbNCVNTVnd/53WMaDzopnam/NCJNDGp2 +RVhf+KuOWLTURXFYN6j1z+f/1BNa4QW+WtofzYkAWEcvCc8zeXUhwL6xE5gDyq1N +kQ/ejqQq+iIJLd1FUFOH1jPSgmW53CiWih2Is6VA0hCzDirdFtAHTui/OekCAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQAK3+eAfReXoGP3CQvTE/Bd6u+u5kG65stV +DONrBzhMQ4R36X+Q6q65qJ0rmvwZcUfkIauQzdNv9ZfCDT7pO1VtNT0R+H6+shz9 +JiwGOudAlFSt31Ps0+lDm6WjA6J1Nmr9N7XrsmfdW4z2n1UZSPS9mOZIj+PpUtQw +OzIwJ/+btS/RVO0cGGFkoFwhrYKilAbq+SsMxMVxPcXUP+xLFYn6FCNFbf5uBpLz +ZM7HBDh2uVfwsaptnY3v+EIELCsXsFm9uj4zG45fJmu4KARY6FAi9sEvfA1ieZuU +8hmovXhKq6eSU2fPoeurRV1gxuanuFObd39LRoCTy3fCnqTZFxXg +-----END CERTIFICATE REQUEST----- diff --git a/metricbeat/module/mysql/_meta/config.reference.yml b/metricbeat/module/mysql/_meta/config.reference.yml index 03880a5ad6a..f31774cb1be 100644 --- a/metricbeat/module/mysql/_meta/config.reference.yml +++ b/metricbeat/module/mysql/_meta/config.reference.yml @@ -21,3 +21,15 @@ # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file diff --git a/metricbeat/module/mysql/_meta/config.yml b/metricbeat/module/mysql/_meta/config.yml index 367b32e9173..a86258fca3b 100644 --- a/metricbeat/module/mysql/_meta/config.yml +++ b/metricbeat/module/mysql/_meta/config.yml @@ -18,3 +18,15 @@ # Password of hosts. Empty by default. #password: secret + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file diff --git a/metricbeat/module/mysql/_meta/test.cnf b/metricbeat/module/mysql/_meta/test.cnf index f759a49631d..24eec52dd05 100644 --- a/metricbeat/module/mysql/_meta/test.cnf +++ b/metricbeat/module/mysql/_meta/test.cnf @@ -1,2 +1,6 @@ [mysqld] bind-address = 0.0.0.0 +require_secure_transport = OFF +ssl-ca = /etc/certs/root-ca.pem +ssl-cert = /etc/certs/server-cert.pem +ssl-key = /etc/certs/server-key.pem \ No newline at end of file diff --git a/metricbeat/module/mysql/config.go b/metricbeat/module/mysql/config.go new file mode 100644 index 00000000000..96704bef479 --- /dev/null +++ b/metricbeat/module/mysql/config.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mysql + +import ( + "crypto/tls" + + "github.com/elastic/elastic-agent-libs/transport/tlscommon" +) + +type Config struct { + Hosts []string `config:"hosts" validate:"required"` + Username string `config:"username"` + Password string `config:"password"` + TLS *tlscommon.Config `config:"ssl"` + TLSConfig *tls.Config +} diff --git a/metricbeat/module/mysql/docker-compose.yml b/metricbeat/module/mysql/docker-compose.yml index e112587fccd..0644d9568ad 100644 --- a/metricbeat/module/mysql/docker-compose.yml +++ b/metricbeat/module/mysql/docker-compose.yml @@ -2,10 +2,10 @@ version: '2.3' services: mysql: - image: docker.elastic.co/integrations-ci/beats-mysql:${MYSQL_VARIANT:-mysql}-${MYSQL_VERSION:-5.7.12}-1 + image: docker.elastic.co/integrations-ci/beats-mysql:${MYSQL_VARIANT:-mysql}-${MYSQL_VERSION:-8.0}-1 build: context: ./_meta args: - MYSQL_IMAGE: ${MYSQL_VARIANT:-mysql}:${MYSQL_VERSION:-5.7.12} + MYSQL_IMAGE: ${MYSQL_VARIANT:-mysql}:${MYSQL_VERSION:-8.0} ports: - 3306 diff --git a/metricbeat/module/mysql/galera_status/status.go b/metricbeat/module/mysql/galera_status/status.go index d1dc68cd0a2..6f27b8d4e8f 100644 --- a/metricbeat/module/mysql/galera_status/status.go +++ b/metricbeat/module/mysql/galera_status/status.go @@ -42,7 +42,7 @@ func init() { // MetricSet for fetching Galera-MySQL server status type MetricSet struct { - mb.BaseMetricSet + *mysql.Metricset db *sql.DB } @@ -50,7 +50,13 @@ type MetricSet struct { // Loads query_mode config setting from the config file func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The galera_status metricset is experimental.") - return &MetricSet{BaseMetricSet: base}, nil + + ms, err := mysql.NewMetricset(base) + if err != nil { + return nil, err + } + + return &MetricSet{Metricset: ms, db: nil}, nil } // Fetch methods implements the data gathering and data conversion to the right format @@ -58,7 +64,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.db == nil { var err error - m.db, err = mysql.NewDB(m.HostData().URI) + m.db, err = mysql.NewDB(m.HostData().URI, m.Metricset.Config.TLSConfig) if err != nil { return fmt.Errorf("Galera-status fetch failed: %w", err) } diff --git a/metricbeat/module/mysql/mysql.go b/metricbeat/module/mysql/mysql.go index 35388a9a1bd..23c0f8dda10 100644 --- a/metricbeat/module/mysql/mysql.go +++ b/metricbeat/module/mysql/mysql.go @@ -21,14 +21,18 @@ Package mysql is Metricbeat module for MySQL server. package mysql import ( + "crypto/tls" "database/sql" "fmt" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" "github.com/go-sql-driver/mysql" ) +const TLSConfigKey = "custom" + func init() { // Register the ModuleFactory function for the "mysql" module. if err := mb.Registry.AddModule("mysql", NewModule); err != nil { @@ -38,16 +42,37 @@ func init() { func NewModule(base mb.BaseModule) (mb.Module, error) { // Validate that at least one host has been specified. - config := struct { - Hosts []string `config:"hosts" validate:"required"` - }{} - if err := base.UnpackConfig(&config); err != nil { + var c Config + if err := base.UnpackConfig(&c); err != nil { return nil, err } return &base, nil } +type Metricset struct { + mb.BaseMetricSet + Config Config +} + +func NewMetricset(base mb.BaseMetricSet) (*Metricset, error) { + var c Config + if err := base.Module().UnpackConfig(&c); err != nil { + return nil, fmt.Errorf("could not read config: %w", err) + } + + if c.TLS.IsEnabled() { + tlsConfig, err := tlscommon.LoadTLSConfig(c.TLS) + if err != nil { + return nil, fmt.Errorf("could not load provided TLS configuration: %w", err) + } + + c.TLSConfig = tlsConfig.ToConfig() + } + + return &Metricset{Config: c, BaseMetricSet: base}, nil +} + // ParseDSN creates a DSN (data source name) string by parsing the host. // It validates the resulting DSN and returns an error if the DSN is invalid. // @@ -55,9 +80,11 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { // Example: root:test@tcp(127.0.0.1:3306)/ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { c := struct { - Username string `config:"username"` - Password string `config:"password"` + Username string `config:"username"` + Password string `config:"password"` + TLS *tlscommon.Config `config:"ssl"` }{} + if err := mod.UnpackConfig(&c); err != nil { return mb.HostData{}, err } @@ -86,6 +113,10 @@ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { noCredentialsConfig.User = "" noCredentialsConfig.Passwd = "" + if c.TLS.IsEnabled() { + config.TLSConfig = TLSConfigKey + } + return mb.HostData{ URI: config.FormatDSN(), SanitizedURI: noCredentialsConfig.FormatDSN(), @@ -99,10 +130,18 @@ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { // must be valid, otherwise an error will be returned. // // DSN Format: [username[:password]@][protocol[(address)]]/ -func NewDB(dsn string) (*sql.DB, error) { +func NewDB(dsn string, tlsConfig *tls.Config) (*sql.DB, error) { + if tlsConfig != nil { + err := mysql.RegisterTLSConfig(TLSConfigKey, tlsConfig) + if err != nil { + return nil, fmt.Errorf("registering custom tls config failed: %w", err) + } + } + db, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("sql open failed: %w", err) } + return db, nil } diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go index 5713a582149..2fc96475646 100644 --- a/metricbeat/module/mysql/mysql_integration_test.go +++ b/metricbeat/module/mysql/mysql_integration_test.go @@ -20,6 +20,9 @@ package mysql import ( + "crypto/tls" + "crypto/x509" + "os" "testing" "github.com/stretchr/testify/assert" @@ -31,9 +34,58 @@ import ( func TestNewDB(t *testing.T) { service := compose.EnsureUp(t, "mysql") - db, err := NewDB(GetMySQLEnvDSN(service.Host())) + db, err := NewDB(GetMySQLEnvDSN(service.Host()), nil) assert.NoError(t, err) err = db.Ping() assert.NoError(t, err) } + +func loadTLSConfig(caCertPath, clientCertPath, clientKeyPath string) (*tls.Config, error) { + caCert, err := os.ReadFile(caCertPath) + if err != nil { + return nil, err + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + cert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath) + if err != nil { + return nil, err + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + MinVersion: tls.VersionTLS12, + } + + return tlsConfig, nil +} + +func TestNewDBWithSSL(t *testing.T) { + service := compose.EnsureUp(t, "mysql") + + tlsConfig, err := loadTLSConfig("_meta/certs/root-ca.pem", "_meta/certs/client-cert.pem", "_meta/certs/client-key.pem") + tlsConfig.InsecureSkipVerify = true + assert.NoError(t, err) + + db, err := NewDB(GetMySQLEnvDSN(service.Host())+"?tls=custom", tlsConfig) + assert.NoError(t, err) + + err = db.Ping() + assert.NoError(t, err) + + // Check if the current connection is using SSL + var sslCipher, variableName, value string + err = db.QueryRow(`show status like 'Ssl_cipher'`).Scan(&variableName, &sslCipher) + assert.NoError(t, err) + + // If sslCipher is not empty, then SSL is being used for the connection + assert.NotEmpty(t, variableName) + assert.NotEmpty(t, sslCipher) + + err = db.QueryRow(`show variables like 'have_ssl'`).Scan(&variableName, &value) + assert.NoError(t, err) + assert.Equal(t, "YES", value) +} diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index 35881d76401..d7bbaaa4cd7 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -25,13 +25,17 @@ package query import ( "context" + "crypto/tls" "fmt" + mysqlDriver "github.com/go-sql-driver/mysql" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/mysql" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) func init() { @@ -57,8 +61,10 @@ type MetricSet struct { mb.BaseMetricSet db *sql.DbClient Config struct { - Queries []query `config:"queries" validate:"nonzero,required"` - Namespace string `config:"namespace" validate:"nonzero,required"` + Queries []query `config:"queries" validate:"nonzero,required"` + Namespace string `config:"namespace" validate:"nonzero,required"` + TLS *tlscommon.Config `config:"ssl"` + TLSConfig *tls.Config } } @@ -72,16 +78,31 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + if b.Config.TLS.IsEnabled() { + tlsConfig, err := tlscommon.LoadTLSConfig(b.Config.TLS) + if err != nil { + return nil, fmt.Errorf("could not load provided TLS configuration: %w", err) + } + + b.Config.TLSConfig = tlsConfig.ToConfig() + } + return b, nil } // Fetch fetches status messages from a mysql host. func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { if m.db == nil { + if m.Config.TLSConfig != nil { + err := mysqlDriver.RegisterTLSConfig(mysql.TLSConfigKey, m.Config.TLSConfig) + if err != nil { + return fmt.Errorf("registering custom tls config failed: %w", err) + } + } var err error m.db, err = sql.NewDBClient("mysql", m.HostData().URI, m.Logger()) if err != nil { - return fmt.Errorf("mysql-status fetch failed: %w", err) + return fmt.Errorf("mysql-query fetch failed: %w", err) } } diff --git a/metricbeat/module/mysql/status/status.go b/metricbeat/module/mysql/status/status.go index dd57f7e23c9..ac3e5b83a18 100644 --- a/metricbeat/module/mysql/status/status.go +++ b/metricbeat/module/mysql/status/status.go @@ -40,20 +40,25 @@ func init() { // MetricSet for fetching MySQL server status. type MetricSet struct { - mb.BaseMetricSet + *mysql.Metricset db *sql.DB } // New creates and returns a new MetricSet instance. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return &MetricSet{BaseMetricSet: base}, nil + ms, err := mysql.NewMetricset(base) + if err != nil { + return nil, err + } + + return &MetricSet{Metricset: ms, db: nil}, nil } // Fetch fetches status messages from a mysql host. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.db == nil { var err error - m.db, err = mysql.NewDB(m.HostData().URI) + m.db, err = mysql.NewDB(m.HostData().URI, m.Metricset.Config.TLSConfig) if err != nil { return fmt.Errorf("mysql-status fetch failed: %w", err) } diff --git a/metricbeat/modules.d/mysql.yml.disabled b/metricbeat/modules.d/mysql.yml.disabled index 2913f5af8bc..27dcc1e59ea 100644 --- a/metricbeat/modules.d/mysql.yml.disabled +++ b/metricbeat/modules.d/mysql.yml.disabled @@ -21,3 +21,15 @@ # Password of hosts. Empty by default. #password: secret + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1e6abf11a60..313fc5491e0 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1172,6 +1172,17 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: From ffd9fd899b7a1955a6da14e5ca561d4b4e5780a6 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Sun, 17 Mar 2024 13:50:49 -0400 Subject: [PATCH 11/31] chore: Update snapshot.yml (#38352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3e0cd8a6d90..693e6d2f748 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-a2d5464c-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-a2d5464c-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-a2d5464c-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 093ab5e824dd601c39a84cb50ef6c5934af2c3ff Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:28:37 -0400 Subject: [PATCH 12/31] Fix docs build break from SSL module page (#38367) * Fix docs build break from SSL module page * Update metricbeat/docs/modules/mysql.asciidoc Co-authored-by: Denis * Fix the config reference, so we can build the docs --------- Co-authored-by: Denis Co-authored-by: Denis Rechkunov --- metricbeat/docs/modules/mysql.asciidoc | 5 +++-- metricbeat/metricbeat.reference.yml | 3 ++- metricbeat/module/mysql/_meta/config.reference.yml | 4 ++-- x-pack/metricbeat/metricbeat.reference.yml | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 5433fe94224..8711359bf5f 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -98,9 +98,10 @@ metricbeat.modules: # Certificate for SSL/TLS client authentication #ssl.certificate: "/etc/pki/client/cert.crt" - + # Client certificate key file - #ssl.key: "/etc/pki/client/cert.key"---- + #ssl.key: "/etc/pki/client/cert.key" +---- [float] === Metricsets diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 59a8d5d6c38..3f9ccb0a9db 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -772,9 +772,10 @@ metricbeat.modules: # Certificate for SSL/TLS client authentication #ssl.certificate: "/etc/pki/client/cert.crt" - + # Client certificate key file #ssl.key: "/etc/pki/client/cert.key" + #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: diff --git a/metricbeat/module/mysql/_meta/config.reference.yml b/metricbeat/module/mysql/_meta/config.reference.yml index f31774cb1be..4e5cc470aca 100644 --- a/metricbeat/module/mysql/_meta/config.reference.yml +++ b/metricbeat/module/mysql/_meta/config.reference.yml @@ -30,6 +30,6 @@ # Certificate for SSL/TLS client authentication #ssl.certificate: "/etc/pki/client/cert.crt" - + # Client certificate key file - #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file + #ssl.key: "/etc/pki/client/cert.key" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 313fc5491e0..f71e58904fd 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1180,9 +1180,10 @@ metricbeat.modules: # Certificate for SSL/TLS client authentication #ssl.certificate: "/etc/pki/client/cert.crt" - + # Client certificate key file #ssl.key: "/etc/pki/client/cert.key" + #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: From 8ccfa43ef62c8022fc98be976cbdc8a3b450f6f5 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:30:49 +0200 Subject: [PATCH 13/31] migrate xpack-winlogbeat pipeline (#38348) * migrate xpack-winlogbeat pipeline * add generate_xpack_winlogbeat_pipeline.sh and cleanup setenv.sh * full test * full test * test xpack-packetbeat packaging * update comment --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/cloud_tests.sh | 3 - .buildkite/scripts/common.sh | 12 ++ .../generate_xpack_metricbeat_pipeline.sh | 6 + .../generate_xpack_winlogbeat_pipeline.sh | 110 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 2 +- .../x-pack/pipeline.xpack.metricbeat.yml | 2 +- .../x-pack/pipeline.xpack.winlogbeat.yml | 38 +++++- 8 files changed, 167 insertions(+), 8 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c23f5c3af64..1111caf5ac0 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -34,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 9e7ce95be66..3baeb1be1a3 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -9,9 +9,6 @@ set -euo pipefail trap 'teardown || true; unset_secrets' EXIT -# Set the MODULE env variable if possible -defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" - # Prepare the cloud resources using Terraform startCloudTestEnv "${MODULE_DIR}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d6a91a48243..9e75d3535cd 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -15,6 +15,10 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" [ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" +[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" +[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" +[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" +[ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" @@ -110,6 +114,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-packetbeat") BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} ;; + "beats-xpack-winlogbeat") + BEAT_CHANGESET_REFERENCE=${xpack_winlogbeat_changeset[@]} + ;; *) echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; @@ -501,4 +508,9 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then + # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. + defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index af116b17209..ddc3ce2c8f2 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,6 +27,8 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -36,6 +38,8 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: ".buildkite/scripts/py_int_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -131,6 +135,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" diff --git a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh new file mode 100755 index 00000000000..108a70c1562 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-winlogbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + + - label: ":windows: Windows 2019 Unit (MODULE) Tests" + key: "mandatory-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# echo "Add the extended windows tests into the pipeline" +# TODO: ADD conditions from the main pipeline + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 29a8a05446e..db86a7c9112 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -56,7 +56,7 @@ exportVars() { fi } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 81756564919..216f3134344 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -54,7 +54,7 @@ steps: allow_dependency_failure: false - label: ":linux: Load dynamic x-pack metricbeat pipeline" - key: "metricbeat-pipeline" + key: "xpack-metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: - github_commit_status: diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 34321b61161..5c8acefd698 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -1,5 +1,39 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-winlogbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + BEATS_PROJECT_NAME: "x-pack/winlogbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Winlogbeat - run_xpack_winlogbeat" + key: "run_xpack_winlogbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack winlogbeat pipeline" + key: "xpack-winlogbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From 202b4904d61c99463d122c92efd6a05a711f7e7a Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 18 Mar 2024 18:09:34 +0200 Subject: [PATCH 14/31] disabled buildvcs (#38366) Force merging this to unblock buildkite migration. --- dev-tools/mage/crossbuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index f500611e5cc..94b972ea25e 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -327,7 +327,7 @@ func (b GolangCrossBuilder) Build() error { args = append(args, "--rm", - "--env", "GOFLAGS=-mod=readonly", + "--env", "GOFLAGS=-mod=readonly -buildvcs=false", "--env", "MAGEFILE_VERBOSE="+verbose, "--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""), "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), From 8289144f4c24ca591f56fb5b3ac909eb11e6d3c0 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 18 Mar 2024 17:14:43 +0100 Subject: [PATCH 15/31] [Kafka output] Add config validation for topic and topics fields (#38058) This commit adds configuration validation for `topic` and `topics` fields from the Kafka output. Validation is performed for both cases: standalone Beat and running under Elastic-Agent. The latter does not support dynamic topics, hence setting `topics` is rejected by the validation. When running under Elastic-Agent the topic is validated using a regexpt taken from the Kafka source code: https://github.com/apache/kafka/blob/a126e3a622f2b7142f3543b9dbee54b6412ba9d8/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L33 --- CHANGELOG.next.asciidoc | 1 + libbeat/outputs/kafka/config.go | 30 +++++++++ libbeat/outputs/kafka/config_test.go | 95 +++++++++++++++++++++++++++- 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d06ffb5f705..6b2023ddcec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -146,6 +146,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Upgrade go-sysinfo from 1.12.0 to 1.13.1. {pull}37996[37996] - Make `range` condition work with numeric values as strings. {pull}38080[38080] - Allow users to configure number of output workers (for outputs that support workers) with either `worker` or `workers`. {pull}38257[38257] +- Kafka output now validates the `topics` and `topic` configuration values {pull}38058[38058] *Auditbeat* diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 8fff8dad0d5..3e2c836a06f 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -22,6 +22,7 @@ import ( "fmt" "math" "math/rand" + "regexp" "strings" "time" @@ -31,6 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/common/kafka" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/outputs/codec" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -77,6 +79,11 @@ type kafkaConfig struct { Sasl kafka.SaslConfig `config:"sasl"` EnableFAST bool `config:"enable_krb5_fast"` Queue config.Namespace `config:"queue"` + + // Currently only used for validation. Those values are later + // unpacked into temporary structs whenever they're necessary. + Topic string `config:"topic"` + Topics []any `config:"topics"` } type metaConfig struct { @@ -102,6 +109,11 @@ var compressionModes = map[string]sarama.CompressionCodec{ "snappy": sarama.CompressionSnappy, } +// validTopicRegExp is used to validate the topic contains only valid characters +// when running under Elastic-Agent. The regexp is taken from: +// https://github.com/apache/kafka/blob/a126e3a622f2b7142f3543b9dbee54b6412ba9d8/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L33 +var validTopicRegExp = regexp.MustCompile("^[a-zA-Z0-9._-]+$") + func defaultConfig() kafkaConfig { return kafkaConfig{ Hosts: nil, @@ -169,6 +181,24 @@ func (c *kafkaConfig) Validate() error { return fmt.Errorf("compression_level must be between 0 and 9") } } + + if c.Topic == "" && len(c.Topics) == 0 { + return errors.New("either 'topic' or 'topics' must be defined") + } + + // When running under Elastic-Agent we do not support dynamic topic + // selection, so `topics` is not supported and `topic` is treated as an + // plain string + if management.UnderAgent() { + if len(c.Topics) != 0 { + return errors.New("'topics' is not supported when running under Elastic-Agent") + } + + if !validTopicRegExp.MatchString(c.Topic) { + return fmt.Errorf("topic '%s' is invalid, it must match '[a-zA-Z0-9._-]'", c.Topic) + } + } + return nil } diff --git a/libbeat/outputs/kafka/config_test.go b/libbeat/outputs/kafka/config_test.go index 25c0c5dce99..2435b274f6e 100644 --- a/libbeat/outputs/kafka/config_test.go +++ b/libbeat/outputs/kafka/config_test.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/internal/testutil" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -32,16 +33,18 @@ import ( func TestConfigAcceptValid(t *testing.T) { tests := map[string]mapstr.M{ - "default config is valid": mapstr.M{}, "lz4 with 0.11": mapstr.M{ "compression": "lz4", "version": "0.11", + "topic": "foo", }, "lz4 with 1.0": mapstr.M{ "compression": "lz4", "version": "1.0.0", + "topic": "foo", }, "Kerberos with keytab": mapstr.M{ + "topic": "foo", "kerberos": mapstr.M{ "auth_type": "keytab", "username": "elastic", @@ -52,6 +55,7 @@ func TestConfigAcceptValid(t *testing.T) { }, }, "Kerberos with user and password pair": mapstr.M{ + "topic": "foo", "kerberos": mapstr.M{ "auth_type": "password", "username": "elastic", @@ -67,7 +71,9 @@ func TestConfigAcceptValid(t *testing.T) { test := test t.Run(name, func(t *testing.T) { c := config.MustNewConfigFrom(test) - c.SetString("hosts", 0, "localhost") + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } cfg, err := readConfig(c) if err != nil { t.Fatalf("Can not create test configuration: %v", err) @@ -89,13 +95,17 @@ func TestConfigInvalid(t *testing.T) { "realm": "ELASTIC", }, }, + // The default config does not set `topic` nor `topics`. + "No topics or topic provided": mapstr.M{}, } for name, test := range tests { test := test t.Run(name, func(t *testing.T) { c := config.MustNewConfigFrom(test) - c.SetString("hosts", 0, "localhost") + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } _, err := readConfig(c) if err == nil { t.Fatalf("Can create test configuration from invalid input") @@ -104,6 +114,84 @@ func TestConfigInvalid(t *testing.T) { } } +func TestConfigUnderElasticAgent(t *testing.T) { + oldUnderAgent := management.UnderAgent() + t.Cleanup(func() { + // Restore the previous value + management.SetUnderAgent(oldUnderAgent) + }) + + management.SetUnderAgent(true) + + tests := []struct { + name string + cfg mapstr.M + expectError bool + }{ + { + name: "topic with all valid characters", + cfg: mapstr.M{ + "topic": "abcdefghijklmnopqrstuvxz-ABCDEFGHIJKLMNOPQRSTUVXZ_01234567890.", + }, + }, + { + name: "topics is provided", + cfg: mapstr.M{ + "topics": []string{"foo", "bar"}, + }, + expectError: true, + }, + { + name: "topic cannot contain invalid characters", + cfg: mapstr.M{ + "topic": "foo bar", + }, + expectError: true, + }, + { + name: "topic with invalid characters", + cfg: mapstr.M{ + "topic": "foo + bar", + }, + expectError: true, + }, + { + name: "topic with invalid characters from dynamic topic selection", + cfg: mapstr.M{ + "topic": "%{event.field}", + }, + expectError: true, + }, + + // The default config does not set `topic` not `topics`. + { + name: "empty config is invalid", + cfg: mapstr.M{}, + expectError: true, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + c := config.MustNewConfigFrom(test.cfg) + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } + + _, err := readConfig(c) + + if test.expectError && err == nil { + t.Fatalf("invalid configuration must not be created") + } + + if !test.expectError && err != nil { + t.Fatalf("could not create config: %s", err) + } + }) + } +} + func TestBackoffFunc(t *testing.T) { testutil.SeedPRNG(t) tests := map[int]backoffConfig{ @@ -178,6 +266,7 @@ func TestTopicSelection(t *testing.T) { for name, test := range cases { t.Run(name, func(t *testing.T) { + test := test selector, err := buildTopicSelector(config.MustNewConfigFrom(test.cfg)) if err != nil { t.Fatalf("Failed to parse configuration: %v", err) From 3809f4b270a3264ec8c9447b42d41bc5d2e02b01 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Mon, 18 Mar 2024 19:15:37 +0200 Subject: [PATCH 16/31] migrate xpack dockerlogbeat pipeline (#38365) * migrate xpack dockerlogbeat pipeline * add pre-command * add common.sh to setenv.sh * add common.sh to setenv.sh * add arm image type --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/common.sh | 9 +- .../generate_xpack_dockerlogbeat_pipeline.sh | 83 +++++++++++++++++++ .buildkite/scripts/setenv.sh | 6 ++ .../x-pack/pipeline.xpack.dockerlogbeat.yml | 36 +++++++- 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1111caf5ac0..465b0a7e85e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -34,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9e75d3535cd..3b9bd5e1585 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -45,6 +45,10 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) +xpack_dockerlogbeat_changeset=( + "^x-pack/dockerlogbeat/.*" + ) + xpack_libbeat_changeset=( "^x-pack/libbeat/.*" ) @@ -105,6 +109,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; + "beats-xpack-dockerlogbeat") + BEAT_CHANGESET_REFERENCE=${xpack_dockerlogbeat_changeset[@]} + ;; "beats-xpack-libbeat") BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} ;; @@ -508,7 +515,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh new file mode 100755 index 00000000000..46e92f8ddf9 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-dockerlogbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index db86a7c9112..29acf0ec170 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -63,3 +63,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export DOCKER_PULL="0" export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi + +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then + # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. + source .buildkite/scripts/common.sh + defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" +fi diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index c4a0805615b..bcc2610e175 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -1,6 +1,38 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-dockerlogbeat" +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/dockerlogbeat" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Dockerlogbeat - run_xpack_dockerlogbeat" + key: "run_xpack_dockerlogbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack dockerlogbeat pipeline" + key: "xpack-dockerlogbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From 6826980c55e4e89bff53821971e482d4f4c7e98d Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Tue, 19 Mar 2024 06:13:45 +1030 Subject: [PATCH 17/31] x-pack/filebeat/input/httpjson: fix handling of POST requests with empty object bodies (#38290) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/httpjson/encoding.go | 6 ++++-- x-pack/filebeat/input/httpjson/input_test.go | 13 ++++++++++++- x-pack/filebeat/input/httpjson/request.go | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6b2023ddcec..1422424e423 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -101,6 +101,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] - Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] - Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] +- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] *Heartbeat* diff --git a/x-pack/filebeat/input/httpjson/encoding.go b/x-pack/filebeat/input/httpjson/encoding.go index 5dd62f10535..a7da4f25c0d 100644 --- a/x-pack/filebeat/input/httpjson/encoding.go +++ b/x-pack/filebeat/input/httpjson/encoding.go @@ -16,6 +16,7 @@ import ( "net/http" "unicode" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/mito/lib/xml" ) @@ -64,13 +65,14 @@ type decoderFunc func(p []byte, dst *response) error // encodeAsJSON encodes trReq as a JSON message. func encodeAsJSON(trReq transformable) ([]byte, error) { - if len(trReq.body()) == 0 { + body, err := trReq.GetValue("body") + if err == mapstr.ErrKeyNotFound { return nil, nil } header := trReq.header() header.Set("Content-Type", "application/json") trReq.setHeader(header) - return json.Marshal(trReq.body()) + return json.Marshal(body) } // decodeAsJSON decodes the JSON message in p into dst. diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 498ccc86183..6604c999d44 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -93,6 +93,17 @@ var testCases = []struct { handler: defaultHandler(http.MethodPost, `{"test":"abc"}`, ""), expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, }, + { + name: "POST_request_with_empty_object_body", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": http.MethodPost, + "request.body": map[string]interface{}{}, + }, + handler: defaultHandler(http.MethodPost, `{}`, ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, { name: "repeated_POST_requests", setupServer: newTestServer(httptest.NewServer), @@ -1516,7 +1527,7 @@ func defaultHandler(expectedMethod, expectedBody, msg string) http.HandlerFunc { r.Body.Close() if expectedBody != string(body) { w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected body was %q"}`, expectedBody) + msg = fmt.Sprintf(`{"error":"expected body was %q, but got %q"}`, expectedBody, body) } } diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 9e60d22ac49..3e63f026716 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -440,7 +440,7 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro req := transformable{} req.setURL(rf.url) - if rf.body != nil && len(*rf.body) > 0 { + if rf.body != nil { req.setBody(rf.body.Clone()) } From fe092972a481354a7c84c553ddad07f2ea5bbf34 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:02:38 -0400 Subject: [PATCH 18/31] chore: Update snapshot.yml (#38372) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 693e6d2f748..5cae69cbdbe 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-a2d5464c-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-09a11c1c-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-a2d5464c-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-09a11c1c-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-a2d5464c-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-09a11c1c-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 51974d9f8857a0e52da7367054581d78eb3fe3d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 22:30:10 +0000 Subject: [PATCH 19/31] build(deps): bump github.com/elastic/go-ucfg from 0.8.6 to 0.8.7 (#38251) * build(deps): bump github.com/elastic/go-ucfg from 0.8.6 to 0.8.7 Bumps [github.com/elastic/go-ucfg](https://github.com/elastic/go-ucfg) from 0.8.6 to 0.8.7. - [Release notes](https://github.com/elastic/go-ucfg/releases) - [Changelog](https://github.com/elastic/go-ucfg/blob/main/CHANGELOG.md) - [Commits](https://github.com/elastic/go-ucfg/compare/v0.8.6...v0.8.7) --- updated-dependencies: - dependency-name: github.com/elastic/go-ucfg dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Craig MacKenzie --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 2f66d290a43..ca91143b72c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -15223,11 +15223,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-ucfg -Version: v0.8.6 +Version: v0.8.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.7/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index fc7c97929b2..a54aa532781 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/elastic/go-seccomp-bpf v1.4.0 github.com/elastic/go-structform v0.0.10 github.com/elastic/go-sysinfo v1.13.1 - github.com/elastic/go-ucfg v0.8.6 + github.com/elastic/go-ucfg v0.8.7 github.com/elastic/gosigar v0.14.2 github.com/fatih/color v1.15.0 github.com/fearful-symmetry/gorapl v0.0.4 diff --git a/go.sum b/go.sum index f95a7973ed8..11a7f4c415b 100644 --- a/go.sum +++ b/go.sum @@ -710,8 +710,8 @@ github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gn github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.13.1 h1:U5Jlx6c/rLkR72O8wXXXo1abnGlWGJU/wbzNJ2AfQa4= github.com/elastic/go-sysinfo v1.13.1/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ= -github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= -github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= +github.com/elastic/go-ucfg v0.8.7 h1:/bKaN553LY3MsfEIz2XOEEs+tRw03TzJCARrnVPpOyc= +github.com/elastic/go-ucfg v0.8.7/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11iyX5t3Dd9dDMMNqPfEZNc//JsWGIhEgQ= From c29075eea0e7a21500b15634ffa160080eb1a4e9 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:11:43 +1030 Subject: [PATCH 20/31] x-pack/filebeat/input/{cel,httpjson}: fix PEM key validation (#38405) Previously the validation was attempting to parse the PEM text as a key and was also attempting to parse the data as the wrong kind of key. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/cel/config_auth.go | 6 ++- x-pack/filebeat/input/cel/config_okta_auth.go | 20 ++++++--- x-pack/filebeat/input/cel/config_test.go | 41 +++++++++++++++++++ x-pack/filebeat/input/httpjson/config_auth.go | 8 ++-- .../input/httpjson/config_okta_auth.go | 20 ++++++--- x-pack/filebeat/input/httpjson/config_test.go | 41 +++++++++++++++++++ 7 files changed, 120 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1422424e423..1cd8999245b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -102,6 +102,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] - Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] - Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] +- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405] *Heartbeat* diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index d6b35d633e6..ac187f4ffa1 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -6,7 +6,6 @@ package cel import ( "context" - "crypto/x509" "encoding/json" "errors" "fmt" @@ -341,7 +340,10 @@ func (o *oAuth2Config) validateOktaProvider() error { } // jwk_pem if o.OktaJWKPEM != "" { - _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) + _, err := pemPKCS8PrivateKey([]byte(o.OktaJWKPEM)) + if err != nil { + return fmt.Errorf("okta validation error: %w", err) + } return err } // jwk_file diff --git a/x-pack/filebeat/input/cel/config_okta_auth.go b/x-pack/filebeat/input/cel/config_okta_auth.go index 74366afd3d5..0f18b12e66c 100644 --- a/x-pack/filebeat/input/cel/config_okta_auth.go +++ b/x-pack/filebeat/input/cel/config_okta_auth.go @@ -12,6 +12,7 @@ import ( "encoding/base64" "encoding/json" "encoding/pem" + "errors" "fmt" "math/big" "net/http" @@ -160,17 +161,24 @@ func (i *base64int) UnmarshalJSON(b []byte) error { } func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { - blk, rest := pem.Decode([]byte(pemdata)) - if rest := bytes.TrimSpace(rest); len(rest) != 0 { - return "", fmt.Errorf("PEM text has trailing data: %s", rest) - } - key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) + key, err := pemPKCS8PrivateKey([]byte(pemdata)) if err != nil { return "", err } return signJWT(cnf, key) } +func pemPKCS8PrivateKey(pemdata []byte) (any, error) { + blk, rest := pem.Decode(pemdata) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return nil, fmt.Errorf("PEM text has trailing data: %d bytes", len(rest)) + } + if blk == nil { + return nil, errors.New("no PEM data") + } + return x509.ParsePKCS8PrivateKey(blk.Bytes) +} + // signJWT creates a JWT token using required claims and sign it with the // private key. func signJWT(cnf *oauth2.Config, key any) (string, error) { @@ -182,7 +190,7 @@ func signJWT(cnf *oauth2.Config, key any) (string, error) { Expiration(now.Add(time.Hour)). Build() if err != nil { - return "", err + return "", fmt.Errorf("failed to create token: %w", err) } signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index 7acf74df08c..0a686df099c 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -539,6 +539,47 @@ var oAuth2ValidationTests = []struct { }, }, }, + { + name: "okta_successful_pem_oauth2_validation", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "okta", + "client.id": "a_client_id", + "token_url": "localhost", + "scopes": []string{"foo"}, + "okta.jwk_pem": ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +`, + }, + }, + }, } func TestConfigOauth2Validation(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index d05592dfa50..b25bab03dd3 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -6,7 +6,6 @@ package httpjson import ( "context" - "crypto/x509" "encoding/json" "errors" "fmt" @@ -309,8 +308,11 @@ func (o *oAuth2Config) validateOktaProvider() error { } // jwk_pem if o.OktaJWKPEM != "" { - _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) - return err + _, err := pemPKCS8PrivateKey([]byte(o.OktaJWKPEM)) + if err != nil { + return fmt.Errorf("okta validation error: %w", err) + } + return nil } // jwk_file if o.OktaJWKFile != "" { diff --git a/x-pack/filebeat/input/httpjson/config_okta_auth.go b/x-pack/filebeat/input/httpjson/config_okta_auth.go index c2b4289d9c9..8d2a8415c2e 100644 --- a/x-pack/filebeat/input/httpjson/config_okta_auth.go +++ b/x-pack/filebeat/input/httpjson/config_okta_auth.go @@ -12,6 +12,7 @@ import ( "encoding/base64" "encoding/json" "encoding/pem" + "errors" "fmt" "math/big" "net/http" @@ -158,17 +159,24 @@ func (i *base64int) UnmarshalJSON(b []byte) error { } func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { - blk, rest := pem.Decode([]byte(pemdata)) - if rest := bytes.TrimSpace(rest); len(rest) != 0 { - return "", fmt.Errorf("PEM text has trailing data: %s", rest) - } - key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) + key, err := pemPKCS8PrivateKey([]byte(pemdata)) if err != nil { return "", err } return signJWT(cnf, key) } +func pemPKCS8PrivateKey(pemdata []byte) (any, error) { + blk, rest := pem.Decode(pemdata) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return nil, fmt.Errorf("PEM text has trailing data: %d bytes", len(rest)) + } + if blk == nil { + return nil, errors.New("no PEM data") + } + return x509.ParsePKCS8PrivateKey(blk.Bytes) +} + // signJWT creates a JWT token using required claims and sign it with the private key. func signJWT(cnf *oauth2.Config, key any) (string, error) { now := time.Now() @@ -179,7 +187,7 @@ func signJWT(cnf *oauth2.Config, key any) (string, error) { Expiration(now.Add(time.Hour)). Build() if err != nil { - return "", err + return "", fmt.Errorf("failed to create token: %w", err) } signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index d88c6ac4a62..910510b6e9c 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -499,6 +499,47 @@ func TestConfigOauth2Validation(t *testing.T) { }, }, }, + { + name: "okta successful pem oauth2 validation", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "okta", + "client.id": "a_client_id", + "token_url": "localhost", + "scopes": []string{"foo"}, + "okta.jwk_pem": ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +`, + }, + }, + }, } for _, c := range cases { From 51fb8b267ec9513dee87bd5963e7b3b7314c538d Mon Sep 17 00:00:00 2001 From: Kurt Date: Tue, 19 Mar 2024 04:50:09 -0400 Subject: [PATCH 21/31] Add Saved Object name to Kibana audit logging (#38307) * Initial commit to add name to beats * adding PR * Output from go/mage commands? * changing test data * updating log offset * log offset of last event --------- Co-authored-by: Milton Hultgren Co-authored-by: sharbuz <87968844+sharbuz@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 11 ++ filebeat/module/kibana/_meta/fields.yml | 4 + .../kibana/audit/test/test-audit-814.log | 5 + .../test/test-audit-814.log-expected.json | 171 ++++++++++++++++++ filebeat/module/kibana/fields.go | 2 +- 6 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/kibana/audit/test/test-audit-814.log create mode 100644 filebeat/module/kibana/audit/test/test-audit-814.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1cd8999245b..f494dd0eaf7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -157,6 +157,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Filebeat* +- Adding Saved Object name field to Kibana audit logs {pull}38307[38307] - Update SQL input documentation regarding Oracle DSNs {pull}37590[37590] - add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] - httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index ddc887d246f..4cf4b99b5e7 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -86793,6 +86793,17 @@ example: 6295bdd0-0a0e-11e7-825f-6748cda7d858 -- +*`kibana.saved_object.name`*:: ++ +-- +The name of the saved object associated with this event. + +type: keyword + +example: my-saved-object + +-- + *`kibana.add_to_spaces`*:: + -- diff --git a/filebeat/module/kibana/_meta/fields.yml b/filebeat/module/kibana/_meta/fields.yml index d4e664ade58..aed9252122c 100644 --- a/filebeat/module/kibana/_meta/fields.yml +++ b/filebeat/module/kibana/_meta/fields.yml @@ -27,6 +27,10 @@ description: "The id of the saved object associated with this event." example: "6295bdd0-0a0e-11e7-825f-6748cda7d858" type: keyword + - name: saved_object.name + description: "The name of the saved object associated with this event." + example: "my-saved-object" + type: keyword - name: add_to_spaces description: "The set of space ids that a saved object was shared to." example: "['default', 'marketing']" diff --git a/filebeat/module/kibana/audit/test/test-audit-814.log b/filebeat/module/kibana/audit/test/test-audit-814.log new file mode 100644 index 00000000000..97127ddcbf0 --- /dev/null +++ b/filebeat/module/kibana/audit/test/test-audit-814.log @@ -0,0 +1,5 @@ +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"fleet-default-settings","type":"ingest_manager_settings"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:47.298+00:00","message":"User is accessing ingest_manager_settings [id=fleet-default-settings]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"a09a5397-7b9a-5a73-a622-e29f4c635658","type":"ingest-outputs"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:48.987+00:00","message":"User is accessing ingest-outputs [id=a09a5397-7b9a-5a73-a622-e29f4c635658]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"synthetics","type":"epm-packages"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:53.426+00:00","message":"User is accessing epm-packages [id=synthetics]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"kibana","path":"/api/features","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default"},"trace":{"id":"e2792f3f-4cf1-4f6d-b4eb-5b491724c295"},"client":{"ip":"172.22.0.2"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:19:18.882+00:00","message":"User is requesting [/api/features] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"cf44f52888b9ec5a"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"abcde-fghijk","type":"ingest_manager_settings","name":"fleet-object-name"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T16:18:47.298+00:00","message":"User is accessing ingest_manager_settings [id=fleet-default-settings]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} diff --git a/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json b/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json new file mode 100644 index 00000000000..9ab233fea60 --- /dev/null +++ b/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json @@ -0,0 +1,171 @@ +[ + { + "@timestamp": "2023-06-19T15:18:47.298+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "fleet-default-settings", + "kibana.saved_object.type": "ingest_manager_settings", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 0, + "message": "User is accessing ingest_manager_settings [id=fleet-default-settings]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:18:48.987+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "a09a5397-7b9a-5a73-a622-e29f4c635658", + "kibana.saved_object.type": "ingest-outputs", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 616, + "message": "User is accessing ingest-outputs [id=a09a5397-7b9a-5a73-a622-e29f4c635658]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:18:53.426+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "synthetics", + "kibana.saved_object.type": "epm-packages", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 1242, + "message": "User is accessing epm-packages [id=synthetics]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:19:18.882+00:00", + "client.ip": "172.22.0.2", + "event.action": "http_request", + "event.category": [ + "web" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "fileset.name": "audit", + "http.request.method": "get", + "input.type": "log", + "kibana.space_id": "default", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 1812, + "message": "User is requesting [/api/features] endpoint", + "process.pid": 7, + "related.user": [ + "elastic" + ], + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "e2792f3f-4cf1-4f6d-b4eb-5b491724c295", + "transaction.id": "cf44f52888b9ec5a", + "url.domain": "kibana", + "url.path": "/api/features", + "url.port": 5601, + "url.scheme": "http", + "user.name": "elastic", + "user.roles": [ + "superuser" + ] + }, + { + "@timestamp": "2023-06-19T16:18:47.298+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "abcde-fghijk", + "kibana.saved_object.type": "ingest_manager_settings", + "kibana.saved_object.name": "fleet-object-name", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 2466, + "message": "User is accessing ingest_manager_settings [id=fleet-default-settings]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + } +] diff --git a/filebeat/module/kibana/fields.go b/filebeat/module/kibana/fields.go index 504d1f6283e..fce968bbf78 100644 --- a/filebeat/module/kibana/fields.go +++ b/filebeat/module/kibana/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKibana returns asset data. // This is the base64 encoded zlib format compressed contents of module/kibana. func AssetKibana() string { - return "eJy8ls9ymzAQxu9+ih1ffCmM7fhffMilzaHT6a23TodZowVUC4lIi928fUfguGBj4rhpOe7ut/v7hJAIYEvPa9jKDWocALBkRWsY1oHhAMCSInS0htTnBbnYyoKl0Wt4GADAQQtfjSgVDQASSUq4dZULQGNOa3BkdzKmUBtBoTWKXJUG4OeC1p5hb6xoSY5If8pSa8riEOkA8U9NAYmxUKB1UqfwpeZTJnXhoa6J2MZ0ThodSXFMnUz6lhF8/gQmAc4ISkf2RQTonIklMgnYS86AM+mAdqQ5hEeMM08gNSAz5QWDJVcqduBDUGr5VNKxlRRhA4B+YV5Ub2UyvaPZfLEMaHW/CSZTcRfgbL4IZtPFYjKbLGfj8XjYUJ6vbsttgTH1eB16s1K8mK3K+1wOO5kFJVgqfgMW7khEZvOTYg59aS+fLzgSeiXUyhtA0WUbg1bciHr9Qv4V5mJ6P98IMQ7GOKZgMqFlsJrOk2CxnK1igUuxmq+ud4BCRGyi6t26Xn5H7A3Uu0AKB5whA7bd7NGBy9CSADYXDHwfHbbE6AOMcrRbYqnT0Y/roQUpYooSa/J3JbeUGx/yjf8ZPJackWYZo0eMCmt2UpDtddDWwIvmbPPg4Yzp2z8bdDKe3Mz76gd5ibX6UG8GvpnXEqq8F/hRoWMZO0IbZ6f4lbzqDPtMxhkkpUqkUiSuYtfIckfXwytjtmXxZuhadoD9P5zpWW3zdj6DfmgkoHElg5KaXPO2O72bm2MZU9dKXOJ8laBNkfr/BN887BzrGJneb+7H0lrSXLf1J1MN0j07J8bO0fW5dZKqg1E33MX+oaWnMCMUZLuXN1H+r0WT6GvhrmzxOwAA//+jaglP" + return "eJy8ls1y2kAMx+88hYYLl9oDhK9wyKXNodPprbdOxyO8sr1lvevsytC8fWdtQm0wDqFpOUr6Sz8tQiiALT2vYSs3qHEAwJIVrWFYG4YDAEuK0NEaUu8X5GIrC5ZGr+FhAAAHLXw1olQ0AEgkKeHWlS8AjTmtwZHdyZhCbQSF1ihylRuAnwtae4a9saIlOSL9CUutKYuDpQPEf2oKSIyFAq2TOoUvNZ8yqQsPcU3ENqZz0uhIiqPrpNK3jODzJzAJcEZQOrIvIkDnTCyRScBecgacSQe0I80hPGKceQKpAZkpLxgsuVKxA2+CUsunko6ppAgbAPQL86L6VibTO5rNF8uAVvebYDIVdwHO5otgNl0sJrPJcjYej4cN5fnrtrotMKaeXoe+WSlemq3C+7ocdjILSrBU/AYs3JGIzOYnxRz60F4+H3Ak9EqolTeAoss2Bq24EfX6h/wrzMX0fr4RYhyMcUzBZELLYDWdJ8FiOVvFApdiNV/d2IE39fbgA96ni/w5qBIEdYLrgVGIiE1UDaPrhXXEnrUeWykccIYM2AbfowOXoSUBbC6wfh8dZnj0AUY52i2x1Onox/XQghQxRYk1+buSW8qNN/nE/wweS85Is4zRI0aFNTspyPZ20NbAi+ZsTvCwFPtGZYNOxpObeV/dIJdYq81yM/DNvJZQ5b3Ajwody9gR2jg7xa/k9Q91n8k4g6RUiVSKxFXsGlnu6Hp4Zcy2LN4MXcsOsP+HMz2LbZ4TZ9APDQc0bghQUpNr/j2fHhPNsoypazkucb5K0KZI/WHjk4edZR0j0/vV/VhaS5rrtH4z1SDdtXNi7Cxd760TV22MuuEu5g8tPYUZoSDb/byJ8meWJtGXwl2Z4ncAAAD//zz3P1Q=" } From 1b13c648feef40302b7f23f20f0d8780e48800cd Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Tue, 19 Mar 2024 08:00:44 -0500 Subject: [PATCH 22/31] [filebeat] Deprecate syslog input (#38277) - The syslog input has been deprecated in favor of the syslog processor. --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/inputs/input-syslog.asciidoc | 4 ++++ filebeat/input/syslog/input.go | 13 ++++++++++--- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f494dd0eaf7..a1b1f8e84eb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -273,6 +273,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Filebeat* +- Deprecate `syslog` input in favor of `syslog` processor. {issue}37555[37555] {pull}38277[38277] *Heartbeat* diff --git a/filebeat/docs/inputs/input-syslog.asciidoc b/filebeat/docs/inputs/input-syslog.asciidoc index e43eabea378..3e0555d03b9 100644 --- a/filebeat/docs/inputs/input-syslog.asciidoc +++ b/filebeat/docs/inputs/input-syslog.asciidoc @@ -3,6 +3,10 @@ [id="{beatname_lc}-input-{type}"] === Syslog input +deprecated:[8.14.0] + +The syslog input is deprecated. Please use the <> processor for processing syslog messages. + ++++ Syslog ++++ diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go index 702472794dd..c91158c0f89 100644 --- a/filebeat/input/syslog/input.go +++ b/filebeat/input/syslog/input.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -85,6 +86,8 @@ var ( "local6", "local7", } + + deprecatedNotificationOnce sync.Once ) func init() { @@ -112,6 +115,10 @@ func NewInput( ) (input.Input, error) { log := logp.NewLogger("syslog") + deprecatedNotificationOnce.Do(func() { + cfgwarn.Deprecate("", "Syslog input. Use Syslog processor instead.") + }) + out, err := outlet.Connect(cfg) if err != nil { return nil, err @@ -180,7 +187,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) case syslogFormatRFC5424: return func(data []byte, metadata inputsource.NetworkMetadata) { ev := parseAndCreateEvent5424(data, metadata, cfg.Timezone.Location(), log) - forwarder.Send(ev) + _ = forwarder.Send(ev) } case syslogFormatAuto: @@ -191,7 +198,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) } else { ev = parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) } - forwarder.Send(ev) + _ = forwarder.Send(ev) } case syslogFormatRFC3164: break @@ -199,7 +206,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) return func(data []byte, metadata inputsource.NetworkMetadata) { ev := parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) - forwarder.Send(ev) + _ = forwarder.Send(ev) } } From 45d5142abf39c0141ad9e75a993f7f8288da4bf1 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:06:43 +0200 Subject: [PATCH 23/31] Migrate xpack auditbeat pipeline (#38349) * migrate xpack-winlogbeat pipeline * add generate_xpack_winlogbeat_pipeline.sh and cleanup setenv.sh * full test * migrate xpack-auditbeat pipeline * add rpm-devel * add librpm-dev for ubuntu * debug * debug * debug * debug * debug * debug * debug python * debug python * debug python * debug python * debug python * add default ununtu image for tests * revert test changes * revert test changes * revert test changes --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/common.sh | 33 +++- .../generate_xpack_auditbeat_pipeline.sh | 166 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 2 +- .../x-pack/pipeline.xpack.auditbeat.yml | 60 ++++++- 5 files changed, 251 insertions(+), 12 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_auditbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 465b0a7e85e..7be1f965230 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -34,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3b9bd5e1585..b6170070836 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -11,6 +11,7 @@ GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +# define if needed run the whole pipeline for the particular beat [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -19,16 +20,25 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" +[ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" + +# define if needed run ARM platform-specific tests for the particular beat [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" +[ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" +[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" +[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" + +# define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" -[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" +[ -z "${run_xpack_auditbeat_macos_tests+x}" ] && run_xpack_auditbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" -[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" +# define if needed run cloud-specific tests for the particular beat +[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" + metricbeat_changeset=( "^metricbeat/.*" ) @@ -45,6 +55,10 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) +xpack_auditbeat_changeset=( + "^x-pack/auditbeat/.*" + ) + xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) @@ -109,6 +123,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; + "beats-xpack-auditbeat") + BEAT_CHANGESET_REFERENCE=${xpack_auditbeat_changeset[@]} + ;; "beats-xpack-dockerlogbeat") BEAT_CHANGESET_REFERENCE=${xpack_dockerlogbeat_changeset[@]} ;; @@ -287,10 +304,10 @@ with_dependencies() { if [ "${platform_type}" == "Linux" ]; then if [ "${linuxType}" = "ubuntu" ]; then sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev + sudo apt-get install -y libsystemd-dev libpcap-dev librpm-dev elif [ "${linuxType}" = "rhel" ]; then # sudo dnf update -y - sudo dnf install -y systemd-devel + sudo dnf install -y systemd-devel rpm-devel wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image fi @@ -371,7 +388,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -382,7 +399,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -515,7 +532,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-auditbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh new file mode 100755 index 00000000000..f29e6152e60 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-auditbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + + - label: ":linux: Ubuntu Unit (MODULE) Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 29acf0ec170..e40f858f1c4 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -56,7 +56,7 @@ exportVars() { fi } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 2343eb6a4dd..d88bf2a4ff0 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -1,6 +1,62 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-auditbeat" +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/auditbeat" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Auditbeat - run_xpack_auditbeat" + key: "run_xpack_auditbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Auditbeat - run_xpack_auditbeat_macos_tests" + key: "run_xpack_auditbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Auditbeat - run_xpack_auditbeat_arm_tests" + key: "run_xpack_auditbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack auditbeat pipeline" + key: "xpack-auditbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From 15d6fd7dcaab8ed83f87eb4e9e0ad884a3f10244 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:12:14 +0000 Subject: [PATCH 24/31] build(deps): bump github.com/elastic/elastic-agent-client/v7 from 7.8.0 to 7.8.1 (#38412) * build(deps): bump github.com/elastic/elastic-agent-client/v7 Bumps [github.com/elastic/elastic-agent-client/v7](https://github.com/elastic/elastic-agent-client) from 7.8.0 to 7.8.1. - [Release notes](https://github.com/elastic/elastic-agent-client/releases) - [Commits](https://github.com/elastic/elastic-agent-client/compare/v7.8.0...v7.8.1) --- updated-dependencies: - dependency-name: github.com/elastic/elastic-agent-client/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- NOTICE.txt | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index ca91143b72c..2ce6e0e15c9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12524,11 +12524,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.8.0 +Version: v7.8.1 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.8.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.8.1/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -25852,11 +25852,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.58.3/LIC -------------------------------------------------------------------------------- Dependency : google.golang.org/protobuf -Version: v1.32.0 +Version: v1.33.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.32.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.33.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index a54aa532781..afe0c99b6d2 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.8.0 + github.com/elastic/elastic-agent-client/v7 v7.8.1 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.5.0 github.com/elastic/go-licenser v0.4.1 @@ -164,7 +164,7 @@ require ( google.golang.org/api v0.128.0 google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/grpc v1.58.3 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/inf.v0 v0.9.1 gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index 11a7f4c415b..47d892296a0 100644 --- a/go.sum +++ b/go.sum @@ -675,8 +675,8 @@ github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= -github.com/elastic/elastic-agent-client/v7 v7.8.0 h1:GHFzDJIWpdgI0qDk5EcqbQJGvwTsl2E2vQK3/xe+MYQ= -github.com/elastic/elastic-agent-client/v7 v7.8.0/go.mod h1:ihtjqJzYiIltlRhNruaSSc0ogxIhqPD5hOMKq16cI1s= +github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM= +github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY= github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg= github.com/elastic/elastic-agent-libs v0.7.5/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= @@ -2682,8 +2682,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 4a3da808e4760eedf8016a2d7d876bb3bfa40aa9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 19 Mar 2024 12:47:21 -0600 Subject: [PATCH 25/31] [AWS] Support VPC endpoint for aws-s3 input SQS queue url (#38189) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/awss3/input.go | 24 +++++++++++--- x-pack/filebeat/input/awss3/input_test.go | 39 ++++------------------- 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a1b1f8e84eb..0b33bffc31a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -205,6 +205,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] - Add logging for cache processor file reads and writes. {pull}38052[38052] - Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] +- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189] - Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] - Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] - Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 0b33ae042f9..5fc1c1f0491 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -317,18 +317,20 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli return s3Poller, nil } -var errBadQueueURL = errors.New("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") +var errBadQueueURL = errors.New("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME} or https://{VPC_ENDPOINT}.sqs.{REGION_ENDPOINT}.vpce.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (region string, err error) { // get region from queueURL - // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs + // Example for sqs queue: https://sqs.us-east-1.amazonaws.com/12345678912/test-s3-logs + // Example for vpce: https://vpce-test.sqs.us-east-1.vpce.amazonaws.com/12345678912/sqs-queue u, err := url.Parse(queueURL) if err != nil { return "", fmt.Errorf(queueURL + " is not a valid URL") } if (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" { queueHostSplit := strings.SplitN(u.Host, ".", 3) - if len(queueHostSplit) == 3 { + // check for sqs queue url + if len(queueHostSplit) == 3 && queueHostSplit[0] == "sqs" { if queueHostSplit[2] == endpoint || (endpoint == "" && strings.HasPrefix(queueHostSplit[2], "amazonaws.")) { region = queueHostSplit[1] if defaultRegion != "" && region != defaultRegion { @@ -336,7 +338,21 @@ func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (reg } return region, nil } - } else if defaultRegion != "" { + } + + // check for vpce url + queueHostSplitVPC := strings.SplitN(u.Host, ".", 5) + if len(queueHostSplitVPC) == 5 && queueHostSplitVPC[1] == "sqs" { + if queueHostSplitVPC[4] == endpoint || (endpoint == "" && strings.HasPrefix(queueHostSplitVPC[4], "amazonaws.")) { + region = queueHostSplitVPC[2] + if defaultRegion != "" && region != defaultRegion { + return defaultRegion, regionMismatchError{queueURLRegion: region, defaultRegion: defaultRegion} + } + return region, nil + } + } + + if defaultRegion != "" { return defaultRegion, nil } } diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 8a195eb3084..abc9f5c9a6a 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -5,7 +5,6 @@ package awss3 import ( - "errors" "testing" "github.com/stretchr/testify/assert" @@ -76,40 +75,16 @@ func TestGetRegionFromQueueURL(t *testing.T) { wantErr: errBadQueueURL, }, { - name: "abc.xyz_and_domain_with_different_endpoint", - queueURL: "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", - endpoint: "googlecloud.com", - wantErr: errBadQueueURL, - }, - { - name: "mismatch_regions_no_default", - queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + name: "vpce_endpoint", + queueURL: "https://vpce-test.sqs.us-east-2.vpce.amazonaws.com/12345678912/sqs-queue", deflt: "", - want: "us-east-1", - }, - { - name: "mismatch_regions", - queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", - deflt: "ap-west-1", - want: "ap-west-1", - wantErr: regionMismatchError{queueURLRegion: "us-east-1", defaultRegion: "ap-west-1"}, - }, - { - name: "localstack", - queueURL: "http://localhost:4566/000000000000/filebeat-s3-integtest-d9clk9", - deflt: "localstack", - want: "localstack", + want: "us-east-2", }, { - name: "localstack_sns", - queueURL: "http://localhost:4566/000000000000/filebeat-s3-integtest-sns-d9clk9", - deflt: "localstack_sns", - want: "localstack_sns", - }, - { - name: "invalid_queue_url", - queueURL: ":foo", - wantErr: errors.New(":foo is not a valid URL"), + name: "vpce_endpoint_with_endpoint", + queueURL: "https://vpce-test.sqs.us-east-1.vpce.amazonaws.com/12345678912/sqs-queue", + endpoint: "amazonaws.com", + want: "us-east-1", }, } From 08e59363b6410f1701c9902858b3c0149556ff10 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Mar 2024 15:53:53 -0400 Subject: [PATCH 26/31] auditbeat - skip TestExeObjParser on CI (#38441) TestExeObjParser fails on Windows because Defender is removing the testdata that is required by the test. This adds a conditional test skip for CI when those files are missing. Closes #38211 --- auditbeat/module/file_integrity/exeobjparser_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/auditbeat/module/file_integrity/exeobjparser_test.go b/auditbeat/module/file_integrity/exeobjparser_test.go index 0958305afb8..f8ca144e4ed 100644 --- a/auditbeat/module/file_integrity/exeobjparser_test.go +++ b/auditbeat/module/file_integrity/exeobjparser_test.go @@ -19,8 +19,11 @@ package file_integrity import ( + "errors" "fmt" + "io/fs" "math" + "os" "reflect" "strconv" "testing" @@ -44,6 +47,12 @@ func TestExeObjParser(t *testing.T) { t.Skip("skipping test on garbled PE file: see https://github.com/elastic/beats/issues/35705") } + if _, ci := os.LookupEnv("CI"); ci { + if _, err := os.Stat(target); err != nil && errors.Is(fs.ErrNotExist, err) { + t.Skip("skipping test because target binary was not found: see https://github.com/elastic/beats/issues/38211") + } + } + got := make(mapstr.M) err := exeObjParser(nil).Parse(got, target) if err != nil { From d64ce48f566954b68eb84711ec1c61dffcf2899d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 19 Mar 2024 18:02:34 -0400 Subject: [PATCH 27/31] go.mod - bump to github.com/lestrrat-go/jwx/v2 v2.0.21 (#38346) Bump to github.com/lestrrat-go/jwx/v2 v2.0.21. Filebeat is unaffected by CVE-2024-28122 which affected prior versions of github.com/lestrrat-go/jwx/v2, it does not use the affected github.com/lestrrat-go/jwx/v2/jwe package nor does process any JWE tokens. --- NOTICE.txt | 28 ++++++++++++++-------------- go.mod | 17 +++++++++-------- go.sum | 27 ++++++++++++++------------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 2ce6e0e15c9..f2e34a27813 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -20695,11 +20695,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/lestrrat-go/jwx/v2 -Version: v2.0.19 +Version: v2.0.21 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/jwx/v2@v2.0.19/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/jwx/v2@v2.0.21/LICENSE: The MIT License (MIT) @@ -24984,11 +24984,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.17.0 +Version: v0.21.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.17.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.21.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25095,11 +25095,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.19.0 +Version: v0.21.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.19.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.21.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25206,11 +25206,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.15.0 +Version: v0.18.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.15.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.18.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -46531,11 +46531,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/lestrrat-go/httprc -Version: v1.0.4 +Version: v1.0.5 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/httprc@v1.0.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/httprc@v1.0.5/LICENSE: MIT License @@ -50813,11 +50813,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx -Version: v0.5.0 +Version: v0.5.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.5.2/LICENSE: The MIT License @@ -53894,11 +53894,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.15.0 +Version: v0.18.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.15.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.18.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index afe0c99b6d2..cb6342852c4 100644 --- a/go.mod +++ b/go.mod @@ -139,7 +139,7 @@ require ( github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/ugorji/go/codec v1.1.8 github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 @@ -151,13 +151,13 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.21.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.14.0 - golang.org/x/net v0.19.0 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.10.0 golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.16.0 @@ -218,7 +218,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/icholy/digest v0.1.22 - github.com/lestrrat-go/jwx/v2 v2.0.19 + github.com/lestrrat-go/jwx/v2 v2.0.21 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 @@ -330,7 +330,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc v1.0.4 // indirect + github.com/lestrrat-go/httprc v1.0.5 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -359,7 +359,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect @@ -376,7 +376,7 @@ require ( go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/otel/trace v1.21.0 // indirect golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect @@ -420,6 +420,7 @@ replace ( github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 + github.com/stretchr/testify v1.9.0 => github.com/stretchr/testify v1.8.4 // Temporary fix for https://github.com/elastic/beats/pull/38436. github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 ) diff --git a/go.sum b/go.sum index 47d892296a0..709e6f12d53 100644 --- a/go.sum +++ b/go.sum @@ -1379,12 +1379,12 @@ github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= -github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= +github.com/lestrrat-go/httprc v1.0.5 h1:bsTfiH8xaKOJPrg1R+E3iE/AWZr/x0Phj9PBTG/OLUk= +github.com/lestrrat-go/httprc v1.0.5/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.0.19 h1:ekv1qEZE6BVct89QA+pRF6+4pCpfVrOnEJnTnT4RXoY= -github.com/lestrrat-go/jwx/v2 v2.0.19/go.mod h1:l3im3coce1lL2cDeAjqmaR+Awx+X8Ih+2k8BuHNJ4CU= +github.com/lestrrat-go/jwx/v2 v2.0.21 h1:jAPKupy4uHgrHFEdjVjNkUgoBKtVDgrQPB/h55FHrR0= +github.com/lestrrat-go/jwx/v2 v2.0.21/go.mod h1:09mLW8zto6bWL9GbwnqAli+ArLf+5M33QLQPDggkUWM= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1833,8 +1833,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -2067,8 +2068,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2210,8 +2211,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2379,8 +2380,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2390,8 +2391,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 4df595ff4f416586404d2040eb1e918c3e3af96d Mon Sep 17 00:00:00 2001 From: Denis Date: Wed, 20 Mar 2024 07:53:19 +0100 Subject: [PATCH 28/31] Make log messages in the file scanner less noisy (#38421) * Demote warning about empty files to the Debug level * Demote warnings about files too small to the Debug level * Improve wording, so it does not sound like an error --- CHANGELOG-developer.next.asciidoc | 1 + filebeat/input/filestream/fswatch.go | 13 +++++++-- filebeat/input/filestream/fswatch_test.go | 34 +++++++++++++++++++---- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 14901ead1bc..b617edfaf04 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -181,6 +181,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Elide retryable HTTP client construction in Filebeat HTTPJSON and CEL inputs if not needed. {pull}36916[36916] - Allow assignment of packetbeat protocols to interfaces. {issue}36574[36564] {pull}36852[36852] - Add Active Directory entity collector for Filebeat entity analytics. {pull}37854[37854] +- Make logs for empty and small files less noisy when using fingerprint file identity in filestream. {pull}38421[38421] ==== Deprecated diff --git a/filebeat/input/filestream/fswatch.go b/filebeat/input/filestream/fswatch.go index 454a5b428b0..c51d850bbd2 100644 --- a/filebeat/input/filestream/fswatch.go +++ b/filebeat/input/filestream/fswatch.go @@ -20,6 +20,7 @@ package filestream import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "hash" "io" @@ -45,6 +46,10 @@ const ( watcherDebugKey = "file_watcher" ) +var ( + errFileTooSmall = errors.New("file size is too small for ingestion") +) + type fileWatcherConfig struct { // Interval is the time between two scans. Interval time.Duration `config:"check_interval"` @@ -202,7 +207,7 @@ func (w *fileWatcher) watch(ctx unison.Canceler) { for path, fd := range newFilesByName { // no need to react on empty new files if fd.Info.Size() == 0 { - w.log.Warnf("file %q has no content yet, skipping", fd.Filename) + w.log.Debugf("file %q has no content yet, skipping", fd.Filename) delete(paths, path) continue } @@ -385,6 +390,10 @@ func (s *fileScanner) GetFiles() map[string]loginp.FileDescriptor { } fd, err := s.toFileDescriptor(&it) + if errors.Is(err, errFileTooSmall) { + s.log.Debugf("cannot start ingesting from file %q: %s", filename, err) + continue + } if err != nil { s.log.Warnf("cannot create a file descriptor for an ingest target %q: %s", filename, err) continue @@ -473,7 +482,7 @@ func (s *fileScanner) toFileDescriptor(it *ingestTarget) (fd loginp.FileDescript // we should not open the file if we know it's too small minSize := s.cfg.Fingerprint.Offset + s.cfg.Fingerprint.Length if fileSize < minSize { - return fd, fmt.Errorf("filesize of %q is %d bytes, expected at least %d bytes for fingerprinting", fd.Filename, fileSize, minSize) + return fd, fmt.Errorf("filesize of %q is %d bytes, expected at least %d bytes for fingerprinting: %w", fd.Filename, fileSize, minSize, errFileTooSmall) } file, err := os.Open(it.originalFilename) diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go index 6c9d88b858e..3fab8bfd2bd 100644 --- a/filebeat/input/filestream/fswatch_test.go +++ b/filebeat/input/filestream/fswatch_test.go @@ -276,17 +276,20 @@ scanner: err := os.WriteFile(filename, nil, 0777) require.NoError(t, err) - t.Run("issues a warning in logs", func(t *testing.T) { - var lastWarning string + t.Run("issues a debug message in logs", func(t *testing.T) { expLogMsg := fmt.Sprintf("file %q has no content yet, skipping", filename) require.Eventually(t, func() bool { - logs := logp.ObserverLogs().FilterLevelExact(logp.WarnLevel.ZapLevel()).TakeAll() + logs := logp.ObserverLogs().FilterLevelExact(logp.DebugLevel.ZapLevel()).TakeAll() if len(logs) == 0 { return false } - lastWarning = logs[len(logs)-1].Message - return strings.Contains(lastWarning, expLogMsg) - }, 100*time.Millisecond, 10*time.Millisecond, "required a warning message %q but got %q", expLogMsg, lastWarning) + for _, l := range logs { + if strings.Contains(l.Message, expLogMsg) { + return true + } + } + return false + }, 100*time.Millisecond, 10*time.Millisecond, "required a debug message %q but never found", expLogMsg) }) t.Run("emits a create event once something is written to the empty file", func(t *testing.T) { @@ -797,6 +800,25 @@ scanner: }) } + t.Run("does not issue warnings when file is too small", func(t *testing.T) { + cfgStr := ` +scanner: + fingerprint: + enabled: true + offset: 0 + length: 1024 +` + logp.DevelopmentSetup(logp.ToObserverOutput()) + + // this file is 128 bytes long + paths := []string{filepath.Join(dir, undersizedBasename)} + s := createScannerWithConfig(t, paths, cfgStr) + files := s.GetFiles() + require.Empty(t, files) + logs := logp.ObserverLogs().FilterLevelExact(logp.WarnLevel.ZapLevel()).TakeAll() + require.Empty(t, logs, "there must be no warning logs for files too small") + }) + t.Run("returns error when creating scanner with a fingerprint too small", func(t *testing.T) { cfgStr := ` scanner: From 87b16ce0cc5f8087a2f4ad3b79e98be1e180e4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Wed, 20 Mar 2024 07:54:00 +0100 Subject: [PATCH 29/31] Add new slowlog fields to filebeat (#38295) --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 55 ++++++++ filebeat/module/elasticsearch/fields.go | 2 +- .../elasticsearch/slowlog/_meta/fields.yml | 20 +++ .../slowlog/ingest/pipeline-json.yml | 20 +++ .../es814_index_indexing_slowlog-json.log | 4 + ...ex_indexing_slowlog-json.log-expected.json | 130 ++++++++++++++++++ .../test/es814_index_search_slowlog-json.log | 3 + ...ndex_search_slowlog-json.log-expected.json | 104 ++++++++++++++ 9 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0b33bffc31a..8a24d42a08b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -209,6 +209,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] - Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] - Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] +- Parse more fields from Elasticsearch slowlogs {pull}38295[38295] *Auditbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4cf4b99b5e7..5b03cdfb0a9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -50730,6 +50730,61 @@ type: keyword -- +*`elasticsearch.slowlog.user.realm`*:: ++ +-- +The authentication realm the user was authenticated against + +type: keyword + +example: default_file + +-- + +*`elasticsearch.slowlog.user.effective.realm`*:: ++ +-- +The authentication realm the effective user was authenticated against + +type: keyword + +example: default_file + +-- + +*`elasticsearch.slowlog.auth.type`*:: ++ +-- +The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY + +type: keyword + +example: REALM + +-- + +*`elasticsearch.slowlog.apikey.id`*:: ++ +-- +The id of the API key used + +type: keyword + +example: WzL_kb6VSvOhAq0twPvHOQ + +-- + +*`elasticsearch.slowlog.apikey.name`*:: ++ +-- +The name of the API key used + +type: keyword + +example: my-api-key + +-- + [[exported-fields-envoyproxy]] == Envoyproxy fields diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 525d0c50eac..4f27bd426ff 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUWltz2zb2f8+nOKOXfztj8y9f6taa2Z1plcRxprk0sp1tFQ/nCDyiEIEADYCS1U6++w5AShYpkrpsm+3qxSZxOb9zPwfEMUxp0QMSaCxnhlCzyTMAy62gHnRerL/vPAPQJAgN9SDGZwARGaZ5armSPfjnMwAo7wRvVJQJegYw5iQi0/NTjkFiQptE3c8uUre5VllavKmhUd5ufUumklRJknY1UtmgzNHTfBhrlcB8QprATgiEioFmbkBpHnOJlqLO2qb0iEnqRaQCCliQBG/I4nO02NeElq5lRI8D0jPOaH1dzt+UFnOlo034IjOWdJBlPGrk4Pb2+jmosYdZLKhHdpXM9OiVeHvDB3e/8vfjH6aP8WW8Pxr31IjmLSa0E5pIsSnp45o57SikiihoEceTMNzMetrPB/wju1nQzeSjvf3Xzz9dvu7+9Ga+J4adxdCMY/bx7Wvz29nuhLkzo3bK3tL89HqaYy5oRGiPLRl7zGWa2X3pt0nfU+cNvoHvruLn89Hth3H/7rvvfxywh1E/3kPuZoI6aiUfLYXup9aj6O5OsAhJYapVlDEb5s6/dfmGixqKYLSAItiAscimYBXwiKTl4wXMJ7wUfpZc+InGvXGPmh4yMraerSkfocTOBg8Ta9OgWBk8hirFh4zCtoBSRluERqvATrSyVhCgjCCiKEsFZ2gJIko1MXTrYY5achmbBov/HtN0D/m7kBs4GrHSi0bE/WLCUmzrePwW9WCcuNHykaAQU747KswibjdmrycpqMlJ6zsIXJAujVRZunFJx81aZiJnH3bCzUYa6oEmY4/AapQmVdqNAU/DMReVwFrmXlcsqc2cKxoJ3byt8N0kp5EcsJ2gBcVYprXDjFLJRaIyEyJjZEwYkeQUHQFmduKcIldfOEYu/OvKrPwx1iite2ZKSmJ+Rd275TKLSUqaorDwhyPQmQxxbaPiOV/QLLwy/f3FmKsv2CrHj6sKpEC8oXj4ZnMktxmEDy8GN/Dj++vl4m/XrWS1bo4GNDHiM4pASU/taRqboJQkvj0CoRiK0GUz+CaviRgKn92AG5NRtI7z22bZPe2zv9w0oUi2Wl7ZhvJFHlxlwHE+Q8EjLzSMkctNnyiAd1zNQmPMhHWudQD2zJAOdmPATf0/U8vHEfDx+kCjlXa8mVo+ozDimphVenEoaCXItIL+4Ga4HLEMVASp5pLxFAWMSKhKRihZxHCZvEKMEi47R9BxFYopHuH+QNTelavV0v4b1Kls+w51rcduK9HHrK02ItdKzHxJHmOdUdMjsazZNFzuy+vuXqIkt0r/f4JcHmAdWgQpaky2WIeLQ7cfrsHPJUu62Rg6fzjZu+3/8RnZVHI2Of3SqaXOZcTZFsO8zucUCSMvbHJptZnjWKnj0+7JZdA9CbrnziBLb8423lwcYqXLooxHrSzcSv6QEeSNTU0JWBbfx99/Dqeji7vB7N3kx4eunb+fvXr3yyGRNgdX4z7NqX6ZUvYwxL4g1AOmlRAf6nnbGWs4UtGidjEKjlU7SdFOKtWxWx8wJe2m2yY81phzbHVGLUk9xCjSZKrktgExKtOMAp4eQDjTfE9qznGLekEcQHAV2/clazbPD3almZAxGNeHckuPtiFELPN7gCmf0sIEai4pCkeLsJREQwetdu+RUoJQ1ofqUoKuK+d2KAVrGrIGzpZL1vqbrW1I7XkZtJyZxezwPQGu+uCqI0O2IBDs2BKlEzT1wqtS34LA/V56QmBSYnzMmStMrvo5iaAyuQ7TOq4aa4VWve4E0P3Wz4iu+sCUEHm7Ug90Tf1Z7iyhIdYIbSwUVoPYjsD6FSQrgi6LKh1xGefHAQSvcYYw49pmKCBBNuGyBbhhOhuFZpGMlAgtusbb8oT+Kj7gPWaGwJEALsEQUzIywJw7Ox6yFHIs4LGYrcCt5jL+CsB3wO2hbMU9J5yGmsYmTLVyBZDH/xciv3GYTeoa/ieKHgZoGpMm6YqxJ6aaobtSUQgSoSbDUH4t1GvyTlBPHXrBZwRq9JmYNa6/EQSY5kdfzie4AWNVmlLUzAwTaEyYSaEw+lqc5NS8vcjMVb4exI7SZ2nmcTZirAvKO2J8nxsG9N/f5jZe2AvpsdKJA/wUCmsgNodsqDRwDUKGrYLekRH3qzChMmt4lJ/YTElLEnUMrAWWhfkvoOSyChJaUboO+GvAvFEWBZDA1NlrBbRVvqcWZHPka/nSH3cZi9rPGnPJzSSorTI+z5JQZ7LBBZsZ2cKA74EcVI/k9d2bAk2WrnnbEaABzLd3Vp4qLi3ILBmRrkdrJ5owMqF1cgldlGkKHgcjv0I9wrgkzYIqeKo+thVqqAsaK0N2IdBnlyXmP1vEDoJVyn83yUEVOFtxWYzrO7H60m2btPogVBznqTduIDkhrEbGgwvZV4QpoBCqSDYoo6Ve+O9717JuTTgdNQZ1Li3FG23QDjBh5byOeU/HGf6UCzVa2LYKxWWmvwyS/5LlETWDWbVhIgpjqh67Hay4dyKCmCQVhbNiLEtRssXfX4NeeWrsBLLOwd9AnY0y3a7dhcpk/Gfq91e34f+4hhdVHv4GOm6Raz26ldxIz0pEy6eUAz/s7/FUv97s+vn2KdUhm1qNrFwdr9Hr9GCQf+J3sxxw5tpoNQbSWulyQvJ3KnowRlE6/6g9jqlyleej8oFpk0m3Hb54S2hzgE6ul6t+80Fu/bFtnWvVu8AqEMvNrqOMpUqpDcUSh1AbDK5KhLn6GgRX/M1ITwij0NBDq8gH9JC5frkoERslf3Z+fnl5eVor/kYUT/VeuDzdCbZ8xSl3yVf9I/cn4ULwogJrRHhy0e3uWAeupDRyDo37AfTRzdeqTsirSzWrynaOptiYoj3Q/7AT+lV4EGouVNwcifLx/FqEyTuGjXuVGyA6w9PuyQ/H3Yvj08ubk26ve9E7OT+6PDu7H16/ffkO7of53ax8i6AAETxkpBf3MJyFd68nn+/uYZiQ1Zz5G2AXwVnQPXb7Bt2L4PTifti99yX28Dz4LjH3R/4hzIU0PPfPrhGZcGuGJ5fnZ9+5V4uUzPD+yIVFm//jIfg7IsNfbl98+DW8efXibfjyxU3/1WoPfz/LDE/cfP/lY/jHp45H+6nT++NTJ0HLJiEKkT+OlDL2U6d3EnS/fPlyf/SfxG9XwVfSU1lDP/sJG3fo1rVRK+wx2bL2mnuNVexRatqCxLsct6u+p/ic5vtfL6wmfGfdbmL2hOIU2YbFjTfR24+UN5UWUgM3nmu0kaIfPdmT7pNltlEvLrwt0kb1V816Txje4EOvwDYcQs3btbyHy+yHkB6txjDH2YLwhZtWsANcjpVOcPPT+qFW8hRs2qwy7zq5bTKU89MDiObRaStZJ3xOUX7ZtAnA6X4AtMosryTt6l0bP6NJyKZ78uq3019+ml5+np/HNsaXVu4n+MqnyBL16+jP0W27C960+F6k2CHu1kxtkNuvGkOkWJasLiu6asHHeYpa6P07AAD//wFdWko=" + return "eJzUmt1z2zYSwN/zV+zo5doZmyd/1K09czeTKo7jNB9ubCeXKh7OClxRiECABkDJapv//QYgJYsUSX20zfX8klAEsL9dLHYXIPZhTLMzIIHGcmYINRs9AbDcCjqDzvny750nAJoEoaEziPEJQESGaZ5aruQZ/PsJAJRHgtcqygQ9ARhyEpE58032QWJCq0Ldn52lbnCtsrT4pUZGebjlIZlKUiVJ2sWbygBljR7bw1CrBKYj0gR2RCBUDDRxL5TmMZdoKeosDUoPmKTeRCqggAVJ8JosPkOLPU1o6VJG9HBNesIZLffL9RvTbKp0tIovMmNJB1nGo0YNbm8vn4EaesyiQz3ZRTLRgxfizQ2/fv+RXw1/GD/Ep/H2NO6pkeYNJrQRTaTYmPR+TZt2CqkiClrM8WgM17Je9rNr/oHdzOhm9MHe/ufVj6cvuz++nm7JsLEZmjkmH968NL8cbS6YOzdql+w9zTevlznkggaEdt+SsftcppndVn6b9b103rA28O1F/Gw6uH037L3/7vun1+x+0Iu3sLsZoY5axUdzo/um9RTdzQUWISlMtYoyZsN88a/tvrJEDUUwmEERbMBYZGOwCnhE0vLhDKYjXgo/cy18Q+N+cY+a7jMytl6tMR+gxM6KDiNr06DoGTyEKsX7jMK2gFKmLUKjVWBHWlkrCFBGEFGUpYIztAQRpZoYuv4wRS25jE2Dx3+PabqF/V3IDZyMWOlZI3GvaDA32zKPH6IexpkbLR8ICjHlm1NhFnG70no5SUFNTloeQeCMdOlNVaUbl3Rcq3kmcv5hR9yspKEz0GTsHliN0qRKu3fA03DIRSWwlrXXFU9qc+fKjISu3Vp818jNSA5sR2hBMZZp7ZhRKjlLVGZCZIyMCSOSnKI9wMyO3KLIpy8cIhf+50qr/DHWKK17ZkpKYr5H3W/zbhaTlDRFYbEe9kBnMsSlgYrnvEOz8crytzdjPn3BWjt+WFQgBfHKxMM3q29yn0F4d359A0+vLuedv132kkW/KRrQxIhPKAIlvbTHZmyEUpL4dg+EYihCl83gm7wmYih8dgNuTEbRMue3zbZ7HGd7u2lCkaz1vLIP5Z08XOWF03yCgkfeaBgjl6trogDvuJqFhpgJ65bWDuyZIR1spoBr+g9Tq8ce8OHyi0Yv7Xg3tXxCYcQ1Mav0bFdoJci0Qr9zLVyOmAcqglRzyXiKAgYkVCUjlDyiP09eIUYJl5096LgKxRSPcLcjtV/K1Wpp+wHqpmz9CHVbj816oo9Za31ELpWYeZc8xjqnpgdiWbNruNyX191niZLcKv3PBLncwTu0CFLUmKzxDheHbt9dgm9LlnSzM3R+c7Z3w//rM7Kx5Gx0+KVTK53LiLM1jnmZtykSRl7Y5NZqc8ehUvuH3YPToHsQdI+dQ5Z+OVr55WQXL50XZTxqVeFW8vuMIN/Y1JSAZfN9+PVVOB6cvL+evB09ve/a6dXkxdufd4m0OVzN8mlO9fOUsoUj9gShvmZaCfGuXreNWcOBima1nVFwrPpJinZUqY5d/4ApaVeXbcJjjbnGVmfUktRDjCJNpipuHYhRmWYU8HQHwZnmW0pzC7eoF8QOAhexfVuxZvX8YFOZCRmDcX0ot/RgG0LEPL8HmPIxzUygppKicDALS0k0dGi1Yw+UEoSyPlSXEnRdObdBKVizIWvQbN5laX+zdhtSe14GLWdmMdt9TICLHrjqyJAtBAQbbonSEZp641WlryFwf8+9IDApMT7kzBUmF71cRFBpXMe0zFXjrdA6rxsBur/lM6KLHjAlRL5dqQddmv4sXyyhIdaINhQKq0FsQ7BehWQh0GVRpSMu4/w4gOAlThAmXNsMBSTIRly2gBums0FoZslAidCi23hbntBfpQdcYWYInAjgEgwxJSMDzC1np0OWQs4CnsWsBbeay/grgG/A7VHWck8Jx6GmoQlTrVwB5Pn/QvIbx2xSt+F/lOgxQNOQNElXjD0q1YzuSkUhSISaDEP5taiX7J2gHjt6wScEavCZmDVufyMIMM2Pvtya4AaMVWlKUbMyTKAxYSaFwuhraZJL8/4iM1f5eogNrc/SzHM2MtYF5Q0Zr3LHgN7Vbe7jhb+QHiqdOODHUFiD2ByyobKBazAyrDX0hoq4v4oSKrOGR/mJzZi0JFGnwFJgmZn/ASWXVUhopXQ74K+BeaMsCiCBqfPXCrRVfk8tyObkS/nSH3cZi9q3GnLJzSiorTI+T5JQZ7JhCTYrskYBvwdyqJ7k5fvXBU2WLq22PUADmA/vvDxVXFqQWTIgXU9rR5owMqF1dgldlGkKHjuTX6AeYFyyZiEVvFQf24ppqAsaC0d2IdBnlznzn21ih2CV8t9NcqiCs5XLYly/E6sv3dZZqwdCxXGeeuMGkSPCamTcuZB9QZgCCqGKZIMyms8L/3XrWtb1CceDxqDOpaV4ZRu0ASYsFq9T3stxjj/mQg1mtq1CcZnpL0PyX7I8UTPMYhsmojCm6rHbzhP3VkQQk6SicFaMZSlKNvv7z6CfPDV0BlnW4G8wnY02XT+7M5XJ+M+c349uwP/zGZ5VdfgbzHGLXevpFnYjPSkJLZ9SXvvX/h5P9evNpp9vH1MdsrHVyMrV8ZK8zhlc55/4XSsHztw2Wg2BtFa6nJD8nYozGKIonX/UHsdUtcrzUfnAtMml2w5fvCe0LYBOPi8XveaD3Ppj27qlVb8EFoFYru46yixVSW0Ucw6hVhRclAhT9TUELvSbkB4RRqGh+1aTX9N95vbLRYnYaPmj4+PT09PDWvM3UjzWe+H8dCdY8xWnvEu+6O25fxIuBC8qsEbCg5Nud8M6cGGlgVvQuB2gj26+VnVGXlyqWVS2UzTFwBRtQf/DRvSL8CDUVKi4ORLl7/NrESbfMazcq1yB6PQPuwc/7HdP9g9Pbw66Z92Ts4PjvdOjo7v+5Zvnb+Gun9/NyocICojgPiM9u4P+JHz/cvT5/R30E7KaM38D7CQ4Crr7btygexIcntz1u3e+xO4fB98l5m7PP4S5kfrH/tltREbcmv7B6fHRd+6nWUqmf7fnwqLN/+MR/B2R/s+35+8+hjcvzt+Ez89vei8WY/j7WaZ/4Nr7Lx/93z51PO2nztlvnzoJWjYKUYj8caCUsZ86ZwdB98uXL3d7fyR+uwq+kp7KM/TKN1i5Q7c8G7XGHpItz17zXmMRe5Qat5D4JcftYt9TfE7z+19vrCa+o243MVuiuIlsY3Hvm+RtJ8q7Souoa/c+n9FGif7twZZyHz2zTXpx4W2WNk5/1a23xPAOH/oJbOMQato+y1ssme0I6cFqDHPOFsJz16xQB7gcKp3g6qf1Xb3kMdi0eWW+6+S2yVGOD3cQmkentWKd8TlF+WXTJoDD7QC0yiyvJO3qXRvfosnIpnvw4pfDn38cn36eHsc2xudWbmf4yqfIkvTL6M+Z2/YleNOy9iLFdlluzdKuc/9VQ4gUy5LFZUVXLfg4T9FW8hrueZVktt5ScwN46aUv1PP7aU1mabybthEvDYfkL4v9IfLFKF9bBydl9fP7OnB/bcV/G7GqBLqYhgDeSu8aN29/On8Dv8O786evXsPv8PTqMvzp/GO9Ir7Rdvz+ckL1JtAKP1/cb396demG8/QN1tzkJtCGXCsfwVfIlm+jbcCWzPYx5ftjmrXx/DcAAP//7NJVlQ==" } diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml index fa251b39789..0055a7df364 100644 --- a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml +++ b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml @@ -54,3 +54,23 @@ - name: source description: Source of document that was indexed type: keyword + - name: user.realm + description: The authentication realm the user was authenticated against + example: "default_file" + type: keyword + - name: user.effective.realm + description: The authentication realm the effective user was authenticated against + example: "default_file" + type: keyword + - name: auth.type + description: The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY + example: REALM + type: keyword + - name: apikey.id + description: The id of the API key used + example: "WzL_kb6VSvOhAq0twPvHOQ" + type: keyword + - name: apikey.name + description: The name of the API key used + example: "my-api-key" + type: keyword diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 614c9f7aa43..8a3c8e4f6f0 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -10,3 +10,23 @@ processors: - pipeline: if: 'ctx.message.contains("ecs.version")' name: '{< IngestPipeline "pipeline-json-8" >}' + - rename: + field: auth.type + target_field: elasticsearch.slowlog.auth.type + ignore_missing: true + - rename: + field: user.realm + target_field: elasticsearch.slowlog.user.realm + ignore_missing: true + - rename: + field: user.effective.realm + target_field: elasticsearch.slowlog.user.effective.realm + ignore_missing: true + - rename: + field: apikey.id + target_field: elasticsearch.slowlog.user.apikey.id + ignore_missing: true + - rename: + field: apikey.name + target_field: elasticsearch.slowlog.user.apikey.name + ignore_missing: true diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log new file mode 100644 index 00000000000..920951b8caf --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log @@ -0,0 +1,4 @@ +{"@timestamp":"2024-03-13T10:34:33.289Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"2","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"12.3ms","elasticsearch.slowlog.took_millis":"12","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#7]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:36.139Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"3","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"5.9ms","elasticsearch.slowlog.took_millis":"5","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#9]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:37.257Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"4","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.5ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#12]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:38.373Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"5","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.2ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#3]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json new file mode 100644 index 00000000000..b24a197b41c --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json @@ -0,0 +1,130 @@ +[ + { + "@timestamp": "2024-03-13T10:34:33.289Z", + "log.level": "WARN", + "log.offset": 0, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "2", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "12.3ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 12000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#7]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:36.139Z", + "log.level": "WARN", + "log.offset": 980, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "3", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "5.9ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 5000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#9]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:37.257Z", + "log.level": "WARN", + "log.offset": 1958, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "4", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "2.5ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 2000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#12]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:38.373Z", + "log.level": "WARN", + "log.offset": 2937, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "5", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "2.2ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 2000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#3]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + } +] diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log new file mode 100644 index 00000000000..40e1a31906f --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log @@ -0,0 +1,3 @@ +{"@timestamp":"2024-03-13T09:42:41.350Z", "log.level": "WARN", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"7.7ms","elasticsearch.slowlog.took_millis":7,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#3]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T09:43:56.663Z", "log.level": "WARN", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"946.6micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#6]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T09:44:20.724Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"509.5micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1,"user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#8]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json new file mode 100644 index 00000000000..651f6ce267f --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json @@ -0,0 +1,104 @@ +[ + { + "@timestamp": "2024-03-13T09:42:41.350Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "7.7ms", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.total_shards": 1, + "event.dataset": "elasticsearch.index_search_slowlog", + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 0, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "event.category": "database", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 7000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#3]", + "log.logger": "index.search.slowlog.query" + }, + { + "@timestamp": "2024-03-13T09:43:56.663Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "946.6micros", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.total_shards": 1, + "elasticsearch.slowlog.stats": "[]", + "event.dataset": "elasticsearch.index_search_slowlog", + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 869, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "event.category": "database", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 0, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#6]", + "log.logger": "index.search.slowlog.query" + }, + { + "@timestamp": "2024-03-13T09:44:20.724Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "509.5micros", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.total_shards": 1, + "event.type": "info", + "event.kind": "event", + "event.dataset": "elasticsearch.index_search_slowlog", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 1744, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "elasticsearch.slowlog.auth.type": "REALM", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 0, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#8]", + "log.logger": "index.search.slowlog.query" + } +] From 01f5a90209120459cfbbd090cb29fd803618b11a Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 20 Mar 2024 09:45:33 +0200 Subject: [PATCH 30/31] Snapshot packaging stage migration (#37827) --- .buildkite/auditbeat/auditbeat-pipeline.yml | 28 ++++++---- .buildkite/auditbeat/scripts/crosscompile.sh | 2 - .buildkite/auditbeat/scripts/package.sh | 13 ----- .../auditbeat/scripts/unit-tests-win.ps1 | 51 ------------------- .buildkite/auditbeat/scripts/unit-tests.sh | 2 - .buildkite/env-scripts/env.sh | 42 +++++++++++++-- .buildkite/env-scripts/linux-env.sh | 45 ---------------- .buildkite/env-scripts/util.sh | 26 +++++----- .buildkite/filebeat/filebeat-pipeline.yml | 31 ++++++----- .../filebeat/scripts/integration-gotests.sh | 2 - .../filebeat/scripts/integration-pytests.sh | 2 - .buildkite/filebeat/scripts/package-step.sh | 51 ------------------- .buildkite/filebeat/scripts/package.sh | 10 ---- .../filebeat/scripts/unit-tests-win.ps1 | 51 ------------------- .buildkite/filebeat/scripts/unit-tests.sh | 2 - .buildkite/heartbeat/heartbeat-pipeline.yml | 28 ++++++---- .../heartbeat/scripts/integration-gotests.sh | 3 -- .../heartbeat/scripts/integration-pytests.sh | 3 -- .buildkite/heartbeat/scripts/package-step.sh | 51 ------------------- .buildkite/heartbeat/scripts/package.sh | 13 ----- .../heartbeat/scripts/unit-tests-win.ps1 | 51 ------------------- .buildkite/heartbeat/scripts/unit-tests.sh | 3 -- .buildkite/hooks/post-checkout | 10 ++-- .buildkite/hooks/pre-command | 35 +++++++++++-- .buildkite/hooks/pre-exit | 15 ++++++ .buildkite/hooks/scripts/util.sh | 33 ++++++++++++ .../packaging}/package-step.sh | 27 +++++----- .buildkite/scripts/packaging/package-util.sh | 22 ++++++++ .buildkite/scripts/packaging/package.sh | 51 +++++++++++++++++++ .../module/file_integrity/metricset_test.go | 9 ++++ 30 files changed, 285 insertions(+), 427 deletions(-) delete mode 100755 .buildkite/auditbeat/scripts/package.sh delete mode 100644 .buildkite/auditbeat/scripts/unit-tests-win.ps1 delete mode 100644 .buildkite/env-scripts/linux-env.sh delete mode 100755 .buildkite/filebeat/scripts/package-step.sh delete mode 100755 .buildkite/filebeat/scripts/package.sh delete mode 100644 .buildkite/filebeat/scripts/unit-tests-win.ps1 delete mode 100755 .buildkite/heartbeat/scripts/package-step.sh delete mode 100755 .buildkite/heartbeat/scripts/package.sh delete mode 100644 .buildkite/heartbeat/scripts/unit-tests-win.ps1 create mode 100644 .buildkite/hooks/pre-exit create mode 100755 .buildkite/hooks/scripts/util.sh rename .buildkite/{auditbeat/scripts => scripts/packaging}/package-step.sh (55%) create mode 100755 .buildkite/scripts/packaging/package-util.sh create mode 100755 .buildkite/scripts/packaging/package.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 798939bbf32..be03d7843b2 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,13 +1,18 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "auditbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - group: "Auditbeat Mandatory Testing" @@ -24,6 +29,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" @@ -37,19 +43,20 @@ steps: agents: provider: "gcp" image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Auditbeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" matrix: @@ -72,6 +79,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - group: "Extended Testing" key: "extended-tests" @@ -88,8 +96,8 @@ steps: context: "Auditbeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "auditbeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -112,14 +120,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Auditbeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" artifact_paths: @@ -134,4 +142,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/auditbeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh index 866d6be4223..da8452d5380 100755 --- a/.buildkite/auditbeat/scripts/crosscompile.sh +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -2,7 +2,5 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Crosscompile" make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh deleted file mode 100755 index 71872ca15a3..00000000000 --- a/.buildkite/auditbeat/scripts/package.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Docker Version: $(docker --version)" - -echo "--- Start Packaging" -cd auditbeat -umask 0022 -mage package - diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 200627d518f..00000000000 --- a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "--- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "--- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path auditbeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh index c1f5685c77f..8a7514d747a 100755 --- a/.buildkite/auditbeat/scripts/unit-tests.sh +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Running Unit Tests" sudo chmod -R go-w auditbeat/ diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index 4dfc01bafc3..f28658a107d 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -1,19 +1,53 @@ #!/usr/bin/env bash -SETUP_GVM_VERSION="v0.5.1" +source .buildkite/env-scripts/util.sh + +DOCS_CHANGESET="^.*\.(asciidoc|md)$ +deploy/kubernetes/.*-kubernetes.yaml" +PACKAGING_CHANGESET="^dev-tools/packaging/ +^.go-version" + +REPO="beats" WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" -REPO="beats" TMP_FOLDER="tmp.${REPO}" +SNAPSHOT="true" +PYTEST_ADDOPTS="" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" + +SETUP_GVM_VERSION="v0.5.1" +ASDF_MAGE_VERSION="1.14.0" +SETUP_WIN_PYTHON_VERSION="3.11.0" + +# Docker & DockerHub +DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION +ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET") +PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET") +GO_MOD_CHANGES=$(changeset_applies "^go.mod") + +export REPO export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE -export REPO export TMP_FOLDER +export SNAPSHOT +export PYTEST_ADDOPTS +export OSS_MODULE_PATTERN +export XPACK_MODULE_PATTERN + +export SETUP_GVM_VERSION +export ASDF_MAGE_VERSION +export SETUP_WIN_PYTHON_VERSION + +export DOCKER_COMPOSE_VERSION export DOCKER_REGISTRY + +export ONLY_DOCS +export PACKAGING_CHANGES +export GO_MOD_CHANGES diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh deleted file mode 100644 index 1365aaace4a..00000000000 --- a/.buildkite/env-scripts/linux-env.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -DEBIAN_FRONTEND="noninteractive" - -sudo mkdir -p /etc/needrestart -echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null - -if [[ $PLATFORM_TYPE == "Linux" ]]; then - # Remove this code once beats specific agent is set up - if grep -q 'Ubuntu' /etc/*release; then - export DEBIAN_FRONTEND - - echo "--- Ubuntu - Installing libs" - sudo apt-get update - sudo apt-get install -y libsystemd-dev - sudo apt install -y python3-pip - sudo apt-get install -y python3-venv - fi - - # Remove this code once beats specific agent is set up - if grep -q 'Red Hat' /etc/*release; then - echo "--- RHL - Installing libs" - sudo yum update -y - sudo yum install -y systemd-devel - sudo yum install -y python3-pip - sudo yum install -y python3 - pip3 install virtualenv - fi -fi - -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo "--- Setting larger ulimit on MacOS" - # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 50000 - echo "--- ULIMIT: $(ulimit -n)" -fi - -echo "--- Setting up environment" -add_bin_path -with_go -with_mage diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 6a5c36bcd04..68fd08a75df 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -9,11 +9,11 @@ add_bin_path() { } with_go() { - local go_version="${GOLANG_VERSION}" + local go_version="${GO_VERSION}" echo "Setting up the Go environment..." create_bin check_platform_architecture - retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" + retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" export PATH="${PATH}:${BIN}" chmod +x ${BIN}/gvm eval "$(gvm "$go_version")" @@ -60,7 +60,7 @@ check_platform_architecture() { esac } -retry() { +retry_with_count() { local retries=$1 shift local count=0 @@ -89,16 +89,16 @@ are_files_changed() { fi } -cleanup() { - echo "Deleting temporary files..." - rm -rf ${BIN}/${TMP_FOLDER}.* - echo "Done." +changeset_applies() { + local changeset=$1 + if are_files_changed "$changeset"; then + echo true + else + echo false + fi } -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then - unset "$var" - fi - done +set_git_config() { + git config user.name "${GITHUB_USERNAME_SECRET}" + git config user.email "${GITHUB_EMAIL_SECRET}" } diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e811d286953..ae22629e6ec 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,12 +1,17 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "filebeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - group: "Filebeat Mandatory Testing" @@ -23,7 +28,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" @@ -37,7 +42,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" @@ -49,22 +54,22 @@ steps: - github_commit_status: context: "Filebeat: Python Integration Tests" agents: - provider: "gcp" + provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Filebeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" matrix: @@ -91,8 +96,8 @@ steps: context: "Filebeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "filebeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -115,14 +120,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Filebeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" artifact_paths: @@ -137,4 +142,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/filebeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index d64ce7c98eb..6de39ff8817 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index b51e8ae18a6..9aff8695c35 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh deleted file mode 100755 index f8fa02db81d..00000000000 --- a/.buildkite/filebeat/scripts/package-step.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -changeset="^filebeat/ -^go.mod -^pytest.ini -^dev-tools/ -^libbeat/ -^testing/ -^\.buildkite/filebeat/" - -if are_files_changed "$changeset"; then - bk_pipeline=$(cat <<-YAML - steps: - - label: ":ubuntu: Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - command: - - ".buildkite/filebeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Filebeat/Packaging: Linux X86" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Packaging Linux ARM" - key: "package-linux-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: - - ".buildkite/filebeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Filebeat/Packaging: ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" -YAML -) - echo "${bk_pipeline}" | buildkite-agent pipeline upload -else - buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' - exit 0 -fi diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh deleted file mode 100755 index 0bb03250348..00000000000 --- a/.buildkite/filebeat/scripts/package.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Start Packaging" -cd filebeat -umask 0022 -mage package diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 8990eb30a09..00000000000 --- a/.buildkite/filebeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "-- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path filebeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index 08ce9d4ea1c..2efb6b1ff8e 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Unit Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index bf645a2b295..2b5f6195f45 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,13 +1,18 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "heartbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - group: "Heartbeat Mandatory Testing" @@ -24,6 +29,7 @@ steps: agents: provider: "gcp" image: "{{matrix.image}}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" matrix: setup: image: @@ -35,14 +41,14 @@ steps: - label: ":windows: Unit Tests / {{matrix.image}}" command: - - ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + - ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Heartbeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" matrix: setup: @@ -62,6 +68,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" @@ -75,6 +82,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" @@ -94,8 +102,8 @@ steps: context: "Heartbeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -118,14 +126,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Heartbeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" artifact_paths: - "heartbeat/build/*.xml" @@ -139,4 +147,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/heartbeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh index 8eab0e8b5d8..c50dbf45347 100755 --- a/.buildkite/heartbeat/scripts/integration-gotests.sh +++ b/.buildkite/heartbeat/scripts/integration-gotests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh index 729df5ae6f6..5875b5460ed 100755 --- a/.buildkite/heartbeat/scripts/integration-pytests.sh +++ b/.buildkite/heartbeat/scripts/integration-pytests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/heartbeat/scripts/package-step.sh b/.buildkite/heartbeat/scripts/package-step.sh deleted file mode 100755 index 03790edfa5f..00000000000 --- a/.buildkite/heartbeat/scripts/package-step.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -changeset="^heartbeat/ -^go.mod -^pytest.ini -^dev-tools/ -^libbeat/ -^testing/ -^\.buildkite/heartbeat/" - -if are_files_changed "$changeset"; then - bk_pipeline=$(cat <<-YAML - steps: - - label: ":ubuntu: Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - command: - - ".buildkite/heartbeat/scripts/package.sh" - notify: - - github_commit_status: - context: "heartbeat/Packaging: Linux X86" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Packaging Linux ARM" - key: "package-linux-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: - - ".buildkite/heartbeat/scripts/package.sh" - notify: - - github_commit_status: - context: "heartbeat/Packaging: ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" -YAML -) - echo "${bk_pipeline}" | buildkite-agent pipeline upload -else - buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' - exit 0 -fi diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh deleted file mode 100755 index 7f51a6b5ca1..00000000000 --- a/.buildkite/heartbeat/scripts/package.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Docker Version: $(docker --version)" - -echo "--- Start Packaging" -cd heartbeat -umask 0022 -mage package - diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 17282813e13..00000000000 --- a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "--- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "--- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path heartbeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh index 4b746da2d57..1d8de945788 100755 --- a/.buildkite/heartbeat/scripts/unit-tests.sh +++ b/.buildkite/heartbeat/scripts/unit-tests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Running Unit Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b6cc7ad60bd..43881f6e2d8 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -8,7 +8,7 @@ checkout_merge() { local merge_branch=$3 if [[ -z "${target_branch}" ]]; then - echo "No pull request target branch" + echo "--- No pull request target branch" exit 1 fi @@ -24,9 +24,9 @@ checkout_merge() { git config user.name "github-merged-pr-post-checkout" git config user.email "auto-merge@buildkite" - git merge --no-edit "${BUILDKITE_COMMIT}" || { + git merge --no-edit "${pr_commit}" || { local merge_result=$? - echo "Merge failed: ${merge_result}" + echo "--- Merge failed: ${merge_result}" git merge --abort exit ${merge_result} } @@ -35,7 +35,7 @@ checkout_merge() { pull_request="${BUILDKITE_PULL_REQUEST:-false}" if [[ "${pull_request}" == "false" ]]; then - echo "Not a pull request, skipping" + echo "--- Not a pull request, skipping" exit 0 fi @@ -46,7 +46,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}" checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" -echo "Commit information" +echo "--- Commit information" git --no-pager log --format=%B -n 1 # Ensure buildkite groups are rendered diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7be1f965230..9df1bed50a7 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,8 +2,13 @@ set -euo pipefail +# Secrets must be redacted +# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" +GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" retry() { local retries=$1 @@ -24,13 +29,35 @@ retry() { return 0 } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh - source .buildkite/env-scripts/win-env.sh - if [[ -z "${GOLANG_VERSION-""}" ]]; then - export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") + if [[ -z "${GO_VERSION-""}" ]]; then + export GO_VERSION=$(cat "${WORKSPACE}/.go-version") + fi + + if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then + ulimit -Sn 30000 + + echo "--- Setting up environment" + add_bin_path + with_go + with_mage + fi + + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then + export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + + github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) + github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) + + git config user.name "$github_username" + git config user.email "$github_email" fi fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 00000000000..d1ff6e0ac1c --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/hooks/scripts/util.sh + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then + docker logout "${DOCKER_REGISTRY}" + fi + + # Ensure that any temporal files created during any step are removed + cleanup + unset_secrets +fi diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh new file mode 100755 index 00000000000..07ab6cf4c9a --- /dev/null +++ b/.buildkite/hooks/scripts/util.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} + +google_cloud_logout_active_account() { + local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null) + if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then + echo "Logging out from GCP for active account" + gcloud auth revoke $active_account > /dev/null 2>&1 + else + echo "No active GCP accounts found." + fi + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then + unset GOOGLE_APPLICATION_CREDENTIALS + cleanup + fi +} + +cleanup() { + echo "Deleting temporary files..." + if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then + rm -rf "${BIN}/${TMP_FOLDER}.*" + fi + echo "Done." +} diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/scripts/packaging/package-step.sh similarity index 55% rename from .buildkite/auditbeat/scripts/package-step.sh rename to .buildkite/scripts/packaging/package-step.sh index cb06895879a..9eddfafcfba 100755 --- a/.buildkite/auditbeat/scripts/package-step.sh +++ b/.buildkite/scripts/packaging/package-step.sh @@ -4,44 +4,45 @@ set -euo pipefail source .buildkite/env-scripts/util.sh -changeset="^auditbeat/ +changeset="^${BEATS_PROJECT_NAME}/ ^go.mod ^pytest.ini ^dev-tools/ ^libbeat/ ^testing/ -^\.buildkite/auditbeat/" +^\.buildkite/${BEATS_PROJECT_NAME}/" if are_files_changed "$changeset"; then bk_pipeline=$(cat <<-YAML steps: - - label: ":ubuntu: Packaging Linux X86" + - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" key: "package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - command: - - ".buildkite/auditbeat/scripts/package.sh" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package.sh" notify: - github_commit_status: - context: "Auditbeat/Packaging: Linux X86" + context: "${BEATS_PROJECT_NAME}/Packaging: Linux X86" agents: - provider: "gcp" + provider: gcp image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - - label: ":linux: Packaging Linux ARM" + - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" key: "package-linux-arm" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - command: - - ".buildkite/auditbeat/scripts/package.sh" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package.sh" notify: - github_commit_status: - context: "Auditbeat/Packaging: ARM" + context: "${BEATS_PROJECT_NAME}/Packaging: ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" YAML ) echo "${bk_pipeline}" | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh new file mode 100755 index 00000000000..4a50457cc9c --- /dev/null +++ b/.buildkite/scripts/packaging/package-util.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +is_pr() { + if [[ $BUILDKITE_PULL_REQUEST != false ]]; then + return 0 + else + return 1 + fi +} + +define_tags() { + aliasVersion="${VERSION%.*}${IMG_POSTFIX}" + tags=("${BUILDKITE_COMMIT}") + + if is_pr; then + tags+=("pr-${GITHUB_PR_NUMBER}") + else + tags+=("${SOURCE_TAG}" "${aliasVersion}") + fi +} diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh new file mode 100755 index 00000000000..5744ee0776b --- /dev/null +++ b/.buildkite/scripts/packaging/package.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/packaging/package-util.sh + +IMG_POSTFIX="-SNAPSHOT" +VARIANTS=("" "-ubi" "-oss") +VERSION="$(make get-version)" +SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" +TARGET="observability-ci/${BEATS_PROJECT_NAME}" + +echo "--- Creating package" +mage -d "${BEATS_PROJECT_NAME}" package + +echo "--- Distribution list" +dir="${BEATS_PROJECT_NAME}/build/distributions" +buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" + +echo "--- Docker image list" +docker images + +define_tags + +targetSuffix="" +if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then + targetSuffix="-arm64" +fi + +for variant in "${VARIANTS[@]}"; do + source="beats/${BEATS_PROJECT_NAME}${variant}" + + for tag in "${tags[@]}"; do + targetTag=$tag${targetSuffix} + + sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}" + targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}" + #TODO Remove following line once beats fully migrated to Buildkite and Jenkins builds will be disabled + #Avoid clashing with the Jenkins produced images + targetName="${targetName}-buildkite" + + if docker image inspect "${sourceName}" &>/dev/null; then + echo "--- Tag & Push with target: $targetName" + echo "Source name: $sourceName" + docker tag "$sourceName" "$targetName" + docker push "$targetName" + else + echo "Docker image ${sourceName} does not exist" + fi + done +done diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 4ad58aa89fa..2a6c33e1798 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -62,6 +62,7 @@ func TestData(t *testing.T) { func TestActions(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -154,6 +155,7 @@ func TestActions(t *testing.T) { func TestExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -201,6 +203,7 @@ func TestExcludedFiles(t *testing.T) { func TestIncludedExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -949,3 +952,9 @@ func skipOnCIForDarwinAMD64(t testing.TB) { t.Skip("Skip test on CI for darwin/amd64") } } + +func skipOnBuildkiteWindows(t testing.TB) { + if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "windows" { + t.Skip("Skip on Buildkite Windows: Shortened TMP problem") + } +} From 10ff99297a608def3f5933e72cd82cd4a2cf79ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:06:34 +0100 Subject: [PATCH 31/31] [Metricbeat][Azure] Azure monitor/storage: Fix 429 Too Many Requests error (#38294) * Fiz azure too many requests error Signed-off-by: constanca * Add CHANGELOG.next.asciidoc entry Signed-off-by: constanca * Run mage check Signed-off-by: constanca * Get metrics 1 time for each namespace/id Signed-off-by: constanca * Add error comparison Signed-off-by: constanca * Move retry to monitor service Signed-off-by: constanca * Remove GetMetricDefinitions without retry Signed-off-by: constanca * fix storage unit tests Signed-off-by: constanca * fix monitor unit tests Signed-off-by: constanca * Remove unnecessary for cycle Signed-off-by: constanca --------- Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 1 + .../metricbeat/module/azure/mock_service.go | 4 +- .../module/azure/monitor/client_helper.go | 20 ++++++-- .../azure/monitor/client_helper_test.go | 6 +-- .../module/azure/monitor_service.go | 51 +++++++++++++++++-- .../module/azure/service_interface.go | 2 +- .../module/azure/storage/client_helper.go | 6 +-- .../azure/storage/client_helper_test.go | 6 +-- 8 files changed, 76 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8a24d42a08b..f3bb1acc9a3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -113,6 +113,7 @@ fields added to events containing the Beats version. {pull}37553[37553] *Metricbeat* +- Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] *Osquerybeat* diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index 65f606dde12..9626952fa6d 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -29,8 +29,8 @@ func (client *MockService) GetResourceDefinitions(id []string, group []string, r return args.Get(0).([]*armresources.GenericResourceExpanded), args.Error(1) } -// GetMetricDefinitions is a mock function for the azure service -func (client *MockService) GetMetricDefinitions(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) { +// GetMetricDefinitionsWithRetry is a mock function for the azure service +func (client *MockService) GetMetricDefinitionsWithRetry(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) { args := client.Called(resourceId, namespace) return args.Get(0).(armmonitor.MetricDefinitionCollection), args.Error(1) } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 9d69f67f687..5fa5b9964e6 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -20,12 +20,24 @@ const missingNamespace = "no metric definitions were found for resource %s and n // mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters func mapMetrics(client *azure.Client, resources []*armresources.GenericResourceExpanded, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric + for _, resource := range resources { + + // We use this map to avoid calling the metrics definition function for the same namespace and same resource + // multiple times. + namespaceMetrics := make(map[string]armmonitor.MetricDefinitionCollection) + for _, metric := range resourceConfig.Metrics { - // get all metrics supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) - if err != nil { - return nil, fmt.Errorf("no metric definitions were found for resource %s and namespace %s %w", *resource.ID, metric.Namespace, err) + + var err error + + metricDefinitions, exists := namespaceMetrics[metric.Namespace] + if !exists { + metricDefinitions, err = client.AzureMonitorService.GetMetricDefinitionsWithRetry(*resource.ID, metric.Namespace) + if err != nil { + return nil, err + } + namespaceMetrics[metric.Namespace] = metricDefinitions } if len(metricDefinitions.Value) == 0 { diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index d5c89bbbd78..782d941166b 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -88,7 +88,7 @@ func TestMapMetric(t *testing.T) { client := azure.NewMockClient() t.Run("return error when no metric definitions were found", func(t *testing.T) { m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(armmonitor.MetricDefinitionCollection{}, fmt.Errorf("invalid resource ID")) + m.On("GetMetricDefinitionsWithRetry", mock.Anything, mock.Anything).Return(armmonitor.MetricDefinitionCollection{}, fmt.Errorf("invalid resource ID")) client.AzureMonitorService = m metric, err := mapMetrics(client, []*armresources.GenericResourceExpanded{resource}, resourceConfig) assert.Error(t, err) @@ -97,7 +97,7 @@ func TestMapMetric(t *testing.T) { }) t.Run("return all metrics when all metric names and aggregations were configured", func(t *testing.T) { m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) + m.On("GetMetricDefinitionsWithRetry", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metricConfig.Name = []string{"*"} resourceConfig.Metrics = []azure.MetricConfig{metricConfig} @@ -112,7 +112,7 @@ func TestMapMetric(t *testing.T) { }) t.Run("return all metrics when specific metric names and aggregations were configured", func(t *testing.T) { m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) + m.On("GetMetricDefinitionsWithRetry", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metricConfig.Name = []string{"TotalRequests", "Capacity"} metricConfig.Aggregations = []string{"Average"} diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index 823a9cdf22a..70d79729920 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -6,8 +6,13 @@ package azure import ( "context" + "errors" "fmt" + "net/http" "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/elastic/elastic-agent-libs/logp" @@ -195,8 +200,43 @@ func (service *MonitorService) GetMetricNamespaces(resourceId string) (armmonito return metricNamespaceCollection, nil } -// GetMetricDefinitions will return all supported metrics based on the resource id and namespace -func (service *MonitorService) GetMetricDefinitions(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) { +// sleepIfPossible will check for the error 429 in the azure response, and look for the retry after header. +// If the header is present, then metricbeat will sleep for that duration, otherwise it will return an error. +func (service *MonitorService) sleepIfPossible(err error, resourceId string, namespace string) error { + errorMsg := "no metric definitions were found for resource " + resourceId + " and namespace " + namespace + + var respError *azcore.ResponseError + ok := errors.As(err, &respError) + if !ok { + return fmt.Errorf("%s, failed to cast error to azcore.ResponseError", errorMsg) + } + // Check for TooManyRequests error and retry if it is the case + if respError.StatusCode != http.StatusTooManyRequests { + return fmt.Errorf("%s, %w", errorMsg, err) + } + + // Check if the error has the header Retry After. + // If it is present, then we should try to make this request again. + retryAfter := respError.RawResponse.Header.Get("Retry-After") + if retryAfter == "" { + return fmt.Errorf("%s %w, failed to find Retry-After header", errorMsg, err) + } + + duration, errD := time.ParseDuration(retryAfter + "s") + if errD != nil { + return fmt.Errorf("%s, failed to parse duration %s from header retry after", errorMsg, retryAfter) + } + + service.log.Infof("%s, metricbeat will try again after %s seconds", errorMsg, retryAfter) + time.Sleep(duration) + service.log.Infof("%s, metricbeat finished sleeping and will try again now", errorMsg) + + return nil +} + +// GetMetricDefinitionsWithRetry will return all supported metrics based on the resource id and namespace +// It will check for an error when moving the pager to the next page, and retry if possible. +func (service *MonitorService) GetMetricDefinitionsWithRetry(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) { opts := &armmonitor.MetricDefinitionsClientListOptions{} if namespace != "" { @@ -210,9 +250,12 @@ func (service *MonitorService) GetMetricDefinitions(resourceId string, namespace for pager.More() { nextPage, err := pager.NextPage(service.context) if err != nil { - return armmonitor.MetricDefinitionCollection{}, err + retryError := service.sleepIfPossible(err, resourceId, namespace) + if retryError != nil { + return armmonitor.MetricDefinitionCollection{}, err + } + continue } - metricDefinitionCollection.Value = append(metricDefinitionCollection.Value, nextPage.Value...) } diff --git a/x-pack/metricbeat/module/azure/service_interface.go b/x-pack/metricbeat/module/azure/service_interface.go index 39a7da63621..cb524c7f6ea 100644 --- a/x-pack/metricbeat/module/azure/service_interface.go +++ b/x-pack/metricbeat/module/azure/service_interface.go @@ -13,7 +13,7 @@ import ( type Service interface { GetResourceDefinitionById(id string) (armresources.GenericResource, error) GetResourceDefinitions(id []string, group []string, rType string, query string) ([]*armresources.GenericResourceExpanded, error) - GetMetricDefinitions(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) + GetMetricDefinitionsWithRetry(resourceId string, namespace string) (armmonitor.MetricDefinitionCollection, error) GetMetricNamespaces(resourceId string) (armmonitor.MetricNamespaceCollection, error) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]armmonitor.Metric, string, error) } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper.go b/x-pack/metricbeat/module/azure/storage/client_helper.go index 393607be7ae..e60b9472a57 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper.go @@ -41,13 +41,13 @@ func mapMetrics(client *azure.Client, resources []*armresources.GenericResourceE } // get all metric definitions supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(resourceID, namespace) + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitionsWithRetry(resourceID, namespace) if err != nil { - return nil, fmt.Errorf("no metric definitions were found for resource %s and namespace %s %w", resourceID, namespace, err) + return nil, err } if len(metricDefinitions.Value) == 0 { - return nil, fmt.Errorf("no metric definitions were found for resource %s and namespace %s %w", resourceID, namespace, err) + return nil, fmt.Errorf("no metric definitions were found for resource %s and namespace %s", resourceID, namespace) } var filteredMetricDefinitions []armmonitor.MetricDefinition diff --git a/x-pack/metricbeat/module/azure/storage/client_helper_test.go b/x-pack/metricbeat/module/azure/storage/client_helper_test.go index ecdf4941ac9..14121c3a0b3 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper_test.go @@ -119,17 +119,17 @@ func TestMapMetric(t *testing.T) { client := azure.NewMockClient() t.Run("return error when no metric definitions were found", func(t *testing.T) { m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) + m.On("GetMetricDefinitionsWithRetry", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) client.AzureMonitorService = m metric, err := mapMetrics(client, []*armresources.GenericResourceExpanded{resource}, resourceConfig) assert.Error(t, err) - assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace Microsoft.Storage/storageAccounts %!w()") + assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace Microsoft.Storage/storageAccounts") assert.Equal(t, metric, []azure.Metric(nil)) m.AssertExpectations(t) }) t.Run("return mapped metrics correctly", func(t *testing.T) { m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) + m.On("GetMetricDefinitionsWithRetry", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metrics, err := mapMetrics(client, []*armresources.GenericResourceExpanded{resource}, resourceConfig) assert.NoError(t, err)