From fa82d1a979d65683db336ad24cf6cfe6a97a25c5 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 15 Jun 2022 01:35:17 -0400 Subject: [PATCH 01/69] [Automation] Update elastic stack version to 8.4.0-40cff009 for testing (#557) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 9a64bdc5d95..a1753983e81 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-7e67f5d9-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-40cff009-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-7e67f5d9-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-40cff009-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 9e5bb1568c018f41e10b870494b220924b5c20d0 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 16 Jun 2022 01:44:50 -0400 Subject: [PATCH 02/69] [Automation] Update elastic stack version to 8.4.0-5e6770b1 for testing (#564) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index a1753983e81..3e0b7d1eb16 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-40cff009-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-5e6770b1-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-40cff009-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-5e6770b1-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 5f66839fc488c7c25376a983de70485ccccb1282 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 16 Jun 2022 09:49:36 -0400 Subject: [PATCH 03/69] Fix regression and use comma separated values (#560) Fix regression from https://github.com/elastic/elastic-agent/pull/509 --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index bcacd5e1be3..be1a3065c8d 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -219,7 +219,7 @@ pipeline { axes { axis { name 'K8S_VERSION' - values "v1.24.0, v1.23.6, v1.22.9, v1.21.12" + values "v1.24.0", "v1.23.6", "v1.22.9", "v1.21.12" } } stages { From 06ee1dd37d45749c96db749cfab3679d4ec11f21 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 16 Jun 2022 16:56:36 -0400 Subject: [PATCH 04/69] Change in Jenkinsfile will trigger k8s run (#568) --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index be1a3065c8d..10b0b97a572 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { dir("${BASE_DIR}"){ setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '(^dev-tools/packaging/.*|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString()) - setEnvVar('K8S_CHANGES', isGitRegionMatch(patterns: [ '(^deploy/kubernetes/.*|^version/docs/version.asciidoc)' ], shouldMatchAll: false).toString()) + setEnvVar('K8S_CHANGES', isGitRegionMatch(patterns: [ '(^deploy/kubernetes/.*|^version/docs/version.asciidoc|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString()) } } } From 86ca6f773ffd16464b32d0650307e8832e0671a1 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 17 Jun 2022 01:41:07 -0400 Subject: [PATCH 05/69] [Automation] Update elastic stack version to 8.4.0-da5a1c6d for testing (#573) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3e0b7d1eb16..f067998ade9 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-5e6770b1-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-da5a1c6d-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-5e6770b1-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-da5a1c6d-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 7b29dbc22fe3f8c0a4ffdf8b4ef9cbb8a4f40313 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Fri, 17 Jun 2022 10:40:47 +0200 Subject: [PATCH 06/69] Add `@metadata.input_id` and `@metadata.stream_id` when injecting streams (#527) These 2 value are going to be used in the shipper to identify where an event came from in order to apply processors accordingly. Also, added test cases for the processor to verify the change and updated test cases with the new processor. --- CHANGELOG.next.asciidoc | 1 + .../testdata/audit_config-auditbeat.yml | 20 ++++ .../testdata/logstash_config-metricbeat.yml | 4 + .../program/testdata/namespace-metricbeat.yml | 6 +- .../testdata/single_config-metricbeat.yml | 4 + .../generated/namespace.metricbeat.golden.yml | 4 + .../single_config.filebeat.golden.yml | 8 ++ .../single_config.metricbeat.golden.yml | 4 + internal/pkg/agent/transpiler/rules.go | 53 +++++++++ internal/pkg/agent/transpiler/rules_test.go | 108 ++++++++++++++++++ 10 files changed, 211 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2c3e563cf21..c911f1bedaf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -183,3 +183,4 @@ - Save the agent configuration and the state encrypted on the disk. {issue}535[535] {pull}398[398] - Bump node.js version for heartbeat/synthetics to 16.15.0 - Support scheduled actions and cancellation of pending actions. {issue}393[393] {pull}419[419] +- Add `@metadata.input_id` and `@metadata.stream_id` when applying the inject stream processor {pull}527[527] diff --git a/internal/pkg/agent/program/testdata/audit_config-auditbeat.yml b/internal/pkg/agent/program/testdata/audit_config-auditbeat.yml index 7ea962de31a..0ebac6eff69 100644 --- a/internal/pkg/agent/program/testdata/audit_config-auditbeat.yml +++ b/internal/pkg/agent/program/testdata/audit_config-auditbeat.yml @@ -24,6 +24,10 @@ auditbeat: fields: dataset: auditd_manager.auditd target: event + - add_fields: + fields: + stream_id: audit/auditd-auditd_manager.auditd-6d35f37c-2243-4229-9fba-ccf39305b536 + target: '@metadata' - add_fields: fields: id: agent-id @@ -59,6 +63,10 @@ auditbeat: fields: dataset: auditd_manager.file_integrity target: event + - add_fields: + fields: + stream_id: fim_1 + target: '@metadata' - add_fields: fields: id: agent-id @@ -89,6 +97,10 @@ auditbeat: fields: dataset: auditd_manager.file_integrity target: event + - add_fields: + fields: + stream_id: fim_2 + target: '@metadata' - add_fields: fields: id: agent-id @@ -117,6 +129,10 @@ auditbeat: fields: dataset: audit_system.socket target: event + - add_fields: + fields: + stream_id: id-auditd-system-socket + target: '@metadata' - add_fields: fields: id: agent-id @@ -143,6 +159,10 @@ auditbeat: fields: dataset: audit_system.process target: event + - add_fields: + fields: + stream_id: id-auditd-system-process + target: '@metadata' - add_fields: fields: id: agent-id diff --git a/internal/pkg/agent/program/testdata/logstash_config-metricbeat.yml b/internal/pkg/agent/program/testdata/logstash_config-metricbeat.yml index a2d429c8108..8b79bc1e060 100644 --- a/internal/pkg/agent/program/testdata/logstash_config-metricbeat.yml +++ b/internal/pkg/agent/program/testdata/logstash_config-metricbeat.yml @@ -58,6 +58,10 @@ metricbeat: - add_fields: fields: should_be: first + - add_fields: + target: "@metadata" + fields: + input_id: apache-metrics-id - add_fields: target: "data_stream" fields: diff --git a/internal/pkg/agent/program/testdata/namespace-metricbeat.yml b/internal/pkg/agent/program/testdata/namespace-metricbeat.yml index d0d4c24f058..46df931eb66 100644 --- a/internal/pkg/agent/program/testdata/namespace-metricbeat.yml +++ b/internal/pkg/agent/program/testdata/namespace-metricbeat.yml @@ -58,6 +58,10 @@ metricbeat: - add_fields: fields: should_be: first + - add_fields: + target: "@metadata" + fields: + input_id: apache-metrics-id - add_fields: target: "data_stream" fields: @@ -83,7 +87,7 @@ output: hosts: [127.0.0.1:9200, 127.0.0.1:9300] headers: h1: test-header - + namespace: test_namespace username: elastic password: changeme diff --git a/internal/pkg/agent/program/testdata/single_config-metricbeat.yml b/internal/pkg/agent/program/testdata/single_config-metricbeat.yml index a2c36b151f0..0580c5454eb 100644 --- a/internal/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/internal/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -58,6 +58,10 @@ metricbeat: - add_fields: fields: should_be: first + - add_fields: + target: "@metadata" + fields: + input_id: apache-metrics-id - add_fields: target: "data_stream" fields: diff --git a/internal/pkg/agent/program/testdata/usecases/generated/namespace.metricbeat.golden.yml b/internal/pkg/agent/program/testdata/usecases/generated/namespace.metricbeat.golden.yml index 3232297227a..2e4f8833947 100644 --- a/internal/pkg/agent/program/testdata/usecases/generated/namespace.metricbeat.golden.yml +++ b/internal/pkg/agent/program/testdata/usecases/generated/namespace.metricbeat.golden.yml @@ -64,6 +64,10 @@ metricbeat: - add_fields: fields: should_be: first + - add_fields: + fields: + input_id: apache-metrics-id + target: '@metadata' - add_fields: fields: dataset: generic diff --git a/internal/pkg/agent/program/testdata/usecases/generated/single_config.filebeat.golden.yml b/internal/pkg/agent/program/testdata/usecases/generated/single_config.filebeat.golden.yml index 22055f9e09e..507efa09af9 100644 --- a/internal/pkg/agent/program/testdata/usecases/generated/single_config.filebeat.golden.yml +++ b/internal/pkg/agent/program/testdata/usecases/generated/single_config.filebeat.golden.yml @@ -6,6 +6,10 @@ filebeat: - /var/log/hello1.log - /var/log/hello2.log processors: + - add_fields: + fields: + input_id: logfile-1 + target: '@metadata' - add_fields: fields: dataset: generic @@ -35,6 +39,10 @@ filebeat: - /var/log/hello3.log - /var/log/hello4.log processors: + - add_fields: + fields: + input_id: logfile-2 + target: '@metadata' - add_fields: fields: dataset: generic diff --git a/internal/pkg/agent/program/testdata/usecases/generated/single_config.metricbeat.golden.yml b/internal/pkg/agent/program/testdata/usecases/generated/single_config.metricbeat.golden.yml index aca14055635..f51381d3394 100644 --- a/internal/pkg/agent/program/testdata/usecases/generated/single_config.metricbeat.golden.yml +++ b/internal/pkg/agent/program/testdata/usecases/generated/single_config.metricbeat.golden.yml @@ -64,6 +64,10 @@ metricbeat: - add_fields: fields: should_be: first + - add_fields: + fields: + input_id: apache-metrics-id + target: '@metadata' - add_fields: fields: dataset: generic diff --git a/internal/pkg/agent/transpiler/rules.go b/internal/pkg/agent/transpiler/rules.go index e4e466ddcd9..ca97cedd707 100644 --- a/internal/pkg/agent/transpiler/rules.go +++ b/internal/pkg/agent/transpiler/rules.go @@ -669,6 +669,42 @@ func (r *InjectStreamProcessorRule) Apply(_ AgentInfo, ast *AST) (err error) { namespace := datastreamNamespaceFromInputNode(inputNode) datastreamType := datastreamTypeFromInputNode(inputNode, r.Type) + var inputID *StrVal + inputIDNode, found := inputNode.Find("id") + if found { + inputID, _ = inputIDNode.Value().(*StrVal) + } + + if inputID != nil { + // get input-level processors node + processorsNode, found := inputNode.Find("processors") + if !found { + processorsNode = &Key{ + name: "processors", + value: &List{value: make([]Node, 0)}, + } + + inputMap, ok := inputNode.(*Dict) + if ok { + inputMap.value = append(inputMap.value, processorsNode) + } + } + + processorsList, ok := processorsNode.Value().(*List) + if !ok { + return errors.New("InjectStreamProcessorRule: input processors is not a list") + } + + // inject `input_id` on the input level + processorMap := &Dict{value: make([]Node, 0)} + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "@metadata"}}) + processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ + &Key{name: "input_id", value: inputID}, + }}}) + addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} + processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + } + streamsNode, ok := inputNode.Find("streams") if !ok { continue @@ -680,6 +716,12 @@ func (r *InjectStreamProcessorRule) Apply(_ AgentInfo, ast *AST) (err error) { } for _, streamNode := range streamsList.value { + var streamID *StrVal + streamIDNode, ok := streamNode.Find("id") + if ok { + streamID, _ = streamIDNode.Value().(*StrVal) + } + streamMap, ok := streamNode.(*Dict) if !ok { continue @@ -722,6 +764,17 @@ func (r *InjectStreamProcessorRule) Apply(_ AgentInfo, ast *AST) (err error) { }}}) addFieldsMap = &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + + if streamID != nil { + // source stream + processorMap = &Dict{value: make([]Node, 0)} + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "@metadata"}}) + processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ + &Key{name: "stream_id", value: streamID.Clone()}, + }}}) + addFieldsMap = &Dict{value: []Node{&Key{"add_fields", processorMap}}} + processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + } } } diff --git a/internal/pkg/agent/transpiler/rules_test.go b/internal/pkg/agent/transpiler/rules_test.go index ab2df9c1bce..840e1442fde 100644 --- a/internal/pkg/agent/transpiler/rules_test.go +++ b/internal/pkg/agent/transpiler/rules_test.go @@ -165,6 +165,114 @@ inputs: }, }, + "inject stream": { + givenYAML: ` +inputs: + - name: No streams, no IDs + type: file + - name: With streams and IDs + id: input-id + type: file + data_stream.namespace: nsns + streams: + - paths: /var/log/mysql/error.log + id: stream-id + data_stream.dataset: dsds + - name: With processors + id: input-id + type: file + data_stream.namespace: nsns + processors: + - add_fields: + target: some + fields: + dataset: value + streams: + - paths: /var/log/mysql/error.log + id: stream-id + data_stream.dataset: dsds + processors: + - add_fields: + target: another + fields: + dataset: value +`, + expectedYAML: ` +inputs: + - name: No streams, no IDs + type: file + - name: With streams and IDs + id: input-id + type: file + data_stream.namespace: nsns + processors: + - add_fields: + target: "@metadata" + fields: + input_id: input-id + streams: + - paths: /var/log/mysql/error.log + id: stream-id + data_stream.dataset: dsds + processors: + - add_fields: + target: data_stream + fields: + type: stream-type + namespace: nsns + dataset: dsds + - add_fields: + target: event + fields: + dataset: dsds + - add_fields: + target: "@metadata" + fields: + stream_id: stream-id + - name: With processors + id: input-id + type: file + data_stream.namespace: nsns + processors: + - add_fields: + target: some + fields: + dataset: value + - add_fields: + target: "@metadata" + fields: + input_id: input-id + streams: + - paths: /var/log/mysql/error.log + id: stream-id + data_stream.dataset: dsds + processors: + - add_fields: + target: another + fields: + dataset: value + - add_fields: + target: data_stream + fields: + type: stream-type + namespace: nsns + dataset: dsds + - add_fields: + target: event + fields: + dataset: dsds + - add_fields: + target: "@metadata" + fields: + stream_id: stream-id +`, + rule: &RuleList{ + Rules: []Rule{ + InjectStreamProcessor("insert_after", "stream-type"), + }, + }, + }, + "inject agent info": { givenYAML: ` inputs: From b728dfe00b576b8f244bd571d6b5cc64bac8709a Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Fri, 17 Jun 2022 09:52:31 -0700 Subject: [PATCH 07/69] Add filemod times to contents of diagnostics collect command (#570) * Add filemod times to contents of diagnostics collect command Add filemod times to the files and directories in the zip archive. Log files (and sub dirs) will use the modtime returned by the fileinfo for the source. Others will use the timestamp from when the zip is created. * Fix linter --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/cmd/diagnostics.go | 136 ++++++++++++++++++++------ 2 files changed, 108 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c911f1bedaf..24b4f09c44a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -102,6 +102,7 @@ - Update API calls that the agent makes to Kibana when running the container command. {pull}253[253] - diagnostics collect log names are fixed on Windows machines, command will ignore failures. AgentID is included in diagnostics(and diagnostics collect) output. {issue}81[81] {issue}92[92] {issue}190[190] {pull}262[262] - Collects stdout and stderr of applications run as a process and logs them. {issue}[88] +- diagnostics collect file mod times are set. {pull}570[570] ==== New features diff --git a/internal/pkg/agent/cmd/diagnostics.go b/internal/pkg/agent/cmd/diagnostics.go index 9095e34bfc3..3f68689930b 100644 --- a/internal/pkg/agent/cmd/diagnostics.go +++ b/internal/pkg/agent/cmd/diagnostics.go @@ -97,10 +97,7 @@ func newDiagnosticsCollectCommandWithArgs(_ []string, streams *cli.IOStreams) *c } output, _ := c.Flags().GetString("output") - switch output { - case "yaml": - case "json": - default: + if _, ok := diagOutputs[output]; !ok { return fmt.Errorf("unsupported output: %s", output) } @@ -454,6 +451,7 @@ func gatherConfig() (AgentConfig, error) { // The passed DiagnosticsInfo and AgentConfig data is written in the specified output format. // Any local log files are collected and copied into the archive. func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentConfig, pprof map[string][]client.ProcPProf, metrics *proto.ProcMetricsResponse, errs []error) error { + ts := time.Now().UTC() f, err := os.Create(fileName) if err != nil { return err @@ -461,7 +459,11 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon zw := zip.NewWriter(f) if len(errs) > 0 { - zf, err := zw.Create("errors.txt") + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "errors.txt", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -470,12 +472,20 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon } } - _, err = zw.Create("meta/") + _, err = zw.CreateHeader(&zip.FileHeader{ + Name: "meta/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } - zf, err := zw.Create("meta/elastic-agent-version." + outputFormat) + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "meta/elastic-agent-version" + outputFormat, + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -485,7 +495,11 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon } for _, m := range diag.ProcMeta { - zf, err = zw.Create("meta/" + m.Name + "-" + m.RouteKey + "." + outputFormat) + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "meta/" + m.Name + "-" + m.RouteKey + "." + outputFormat, + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -495,12 +509,20 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon } } - _, err = zw.Create("config/") + _, err = zw.CreateHeader(&zip.FileHeader{ + Name: "config/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } - zf, err = zw.Create("config/elastic-agent-local." + outputFormat) + zf, err = zw.CreateHeader(&zip.FileHeader{ + Name: "config/elastic-agent-local." + outputFormat, + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -508,7 +530,11 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon return closeHandlers(err, zw, f) } - zf, err = zw.Create("config/elastic-agent-policy." + outputFormat) + zf, err = zw.CreateHeader(&zip.FileHeader{ + Name: "config/elastic-agent-policy." + outputFormat, + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -516,7 +542,11 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon return closeHandlers(err, zw, f) } for name, appCfg := range cfg.AppConfig { - zf, err := zw.Create("config/" + name + "." + outputFormat) + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "config/" + name + "." + outputFormat, + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return closeHandlers(err, zw, f) } @@ -525,19 +555,19 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon } } - if err := zipLogs(zw); err != nil { + if err := zipLogs(zw, ts); err != nil { return closeHandlers(err, zw, f) } if pprof != nil { - err := zipProfs(zw, pprof) + err := zipProfs(zw, pprof, ts) if err != nil { return closeHandlers(err, zw, f) } } if metrics != nil && len(metrics.Result) > 0 { - err := zipMetrics(zw, metrics) + err := zipMetrics(zw, metrics, ts) if err != nil { return closeHandlers(err, zw, f) } @@ -547,8 +577,12 @@ func createZip(fileName, outputFormat string, diag DiagnosticsInfo, cfg AgentCon } // zipLogs walks paths.Logs() and copies the file structure into zw in "logs/" -func zipLogs(zw *zip.Writer) error { - _, err := zw.Create("logs/") +func zipLogs(zw *zip.Writer, ts time.Time) error { + _, err := zw.CreateHeader(&zip.FileHeader{ + Name: "logs/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } @@ -575,7 +609,16 @@ func zipLogs(zw *zip.Writer) error { } if d.IsDir() { - _, err := zw.Create("logs/" + name + "/") + dirTS := ts + di, err := d.Info() + if err == nil { + dirTS = di.ModTime() + } + _, err = zw.CreateHeader(&zip.FileHeader{ + Name: "logs/" + name + "/", + Method: zip.Deflate, + Modified: dirTS, + }) if err != nil { return fmt.Errorf("unable to create log directory in archive: %w", err) } @@ -625,7 +668,15 @@ func saveLogs(name string, logPath string, zw *zip.Writer) error { if err != nil { return fmt.Errorf("unable to open log file: %w", err) } - zf, err := zw.Create("logs/" + name) + lfs, err := lf.Stat() + if err != nil { + return closeHandlers(fmt.Errorf("unable to stat log file: %w", err), lf) + } + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "logs/" + name, + Method: zip.Deflate, + Modified: lfs.ModTime(), + }) if err != nil { return closeHandlers(fmt.Errorf("unable to create log file in archive: %w", err), lf) } @@ -681,20 +732,32 @@ func getAllPprof(ctx context.Context, d time.Duration) (map[string][]client.Proc return daemon.Pprof(ctx, d, pprofTypes, "", "") } -func zipProfs(zw *zip.Writer, pprof map[string][]client.ProcPProf) error { - _, err := zw.Create("pprof/") +func zipProfs(zw *zip.Writer, pprof map[string][]client.ProcPProf, ts time.Time) error { + _, err := zw.CreateHeader(&zip.FileHeader{ + Name: "pprof/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } for pType, profs := range pprof { - _, err := zw.Create("pprof/" + pType + "/") + _, err = zw.CreateHeader(&zip.FileHeader{ + Name: "pprof/" + pType + "/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } for _, p := range profs { if p.Error != "" { - zf, err := zw.Create("pprof/" + pType + "/" + p.Name + "_" + p.RouteKey + "_error.txt") + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "pprof/" + pType + "/" + p.Name + "_" + p.RouteKey + "_error.txt", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } @@ -704,7 +767,11 @@ func zipProfs(zw *zip.Writer, pprof map[string][]client.ProcPProf) error { } continue } - zf, err := zw.Create("pprof/" + pType + "/" + p.Name + "_" + p.RouteKey + ".pprof") + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "pprof/" + pType + "/" + p.Name + "_" + p.RouteKey + ".pprof", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } @@ -717,16 +784,23 @@ func zipProfs(zw *zip.Writer, pprof map[string][]client.ProcPProf) error { return nil } -func zipMetrics(zw *zip.Writer, metrics *proto.ProcMetricsResponse) error { - //nolint:staticcheck,wastedassign // false positive - zf, err := zw.Create("metrics/") +func zipMetrics(zw *zip.Writer, metrics *proto.ProcMetricsResponse, ts time.Time) error { + _, err := zw.CreateHeader(&zip.FileHeader{ + Name: "metrics/", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } for _, m := range metrics.Result { if m.Error != "" { - zf, err = zw.Create("metrics/" + m.AppName + "_" + m.RouteKey + "_error.txt") + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "metrics/" + m.AppName + "_" + m.RouteKey + "_error.txt", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } @@ -736,7 +810,11 @@ func zipMetrics(zw *zip.Writer, metrics *proto.ProcMetricsResponse) error { } continue } - zf, err = zw.Create("metrics/" + m.AppName + "_" + m.RouteKey + ".json") + zf, err := zw.CreateHeader(&zip.FileHeader{ + Name: "metrics/" + m.AppName + "_" + m.RouteKey + ".json", + Method: zip.Deflate, + Modified: ts, + }) if err != nil { return err } From 7db7406cc04d6dd8f2d47f8a37e299da4598056b Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 20 Jun 2022 01:57:36 -0400 Subject: [PATCH 08/69] [Automation] Update elastic stack version to 8.4.0-b13123ee for testing (#581) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f067998ade9..728f4152900 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-da5a1c6d-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-b13123ee-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-da5a1c6d-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-b13123ee-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From d77a91e283b1f92e9d89837b30a20d2eb092341f Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 20 Jun 2022 15:50:55 -0400 Subject: [PATCH 09/69] Fix Agent upgrade 8.2->8.3 (#578) * Fix Agent upgrade 8.2->8.3 * Improve the upgrade encryption handling. Add .yml files cleanup. * Rollback ActionUpgrade to action_id, add MarkerActionUpgrade adapter struct for marker serialization compatibility --- .../agent/application/upgrade/step_mark.go | 77 +++++++++- .../pkg/agent/application/upgrade/upgrade.go | 76 +--------- internal/pkg/agent/cleaner/cleaner.go | 111 ++++++++++++++ internal/pkg/agent/cleaner/cleaner_test.go | 68 +++++++++ internal/pkg/agent/cmd/run.go | 135 ++++++++++++++++++ internal/pkg/fileutil/fileutil.go | 46 ++++++ 6 files changed, 434 insertions(+), 79 deletions(-) create mode 100644 internal/pkg/agent/cleaner/cleaner.go create mode 100644 internal/pkg/agent/cleaner/cleaner_test.go create mode 100644 internal/pkg/fileutil/fileutil.go diff --git a/internal/pkg/agent/application/upgrade/step_mark.go b/internal/pkg/agent/application/upgrade/step_mark.go index e176e4c5b96..66924337699 100644 --- a/internal/pkg/agent/application/upgrade/step_mark.go +++ b/internal/pkg/agent/application/upgrade/step_mark.go @@ -38,6 +38,58 @@ type UpdateMarker struct { Action *fleetapi.ActionUpgrade `json:"action" yaml:"action"` } +// MarkerActionUpgrade adapter struct compatible with pre 8.3 version of the marker file format +type MarkerActionUpgrade struct { + ActionID string `yaml:"id"` + ActionType string `yaml:"type"` + Version string `yaml:"version"` + SourceURI string `yaml:"source_uri,omitempty"` +} + +func convertToMarkerAction(a *fleetapi.ActionUpgrade) *MarkerActionUpgrade { + if a == nil { + return nil + } + return &MarkerActionUpgrade{ + ActionID: a.ActionID, + ActionType: a.ActionType, + Version: a.Version, + SourceURI: a.SourceURI, + } +} + +func convertToActionUpgrade(a *MarkerActionUpgrade) *fleetapi.ActionUpgrade { + if a == nil { + return nil + } + return &fleetapi.ActionUpgrade{ + ActionID: a.ActionID, + ActionType: a.ActionType, + Version: a.Version, + SourceURI: a.SourceURI, + } +} + +type updateMarkerSerializer struct { + Hash string `yaml:"hash"` + UpdatedOn time.Time `yaml:"updated_on"` + PrevVersion string `yaml:"prev_version"` + PrevHash string `yaml:"prev_hash"` + Acked bool `yaml:"acked"` + Action *MarkerActionUpgrade `yaml:"action"` +} + +func newMarkerSerializer(m *UpdateMarker) *updateMarkerSerializer { + return &updateMarkerSerializer{ + Hash: m.Hash, + UpdatedOn: m.UpdatedOn, + PrevVersion: m.PrevVersion, + PrevHash: m.PrevHash, + Acked: m.Acked, + Action: convertToMarkerAction(m.Action), + } +} + // markUpgrade marks update happened so we can handle grace period func (u *Upgrader) markUpgrade(_ context.Context, hash string, action Action) error { prevVersion := release.Version() @@ -46,7 +98,7 @@ func (u *Upgrader) markUpgrade(_ context.Context, hash string, action Action) er prevHash = prevHash[:hashLen] } - marker := UpdateMarker{ + marker := &UpdateMarker{ Hash: hash, UpdatedOn: time.Now(), PrevVersion: prevVersion, @@ -54,7 +106,7 @@ func (u *Upgrader) markUpgrade(_ context.Context, hash string, action Action) er Action: action.FleetAction(), } - markerBytes, err := yaml.Marshal(marker) + markerBytes, err := yaml.Marshal(newMarkerSerializer(marker)) if err != nil { return errors.New(err, errors.TypeConfig, "failed to parse marker file") } @@ -103,16 +155,31 @@ func LoadMarker() (*UpdateMarker, error) { return nil, err } - marker := &UpdateMarker{} + marker := &updateMarkerSerializer{} if err := yaml.Unmarshal(markerBytes, &marker); err != nil { return nil, err } - return marker, nil + return &UpdateMarker{ + Hash: marker.Hash, + UpdatedOn: marker.UpdatedOn, + PrevVersion: marker.PrevVersion, + PrevHash: marker.PrevHash, + Acked: marker.Acked, + Action: convertToActionUpgrade(marker.Action), + }, nil } func saveMarker(marker *UpdateMarker) error { - markerBytes, err := yaml.Marshal(marker) + makerSerializer := &updateMarkerSerializer{ + Hash: marker.Hash, + UpdatedOn: marker.UpdatedOn, + PrevVersion: marker.PrevVersion, + PrevHash: marker.PrevHash, + Acked: marker.Acked, + Action: convertToMarkerAction(marker.Action), + } + markerBytes, err := yaml.Marshal(makerSerializer) if err != nil { return err } diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 81fb7a78444..9d67165d0eb 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -5,7 +5,6 @@ package upgrade import ( - "bytes" "context" "fmt" "io/ioutil" @@ -20,10 +19,8 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/info" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/reexec" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/agent/program" - "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/artifact" "github.com/elastic/elastic-agent/internal/pkg/capabilities" "github.com/elastic/elastic-agent/internal/pkg/core/state" @@ -173,10 +170,6 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree return nil, errors.New(err, "failed to copy action store") } - if err := encryptConfigIfNeeded(u.log, newHash); err != nil { - return nil, errors.New(err, "failed to encrypt the configuration") - } - if err := ChangeSymlink(ctx, newHash); err != nil { rollbackInstall(ctx, newHash) return nil, err @@ -220,6 +213,8 @@ func (u *Upgrader) Ack(ctx context.Context) error { return err } + marker.Acked = true + return saveMarker(marker) } @@ -335,73 +330,6 @@ func copyVault(newHash string) error { return nil } -// Create the key if it doesn't exist and encrypt the fleet.yml and state.yml -func encryptConfigIfNeeded(log *logger.Logger, newHash string) (err error) { - vaultPath := getVaultPath(newHash) - - err = secret.CreateAgentSecret(secret.WithVaultPath(vaultPath)) - if err != nil { - return err - } - - newHome := filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, newHash)) - ymlStateStorePath := filepath.Join(newHome, filepath.Base(paths.AgentStateStoreYmlFile())) - stateStorePath := filepath.Join(newHome, filepath.Base(paths.AgentStateStoreFile())) - - files := []struct { - Src string - Dst string - }{ - { - Src: ymlStateStorePath, - Dst: stateStorePath, - }, - { - Src: paths.AgentConfigYmlFile(), - Dst: paths.AgentConfigFile(), - }, - } - for _, f := range files { - var b []byte - b, err = ioutil.ReadFile(f.Src) - if err != nil { - if os.IsNotExist(err) { - continue - } - return err - } - - // Encrypt yml file - store := storage.NewEncryptedDiskStore(f.Dst, storage.WithVaultPath(vaultPath)) - err = store.Save(bytes.NewReader(b)) - if err != nil { - return err - } - - // Remove yml file if no errors - defer func(fp string) { - if err != nil { - return - } - if rerr := os.Remove(fp); rerr != nil { - log.Warnf("failed to remove file: %s, err: %v", fp, rerr) - } - }(f.Src) - } - - // Do not remove AgentConfigYmlFile lock file if any error happened. - if err != nil { - return err - } - - lockFp := paths.AgentConfigYmlFile() + ".lock" - if rerr := os.Remove(lockFp); rerr != nil { - log.Warnf("failed to remove file: %s, err: %v", lockFp, rerr) - } - - return err -} - // shutdownCallback returns a callback function to be executing during shutdown once all processes are closed. // this goes through runtime directory of agent and copies all the state files created by processes to new versioned // home directory with updated process name to match new version. diff --git a/internal/pkg/agent/cleaner/cleaner.go b/internal/pkg/agent/cleaner/cleaner.go new file mode 100644 index 00000000000..856ae020b89 --- /dev/null +++ b/internal/pkg/agent/cleaner/cleaner.go @@ -0,0 +1,111 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cleaner + +import ( + "context" + "os" + "sync" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent/internal/pkg/fileutil" +) + +// Wait interval. +// If the watchFile was not modified after this interval, then remove all the files in the removeFiles array +const defaultCleanWait = 15 * time.Minute + +type Cleaner struct { + log *logp.Logger + watchFile string + removeFiles []string + cleanWait time.Duration + + mx sync.Mutex +} + +type OptionFunc func(c *Cleaner) + +func New(log *logp.Logger, watchFile string, removeFiles []string, opts ...OptionFunc) *Cleaner { + c := &Cleaner{ + log: log, + watchFile: watchFile, + removeFiles: removeFiles, + cleanWait: defaultCleanWait, + } + + for _, opt := range opts { + opt(c) + } + return c +} + +func WithCleanWait(cleanWait time.Duration) OptionFunc { + return func(c *Cleaner) { + c.cleanWait = cleanWait + } +} + +func (c *Cleaner) Run(ctx context.Context) error { + wait, done, err := c.process() + if err != nil { + return err + } + + if done { + return nil + } + + t := time.NewTimer(wait) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + c.log.Debug("cleaner: timer triggered") + wait, done, err = c.process() + if err != nil { + return err + } + + if done { + return nil + } + t.Reset(wait) + } + } +} + +func (c *Cleaner) process() (wait time.Duration, done bool, err error) { + modTime, err := fileutil.GetModTime(c.watchFile) + if err != nil { + return + } + + c.log.Debugf("cleaner: check file %s mod time: %v", c.watchFile, modTime) + curDur := time.Since(modTime) + if curDur > c.cleanWait { + c.log.Debugf("cleaner: file %s modification expired", c.watchFile) + c.deleteFiles() + return wait, true, nil + } + wait = c.cleanWait - curDur + return wait, false, nil +} + +func (c *Cleaner) deleteFiles() { + c.log.Debugf("cleaner: delete files: %v", c.removeFiles) + c.mx.Lock() + defer c.mx.Unlock() + for _, fp := range c.removeFiles { + c.log.Debugf("cleaner: delete file: %v", fp) + err := os.Remove(fp) + if err != nil { + c.log.Warnf("cleaner: delete file %v failed: %v", fp, err) + } + } +} diff --git a/internal/pkg/agent/cleaner/cleaner_test.go b/internal/pkg/agent/cleaner/cleaner_test.go new file mode 100644 index 00000000000..cf189b784d3 --- /dev/null +++ b/internal/pkg/agent/cleaner/cleaner_test.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cleaner + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestCleaner(t *testing.T) { + // Setup + const watchFileName = "fleet.enc" + removeFiles := []string{"fleet.yml", "fleet.yml.lock"} + + ctx, cn := context.WithCancel(context.Background()) + defer cn() + + dir := t.TempDir() + watchFilePath := filepath.Join(dir, watchFileName) + + removeFilePaths := make([]string, len(removeFiles)) + + checkDir(t, dir, 0) + + // Create files + err := ioutil.WriteFile(watchFilePath, []byte{}, 0600) + if err != nil { + t.Fatal(err) + } + + for i, fn := range removeFiles { + removeFilePaths[i] = filepath.Join(dir, fn) + err := ioutil.WriteFile(removeFilePaths[i], []byte{}, 0600) + if err != nil { + t.Fatal(err) + } + } + + checkDir(t, dir, len(removeFiles)+1) + + log := logp.NewLogger("dynamic") + cleaner := New(log, watchFilePath, removeFilePaths, WithCleanWait(500*time.Millisecond)) + err = cleaner.Run(ctx) + if err != nil { + t.Fatal(err) + } + checkDir(t, dir, 1) +} + +func checkDir(t *testing.T, dir string, expectedCount int) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + if len(entries) != expectedCount { + t.Fatalf("Dir %s expected %d entries, found %d", dir, expectedCount, len(entries)) + } +} diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index b584baf2f09..723631b7960 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -5,6 +5,7 @@ package cmd import ( + "bytes" "context" "fmt" "io/ioutil" @@ -20,6 +21,7 @@ import ( "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent-libs/api" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" "github.com/elastic/elastic-agent-libs/service" "github.com/elastic/elastic-agent-system-metrics/report" @@ -31,6 +33,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/reexec" "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade" + "github.com/elastic/elastic-agent/internal/pkg/agent/cleaner" "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/control/server" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" @@ -41,6 +44,7 @@ import ( monitoringCfg "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config" monitoringServer "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/server" "github.com/elastic/elastic-agent/internal/pkg/core/status" + "github.com/elastic/elastic-agent/internal/pkg/fileutil" "github.com/elastic/elastic-agent/internal/pkg/release" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/version" @@ -126,6 +130,19 @@ func run(override cfgOverrider) error { return err } + // Check if the fleet.yml or state.yml exists and encrypt them. + // This is needed to handle upgrade properly. + // On agent upgrade the older version for example 8.2 unpacks the 8.3 agent + // and tries to run it. + // The new version of the agent requires encrypted configuration files or it will not start and upgrade will fail and revert. + err = encryptConfigIfNeeded(logger) + if err != nil { + return err + } + + // Start the old unencrypted agent configuration file cleaner + startOldAgentConfigCleaner(ctx, logger) + agentInfo, err := info.NewAgentInfoWithLog(defaultLogLevel(cfg), createAgentID) if err != nil { return errors.New(err, @@ -476,3 +493,121 @@ func initTracer(agentName, version string, mcfg *monitoringCfg.MonitoringConfig) Transport: ts, }) } + +// encryptConfigIfNeeded encrypts fleet.yml or state.yml if fleet.enc or state.enc does not exist already. +func encryptConfigIfNeeded(log *logger.Logger) (err error) { + log.Debug("encrypt config if needed") + + files := []struct { + Src string + Dst string + }{ + { + Src: paths.AgentStateStoreYmlFile(), + Dst: paths.AgentStateStoreFile(), + }, + { + Src: paths.AgentConfigYmlFile(), + Dst: paths.AgentConfigFile(), + }, + } + for _, f := range files { + var b []byte + + // Check if .yml file modification timestamp and existence + log.Debugf("check if the yml file %v exists", f.Src) + ymlModTime, ymlExists, err := fileutil.GetModTimeExists(f.Src) + if err != nil { + log.Errorf("failed to access yml file %v: %v", f.Src, err) + return err + } + + if !ymlExists { + log.Debugf("yml file %v doesn't exists, continue", f.Src) + continue + } + + // Check if .enc file modification timestamp and existence + log.Debugf("check if the enc file %v exists", f.Dst) + encModTime, encExists, err := fileutil.GetModTimeExists(f.Dst) + if err != nil { + log.Errorf("failed to access enc file %v: %v", f.Dst, err) + return err + } + + // If enc file exists and the yml file modification time is before enc file modification time then skip encryption. + // The reasoning is that the yml was not modified since the last time it was migrated to the encrypted file. + // The modification of the yml is possible in the cases where the agent upgrade failed and rolled back, leaving .enc file on the disk for example + if encExists && ymlModTime.Before(encModTime) { + log.Debugf("enc file %v already exists, and the yml was not modified after migration, yml mod time: %v, enc mod time: %v", f.Dst, ymlModTime, encModTime) + continue + } + + log.Debugf("read file: %v", f.Src) + b, err = ioutil.ReadFile(f.Src) + if err != nil { + log.Debugf("read file: %v, err: %v", f.Src, err) + return err + } + + // Encrypt yml file + log.Debugf("encrypt file %v into %v", f.Src, f.Dst) + store := storage.NewEncryptedDiskStore(f.Dst) + err = store.Save(bytes.NewReader(b)) + if err != nil { + log.Debugf("failed to encrypt file: %v, err: %v", f.Dst, err) + return err + } + } + + if err != nil { + return err + } + + // Remove state.yml file if no errors + fp := paths.AgentStateStoreYmlFile() + // Check if state.yml exists + exists, err := fileutil.FileExists(fp) + if err != nil { + log.Warnf("failed to check if file %s exists, err: %v", fp, err) + } + if exists { + if err := os.Remove(fp); err != nil { + // Log only + log.Warnf("failed to remove file: %s, err: %v", fp, err) + } + } + + // The agent can't remove fleet.yml, because it can be rolled back by the older version of the agent "watcher" + // and pre 8.3 version needs unencrypted fleet.yml file in order to start. + // The fleet.yml file removal is performed by the cleaner on the agent start after the .enc configuration was stable for the grace period after upgrade + + return nil +} + +// startOldAgentConfigCleaner starts the cleaner that removes fleet.yml and fleet.yml.lock files after 15 mins by default +// The interval is calculated from the last modified time of fleet.enc. It's possible that the fleet.enc +// will be modified again during that time, the assumption is that at some point there will be 15 mins interval when the fleet.enc is not modified. +// The modification time is used because it's the most cross-patform compatible timestamp on the files. +// This is tied to grace period, default 10 mins, when the agent is considered "stable" after the upgrade. +// The old agent watcher doesn't know anything about configuration encryption so we have to delete the old configuration files here. +// The cleaner is only started if fleet.yml exists +func startOldAgentConfigCleaner(ctx context.Context, log *logp.Logger) { + // Start cleaner only when fleet.yml exists + fp := paths.AgentConfigYmlFile() + exists, err := fileutil.FileExists(fp) + if err != nil { + log.Warnf("failed to check if file %s exists, err: %v", fp, err) + } + if !exists { + return + } + + c := cleaner.New(log, paths.AgentConfigFile(), []string{fp, fmt.Sprintf("%s.lock", fp)}) + go func() { + err := c.Run(ctx) + if err != nil { + log.Warnf("failed running the old configuration files cleaner, err: %v", err) + } + }() +} diff --git a/internal/pkg/fileutil/fileutil.go b/internal/pkg/fileutil/fileutil.go new file mode 100644 index 00000000000..86d1db249aa --- /dev/null +++ b/internal/pkg/fileutil/fileutil.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fileutil + +import ( + "errors" + "io/fs" + "os" + "time" +) + +// FileExists returns true if file/dir exists +func FileExists(fp string) (bool, error) { + _, err := os.Stat(fp) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + return true, nil +} + +// GetModTime returns file modification time +func GetModTime(fp string) (time.Time, error) { + fi, err := os.Stat(fp) + if err != nil { + return time.Time{}, err + } + return fi.ModTime(), nil +} + +// GetModTimeExists returns file modification time and existence status +// Returns no error if the file doesn't exists +func GetModTimeExists(fp string) (time.Time, bool, error) { + modTime, err := GetModTime(fp) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return modTime, false, nil + } + return modTime, false, err + } + return modTime, true, nil +} From e043f36d3c22938bfda20a9f791eac961599ce91 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 21 Jun 2022 08:27:45 -0700 Subject: [PATCH 10/69] Update containerd (#577) --- NOTICE.txt | 12 ++++++------ go.mod | 5 +++-- go.sum | 13 ++++++++----- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 4b402c790f4..e80beb85357 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5274,11 +5274,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.0.0-20220405052023-b1e9470b6e64 +Version: v0.0.0-20220412211240-33da011f77ad Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220405052023-b1e9470b6e64/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220412211240-33da011f77ad/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -7110,11 +7110,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/containerd/containerd -Version: v1.5.10 +Version: v1.5.13 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/containerd/containerd@v1.5.10/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/containerd/containerd@v1.5.13/LICENSE: Apache License @@ -15120,11 +15120,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/goleak -Version: v1.1.11 +Version: v1.1.12 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.1.11/LICENSE: +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.1.12/LICENSE: The MIT License (MIT) diff --git a/go.mod b/go.mod index 37c11d9bfb6..5bc60abc07b 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad golang.org/x/tools v0.1.9 google.golang.org/grpc v1.42.0 google.golang.org/protobuf v1.27.1 @@ -65,7 +65,7 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect - github.com/containerd/containerd v1.5.10 // indirect + github.com/containerd/containerd v1.5.13 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dnephin/pflag v1.0.7 // indirect @@ -119,6 +119,7 @@ require ( go.elastic.co/apm/v2 v2.0.0 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.uber.org/atomic v1.9.0 // indirect + go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect diff --git a/go.sum b/go.sum index 91a2ec0da3a..7198e357fbf 100644 --- a/go.sum +++ b/go.sum @@ -92,7 +92,7 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.8.24/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -229,6 +229,7 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -250,8 +251,8 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.10 h1:3cQ2uRVCkJVcx5VombsE7105Gl9Wrl7ORAO3+4+ogf4= -github.com/containerd/containerd v1.5.10/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= +github.com/containerd/containerd v1.5.13 h1:XqvKw9i4P7/mFrC3TSM7yV5cwFZ9avXe6M3YANKnzEE= +github.com/containerd/containerd v1.5.13/go.mod h1:3AlCrzKROjIuP3JALsY14n8YtntaUDBu7vek+rPN5Vc= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -1260,8 +1261,9 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1562,8 +1564,9 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 86ea452716a6299e3d877bb73c8a2d8a8deb5805 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 22 Jun 2022 01:43:51 -0400 Subject: [PATCH 11/69] [Automation] Update elastic stack version to 8.4.0-4fe26f2a for testing (#591) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 728f4152900..f7f7fd7d996 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-b13123ee-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-4fe26f2a-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-b13123ee-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-4fe26f2a-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From c876ac4879fc901210b338d09b3b87a596bb7969 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Wed, 22 Jun 2022 20:34:47 -0400 Subject: [PATCH 12/69] Set explicit ExitTimeOut for MacOS agent launchd plist (#594) * Set explicit ExitTimeOut for MacOS agent launchd plist --- internal/pkg/agent/install/svc.go | 63 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/internal/pkg/agent/install/svc.go b/internal/pkg/agent/install/svc.go index 7148f4acca0..1a3bf50c896 100644 --- a/internal/pkg/agent/install/svc.go +++ b/internal/pkg/agent/install/svc.go @@ -6,6 +6,7 @@ package install import ( "path/filepath" + "runtime" "github.com/kardianos/service" @@ -18,6 +19,12 @@ const ( // ServiceDescription is the description for the service. ServiceDescription = "Elastic Agent is a unified agent to observe, monitor and protect your system." + + // Set the launch daemon ExitTimeOut to 60 seconds in order to allow the agent to shutdown gracefully + // At the moment the version 8.3 & 8.4 of the agent are taking about 11 secs to shutdown + // and the launchd sends SIGKILL after 5 secs which causes the beats processes to be left running orphaned + // depending on the shutdown timing. + darwinServiceExitTimeout = 60 ) // ExecutablePath returns the path for the installed Agents executable. @@ -30,7 +37,7 @@ func ExecutablePath() string { } func newService() (service.Service, error) { - return service.New(nil, &service.Config{ + cfg := &service.Config{ Name: paths.ServiceName, DisplayName: ServiceDisplayName, Description: ServiceDescription, @@ -45,5 +52,57 @@ func newService() (service.Service, error) { "OnFailureDelayDuration": "1s", "OnFailureResetPeriod": 10, }, - }) + } + + if runtime.GOOS == "darwin" { + // The github.com/kardianos/service library doesn't support ExitTimeOut in their prebuilt template. + // This option allows to pass our own template for the launch daemon plist, which is a copy + // of the prebuilt template with added ExitTimeOut option + cfg.Option["LaunchdConfig"] = darwinLaunchdConfig + cfg.Option["ExitTimeOut"] = darwinServiceExitTimeout + } + + return service.New(nil, cfg) } + +// A copy of the launchd plist template from github.com/kardianos/service +// with added .Config.Option.ExitTimeOut option +const darwinLaunchdConfig = ` + + + + Label + {{html .Name}} + ProgramArguments + + {{html .Path}} + {{range .Config.Arguments}} + {{html .}} + {{end}} + + {{if .UserName}}UserName + {{html .UserName}}{{end}} + {{if .ChRoot}}RootDirectory + {{html .ChRoot}}{{end}} + {{if .Config.Option.ExitTimeOut}}ExitTimeOut + {{html .Config.Option.ExitTimeOut}}{{end}} + {{if .WorkingDirectory}}WorkingDirectory + {{html .WorkingDirectory}}{{end}} + SessionCreate + <{{bool .SessionCreate}}/> + KeepAlive + <{{bool .KeepAlive}}/> + RunAtLoad + <{{bool .RunAtLoad}}/> + Disabled + + + StandardOutPath + /usr/local/var/log/{{html .Name}}.out.log + StandardErrorPath + /usr/local/var/log/{{html .Name}}.err.log + + + +` From 89802c6867264a71876a43058081a4aad5ba9b55 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 23 Jun 2022 02:00:47 -0400 Subject: [PATCH 13/69] [Automation] Update elastic stack version to 8.4.0-2e32a640 for testing (#599) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f7f7fd7d996..1f7885d2eb6 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-4fe26f2a-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-2e32a640-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-4fe26f2a-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-2e32a640-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 11ce21478f207439e109246510cb7c4d9ae2b00c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 23 Jun 2022 09:49:20 +0100 Subject: [PATCH 14/69] ci: enable build notifications as GitHub issues (#595) --- .ci/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 10b0b97a572..68b3c8a7b51 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -262,7 +262,9 @@ pipeline { } post { cleanup { - notifyBuildResult(prComment: true) + notifyBuildResult(prComment: true, + githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS", + githubLabels: 'Team:Elastic-Agent-Control-Plane') } } } From 09d924f7a7ec48fbca1dc6c6edf8ec8dbc9d51f0 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Thu, 23 Jun 2022 10:07:29 -0700 Subject: [PATCH 15/69] status identifies failing component, fleet gateway may report degraded, liveness endpoint added (#569) * Add liveness endpoint Add /liveness route to metrics server. This route will report the status from pkg/core/status. fleet-gateway will now report a degraded state if a checkin fails. This may not propogate to fleet-server as a failed checkin means communications between the agent and the server are not working. It may also lead to the server reporting degraded for up to 30s (fleet-server polling time) when teh agent is able to successfully connect. * linter fix * add nolint direcrtive * Linter fix * Review feedback, add doc strings * Rename noop controller file to _test file --- CHANGELOG.next.asciidoc | 1 + .../gateway/fleet/fleet_gateway.go | 46 ++++--- .../gateway/fleet/fleet_gateway_test.go | 130 +++++++++++------- ...ller.go => noop_status_controller_test.go} | 14 +- internal/pkg/agent/cmd/run.go | 6 +- internal/pkg/core/monitoring/server/server.go | 6 +- internal/pkg/core/status/handler.go | 39 ++++++ internal/pkg/core/status/reporter.go | 32 ++++- internal/pkg/core/status/reporter_test.go | 53 +++++++ internal/pkg/testutils/status_reporter.go | 67 +++++++++ 10 files changed, 319 insertions(+), 75 deletions(-) rename internal/pkg/agent/application/gateway/fleet/{noop_status_controller.go => noop_status_controller_test.go} (77%) create mode 100644 internal/pkg/core/status/handler.go create mode 100644 internal/pkg/testutils/status_reporter.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 24b4f09c44a..cbe894d63c5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -185,3 +185,4 @@ - Bump node.js version for heartbeat/synthetics to 16.15.0 - Support scheduled actions and cancellation of pending actions. {issue}393[393] {pull}419[419] - Add `@metadata.input_id` and `@metadata.stream_id` when applying the inject stream processor {pull}527[527] +- Add liveness endpoint, allow fleet-gateway component to report degraded state, add update time and messages to status output. {issue}390[390] {pull}569[569] diff --git a/internal/pkg/agent/application/gateway/fleet/fleet_gateway.go b/internal/pkg/agent/application/gateway/fleet/fleet_gateway.go index 4ff4c34ad42..f5c02d3356a 100644 --- a/internal/pkg/agent/application/gateway/fleet/fleet_gateway.go +++ b/internal/pkg/agent/application/gateway/fleet/fleet_gateway.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// Package fleet handles interactions between the elastic-agent and fleet-server. +// Specifically it will handle agent checkins, and action queueing/dispatch. package fleet import ( @@ -75,23 +77,24 @@ type actionQueue interface { } type fleetGateway struct { - bgContext context.Context - log *logger.Logger - dispatcher pipeline.Dispatcher - client client.Sender - scheduler scheduler.Scheduler - backoff backoff.Backoff - settings *fleetGatewaySettings - agentInfo agentInfo - reporter fleetReporter - done chan struct{} - wg sync.WaitGroup - acker store.FleetAcker - unauthCounter int - statusController status.Controller - statusReporter status.Reporter - stateStore stateStore - queue actionQueue + bgContext context.Context + log *logger.Logger + dispatcher pipeline.Dispatcher + client client.Sender + scheduler scheduler.Scheduler + backoff backoff.Backoff + settings *fleetGatewaySettings + agentInfo agentInfo + reporter fleetReporter + done chan struct{} + wg sync.WaitGroup + acker store.FleetAcker + unauthCounter int + checkinFailCounter int + statusController status.Controller + statusReporter status.Reporter + stateStore stateStore + queue actionQueue } // New creates a new fleet gateway @@ -286,6 +289,7 @@ func (f *fleetGateway) doExecute() (*fleetapi.CheckinResponse, error) { f.log.Debugf("Checking started") resp, err := f.execute(f.bgContext) if err != nil { + f.checkinFailCounter++ f.log.Errorf("Could not communicate with fleet-server Checking API will retry, error: %s", err) if !f.backoff.Wait() { // Something bad has happened and we log it and we should update our current state. @@ -299,8 +303,16 @@ func (f *fleetGateway) doExecute() (*fleetapi.CheckinResponse, error) { f.statusReporter.Update(state.Failed, err.Error(), nil) return nil, err } + if f.checkinFailCounter > 1 { + // Update status reporter for gateway to degraded when there are two consecutive failures. + // Note that this may not propagate to fleet-server as the agent is having issues checking in. + // It may also (falsely) report a degraded session for 30s if it is eventually successful. + // However this component will allow the agent to report fleet gateway degredation locally. + f.statusReporter.Update(state.Degraded, fmt.Sprintf("checkin failed: %v", err), nil) + } continue } + f.checkinFailCounter = 0 // Request was successful, return the collected actions. return resp, nil } diff --git a/internal/pkg/agent/application/gateway/fleet/fleet_gateway_test.go b/internal/pkg/agent/application/gateway/fleet/fleet_gateway_test.go index a9b9380519f..6ce62448276 100644 --- a/internal/pkg/agent/application/gateway/fleet/fleet_gateway_test.go +++ b/internal/pkg/agent/application/gateway/fleet/fleet_gateway_test.go @@ -26,12 +26,14 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/agent/storage/store" + "github.com/elastic/elastic-agent/internal/pkg/core/state" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" noopacker "github.com/elastic/elastic-agent/internal/pkg/fleetapi/acker/noop" repo "github.com/elastic/elastic-agent/internal/pkg/reporter" fleetreporter "github.com/elastic/elastic-agent/internal/pkg/reporter/fleet" fleetreporterConfig "github.com/elastic/elastic-agent/internal/pkg/reporter/fleet/config" "github.com/elastic/elastic-agent/internal/pkg/scheduler" + "github.com/elastic/elastic-agent/internal/pkg/testutils" "github.com/elastic/elastic-agent/pkg/core/logger" ) @@ -705,59 +707,95 @@ func TestRetriesOnFailures(t *testing.T) { Backoff: backoffSettings{Init: 100 * time.Millisecond, Max: 5 * time.Second}, } - t.Run("When the gateway fails to communicate with the checkin API we will retry", - withGateway(agentInfo, settings, func( - t *testing.T, - gateway gateway.FleetGateway, - client *testingClient, - dispatcher *testingDispatcher, - scheduler *scheduler.Stepper, - rep repo.Backend, - ) { - fail := func(_ http.Header, _ io.Reader) (*http.Response, error) { - return wrapStrToResp(http.StatusInternalServerError, "something is bad"), nil - } - clientWaitFn := client.Answer(fail) - err := gateway.Start() - require.NoError(t, err) + t.Run("When the gateway fails to communicate with the checkin API we will retry", func(t *testing.T) { + scheduler := scheduler.NewStepper() + client := newTestingClient() + dispatcher := newTestingDispatcher() + log, _ := logger.New("fleet_gateway", false) + rep := getReporter(agentInfo, log, t) - _ = rep.Report(context.Background(), &testStateEvent{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - // Initial tick is done out of bound so we can block on channels. - scheduler.Next() + diskStore := storage.NewDiskStore(paths.AgentStateStoreFile()) + stateStore, err := store.NewStateStore(log, diskStore) + require.NoError(t, err) - // Simulate a 500 errors for the next 3 calls. - <-clientWaitFn - <-clientWaitFn - <-clientWaitFn - - // API recover - waitFn := ackSeq( - client.Answer(func(_ http.Header, body io.Reader) (*http.Response, error) { - cr := &request{} - content, err := ioutil.ReadAll(body) - if err != nil { - t.Fatal(err) - } - err = json.Unmarshal(content, &cr) - if err != nil { - t.Fatal(err) - } + queue := &mockQueue{} + queue.On("DequeueActions").Return([]fleetapi.Action{}) + queue.On("Actions").Return([]fleetapi.Action{}) + + fleetReporter := &testutils.MockReporter{} + fleetReporter.On("Update", state.Degraded, mock.Anything, mock.Anything).Times(2) + fleetReporter.On("Update", mock.Anything, mock.Anything, mock.Anything).Maybe() + fleetReporter.On("Unregister").Maybe() + + statusController := &testutils.MockController{} + statusController.On("RegisterComponent", "gateway").Return(fleetReporter).Once() + statusController.On("StatusString").Return("string") + + gateway, err := newFleetGatewayWithScheduler( + ctx, + log, + settings, + agentInfo, + client, + dispatcher, + scheduler, + rep, + noopacker.NewAcker(), + statusController, + stateStore, + queue, + ) + require.NoError(t, err) + + fail := func(_ http.Header, _ io.Reader) (*http.Response, error) { + return wrapStrToResp(http.StatusInternalServerError, "something is bad"), nil + } + clientWaitFn := client.Answer(fail) + err = gateway.Start() + require.NoError(t, err) - require.Equal(t, 1, len(cr.Events)) + _ = rep.Report(context.Background(), &testStateEvent{}) - resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`) - return resp, nil - }), + // Initial tick is done out of bound so we can block on channels. + scheduler.Next() - dispatcher.Answer(func(actions ...fleetapi.Action) error { - require.Equal(t, 0, len(actions)) - return nil - }), - ) + // Simulate a 500 errors for the next 3 calls. + <-clientWaitFn + <-clientWaitFn + <-clientWaitFn - waitFn() - })) + // API recover + waitFn := ackSeq( + client.Answer(func(_ http.Header, body io.Reader) (*http.Response, error) { + cr := &request{} + content, err := ioutil.ReadAll(body) + if err != nil { + t.Fatal(err) + } + err = json.Unmarshal(content, &cr) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, 1, len(cr.Events)) + + resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`) + return resp, nil + }), + + dispatcher.Answer(func(actions ...fleetapi.Action) error { + require.Equal(t, 0, len(actions)) + return nil + }), + ) + + waitFn() + statusController.AssertExpectations(t) + fleetReporter.AssertExpectations(t) + }) t.Run("The retry loop is interruptible", withGateway(agentInfo, &fleetGatewaySettings{ diff --git a/internal/pkg/agent/application/gateway/fleet/noop_status_controller.go b/internal/pkg/agent/application/gateway/fleet/noop_status_controller_test.go similarity index 77% rename from internal/pkg/agent/application/gateway/fleet/noop_status_controller.go rename to internal/pkg/agent/application/gateway/fleet/noop_status_controller_test.go index d5097655a63..bbae6958ab6 100644 --- a/internal/pkg/agent/application/gateway/fleet/noop_status_controller.go +++ b/internal/pkg/agent/application/gateway/fleet/noop_status_controller_test.go @@ -5,21 +5,25 @@ package fleet import ( + "net/http" + "github.com/elastic/elastic-agent/internal/pkg/core/state" "github.com/elastic/elastic-agent/internal/pkg/core/status" ) type noopController struct{} +func (*noopController) SetAgentID(_ string) {} func (*noopController) RegisterComponent(_ string) status.Reporter { return &noopReporter{} } func (*noopController) RegisterComponentWithPersistance(_ string, _ bool) status.Reporter { return &noopReporter{} } -func (*noopController) RegisterApp(_ string, _ string) status.Reporter { return &noopReporter{} } -func (*noopController) Status() status.AgentStatus { return status.AgentStatus{Status: status.Healthy} } -func (*noopController) StatusCode() status.AgentStatusCode { return status.Healthy } -func (*noopController) UpdateStateID(_ string) {} -func (*noopController) StatusString() string { return "online" } +func (*noopController) RegisterApp(_ string, _ string) status.Reporter { return &noopReporter{} } +func (*noopController) Status() status.AgentStatus { return status.AgentStatus{Status: status.Healthy} } +func (*noopController) StatusCode() status.AgentStatusCode { return status.Healthy } +func (*noopController) UpdateStateID(_ string) {} +func (*noopController) StatusString() string { return "online" } +func (*noopController) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {} type noopReporter struct{} diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index 723631b7960..ad508af9086 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -169,6 +169,7 @@ func run(override cfgOverrider) error { rex := reexec.NewManager(rexLogger, execPath) statusCtrl := status.NewController(logger) + statusCtrl.SetAgentID(agentInfo.AgentID()) tracer, err := initTracer(agentName, release.Version(), cfg.Settings.MonitoringConfig) if err != nil { @@ -199,7 +200,7 @@ func run(override cfgOverrider) error { control.SetRouteFn(app.Routes) control.SetMonitoringCfg(cfg.Settings.MonitoringConfig) - serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app, tracer) + serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app, tracer, statusCtrl) if err != nil { return err } @@ -354,6 +355,7 @@ func setupMetrics( cfg *monitoringCfg.MonitoringConfig, app application.Application, tracer *apm.Tracer, + statusCtrl status.Controller, ) (func() error, error) { if err := report.SetupMetrics(logger, agentName, version.GetDefaultVersion()); err != nil { return nil, err @@ -366,7 +368,7 @@ func setupMetrics( } bufferEnabled := cfg.HTTP.Buffer != nil && cfg.HTTP.Buffer.Enabled - s, err := monitoringServer.New(logger, endpointConfig, monitoring.GetNamespace, app.Routes, isProcessStatsEnabled(cfg.HTTP), bufferEnabled, tracer) + s, err := monitoringServer.New(logger, endpointConfig, monitoring.GetNamespace, app.Routes, isProcessStatsEnabled(cfg.HTTP), bufferEnabled, tracer, statusCtrl) if err != nil { return nil, errors.New(err, "could not start the HTTP server for the API") } diff --git a/internal/pkg/core/monitoring/server/server.go b/internal/pkg/core/monitoring/server/server.go index e5929909158..e0289ae72d0 100644 --- a/internal/pkg/core/monitoring/server/server.go +++ b/internal/pkg/core/monitoring/server/server.go @@ -33,6 +33,7 @@ func New( enableProcessStats bool, enableBuffer bool, tracer *apm.Tracer, + statusController http.Handler, ) (*api.Server, error) { if err := createAgentMonitoringDrop(endpointConfig.Host); err != nil { // log but ignore @@ -44,7 +45,7 @@ func New( return nil, err } - return exposeMetricsEndpoint(log, cfg, ns, routesFetchFn, enableProcessStats, enableBuffer, tracer) + return exposeMetricsEndpoint(log, cfg, ns, routesFetchFn, enableProcessStats, enableBuffer, tracer, statusController) } func exposeMetricsEndpoint( @@ -55,6 +56,7 @@ func exposeMetricsEndpoint( enableProcessStats bool, enableBuffer bool, tracer *apm.Tracer, + statusController http.Handler, ) (*api.Server, error) { r := mux.NewRouter() if tracer != nil { @@ -63,6 +65,8 @@ func exposeMetricsEndpoint( statsHandler := statsHandler(ns("stats")) r.Handle("/stats", createHandler(statsHandler)) + r.Handle("/liveness", statusController) + if enableProcessStats { r.HandleFunc("/processes", processesHandler(routesFetchFn)) r.Handle("/processes/{processID}", createHandler(processHandler(statsHandler))) diff --git a/internal/pkg/core/status/handler.go b/internal/pkg/core/status/handler.go new file mode 100644 index 00000000000..2e7476901c5 --- /dev/null +++ b/internal/pkg/core/status/handler.go @@ -0,0 +1,39 @@ +package status + +import ( + "encoding/json" + "net/http" + "time" +) + +// LivenessResponse is the response body for the liveness endpoint. +type LivenessResponse struct { + ID string `json:"id"` + Status string `json:"status"` + Message string `json:"message"` + UpdateTime time.Time `json:"update_timestamp"` +} + +// ServeHTTP is an HTTP Handler for the status controller. +// Respose code is 200 for a healthy agent, and 503 otherwise. +// Response body is a JSON object that contains the agent ID, status, message, and the last status update time. +func (r *controller) ServeHTTP(wr http.ResponseWriter, req *http.Request) { + s := r.Status() + lr := LivenessResponse{ + ID: r.agentID, + Status: s.Status.String(), + Message: s.Message, + UpdateTime: s.UpdateTime, + } + status := http.StatusOK + if s.Status != Healthy { + status = http.StatusServiceUnavailable + } + + wr.Header().Set("Content-Type", "application/json") + wr.WriteHeader(status) + enc := json.NewEncoder(wr) + if err := enc.Encode(lr); err != nil { + r.log.Errorf("Unable to encode liveness response: %v", err) + } +} diff --git a/internal/pkg/core/status/reporter.go b/internal/pkg/core/status/reporter.go index 04c8251fa92..92632af2ed5 100644 --- a/internal/pkg/core/status/reporter.go +++ b/internal/pkg/core/status/reporter.go @@ -2,11 +2,15 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// Package status handles process status reporting. package status import ( + "fmt" + "net/http" "reflect" "sync" + "time" "github.com/google/uuid" @@ -47,10 +51,12 @@ type AgentStatus struct { Status AgentStatusCode Message string Applications []AgentApplicationStatus + UpdateTime time.Time } // Controller takes track of component statuses. type Controller interface { + SetAgentID(string) RegisterComponent(string) Reporter RegisterComponentWithPersistance(string, bool) Reporter RegisterApp(id string, name string) Reporter @@ -58,15 +64,19 @@ type Controller interface { StatusCode() AgentStatusCode StatusString() string UpdateStateID(string) + ServeHTTP(http.ResponseWriter, *http.Request) } type controller struct { mx sync.Mutex status AgentStatusCode + message string + updateTime time.Time reporters map[string]*reporter appReporters map[string]*reporter log *logger.Logger stateID string + agentID string } // NewController creates a new reporter. @@ -79,6 +89,14 @@ func NewController(log *logger.Logger) Controller { } } +// SetAgentID sets the agentID of the controller +// The AgentID may be used in the handler output. +func (r *controller) SetAgentID(agentID string) { + r.mx.Lock() + defer r.mx.Unlock() + r.agentID = agentID +} + // UpdateStateID cleans health when new configuration is received. // To prevent reporting failures from previous configuration. func (r *controller) UpdateStateID(stateID string) { @@ -175,8 +193,9 @@ func (r *controller) Status() AgentStatus { } return AgentStatus{ Status: r.status, - Message: "", + Message: r.message, Applications: apps, + UpdateTime: r.updateTime, } } @@ -189,12 +208,14 @@ func (r *controller) StatusCode() AgentStatusCode { func (r *controller) updateStatus() { status := Healthy + message := "" r.mx.Lock() for id, rep := range r.reporters { s := statusToAgentStatus(rep.status) if s > status { status = s + message = fmt.Sprintf("component %s: %s", id, rep.message) } r.log.Debugf("'%s' has status '%s'", id, s) @@ -207,6 +228,7 @@ func (r *controller) updateStatus() { s := statusToAgentStatus(rep.status) if s > status { status = s + message = fmt.Sprintf("app %s: %s", id, rep.message) } r.log.Debugf("'%s' has status '%s'", id, s) @@ -217,15 +239,17 @@ func (r *controller) updateStatus() { } if r.status != status { - r.logStatus(status) + r.logStatus(status, message) r.status = status + r.message = message + r.updateTime = time.Now().UTC() } r.mx.Unlock() } -func (r *controller) logStatus(status AgentStatusCode) { +func (r *controller) logStatus(status AgentStatusCode, message string) { logFn := r.log.Infof if status == Degraded { logFn = r.log.Warnf @@ -233,7 +257,7 @@ func (r *controller) logStatus(status AgentStatusCode) { logFn = r.log.Errorf } - logFn("Elastic Agent status changed to: '%s'", status) + logFn("Elastic Agent status changed to %q: %q", status, message) } // StatusString retrieves human readable string of current agent status. diff --git a/internal/pkg/core/status/reporter_test.go b/internal/pkg/core/status/reporter_test.go index 0d44e402798..09a66661fc5 100644 --- a/internal/pkg/core/status/reporter_test.go +++ b/internal/pkg/core/status/reporter_test.go @@ -6,6 +6,7 @@ package status import ( "testing" + "time" "github.com/stretchr/testify/assert" @@ -98,4 +99,56 @@ func TestReporter(t *testing.T) { assert.Equal(t, Degraded, r.StatusCode()) assert.Equal(t, "degraded", r.StatusString()) }) + + t.Run("Check agent status components healthy", func(t *testing.T) { + r := NewController(l) + r1 := r.RegisterComponent("r1") + r2 := r.RegisterComponent("r2") + r3 := r.RegisterComponent("r3") + + r1.Update(state.Healthy, "", nil) + r2.Update(state.Healthy, "", nil) + r3.Update(state.Healthy, "", nil) + + s := r.Status() + assert.Equal(t, Healthy, s.Status) + assert.Equal(t, "", s.Message) + assert.Equal(t, time.Time{}, s.UpdateTime) + }) + + //nolint:dupl // test case + t.Run("Check agent status one component degraded", func(t *testing.T) { + r := NewController(l) + r1 := r.RegisterComponent("r1") + r2 := r.RegisterComponent("r2") + r3 := r.RegisterComponent("r3") + + r1.Update(state.Healthy, "", nil) + r2.Update(state.Degraded, "degraded", nil) + r3.Update(state.Healthy, "", nil) + + s := r.Status() + assert.Equal(t, Degraded, s.Status) + assert.Contains(t, s.Message, "component r2") + assert.Contains(t, s.Message, "degraded") + assert.NotEqual(t, time.Time{}, s.UpdateTime) + }) + + //nolint:dupl // test case + t.Run("Check agent status one component failed", func(t *testing.T) { + r := NewController(l) + r1 := r.RegisterComponent("r1") + r2 := r.RegisterComponent("r2") + r3 := r.RegisterComponent("r3") + + r1.Update(state.Healthy, "", nil) + r2.Update(state.Failed, "failed", nil) + r3.Update(state.Degraded, "degraded", nil) + + s := r.Status() + assert.Equal(t, Failed, s.Status) + assert.Contains(t, s.Message, "component r2") + assert.Contains(t, s.Message, "failed") + assert.NotEqual(t, time.Time{}, s.UpdateTime) + }) } diff --git a/internal/pkg/testutils/status_reporter.go b/internal/pkg/testutils/status_reporter.go new file mode 100644 index 00000000000..a045e50304a --- /dev/null +++ b/internal/pkg/testutils/status_reporter.go @@ -0,0 +1,67 @@ +package testutils + +import ( + "net/http" + + "github.com/elastic/elastic-agent/internal/pkg/core/state" + "github.com/elastic/elastic-agent/internal/pkg/core/status" + "github.com/stretchr/testify/mock" +) + +type MockController struct { + mock.Mock +} + +func (m *MockController) SetAgentID(id string) { + m.Called(id) +} + +func (m *MockController) RegisterComponent(id string) status.Reporter { + args := m.Called(id) + return args.Get(0).(status.Reporter) +} + +func (m *MockController) RegisterComponentWithPersistance(id string, b bool) status.Reporter { + args := m.Called(id, b) + return args.Get(0).(status.Reporter) +} + +func (m *MockController) RegisterApp(id, name string) status.Reporter { + args := m.Called(id, name) + return args.Get(0).(status.Reporter) +} + +func (m *MockController) Status() status.AgentStatus { + args := m.Called() + return args.Get(0).(status.AgentStatus) +} + +func (m *MockController) StatusCode() status.AgentStatusCode { + args := m.Called() + return args.Get(0).(status.AgentStatusCode) +} + +func (m *MockController) StatusString() string { + args := m.Called() + return args.String(0) +} + +func (m *MockController) UpdateStateID(id string) { + m.Called(id) +} + +func (m *MockController) ServeHTTP(wr http.ResponseWriter, req *http.Request) { + m.Called(wr, req) +} + +type MockReporter struct { + mock.Mock +} + +func (m *MockReporter) Update(state state.Status, message string, meta map[string]interface{}) { + m.Called(state, message, meta) +} + +func (m *MockReporter) Unregister() { + m.Called() +} From daa5c5f4ad194f9d8226adcf084373e3fd582418 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 24 Jun 2022 01:44:50 -0400 Subject: [PATCH 16/69] [Automation] Update elastic stack version to 8.4.0-722a7d79 for testing (#607) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 1f7885d2eb6..8be3d1e8d1b 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-2e32a640-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-722a7d79-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-2e32a640-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-722a7d79-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 90a1f582dcf5d073bf8ddce387fd78e2261e16b4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 24 Jun 2022 12:38:25 +0100 Subject: [PATCH 17/69] ci: enable flaky test detector (#605) --- .ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 68b3c8a7b51..6c98375e5fb 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -263,6 +263,7 @@ pipeline { post { cleanup { notifyBuildResult(prComment: true, + analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: (isPR() ? env.CHANGE_TARGET : env.BRANCH_NAME)), githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS", githubLabels: 'Team:Elastic-Agent-Control-Plane') } From 79bb67ec61db59032cbbfd352b29ba867dc4efcf Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 27 Jun 2022 01:47:56 -0400 Subject: [PATCH 18/69] [Automation] Update elastic stack version to 8.4.0-210dd487 for testing (#620) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 8be3d1e8d1b..2a4560af874 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-722a7d79-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-210dd487-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-722a7d79-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-210dd487-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From f69d5b1fd260e7d64906076c28d98491fd90e346 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 27 Jun 2022 11:13:24 +0100 Subject: [PATCH 19/69] mergify: remove backport automation for non active branches (#615) --- .mergify.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 6e1e3c5f651..00b49806e47 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -181,32 +181,6 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - - name: backport patches to 8.0 branch - conditions: - - merged - - label=backport-v8.0.0 - actions: - backport: - assignees: - - "{{ author }}" - branches: - - "8.0" - labels: - - "backport" - title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - - name: backport patches to 8.1 branch - conditions: - - merged - - label=backport-v8.1.0 - actions: - backport: - assignees: - - "{{ author }}" - branches: - - "8.1" - labels: - - "backport" - title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - name: backport patches to 8.2 branch conditions: - merged From 43ba98daa1f4acafa89872419ead60f095b64969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 27 Jun 2022 15:08:18 +0200 Subject: [PATCH 20/69] chore: use elastic-agent profile to run the E2E tests (#610) --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 6c98375e5fb..10c6c1a4347 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -251,7 +251,7 @@ pipeline { } steps { // TODO: what's the testMatrixFile to be used if any - runE2E(testMatrixFile: '', + runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml', beatVersion: "${env.BEAT_VERSION}-SNAPSHOT", elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT", gitHubCheckName: "e2e-tests", From 662a07ac5037f9b098bb28899df4500379759d71 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 28 Jun 2022 01:35:09 -0400 Subject: [PATCH 21/69] [Automation] Update elastic stack version to 8.4.0-a6aa9f3b for testing (#631) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 2a4560af874..70d8eb97f16 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-210dd487-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-a6aa9f3b-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-210dd487-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-a6aa9f3b-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 5c16e0862d357a54269974f443a32fddd6092cf5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 29 Jun 2022 15:05:34 +0200 Subject: [PATCH 22/69] add macros pointing to new agent's repo and fix old macro calls (#458) --- CHANGELOG.asciidoc | 215 +++++++++++++-------------- CHANGELOG.next.asciidoc | 315 ++++++++++++++++++++-------------------- 2 files changed, 268 insertions(+), 262 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index fad7186655f..a2b19fb1a90 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -1,71 +1,74 @@ // Use these for links to issue and pulls. Note issues and pulls redirect one to // each other on Github, so don't worry too much on using the right prefix. -:issue: https://github.com/elastic/beats/issues/ -:pull: https://github.com/elastic/beats/pull/ +:issue-beats: https://github.com/elastic/beats/issues/ +:pull-beats: https://github.com/elastic/beats/pull/ + +:issue: https://github.com/elastic/elastic-agent/issues/ +:pull: https://github.com/elastic/elastic-agent/pull/ [[release-notes-7.9.0]] === Elastic Agent version 7.9.0 ==== Breaking changes -- Change fleet.yml structure, causes upgraded agent to register as new agent {pull}19248[19248] -- Remove obfuscation of fleet.yml, causes re-enroll of agent to Fleet {pull}19678[19678] -- Rename enroll --ca_sha256 to --ca-sha256 {pull}19900[19900] -- Rename enroll --certificate_authorities to --certificate-authorities {pull}19900[19900] -- Don't build 32 bits version of Elastic Agent. {issue}25533[25533] +- Change fleet.yml structure, causes upgraded agent to register as new agent {pull-beats}[19248] +- Remove obfuscation of fleet.yml, causes re-enroll of agent to Fleet {pull-beats}[19678] +- Rename enroll --ca_sha256 to --ca-sha256 {pull-beats}[19900] +- Rename enroll --certificate_authorities to --certificate-authorities {pull-beats}[19900] +- Don't build 32 bits version of Elastic Agent. {issue-beats}[25533] ==== Bugfixes -- Fix install service script for windows {pull}18814[18814] -- Properly stops subprocess on shutdown {pull}19567[19567] -- Forward revision number of the configuration to the endpoint. {pull}19759[19759] -- Remove support for logs type and use logfile {pull}19761[19761] -- Avoid comparing uncomparable types on enroll {issue}19976[19976] -- Fix issues with merging of elastic-agent.yml and fleet.yml {pull}20026[20026] -- Unzip failures on Windows 8/Windows server 2012 {pull}20088[20088] -- Fix failing unit tests on windows {pull}20127[20127] -- Prevent closing closed reader {pull}20214[20214] -- Improve GRPC stop to be more relaxed {pull}20118[20118] -- Fix Windows service installation script {pull}20203[20203] -- Fix timeout issue stopping service applications {pull}20256[20256] -- Fix incorrect hash when upgrading agent {pull}22322[22322] -- Fix refresh of monitoring configuration {pull}23619[23619] -- Fixed nil pointer during unenroll {pull}23609[23609] -- Fixed reenroll scenario {pull}23686[23686] -- Fixed Monitoring filebeat and metricbeat not connecting to Agent over GRPC {pull}23843[23843] -- Fixed make status readable in the log. {pull}23849[23849] -- Windows agent doesn't uninstall with a lowercase `c:` drive in the path {pull}23998[23998] -- Fix reloading of log level for services {pull}24055[24055] -- Fix: Successfully installed and enrolled agent running standalone{pull}24128[24128] -- Make installer atomic on windows {pull}24253[24253] -- Remove installed services on agent uninstall {pull}24151[24151] -- Fix failing installation on windows 7 {pull}24387[24387] -- Fix capabilities resolution in inspect command {pull}24346[24346] -- Fix windows installer during enroll {pull}24343[24343] -- Logging to file disabled on enroll {issue}24173[24173] -- Prevent uninstall failures on empty config {pull}24838[24838] -- Fix issue with FLEET_CA not being used with Fleet Server in container {pull}26529[26529] +- Fix install service script for windows {pull-beats}[18814] +- Properly stops subprocess on shutdown {pull-beats}[19567] +- Forward revision number of the configuration to the endpoint. {pull-beats}[19759] +- Remove support for logs type and use logfile {pull-beats}[19761] +- Avoid comparing uncomparable types on enroll {issue-beats}[19976] +- Fix issues with merging of elastic-agent.yml and fleet.yml {pull-beats}[20026] +- Unzip failures on Windows 8/Windows server 2012 {pull-beats}[20088] +- Fix failing unit tests on windows {pull-beats}[20127] +- Prevent closing closed reader {pull-beats}[20214] +- Improve GRPC stop to be more relaxed {pull-beats}[20118] +- Fix Windows service installation script {pull-beats}[20203] +- Fix timeout issue stopping service applications {pull-beats}[20256] +- Fix incorrect hash when upgrading agent {pull-beats}[22322] +- Fix refresh of monitoring configuration {pull-beats}[23619] +- Fixed nil pointer during unenroll {pull-beats}[23609] +- Fixed reenroll scenario {pull-beats}[23686] +- Fixed Monitoring filebeat and metricbeat not connecting to Agent over GRPC {pull-beats}[23843] +- Fixed make status readable in the log. {pull-beats}[23849] +- Windows agent doesn't uninstall with a lowercase `c:` drive in the path {pull-beats}[23998] +- Fix reloading of log level for services {pull-beats}[24055] +- Fix: Successfully installed and enrolled agent running standalone{pull-beats}[24128] +- Make installer atomic on windows {pull-beats}[24253] +- Remove installed services on agent uninstall {pull-beats}[24151] +- Fix failing installation on windows 7 {pull-beats}[24387] +- Fix capabilities resolution in inspect command {pull-beats}[24346] +- Fix windows installer during enroll {pull-beats}[24343] +- Logging to file disabled on enroll {issue-beats}[24173] +- Prevent uninstall failures on empty config {pull-beats}[24838] +- Fix issue with FLEET_CA not being used with Fleet Server in container {pull-beats}[26529] ==== New features -- Change monitoring defaults for agent {pull}18927[18927] -- Agent verifies packages before using them {pull}18876[18876] -- Change stream.* to dataset.* fields {pull}18967[18967] -- Agent now runs the GRPC server and spawned application connect by to Agent {pull}18973[18973] -- Rename input.type logs to logfile {pull}19360[19360] -- Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248] -- Agent now downloads Elastic Endpoint {pull}19503[19503] -- Refuse invalid stream values in configuration {pull}19587[19587] -- Agent now load balances across multiple Kibana instances {pull}19628[19628] -- Configuration cleanup {pull}19848[19848] -- Agent now sends its own logs to elasticsearch {pull}19811[19811] -- Add --insecure option to enroll command {pull}19900[19900] -- Will retry to enroll if the server return a 429. {pull}19918[19811] -- Add --staging option to enroll command {pull}20026[20026] -- Add `event.dataset` to all events {pull}20076[20076] -- Send datastreams fields {pull}20416[20416] -- Agent supports capabilities definition {pull}23848[23848] -- Restart process on output change {pull}24907[24907] +- Change monitoring defaults for agent {pull-beats}[18927] +- Agent verifies packages before using them {pull-beats}[18876] +- Change stream.* to dataset.* fields {pull-beats}[18967] +- Agent now runs the GRPC server and spawned application connect by to Agent {pull-beats}[18973] +- Rename input.type logs to logfile {pull-beats}[19360] +- Agent now installs/uninstalls Elastic Endpoint {pull-beats}[19248] +- Agent now downloads Elastic Endpoint {pull-beats}[19503] +- Refuse invalid stream values in configuration {pull-beats}[19587] +- Agent now load balances across multiple Kibana instances {pull-beats}[19628] +- Configuration cleanup {pull-beats}[19848] +- Agent now sends its own logs to elasticsearch {pull-beats}[19811] +- Add --insecure option to enroll command {pull-beats}[19900] +- Will retry to enroll if the server return a 429. {pull-beats}[19811] +- Add --staging option to enroll command {pull-beats}[20026] +- Add `event.dataset` to all events {pull-beats}[20076] +- Send datastreams fields {pull-beats}[20416] +- Agent supports capabilities definition {pull-beats}[23848] +- Restart process on output change {pull-beats}[24907] === Docs @@ -75,61 +78,61 @@ === Elastic Agent version 7.8.0 ==== Breaking changes -- Rename agent to elastic-agent {pull}17391[17391] +- Rename agent to elastic-agent {pull-beats}[17391] ==== Bugfixes -- Fixed tests on windows {pull}16922[16922] -- Fixed installers for SNAPSHOTs and windows {pull}17077[17077] -- Fixed merge of config {pull}17399[17399] -- Handle abs paths on windows correctly {pull}17461[17461] -- Improved cancellation of agent {pull}17318[17318] -- Fixed process spawning on Windows {pull}17751[17751] -- Fix issues when running `mage package` for all the platforms. {pull}17767[17767] -- Rename the User-Agent string from Beats Agent to Elastic Agent. {pull}17765[17765] -- Remove the kbn-version on each request to the Kibana API. {pull}17764[17764] -- Fixed injected log path to monitoring beat {pull}17833[17833] -- Make sure that the Elastic Agent connect over TLS in cloud. {pull}17843[17843] -- Moved stream.* fields to top of event {pull}17858[17858] -- Use /tmp for default monitoring endpoint location for libbeat {pull}18131[18131] -- Use default output by default {pull}18091[18091] -- Fix panic and flaky tests for the Agent. {pull}18135[18135] -- Fix default configuration after enroll {pull}18232[18232] -- Fix make sure the collected logs or metrics include streams information. {pull}18261[18261] -- Fix version to 7.8 {pull}18286[18286] -- Fix an issue where the checkin_frequency, jitter, and backoff options where not configurable. {pull}17843[17843] -- Ensure that the beats uses the params prefer_v2_templates on bulk request. {pull}18318[18318] -- Stop monitoring on config change {pull}18284[18284] -- Enable more granular control of monitoring {pull}18346[18346] -- Fix jq: command not found {pull}18408[18408] -- Avoid Chown on windows {pull}18512[18512] -- Clean action store after enrolling to new configuration {pull}18656[18656] -- Avoid watching monitor logs {pull}18723[18723] -- Correctly report platform and family. {issue}18665[18665] -- Guard against empty stream.datasource and namespace {pull}18769[18769] -- Fix install service script for windows {pull}18814[18814] +- Fixed tests on windows {pull-beats}[16922] +- Fixed installers for SNAPSHOTs and windows {pull-beats}[17077] +- Fixed merge of config {pull-beats}[17399] +- Handle abs paths on windows correctly {pull-beats}[17461] +- Improved cancellation of agent {pull-beats}[17318] +- Fixed process spawning on Windows {pull-beats}[17751] +- Fix issues when running `mage package` for all the platforms. {pull-beats}[17767] +- Rename the User-Agent string from Beats Agent to Elastic Agent. {pull-beats}[17765] +- Remove the kbn-version on each request to the Kibana API. {pull-beats}[17764] +- Fixed injected log path to monitoring beat {pull-beats}[17833] +- Make sure that the Elastic Agent connect over TLS in cloud. {pull-beats}[17843] +- Moved stream.* fields to top of event {pull-beats}[17858] +- Use /tmp for default monitoring endpoint location for libbeat {pull-beats}[18131] +- Use default output by default {pull-beats}[18091] +- Fix panic and flaky tests for the Agent. {pull-beats}[18135] +- Fix default configuration after enroll {pull-beats}[18232] +- Fix make sure the collected logs or metrics include streams information. {pull-beats}[18261] +- Fix version to 7.8 {pull-beats}[18286] +- Fix an issue where the checkin_frequency, jitter, and backoff options where not configurable. {pull-beats}[17843] +- Ensure that the beats uses the params prefer_v2_templates on bulk request. {pull-beats}[18318] +- Stop monitoring on config change {pull-beats}[18284] +- Enable more granular control of monitoring {pull-beats}[18346] +- Fix jq: command not found {pull-beats}[18408] +- Avoid Chown on windows {pull-beats}[18512] +- Clean action store after enrolling to new configuration {pull-beats}[18656] +- Avoid watching monitor logs {pull-beats}[18723] +- Correctly report platform and family. {issue-beats}[18665] +- Guard against empty stream.datasource and namespace {pull-beats}[18769] +- Fix install service script for windows {pull-beats}[18814] ==== New features -- Generate index name in a format type-dataset-namespace {pull}16903[16903] -- OS agnostic default configuration {pull}17016[17016] -- Introduced post install hooks {pull}17241[17241] -- Support for config constraints {pull}17112[17112] -- Introduced `mage demo` command {pull}17312[17312] -- Display the stability of the agent at enroll and start. {pull}17336[17336] -- Expose stream.* variables in events {pull}17468[17468] -- Monitoring configuration reloadable {pull}17855[17855] -- Pack ECS metadata to request payload send to fleet {pull}17894[17894] -- Allow CLI overrides of paths {pull}17781[17781] -- Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909] -- Configurable log level {pull}18083[18083] -- Use data subfolder as default for process logs {pull}17960[17960] -- Enable introspecting configuration {pull}18124[18124] -- Follow home path for all config files {pull}18161[18161] -- Do not require unnecessary configuration {pull}18003[18003] -- Use nested objects so fleet can handle metadata correctly {pull}18234[18234] -- Enable debug log level for Metricbeat and Filebeat when run under the Elastic Agent. {pull}17935[17935] -- Pick up version from libbeat {pull}18350[18350] -- More clear output of inspect command {pull}18405[18405] -- When not port are specified and the https is used fallback to 443 {pull}18844[18844] -- Basic upgrade process {pull}21002[21002] +- Generate index name in a format type-dataset-namespace {pull-beats}[16903] +- OS agnostic default configuration {pull-beats}[17016] +- Introduced post install hooks {pull-beats}[17241] +- Support for config constraints {pull-beats}[17112] +- Introduced `mage demo` command {pull-beats}[17312] +- Display the stability of the agent at enroll and start. {pull-beats}[17336] +- Expose stream.* variables in events {pull-beats}[17468] +- Monitoring configuration reloadable {pull-beats}[17855] +- Pack ECS metadata to request payload send to fleet {pull-beats}[17894] +- Allow CLI overrides of paths {pull-beats}[17781] +- Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull-beats}[17909] +- Configurable log level {pull-beats}[18083] +- Use data subfolder as default for process logs {pull-beats}[17960] +- Enable introspecting configuration {pull-beats}[18124] +- Follow home path for all config files {pull-beats}[18161] +- Do not require unnecessary configuration {pull-beats}[18003] +- Use nested objects so fleet can handle metadata correctly {pull-beats}[18234] +- Enable debug log level for Metricbeat and Filebeat when run under the Elastic Agent. {pull-beats}[17935] +- Pick up version from libbeat {pull-beats}[18350] +- More clear output of inspect command {pull-beats}[18405] +- When not port are specified and the https is used fallback to 443 {pull-beats}[18844] +- Basic upgrade process {pull-beats}[21002] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cbe894d63c5..00f91675544 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -1,101 +1,104 @@ // Use these for links to issue and pulls. Note issues and pulls redirect one to // each other on Github, so don't worry too much on using the right prefix. -:issue: https://github.com/elastic/beats/issues/ -:pull: https://github.com/elastic/beats/pull/ +:issue-beats: https://github.com/elastic/beats/issues/ +:pull-beats: https://github.com/elastic/beats/pull/ + +:issue: https://github.com/elastic/elastic-agent/issues/ +:pull: https://github.com/elastic/elastic-agent/pull/ === Elastic Agent version HEAD ==== Breaking changes -- Docker container is not run as root by default. {pull}21213[21213] -- Read Fleet connection information from `fleet.*` instead of `fleet.kibana.*`. {pull}24713[24713] -- Beats build for 32Bit Windows or Linux system will refuse to run on a 64bit system. {pull}25186[25186] -- Remove the `--kibana-url` from `install` and `enroll` command. {pull}25529[25529] -- Default to port 80 and 443 for Kibana and Fleet Server connections. {pull}25723[25723] -- Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006] -- The `/processes/` endpoint proxies to the subprocess's monitoring endpoint, instead of querying its `/stats` endpoint {pull}28165[28165] -- Remove username/password for fleet-server authentication. {pull}29458[29458] +- Docker container is not run as root by default. {pull-beats}[21213] +- Read Fleet connection information from `fleet.*` instead of `fleet.kibana.*`. {pull-beats}[24713] +- Beats build for 32Bit Windows or Linux system will refuse to run on a 64bit system. {pull-beats}[25186] +- Remove the `--kibana-url` from `install` and `enroll` command. {pull-beats}[25529] +- Default to port 80 and 443 for Kibana and Fleet Server connections. {pull-beats}[25723] +- Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull-beats}[28006] +- The `/processes/` endpoint proxies to the subprocess's monitoring endpoint, instead of querying its `/stats` endpoint {pull-beats}[28165] +- Remove username/password for fleet-server authentication. {pull-beats}[29458] ==== Bugfixes -- Fix rename *ConfigChange to *PolicyChange to align on changes in the UI. {pull}20779[20779] -- Thread safe sorted set {pull}21290[21290] -- Copy Action store on upgrade {pull}21298[21298] -- Include inputs in action store actions {pull}21298[21298] -- Fix issue where inputs without processors defined would panic {pull}21628[21628] -- Prevent reporting ecs version twice {pull}21616[21616] -- Partial extracted beat result in failure to spawn beat {issue}21718[21718] -- Use symlink path for reexecutions {pull}21835[21835] -- Use ML_SYSTEM to detect if agent is running as a service {pull}21884[21884] -- Use local temp instead of system one {pull}21883[21883] -- Rename monitoring index from `elastic.agent` to `elastic_agent` {pull}21932[21932] -- Fix issue with named pipes on Windows 7 {pull}21931[21931] -- Fix missing elastic_agent event data {pull}21994[21994] -- Ensure shell wrapper path exists before writing wrapper on install {pull}22144[22144] -- Fix deb/rpm packaging for Elastic Agent {pull}22153[22153] -- Fix composable input processor promotion to fix duplicates {pull}22344[22344] -- Fix sysv init files for deb/rpm installation {pull}22543[22543] -- Fix shell wrapper for deb/rpm packaging {pull}23038[23038] -- Fixed parsing of npipe URI {pull}22978[22978] -- Select default agent policy if no enrollment token provided. {pull}23973[23973] -- Remove artifacts on transient download errors {pull}23235[23235] -- Support for linux/arm64 {pull}23479[23479] -- Skip top level files when unziping archive during upgrade {pull}23456[23456] -- Do not take ownership of Endpoint log path {pull}23444[23444] -- Fixed fetching DBus service PID {pull}23496[23496] -- Fix issue of missing log messages from filebeat monitor {pull}23514[23514] -- Increase checkin grace period to 30 seconds {pull}23568[23568] -- Fix libbeat from reporting back degraded on config update {pull}23537[23537] -- Rewrite check if agent is running with admin rights on Windows {pull}23970[23970] -- Fix issues with dynamic inputs and conditions {pull}23886[23886] -- Fix bad substitution of API key. {pull}24036[24036] -- Fix docker enrollment issue related to Fleet Server change. {pull}24155[24155] -- Improve log on failure of Endpoint Security installation. {pull}24429[24429] -- Verify communication to Kibana before updating Fleet client. {pull}24489[24489] -- Fix nil pointer when null is generated as list item. {issue}23734[23734] -- Add support for filestream input. {pull}24820[24820] -- Add check for URL set when cert and cert key. {pull}24904[24904] -- Fix install command for Fleet Server bootstrap, remove need for --enrollment-token when using --fleet-server {pull}24981[24981] -- Respect host configuration for exposed processes endpoint {pull}25114[25114] -- Set --inscure in container when FLEET_SERVER_ENABLE and FLEET_INSECURE set {pull}25137[25137] -- Fixed: limit for retries to Kibana configurable {issue}25063[25063] -- Fix issue with status and inspect inside of container {pull}25204[25204] -- Remove FLEET_SERVER_POLICY_NAME env variable as it was not used {pull}25149[25149] -- Reduce log level for listener cleanup to debug {pull}25274 -- Passing in policy id to container command works {pull}25352[25352] -- Reduce log level for listener cleanup to debug {pull}25274[25274] -- Delay the restart of application when a status report of failure is given {pull}25339[25339] -- Don't log when upgrade capability doesn't apply {pull}25386[25386] -- Fixed issue when unversioned home is set and invoked watcher failing with ENOENT {issue}25371[25371] -- Fixed Elastic Agent: expecting Dict and received *transpiler.Key for '0' {issue}24453[24453] -- Fix AckBatch to do nothing when no actions passed {pull}25562[25562] -- Add error log entry when listener creation fails {issue}23483[23482] -- Handle case where policy doesn't contain Fleet connection information {pull}25707[25707] -- Fix fleet-server.yml spec to not overwrite existing keys {pull}25741[25741] -- Agent sends wrong log level to Endpoint {issue}25583[25583] -- Fix startup with failing configuration {pull}26057[26057] -- Change timestamp in elatic-agent-json.log to use UTC {issue}25391[25391] -- Fix add support for Logstash output. {pull}24305[24305] -- Do not log Elasticsearch configuration for monitoring output when running with debug. {pull}26583[26583] -- Fix issue where proxy enrollment options broke enrollment command. {pull}26749[26749] -- Remove symlink.prev from previously failed upgrade {pull}26785[26785] -- Fix apm-server supported outputs not being in sync with supported output types. {pull}26885[26885] -- Set permissions during installation {pull}26665[26665] -- Disable monitoring during fleet-server bootstrapping. {pull}27222[27222] -- Fix issue with atomic extract running in K8s {pull}27396[27396] -- Fix issue with install directory in state path in K8s {pull}27396[27396] -- Disable monitoring during fleet-server bootstrapping. {pull}27222[27222] -- Change output.elasticsearch.proxy_disabled flag to output.elasticsearch.proxy_disable so fleet uses it. {issue}27670[27670] {pull}27671[27671] -- Add validation for certificate flags to ensure they are absolute paths. {pull}27779[27779] -- Migrate state on upgrade {pull}27825[27825] -- Add "_monitoring" suffix to monitoring instance names to remove ambiguity with the status command. {issue}25449[25449] -- Ignore ErrNotExists when fixing permissions. {issue}27836[27836] {pull}27846[27846] -- Snapshot artifact lookup will use agent.download proxy settings. {issue}27903[27903] {pull}27904[27904] -- Fix lazy acker to only add new actions to the batch. {pull}27981[27981] -- Allow HTTP metrics to run in bootstrap mode. Add ability to adjust timeouts for Fleet Server. {pull}28260[28260] -- Fix agent configuration overwritten by default fleet config. {pull}29297[29297] -- Allow agent containers to use basic auth to create a service token. {pull}29651[29651] -- Fix issue where a failing artifact verification does not remove the bad artifact. {pull}30281[30281] -- Reduce Elastic Agent shut down time by stopping processes concurrently {pull}29650[29650] +- Fix rename *ConfigChange to *PolicyChange to align on changes in the UI. {pull-beats}[20779] +- Thread safe sorted set {pull-beats}[21290] +- Copy Action store on upgrade {pull-beats}[21298] +- Include inputs in action store actions {pull-beats}[21298] +- Fix issue where inputs without processors defined would panic {pull-beats}[21628] +- Prevent reporting ecs version twice {pull-beats}[21616] +- Partial extracted beat result in failure to spawn beat {issue-beats}[21718] +- Use symlink path for reexecutions {pull-beats}[21835] +- Use ML_SYSTEM to detect if agent is running as a service {pull-beats}[21884] +- Use local temp instead of system one {pull-beats}[21883] +- Rename monitoring index from `elastic.agent` to `elastic_agent` {pull-beats}[21932] +- Fix issue with named pipes on Windows 7 {pull-beats}[21931] +- Fix missing elastic_agent event data {pull-beats}[21994] +- Ensure shell wrapper path exists before writing wrapper on install {pull-beats}[22144] +- Fix deb/rpm packaging for Elastic Agent {pull-beats}[22153] +- Fix composable input processor promotion to fix duplicates {pull-beats}[22344] +- Fix sysv init files for deb/rpm installation {pull-beats}[22543] +- Fix shell wrapper for deb/rpm packaging {pull-beats}[23038] +- Fixed parsing of npipe URI {pull-beats}[22978] +- Select default agent policy if no enrollment token provided. {pull-beats}[23973] +- Remove artifacts on transient download errors {pull-beats}[23235] +- Support for linux/arm64 {pull-beats}[23479] +- Skip top level files when unziping archive during upgrade {pull-beats}[23456] +- Do not take ownership of Endpoint log path {pull-beats}[23444] +- Fixed fetching DBus service PID {pull-beats}[23496] +- Fix issue of missing log messages from filebeat monitor {pull-beats}[23514] +- Increase checkin grace period to 30 seconds {pull-beats}[23568] +- Fix libbeat from reporting back degraded on config update {pull-beats}[23537] +- Rewrite check if agent is running with admin rights on Windows {pull-beats}[23970] +- Fix issues with dynamic inputs and conditions {pull-beats}[23886] +- Fix bad substitution of API key. {pull-beats}[24036] +- Fix docker enrollment issue related to Fleet Server change. {pull-beats}[24155] +- Improve log on failure of Endpoint Security installation. {pull-beats}[24429] +- Verify communication to Kibana before updating Fleet client. {pull-beats}[24489] +- Fix nil pointer when null is generated as list item. {issue-beats}[23734] +- Add support for filestream input. {pull-beats}[24820] +- Add check for URL set when cert and cert key. {pull-beats}[24904] +- Fix install command for Fleet Server bootstrap, remove need for --enrollment-token when using --fleet-server {pull-beats}[24981] +- Respect host configuration for exposed processes endpoint {pull-beats}[25114] +- Set --inscure in container when FLEET_SERVER_ENABLE and FLEET_INSECURE set {pull-beats}[25137] +- Fixed: limit for retries to Kibana configurable {issue-beats}[25063] +- Fix issue with status and inspect inside of container {pull-beats}[25204] +- Remove FLEET_SERVER_POLICY_NAME env variable as it was not used {pull-beats}[25149] +- Reduce log level for listener cleanup to debug {pull-beats} +- Passing in policy id to container command works {pull-beats}[25352] +- Reduce log level for listener cleanup to debug {pull-beats}[25274] +- Delay the restart of application when a status report of failure is given {pull-beats}[25339] +- Don't log when upgrade capability doesn't apply {pull-beats}[25386] +- Fixed issue when unversioned home is set and invoked watcher failing with ENOENT {issue-beats}[25371] +- Fixed Elastic Agent: expecting Dict and received *transpiler.Key for '0' {issue-beats}[24453] +- Fix AckBatch to do nothing when no actions passed {pull-beats}[25562] +- Add error log entry when listener creation fails {issue-beats}[23482] +- Handle case where policy doesn't contain Fleet connection information {pull-beats}[25707] +- Fix fleet-server.yml spec to not overwrite existing keys {pull-beats}[25741] +- Agent sends wrong log level to Endpoint {issue-beats}[25583] +- Fix startup with failing configuration {pull-beats}[26057] +- Change timestamp in elatic-agent-json.log to use UTC {issue-beats}[25391] +- Fix add support for Logstash output. {pull-beats}[24305] +- Do not log Elasticsearch configuration for monitoring output when running with debug. {pull-beats}[26583] +- Fix issue where proxy enrollment options broke enrollment command. {pull-beats}[26749] +- Remove symlink.prev from previously failed upgrade {pull-beats}[26785] +- Fix apm-server supported outputs not being in sync with supported output types. {pull-beats}[26885] +- Set permissions during installation {pull-beats}[26665] +- Disable monitoring during fleet-server bootstrapping. {pull-beats}[27222] +- Fix issue with atomic extract running in K8s {pull-beats}[27396] +- Fix issue with install directory in state path in K8s {pull-beats}[27396] +- Disable monitoring during fleet-server bootstrapping. {pull-beats}[27222] +- Change output.elasticsearch.proxy_disabled flag to output.elasticsearch.proxy_disable so fleet uses it. {issue-beats}[27670] {pull-beats}[27671] +- Add validation for certificate flags to ensure they are absolute paths. {pull-beats}[27779] +- Migrate state on upgrade {pull-beats}[27825] +- Add "_monitoring" suffix to monitoring instance names to remove ambiguity with the status command. {issue-beats}[25449] +- Ignore ErrNotExists when fixing permissions. {issue-beats}[27836] {pull-beats}[27846] +- Snapshot artifact lookup will use agent.download proxy settings. {issue-beats}[27903] {pull-beats}[27904] +- Fix lazy acker to only add new actions to the batch. {pull-beats}[27981] +- Allow HTTP metrics to run in bootstrap mode. Add ability to adjust timeouts for Fleet Server. {pull-beats}[28260] +- Fix agent configuration overwritten by default fleet config. {pull-beats}[29297] +- Allow agent containers to use basic auth to create a service token. {pull-beats}[29651] +- Fix issue where a failing artifact verification does not remove the bad artifact. {pull-beats}[30281] +- Reduce Elastic Agent shut down time by stopping processes concurrently {pull-beats}[29650] - Move `context cancelled` error from fleet gateway into debug level. {pull}187[187] - Update library containerd to 1.5.10. {pull}186[186] - Add fleet-server to output of elastic-agent inspect output command (and diagnostic bundle). {pull}243[243] @@ -106,75 +109,75 @@ ==== New features -- Prepare packaging for endpoint and asc files {pull}20186[20186] -- Improved version CLI {pull}20359[20359] -- Enroll CLI now restarts running daemon {pull}20359[20359] -- Add restart CLI cmd {pull}20359[20359] -- Add new `synthetics/*` inputs to run Heartbeat {pull}20387[20387] -- Users of the Docker image can now pass `FLEET_ENROLL_INSECURE=1` to include the `--insecure` flag with the `elastic-agent enroll` command {issue}20312[20312] {pull}20713[20713] -- Add `docker` composable dynamic provider. {pull}20842[20842] -- Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull}20839[20839] -- Add support for EQL based condition on inputs {pull}20994[20994] -- Send `fleet.host.id` to Endpoint Security {pull}21042[21042] -- Add `install` and `uninstall` subcommands {pull}21206[21206] -- Use new form of fleet API paths {pull}21478[21478] -- Add `kubernetes` composable dynamic provider. {pull}21480[21480] -- Send updating state {pull}21461[21461] -- Add `elastic.agent.id` and `elastic.agent.version` to published events from filebeat and metricbeat {pull}21543[21543] -- Add `upgrade` subcommand to perform upgrade of installed Elastic Agent {pull}21425[21425] -- Update `fleet.yml` and Kibana hosts when a policy change updates the Kibana hosts {pull}21599[21599] -- Update `install` command to perform enroll before starting Elastic Agent {pull}21772[21772] -- Update `fleet.kibana.path` from a POLICY_CHANGE {pull}21804[21804] -- Removed `install-service.ps1` and `uninstall-service.ps1` from Windows .zip packaging {pull}21694[21694] -- Add `priority` to `AddOrUpdate` on dynamic composable input providers communication channel {pull}22352[22352] -- Ship `endpoint-security` logs to elasticsearch {pull}22526[22526] -- Log level reloadable from fleet {pull}22690[22690] -- Push log level downstream {pull}22815[22815] -- Add metrics collection for Agent {pull}22793[22793] -- Add support for Fleet Server {pull}23736[23736] -- Add support for enrollment with local bootstrap of Fleet Server {pull}23865[23865] -- Add TLS support for Fleet Server {pull}24142[24142] -- Add support for Fleet Server running under Elastic Agent {pull}24220[24220] -- Add CA support to Elastic Agent docker image {pull}24486[24486] -- Add k8s secrets provider for Agent {pull}24789[24789] -- Add STATE_PATH, CONFIG_PATH, LOGS_PATH to Elastic Agent docker image {pull}24817[24817] -- Add status subcommand {pull}24856[24856] -- Add leader_election provider for k8s {pull}24267[24267] -- Add --fleet-server-service-token and FLEET_SERVER_SERVICE_TOKEN options {pull}25083[25083] -- Keep http and logging config during enroll {pull}25132[25132] -- Log output of container to $LOGS_PATH/elastic-agent-start.log when LOGS_PATH set {pull}25150[25150] -- Use `filestream` input for internal log collection. {pull}25660[25660] -- Enable agent to send custom headers to kibana/ES {pull}26275[26275] -- Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] {pull}26548[26548] -- Add proxy support to artifact downloader and communication with fleet server. {pull}25219[25219] -- Add proxy support to enroll command. {pull}26514[26514] -- Enable configuring monitoring namespace {issue}26439[26439] -- Communicate with Fleet Server over HTTP2. {pull}26474[26474] -- Pass logging.metrics.enabled to beats to stop beats from adding metrics into their logs. {issue}26758[26758] {pull}26828[26828] -- Support Node and Service autodiscovery in kubernetes dynamic provider. {pull}26801[26801] -- Increase Agent's mem limits in k8s. {pull}27153[27153] -- Add new --enroll-delay option for install and enroll commands. {pull}27118[27118] -- Add link to troubleshooting guide on fatal exits. {issue}26367[26367] {pull}27236[27236] -- Agent now adapts the beats queue size based on output settings. {issue}26638[26638] {pull}27429[27429] -- Support ephemeral containers in Kubernetes dynamic provider. {issue}27020[#27020] {pull}27707[27707] -- Add complete k8s metadata through composable provider. {pull}27691[27691] -- Add diagnostics command to gather beat metadata. {pull}28265[28265] -- Add diagnostics collect command to gather beat metadata, config, policy, and logs and bundle it into an archive. {pull}28461[28461] -- Add `KIBANA_FLEET_SERVICE_TOKEN` to Elastic Agent container. {pull}28096[28096] -- Enable pprof endpoints for beats processes. Allow pprof endpoints for elastic-agent if enabled. {pull}28983[28983] -- Add `--pprof` flag to `elastic-agent diagnostics` and an `elastic-agent pprof` command to allow operators to gather pprof data from the agent and beats running under it. {pull}28798[28798] -- Allow pprof endpoints for elastic-agent or beats if enabled. {pull}28983[28983] {pull}29155[29155] -- Add --fleet-server-es-ca-trusted-fingerprint flag to allow agent/fleet-server to work with elasticsearch clusters using self signed certs. {pull}29128[29128] -- Discover changes in Kubernetes nodes metadata as soon as they happen. {pull}23139[23139] -- Add results of inspect output command into archive produced by diagnostics collect. {pull}29902[29902] -- Add support for loading input configuration from external configuration files in standalone mode. You can load inputs from YAML configuration files under the folder `{path.config}/inputs.d`. {pull}30087[30087] -- Install command will skip install/uninstall steps when installation via package is detected on Linux distros. {pull}30289[30289] -- Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull}30462[30462] -- Add action_input_type for the .fleet-actions-results {pull}30562[30562] -- Add support for enabling the metrics buffer endpoint in the elastic-agent and beats it runs. diagnostics collect command will gather metrics-buffer data if enabled. {pull}30471[30471] +- Prepare packaging for endpoint and asc files {pull-beats}[20186] +- Improved version CLI {pull-beats}[20359] +- Enroll CLI now restarts running daemon {pull-beats}[20359] +- Add restart CLI cmd {pull-beats}[20359] +- Add new `synthetics/*` inputs to run Heartbeat {pull-beats}[20387] +- Users of the Docker image can now pass `FLEET_ENROLL_INSECURE=1` to include the `--insecure` flag with the `elastic-agent enroll` command {issue-beats}[20312] {pull-beats}[20713] +- Add `docker` composable dynamic provider. {pull-beats}[20842] +- Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull-beats}[20839] +- Add support for EQL based condition on inputs {pull-beats}[20994] +- Send `fleet.host.id` to Endpoint Security {pull-beats}[21042] +- Add `install` and `uninstall` subcommands {pull-beats}[21206] +- Use new form of fleet API paths {pull-beats}[21478] +- Add `kubernetes` composable dynamic provider. {pull-beats}[21480] +- Send updating state {pull-beats}[21461] +- Add `elastic.agent.id` and `elastic.agent.version` to published events from filebeat and metricbeat {pull-beats}[21543] +- Add `upgrade` subcommand to perform upgrade of installed Elastic Agent {pull-beats}[21425] +- Update `fleet.yml` and Kibana hosts when a policy change updates the Kibana hosts {pull-beats}[21599] +- Update `install` command to perform enroll before starting Elastic Agent {pull-beats}[21772] +- Update `fleet.kibana.path` from a POLICY_CHANGE {pull-beats}[21804] +- Removed `install-service.ps1` and `uninstall-service.ps1` from Windows .zip packaging {pull-beats}[21694] +- Add `priority` to `AddOrUpdate` on dynamic composable input providers communication channel {pull-beats}[22352] +- Ship `endpoint-security` logs to elasticsearch {pull-beats}[22526] +- Log level reloadable from fleet {pull-beats}[22690] +- Push log level downstream {pull-beats}[22815] +- Add metrics collection for Agent {pull-beats}[22793] +- Add support for Fleet Server {pull-beats}[23736] +- Add support for enrollment with local bootstrap of Fleet Server {pull-beats}[23865] +- Add TLS support for Fleet Server {pull-beats}[24142] +- Add support for Fleet Server running under Elastic Agent {pull-beats}[24220] +- Add CA support to Elastic Agent docker image {pull-beats}[24486] +- Add k8s secrets provider for Agent {pull-beats}[24789] +- Add STATE_PATH, CONFIG_PATH, LOGS_PATH to Elastic Agent docker image {pull-beats}[24817] +- Add status subcommand {pull-beats}[24856] +- Add leader_election provider for k8s {pull-beats}[24267] +- Add --fleet-server-service-token and FLEET_SERVER_SERVICE_TOKEN options {pull-beats}[25083] +- Keep http and logging config during enroll {pull-beats}[25132] +- Log output of container to $LOGS_PATH/elastic-agent-start.log when LOGS_PATH set {pull-beats}[25150] +- Use `filestream` input for internal log collection. {pull-beats}[25660] +- Enable agent to send custom headers to kibana/ES {pull-beats}[26275] +- Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue-beats}[21121] {pull-beats}[26394] {pull-beats}[26548] +- Add proxy support to artifact downloader and communication with fleet server. {pull-beats}[25219] +- Add proxy support to enroll command. {pull-beats}[26514] +- Enable configuring monitoring namespace {issue-beats}[26439] +- Communicate with Fleet Server over HTTP2. {pull-beats}[26474] +- Pass logging.metrics.enabled to beats to stop beats from adding metrics into their logs. {issue-beats}[26758] {pull-beats}[26828] +- Support Node and Service autodiscovery in kubernetes dynamic provider. {pull-beats}[26801] +- Increase Agent's mem limits in k8s. {pull-beats}[27153] +- Add new --enroll-delay option for install and enroll commands. {pull-beats}[27118] +- Add link to troubleshooting guide on fatal exits. {issue-beats}[26367] {pull-beats}[27236] +- Agent now adapts the beats queue size based on output settings. {issue-beats}[26638] {pull-beats}[27429] +- Support ephemeral containers in Kubernetes dynamic provider. {issue-beats}[#27020] {pull-beats}[27707] +- Add complete k8s metadata through composable provider. {pull-beats}[27691] +- Add diagnostics command to gather beat metadata. {pull-beats}[28265] +- Add diagnostics collect command to gather beat metadata, config, policy, and logs and bundle it into an archive. {pull-beats}[28461] +- Add `KIBANA_FLEET_SERVICE_TOKEN` to Elastic Agent container. {pull-beats}[28096] +- Enable pprof endpoints for beats processes. Allow pprof endpoints for elastic-agent if enabled. {pull-beats}[28983] +- Add `--pprof` flag to `elastic-agent diagnostics` and an `elastic-agent pprof` command to allow operators to gather pprof data from the agent and beats running under it. {pull-beats}[28798] +- Allow pprof endpoints for elastic-agent or beats if enabled. {pull-beats}[28983] {pull-beats}[29155] +- Add --fleet-server-es-ca-trusted-fingerprint flag to allow agent/fleet-server to work with elasticsearch clusters using self signed certs. {pull-beats}[29128] +- Discover changes in Kubernetes nodes metadata as soon as they happen. {pull-beats}[23139] +- Add results of inspect output command into archive produced by diagnostics collect. {pull-beats}[29902] +- Add support for loading input configuration from external configuration files in standalone mode. You can load inputs from YAML configuration files under the folder `{path.config}/inputs.d`. {pull-beats}[30087] +- Install command will skip install/uninstall steps when installation via package is detected on Linux distros. {pull-beats}[30289] +- Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull-beats}[30462] +- Add action_input_type for the .fleet-actions-results {pull-beats}[30562] +- Add support for enabling the metrics buffer endpoint in the elastic-agent and beats it runs. diagnostics collect command will gather metrics-buffer data if enabled. {pull-beats}[30471] - Update ack response schema and processing, add retrier for acks {pull}200[200] - Enhance error messages and logs for process start {pull}225[225] -- Changed the default policy selection logic. When the agent has no policy id or name defined, it will fall back to defaults (defined by $FLEET_SERVER_POLICY_ID and $FLEET_DEFAULT_TOKEN_POLICY_NAME environment variables respectively). {issue}29774[29774] {pull}226[226] +- Changed the default policy selection logic. When the agent has no policy id or name defined, it will fall back to defaults (defined by $FLEET_SERVER_POLICY_ID and $FLEET_DEFAULT_TOKEN_POLICY_NAME environment variables respectively). {issue-beats}[29774] {pull}226[226] - Add Elastic APM instrumentation {pull}180[180] - Agent can be built for `darwin/arm64`. When it's built for both `darwin/arm64` and `darwin/adm64` a universal binary is also built and packaged. {pull}203[203] - Add support for Cloudbeat. {pull}179[179] From eb6fa029d0dad53ce56f210af08741167b7e3416 Mon Sep 17 00:00:00 2001 From: Daniel Araujo Almeida Date: Wed, 29 Jun 2022 10:17:37 -0600 Subject: [PATCH 23/69] Add mount of /etc/machine-id for managed Agent in k8s (#530) --- deploy/kubernetes/elastic-agent-managed-kubernetes.yaml | 7 +++++++ .../elastic-agent-managed-daemonset.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml index 882e7b46e21..761d659771a 100644 --- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml @@ -85,6 +85,9 @@ spec: - name: etcsysmd mountPath: /hostfs/etc/systemd readOnly: true + - name: etc-mid + mountPath: /etc/machine-id + readOnly: true volumes: - name: proc hostPath: @@ -113,6 +116,10 @@ spec: - name: etcsysmd hostPath: path: /etc/systemd + - name: etc-mid + hostPath: + path: /etc/machine-id + type: File --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml index 097d9786e03..ee7b3dc3219 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml @@ -85,6 +85,9 @@ spec: - name: etcsysmd mountPath: /hostfs/etc/systemd readOnly: true + - name: etc-mid + mountPath: /etc/machine-id + readOnly: true volumes: - name: proc hostPath: @@ -113,3 +116,7 @@ spec: - name: etcsysmd hostPath: path: /etc/systemd + - name: etc-mid + hostPath: + path: /etc/machine-id + type: File From 5f60fd0d8f2e8da6187ea2d0af6f9ab7e1c1b29e Mon Sep 17 00:00:00 2001 From: Daniel Araujo Almeida Date: Wed, 29 Jun 2022 10:18:10 -0600 Subject: [PATCH 24/69] Set hostPID=true for managed agent in k8s (#528) * Set hostPID=true for managed agent in k8s * Add comment on hostPID. --- deploy/kubernetes/elastic-agent-managed-kubernetes.yaml | 3 +++ .../elastic-agent-managed/elastic-agent-managed-daemonset.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml index 761d659771a..4771cf37727 100644 --- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml @@ -19,6 +19,9 @@ spec: effect: NoSchedule serviceAccountName: elastic-agent hostNetwork: true + # Sharing the host process ID namespace gives visibility of all processes running on the same host. + # This enables the Elastic Security integration to observe all process exec events on the host. + hostPID: true dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml index ee7b3dc3219..231b976fe71 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml @@ -19,6 +19,9 @@ spec: effect: NoSchedule serviceAccountName: elastic-agent hostNetwork: true + # Sharing the host process ID namespace gives visibility of all processes running on the same host. + # This enables the Elastic Security integration to observe all process exec events on the host. + hostPID: true dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent From 7a4fa6bb4f79e48cdab12d540ea8e028e7f0a2f2 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 30 Jun 2022 01:35:53 -0400 Subject: [PATCH 25/69] [Automation] Update elastic stack version to 8.4.0-86cc80f3 for testing (#648) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 70d8eb97f16..94af4334d03 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-a6aa9f3b-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-86cc80f3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-a6aa9f3b-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-86cc80f3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From e194fb3bfd7bab75251bfb7bd52cd05704d9e2fb Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Thu, 30 Jun 2022 12:49:46 +0200 Subject: [PATCH 26/69] Update elastic-agent-libs version: includes restriction on default VerificationMode to `full` (#521) * update version * mage fmt update * update dependency * update changelog --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 8 ++++---- go.mod | 4 ++-- go.sum | 7 ++++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 00f91675544..2085c1976a4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -105,6 +105,7 @@ - Update API calls that the agent makes to Kibana when running the container command. {pull}253[253] - diagnostics collect log names are fixed on Windows machines, command will ignore failures. AgentID is included in diagnostics(and diagnostics collect) output. {issue}81[81] {issue}92[92] {issue}190[190] {pull}262[262] - Collects stdout and stderr of applications run as a process and logs them. {issue}[88] +- Remove VerificationMode option to empty string. Default value is `full`. {issue}[184] - diagnostics collect file mod times are set. {pull}570[570] ==== New features diff --git a/NOTICE.txt b/NOTICE.txt index e80beb85357..8b5c887cbe4 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1061,11 +1061,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.2.3 +Version: v0.2.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.6/LICENSE: Apache License Version 2.0, January 2004 @@ -16187,11 +16187,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 -Version: v3.0.0-20210107192922-496545a6307b +Version: v3.0.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20210107192922-496545a6307b/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.1/LICENSE: This project is covered by two different licenses: MIT and Apache. diff --git a/go.mod b/go.mod index 5bc60abc07b..95044f69abc 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4 github.com/elastic/elastic-agent-autodiscover v0.0.0-20220404145827-89887023c1ab github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 - github.com/elastic/elastic-agent-libs v0.2.3 + github.com/elastic/elastic-agent-libs v0.2.6 github.com/elastic/elastic-agent-system-metrics v0.3.0 github.com/elastic/go-licenser v0.4.0 github.com/elastic/go-sysinfo v1.7.1 @@ -132,7 +132,7 @@ require ( google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect google.golang.org/grpc/examples v0.0.0-20220304170021-431ea809a767 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect diff --git a/go.sum b/go.sum index 7198e357fbf..5026651a90b 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM= github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= -github.com/elastic/elastic-agent-libs v0.2.3 h1:GY8M0fxOs/GBY2nIB+JOB91aoD72S87iEcm2qVGFUqI= -github.com/elastic/elastic-agent-libs v0.2.3/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= +github.com/elastic/elastic-agent-libs v0.2.6 h1:DpcUcCVYZ7lNtHLUlyT1u/GtGAh49wpL15DTH7+8O5o= +github.com/elastic/elastic-agent-libs v0.2.6/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-system-metrics v0.3.0 h1:W8L0E8lWJmdguH+oIR7OzuFgopvw8ucZAE9w6iqVlpE= github.com/elastic/elastic-agent-system-metrics v0.3.0/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o= github.com/elastic/elastic-package v0.32.1/go.mod h1:l1fEnF52XRBL6a5h6uAemtdViz2bjtjUtgdQcuRhEAY= @@ -1884,8 +1884,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v1.7.0 h1:RwpqwwFKBAa2h+F6pMEGpE707Edld0etUD3GhqqhDNc= From ed9d75e9a3bc2d3473c61e969c597d3dfd24dca9 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 30 Jun 2022 13:38:19 +0200 Subject: [PATCH 27/69] redact sensitive information in diagnostics collect command (#566) --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/cmd/diagnostics.go | 84 ++++++++++++++++++++-- internal/pkg/agent/cmd/diagnostics_test.go | 80 ++++++++++++++++++++- 3 files changed, 155 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2085c1976a4..602b0293cc6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -190,3 +190,4 @@ - Support scheduled actions and cancellation of pending actions. {issue}393[393] {pull}419[419] - Add `@metadata.input_id` and `@metadata.stream_id` when applying the inject stream processor {pull}527[527] - Add liveness endpoint, allow fleet-gateway component to report degraded state, add update time and messages to status output. {issue}390[390] {pull}569[569] +- Redact sensitive information on diagnostics collect command. {issue}[241] {pull}[566] diff --git a/internal/pkg/agent/cmd/diagnostics.go b/internal/pkg/agent/cmd/diagnostics.go index 3f68689930b..0fbba44be71 100644 --- a/internal/pkg/agent/cmd/diagnostics.go +++ b/internal/pkg/agent/cmd/diagnostics.go @@ -14,6 +14,7 @@ import ( "io/fs" "os" "path/filepath" + "reflect" "runtime" "strings" "text/tabwriter" @@ -34,10 +35,17 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/config/operations" ) +const ( + HUMAN = "human" + JSON = "json" + YAML = "yaml" + REDACTED = "" +) + var diagOutputs = map[string]outputter{ - "human": humanDiagnosticsOutput, - "json": jsonOutput, - "yaml": yamlOutput, + HUMAN: humanDiagnosticsOutput, + JSON: jsonOutput, + YAML: yamlOutput, } // DiagnosticsInfo a struct to track all information related to diagnostics for the agent. @@ -83,6 +91,7 @@ func newDiagnosticsCommand(s []string, streams *cli.IOStreams) *cobra.Command { } func newDiagnosticsCollectCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ Use: "collect", Short: "Collect diagnostics information from the elastic-agent and write it to a zip archive.", @@ -115,7 +124,7 @@ func newDiagnosticsCollectCommandWithArgs(_ []string, streams *cli.IOStreams) *c } cmd.Flags().StringP("file", "f", "", "name of the output diagnostics zip archive") - cmd.Flags().String("output", "yaml", "Output the collected information in either json, or yaml (default: yaml)") // replace output flag with different options + cmd.Flags().String("output", YAML, "Output the collected information in either json, or yaml (default: yaml)") // replace output flag with different options cmd.Flags().Bool("pprof", false, "Collect all pprof data from all running applications.") cmd.Flags().Duration("pprof-duration", time.Second*30, "The duration to collect trace and profiling data from the debug/pprof endpoints. (default: 30s)") cmd.Flags().Duration("timeout", time.Second*30, "The timeout for the diagnostics collect command, will be either 30s or 30s+pprof-duration by default. Should be longer then pprof-duration when pprof is enabled as the command needs time to process/archive the response.") @@ -690,16 +699,77 @@ func saveLogs(name string, logPath string, zw *zip.Writer) error { // writeFile writes json or yaml data from the interface to the writer. func writeFile(w io.Writer, outputFormat string, v interface{}) error { - if outputFormat == "json" { + redacted, err := redact(v) + if err != nil { + return err + } + + if outputFormat == JSON { je := json.NewEncoder(w) je.SetIndent("", " ") - return je.Encode(v) + return je.Encode(redacted) } + ye := yaml.NewEncoder(w) - err := ye.Encode(v) + err = ye.Encode(redacted) return closeHandlers(err, ye) } +func redact(v interface{}) (map[string]interface{}, error) { + redacted := map[string]interface{}{} + bs, err := yaml.Marshal(v) + if err != nil { + return nil, fmt.Errorf("could not marshal data to redact: %w", err) + } + + err = yaml.Unmarshal(bs, &redacted) + if err != nil { + return nil, fmt.Errorf("could not unmarshal data to redact: %w", err) + } + + return redactMap(redacted), nil +} + +func toMapStr(v interface{}) map[string]interface{} { + mm := map[string]interface{}{} + m, ok := v.(map[interface{}]interface{}) + if !ok { + return mm + } + + for k, v := range m { + mm[k.(string)] = v + } + return mm +} + +func redactMap(m map[string]interface{}) map[string]interface{} { + for k, v := range m { + if v != nil && reflect.TypeOf(v).Kind() == reflect.Map { + v = redactMap(toMapStr(v)) + } + if redactKey(k) { + v = REDACTED + } + m[k] = v + } + return m +} + +func redactKey(k string) bool { + // "routekey" shouldn't be redacted. + // Add any other exceptions here. + if k == "routekey" { + return false + } + + return strings.Contains(k, "certificate") || + strings.Contains(k, "passphrase") || + strings.Contains(k, "password") || + strings.Contains(k, "token") || + strings.Contains(k, "key") +} + // closeHandlers will close all passed closers attaching any errors to the passed err and returning the result func closeHandlers(err error, closers ...io.Closer) error { var mErr *multierror.Error diff --git a/internal/pkg/agent/cmd/diagnostics_test.go b/internal/pkg/agent/cmd/diagnostics_test.go index d55f0a06721..f029697d77b 100644 --- a/internal/pkg/agent/cmd/diagnostics_test.go +++ b/internal/pkg/agent/cmd/diagnostics_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,7 +31,7 @@ var testDiagnostics = DiagnosticsInfo{ BuildTime: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), Snapshot: false, }, - ProcMeta: []client.ProcMeta{client.ProcMeta{ + ProcMeta: []client.ProcMeta{{ Process: "filebeat", Name: "filebeat", Hostname: "test-host", @@ -45,7 +46,7 @@ var testDiagnostics = DiagnosticsInfo{ BinaryArchitecture: "test-architecture", RouteKey: "test", ElasticLicensed: true, - }, client.ProcMeta{ + }, { Process: "filebeat", Name: "filebeat_monitoring", Hostname: "test-host", @@ -60,7 +61,7 @@ var testDiagnostics = DiagnosticsInfo{ BinaryArchitecture: "test-architecture", RouteKey: "test", ElasticLicensed: true, - }, client.ProcMeta{ + }, { Name: "metricbeat", RouteKey: "test", Error: "failed to get metricbeat data", @@ -137,3 +138,76 @@ func Test_collectEndpointSecurityLogs_noEndpointSecurity(t *testing.T) { err := collectEndpointSecurityLogs(zw, specs) assert.NoError(t, err, "collectEndpointSecurityLogs should not return an error") } + +func Test_redact(t *testing.T) { + tests := []struct { + name string + arg interface{} + wantRedacted []string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "tlscommon.Config", + arg: tlscommon.Config{ + Enabled: nil, + VerificationMode: 0, + Versions: nil, + CipherSuites: nil, + CAs: []string{"ca1", "ca2"}, + Certificate: tlscommon.CertificateConfig{ + Certificate: "Certificate", + Key: "Key", + Passphrase: "Passphrase", + }, + CurveTypes: nil, + Renegotiation: 0, + CASha256: nil, + CATrustedFingerprint: "", + }, + wantRedacted: []string{ + "certificate", "key", "key_passphrase", "certificate_authorities"}, + }, + { + name: "some map", + arg: map[string]interface{}{ + "s": "sss", + "some_key": "hey, a key!", + "a_password": "changeme", + "my_token": "a_token", + "nested": map[string]string{ + "4242": "4242", + "4242key": "4242key", + "4242password": "4242password", + "4242certificate": "4242certificate", + }, + }, + wantRedacted: []string{ + "some_key", "a_password", "my_token", "4242key", "4242password", "4242certificate"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := redact(tt.arg) + require.NoError(t, err) + + for k, v := range got { + if contains(tt.wantRedacted, k) { + assert.Equal(t, v, REDACTED) + } else { + assert.NotEqual(t, v, REDACTED) + } + } + }) + } +} + +func contains(list []string, val string) bool { + for _, k := range list { + if val == k { + return true + } + } + + return false +} From 94975cc6f476f528401a30d6ae6396274895d9ac Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 30 Jun 2022 16:35:02 +0300 Subject: [PATCH 28/69] Support Cloudbeat regex input type (#638) * support input type with regex * Update supported.go * Changing the regex to support backward compatible --- internal/pkg/agent/program/supported.go | 2 +- internal/spec/cloudbeat.yml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/pkg/agent/program/supported.go b/internal/pkg/agent/program/supported.go index 4bd218a4ee9..50140f27617 100644 --- a/internal/pkg/agent/program/supported.go +++ b/internal/pkg/agent/program/supported.go @@ -27,7 +27,7 @@ func init() { // internal/spec/metricbeat.yml // internal/spec/osquerybeat.yml // internal/spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzMe1uTo7iW7vv5Gf16zpnNJZ3dTMR+MGRxs5Mq4zISekOSE2wL7E4b2zAx/31C4mLAZFZWdvWe/VARlVjosrQu3/rW4r9+22Sn9WsWsX8cD2vyj+iQ/v/j+vW8fv2PImW//edvODVP6Ps+XgS6Nw88RjLESHzYYrB4dCzzgpdyiaCrIOjMQuhKEUBJqI7+lpFyH4PLPnYM5+QvnaNjuKcQTBKkBCcEJtI8DfIQuEcEFhq1XRktnaOxmcbORjadzSV2UrqFqs5I6jGcLTTXPumrL/J3P3CBH7gvvqTZi3J/fX7SNSc+UCMNHoilFdQKdlCRGbXdQ6g+PzrmceYY000I9dMc1mfaOEeDSTOSBUcEnx/5uvOlvsWqPoGqf4bK9UDUhXjuGNPYsZiEgPToWOiIQCC1z23//HWjH3Cmy9R+nolnxjTGyuQlVLQcpddDJZ/JGatT/vvJseSEPO3bscQypehpH6P0yhBc3J539tY8my/1AgH5TNPgJVKCydd43/5W/dNfEdzx+9iGSlASWUuIxcTYT81ju6ySKcvRpTtGikkanLCKGFRObP39dp7mn5h3o/P7zul0L95BKXuAqieRNEjw9328VqVaJuiAbZ8RpikhuMq9c9sew1awpZZWjMm6XkdaQ53d3kEJtgNGyt6+TkJPF+1ejtQKitvZ9RKBKwtV/0yyO7nfrVvNp8nU1uXqfDfZdO7y5Fgsj9JgS01tj4C5Q9Atv270318WBzWygvzrRj8iMMmoFe9d+1Sv42mz5fT/Ok/TOASTnWMlCZFObL2Md2ulXtOWjo5BGbbMklpsS5QgIam3d4tL7KouQxYr3eLC95BFiplGypdsbkwzbGkZUf2EKHE2W+z/+dv/G3qFnG5OeB2dhk4BBrsI+hsETMlIvRN62sdh75lZoI7BzcHzZi4M7zZmngbHEHpSBJ5PIaDcsPM1kI/O5lK9s1qJd4gSSBRO8whcT11HgVLzSJTVZj7tOZYSQV8mxkRCQL5gy5TQcsJwam6wFey+Aa4MHhuugYGZiz2AIKdGZzz0Cwru1jhihWYRmGTz9MpoGhy/AZ+FWZAN50WK90KsoAi54j9Ju7VkBv4XtlrsAj0wNfu7RJ++br9cnm1puEZCgX+glstCuMgrB8KyeSqfkS2cQY7AJAm5cj3JaQiuJVo6M7g4JCTzDyg1t5TfSRok1H4+9+4mc5m49Mx/4c4Jq4FE7ECCircPwSTj9yuc8GLfcYrjBmtsnuMFuB6dL6aMLCaJ9YzaQIzKEEmhbxCgB7zR1Qj6e8eg1dmMP7J5vB8xeO+FKkyKTK1AgLK1Pa0NsHFE/oRYq8r4bJd93UwzmFbrz4uHV7d2TkQJjgh4EladR254/Izksp85ha4jyy8dgx5w6rP10z4WhlDIvyPLlMJAK6ntshBIufgbeoza0mmt+Am1zBei+gUC5mmeThIMgpJY5hZBKeMBwLGCJFTiOAKTC4WLnMstApM/+T6gwnJkBQ+Nk6E2u3BZi7W7+7J9FRd6giz/BaWMYfFML7DiMaJ63CFxJ3TG6SpGqVY4VqBUQVTsr0RwUTsKLUdLPcWqw+fehdBP2vtbTsTf3Lbmxt1v4t7mhp7gdBEPZSJ8APQuIfBYdX+tU6vHyC/Eds8i6Cha61TvAyHLiRIU1NQSlPmMvC2XJ6xMFARdqblHIEt5cw9EPlH+nFp/1HL3S/670FmYSITbgqkdIyjugPuFstlTE3Cb92o9O1ArOH3d6M15mucSLjtjxdk7AdzWE2rFXV0bC95b50v1e6XHQelY6Ew2ut6xd263WQQeYmEHxjSr7HdxdouH2IeIkYxJEQ/EXE+47Da6FFkml0/ZrmML+UgILOIQPsfUSphj1f5jqechkPn9NUGNyyknyjWhVvCG/fVBirinRSfY3dlinM+KXdZ9V8z7tI/dpy+zDngQe+kBDwsVWJH6tm7c7gADbUfBlXVkmjrWl7i1V0M/kEIviRVsI4AOQgaKltM0KLhfqmUldMZVGrvxeTDGrgB4MnWLh3aPGJivKNASkrlJH+C0utLaU++81h+PjnWbv/7txGWFgaagQBM+rftODWKG833knTZu/ew6re0v9j8GhZYAeUKe/J4joOXtPZguIzA4kHQVhzwuWN4Zp+iACl3Chb7Fiswc29+TVJORElc6+cXMuP9yjCTHhX7EipdgQ08jcGWk2OU/cZYqrphawXUDKuiAraC207dAZQfobe/8QA2ga5upZUNT80hBUHZkwsGk0Hk07T4T679EgGMRWkTQ7wBvKSYqK7nuGPF+63wxd8h+jqlNz45l7pCppREIjvxZ15eSQgDNxq/HDW6KwKjPbwEhVCoZz1O2mxsjzzMvD6FeRpYmjf/OfY53wMrDIKZVv4vYlv7x6NgnrZK3dyZ2c0d6SlLtdAfYq4Spi3NmfO6hP0CZe+Z4o48d+z62srMWG81qvNID9Tc84LZ3N1/212/tvcZEre5xnYfiXVbdVx3zDL0F8o0/qfxsDcwNrvOTNAJUJqkpbGUQMweJwH5GlYThLY9/QY5Ufz8z/N/7Z2/O+ZcSA5EMkJTP+yySgZ9KDAjb53QsMVCuZwoGiUHv2SAx+LcC+Qf6cXA8ANr9M26pxc6YaZXCm1oFhKsMlBvHEavBToDl6X7IPKQRTFqQWSuFUMq3gKRTA22ePWKFcofXH98490C7kFTbIuiVHBTfGA0th4pw1McIeFJlqB7jMggBkhB02oB3B87+pRl7X26DjLpiF0ztgFN0boypt79th2kZcQaN8feDQJv1nyn0OaifGRndI/Dw6FhXhlMqRQY3uNohqdLBeXqIn2vwHFlmuVSCCTfiRpdelpfYVZrk1ysRMItQiYXhfdog1xk97DfZ0B59MNkRi+vhIl4pwZZC90Dt3SxU5F0FGPwLVlhODblEwJNJyqT1QLfRbY4TsgLFELqBDqGyenSeQvXrUzwLgSecG6zlANUK3H4Vzi7Ika2feSA00usZydolhP6+ul8OQFw14vI0nPN3i21IahbrpWY2Tmsu3d6fq54UQp/NlesZFVpn/9Kfcz53wXVVP0ZgIvPEztloZ2Ivzj64JkT1D2Ghmbd3tJIK36EdsULO3XPONhP+bMN1gAp701RS7mNn9/wIzeuCpFpGUvPkfKlABjSv7X7F/5s1zCvh90atgECLn/1KRtdJvT0C3quQn+on2Lo81gkMC2VNgKFh0uKkHblAj4VqIECGU4+rmdc2qDk8iKcsXS+d27ONdOK65bQBcbohqs8DUtE8oxY7IaDJXBeey+mMWFop7CzzJA7y6jt+QMDjCesRfR8kRIY7rmfNPqwqJrTB1nDbubv7mi/l9k7qcSW1fEYyp/PMOc1hcEGqmyBrNXjuMqJoMkk9RoqODN6QY3/85DGC002TCEZAZtyev26myvPTdEZsl0E1yCMw4Tp1xE/72Xyps7UVbCsgumrAkND9r5vppqsH5GabzRoJSWkfZBqujNNWPzZdMDm8x3H5jOz7hwTPDUhBdeCX3/PtdZLQTRKHwJnLp9WLqZAX13MJQfdlOPYNMued5EcfxqITT4gpWIkzYWBehvbUY91tV8ZWb68/TIgEmZEGRS/h2N7LqmuTfab/LvbcscpiHxAdiMLOOB4Hi9Wc/gAc/vpY1Y9JBw50S2q7Cbcj7itxqkkOX091ZZz5BwxWeQjdbWRL8bfvUuwqZoG/h5JbVOu79qmgYCJ0dJ6iBAN2XMN6rGCxk4QapDqP4f9OsiAXvmk5OYXgcCZZPbYk2Ww5nd3FyJcNW49gVp/7LFAlAHVcFP62whmHyg43Ou5VpDKPUTu4zFN2xB/Ang2+5Qkfsa4JtVZjBO6g6iUnOOUJliySvM54iWTBPb61tAzxWFVMjggihp/kHQKujIofVtOs5epqfoCUvsnFmNSJDckbMoYUI3IQ1TizoBZLwwpbCoxMCq0r4wJBHtddCSocQ9Q6b2rbyDJzxHFGbbs/8leiOpMeeELPE+8dgnHjd4cY+W8nLudL/RTC6cDvD5PgOz9TVR4V84hNTcJyRVz2/E2VOA/IyR9WG5tcga0tjxF7wXOwTUuiFuJ+2/urdLxHOm/gYrDHseoo98fQZ/w+esTF5bP7b2W9QQCdSbp6lyQYEDg9wrY966WOzbZfhCKf4mfwX8S+b8TQ4L68IRn8d5LEIlaFYFKK3LZPODb76udMXHbAvJBeFbN+zv1ChcvE/7vvcH2gKc+Ha4wp7IP0K7hNgUH1JGSxXJzZ8g94sI7wOdniTC3vwvcX9vSuO0+wI0/7mAL/MthLhcMtbRspAccUO6x4r938s5pHYP8CAf9AZK3Elqbyc33d6NWzy/3556o3ITxnKPeCeOvI4SPEokyU4OWOFPobydX3ii6fXr9T9PnLpGivCDfM16U4hLScgypfp4ophUrc/w0+dzCO+Uihy+ag0o/IDm64vc75UarJOPWLddcHKRMpBCy/6YcUN7pze59uiSFfKHBLCm7jItuXiKl18Wcz757a/iXKvDO++eQ/sRoUODWP6DbuNQToNbztp2M3UoyhKXXmzBH0U66nt339UXrboCIfu9iuo7/zZd/O+N9r6PXW4bZ204dgR25z8Vzh4fYbPWCeH9/mrgj22if+dZK85eze1ImmqNyN0RWBLWJ1yv1mk3OEinZpCy1NMcgaI31F/l3xbj+Fu2+Y74Ok70+Ttp018kZmL8td/G0zvTgc1xj6PoTeHMHd3rVPNb/kaxxbo4avUD0mMLNBjo5BK/0oiMBzrnJKUHpKRJcH922CG/H2brEbwb5svT6Nd3j5VQ4Sr5r8peZt69zk1OY1N+6W5+IV32bIJ6z4zGHSoAus6o5qu6zigyjK3DhQdFi3uEtqCge9bpy7fPMHeeCobxov2HRzr1Mnb/s16w9t4Z09NDbxrj+vY0Krn/U+m71ARRQeRjlX0XlX6Cm2AkaNSdOVlzdz3Zpw2twuhm1Rt8UaN+6z4qgbfyCKW3hUPsI2cKsHWdNtN7lghWO7nShajazV4OP82WjHNuu2vDSygjSEwZHa41zzPXd8t489Vj1pwPHeyUl0vI13dOWN3sxTLv8qPkBVP5Ns8aO1S6Jc7rr+Gh8x307zoU52+YqOrKr9tvvTuzbWXf+O/x69r7vOvzpeV0XGu3eruZMzUVu96OQ7b2PdN/f5Dp7p2gGFHmtzxOmnOhFHcdlfmmM5EfkIUfUkVFafOtebzUefO+MtL4k/1eU40K3px2ojjZ02ObgVJMgKhB8SsX20dlLZxyA2/uJOyJGYmKyj17HuxqUVJCTrFzGj3rNOIPxoAfMTZNDPtEt/qDvxLkhzcOKXcyXhIEP6PBnzfuHyLqDKmugEhKrLkBI8dFugxwuCLk9M10TljjZh4mKLP/LZZUSRBwD1PVL5fWDbMczmbm6O9r7Q2XRRZQEbGsEYyP07CpN9x9wBr/+KgqWQDa2SoqUAq4Joo5ZZRgY5GPE/74uW6fr0uiEjFvgdBBJJ2bbWyPqjg7qRX6mp2vEPC5rWgAO2pB9TrA2Nm/kMQ/0oSos/bj/4yY8Xrmek8PSL5FhQIBcNWcGGAjKcNwtl7YKgu+Xzflv6v39fBavVjj19gJo9IegXEfDqHpGm72rC4bmgGMfOZbzpEfqyJmnAb7KgpnbGTGtSlZdQSRKcUm6dleZnLZ05DvE/0g9b331N07VW1HgWqHZoycZyPtrCUKUGN11Sn38VnfgO1XtHHQ7gT53uBlqCresLtbQXbLGSPu3v+h47a3bgw93Zc6xoPWoNwWSLoC4JSJ7VkAVWPfkRWFS96o2tGEKfOq0ibskjx2Cv49R083FMv1/5s+e43WEapLc0PHigtptwnRByMjVBxd7KkgPYqbbnnfX3OHg++IBntPT5VuQaKXeSdPXomA/5rNAa2yzd6fu96f/bJdCBvt/pKZc7zjzhJ6FKD9RKXkgaZAgml36Jor6D+/7dgts43MT/WD1dRZT/tnl4nS3vZVTNw9eIHx3D7yKCClJXsaI7d9NneEeN9/oDb5TdKQQnBhWzIKk5GdXj1k8MYHilK+2ef7K3t/PeYvZjWP8WrfCRd/rfLPwS6vjzc4ykSB84gx2UKNA6KcYv7T/txU3x7lC3WvqxFxPzsdjQlkJ7tuzc06hZxyYHuKaRy7t0ZE8GCQuBz/ic1flHPyYb4IN/gx7S/fHPfP1ajME/1btSEBTrflX+TFRTRtCdDCvzP1GV/3no9zMdpB+qxruf7zKtP+fCQ/m8232nlQQGjGS72ee6zJr3mVCdD3eX2f4Bg+BM4eLRefpyMToM8BvJYpdhbz7NGsC26xnVkO1j3a06h6uiUwFbTMgwAoLd464j5aH97e6Av6861t7fB6shH2xlH3TXdlzIuEv4JaxKqGjiU6wQsJzaz4+Dzhwx7mU5Tb4tp9lzGSvuZcQVHCKyW49RMSvL3EZKIPUSQZsHkBNrmvbbRLAgJ79KPX+QBPIxd2Pf7SUXNZZCNqtay/t9Of2xbyZ/2Vsfa5L+mT9Px/xF2qP/gdeblMclBN4rGuH+xrW5uzfng9z9kD/+V2n3T309Pfvtv//P/wQAAP//cJQCCQ==") + unpacked := packer.MustUnpack("eJzce1mToziX9v33M/r2m5lmSWc3E/FeGLLY7CTLuBIJ3SHJCdgCu9PGNp6Y/z4hsRgwuVR29ztvzEVGVGGh5egsz3nO4b9+SfPD6jWP2K/73Yr8Gu2yf9+vXo+r1/8oM/bLf/6CM/OAfmzjRaB788BjJEeMxLs1Bot7xzJPeClfEHQVBJ1ZCF0pAigJ1dHfcnLZxuC0jR3DOfhLZ+8Y7iEEkwQpwQGBiTTPgiIE7h6BhUZtV0ZLZ2+k09hJZdNJT7GT0TVUdUYyj+F8obn2QX/+Jv/wAxf4gfviS5q9uGzPjw+65sQ7amTBHbG0klrBBioyo7a7C9XHe8fczxxjmoZQP8xhfabU2RtMmpE82CP4eM/XnS/1NVb1CVT9I1TOO6IuxHPHmMaOxSQEpHvHQnsEAql9bvvHp1Tf4VyXqf04E8+MaYyVyUuoaAXKzrtKPpMjVqf894NjyQl52LZjiWVK0cM2RtmZIbi4Pu/srXk2X+olAvKRZsFLpASTp3jb/lb96a8Ibvh9rEMluBBZS4jFxNgvzWO7rJIpK9CpO0aKSRYcsIoYVA5s9eN6nuZPzJvq/L4LOt2Kd1DG7qDqSSQLEvxjG69UqZYJ2mHbZ4RpSgjOcu/ctsewFayppZVjsq7XkVZQZ9d3UILtgJFLb18HoaeLdi97agXl9ez6BYEzC1X/SPIbud+sW82nydTW5ep8V9l07vLgWKyIsmBNTW2LgLlB0L08pfpvL4udGllB8ZTqewQmObXirWsf6nU8bbac/n/nYRqHYLJxrCQh0oGtlvFmpdRr2tLeMSjDlnmhFlsTJUhI5m3d8hS7qsuQxS5ueeJ7yCPFzCLlWz43pjm2tJyofkKUOJ8ttv/45d+GXqGg6QGvosPQKcBgE0E/RcCUjMw7oIdtHPaemSXqGNwcPKZzYXjXMfMs2IfQkyLweAgB5YZdrIC8d9JT9c7zs3iHKIFE4bSIwPnQdRQoM/dEeU7n055juSDoy8SYSAjIJ2yZElpOGM7MFFvB5jvgyuCx4RoYmIXYAwgKanTGQ7+k4GaNPVZoHoFJPs/OjGbB/jvwWZgH+XBepHgvxArKkCv+g7RZSWbgf2PPi02gB6Zm/5Dow9P62+nRloZrJBT4O2q5LISLonIgLJ9n8hHZwhkUCEySkCvXg5yF4HxBS2cGF7uE5P4OZeaa8jvJgoTaj8fe3eQuE5ee+y/cOWE1kIgdSFDxtiGY5Px+hRNebDtOcdxgjfQxXoDz3vlmyshikljPqA3EqAyRlHqKAN3hVFcj6G8dg1ZnM37P5/F2xOC9F6owKTK1EgHKVva0NsDGEfkTYj1Xxme77Cmd5jCr1p+Xd69u7ZyIEuwR8CSsOvfc8PgZyWk7c0pdR5Z/cQy6w5nPVg/bWBhCKf+GLFMKA+1CbZeFQCrE/6HHqC0dVoqfUMt8IapfImAe5tkkwSC4EMtcIyjlPAA4VpCEShxHYHKicFFwuUVg8gffB1RYgazgrnEy1GYnLmuxdndftq/iUk+Q5b+gjDEsnuklVjxGVI87JO6Ejjh7jlGmlY4VKFUQFfu7ILioHYVWoKWeYdXhc29C6Cft/S0n4v/ctubGzW/i3uaGnuBsEQ9lInwA9E4h8Fh1f61Tq8fIL8R2jyLoKFrrVG8DISuIEpTU1BKU+4y8LZcHrEwUBF2puUcgS0VzD0Q+UP6cWr/Xcvcv/HehszCRCLcFU9tHUNwB9wuXZk9NwG3eq/VsR63g8JTqzXma5xK+dMaKs3cCuK0n1Iq7ujYWvNfOt+r3So+Di2OhI0l1vWPv3G7zCNzFwg6MaV7Z7+LolnexDxEjOZMiHoi5nnDZpboUWSaXz6VdxxbykRBYxCF8jKmVMMeq/cdSL0Ig8/trghqXU0GUc0Kt4A3764MUcU+LTrC7scW4mJWbvPuumPdhG7sP32Yd8CD20gMeFiqxIvVt3bjeAQbahoIz68g0c6xvcWuvhr4jpX4hVrCOANoJGShaQbOg5H6plpXQGVdp7MbnwRi7AuDJ1C3v2j1iYL6iQEtI7iZ9gNPqSmtPvfNav9871nX++rcDlxUGmoICTfi07js1iBnO95l32rj1s+u0tr/YfgwKLQHyhDz5PUdAK9p7MF1GYLAj2XMc8rhgeUecoR0qdQmX+horMnNsf0syTUZKXOnkNzPn/ssxkgKX+h4rXoINPYvAmZFyU/zEWaq4Ymol1w2ooB22gtpO3wKVHaC3vvEDNYCubaaWDc3MPQXBpSMTDiaFzqNp95lY/yUCHIvQMoJ+B3hLMVHZheuOEW/Xzjdzg+zHmNr06FjmBplaFoFgz591fSkpBdBs/Hrc4KYIjPr8FhBCpZLxPGObuTHyPPeKEOqXyNKk8d+5z/F2WLkbxLTqdxHbst/vHfugVfL2jsRu7kjPSKYdbgB7lTB1cc6Mzz30Byh3jxxv9LFj38dWdtZio1mNV3qg/ooH3Pbu5sv++q2915io1T2u81C8y6r7qmOeobdAvvEnlZ+tgbnBdX6SRYDKJDOFrQxi5iAR2M6okjC85vEvKJDqb2eG/1v/7M05/1RiIJIBkvF5H0Uy8FOJAWHbgo4lBsr5SMEgMeg9GyQG/1Igf0c/D44HQLt7xngnHFCdcQpHsGrBkVQZSVpnktmOOx/uJDYIxp1MVwS+e8c6H5HKFbo/vgE/UO0Aw/X2HdZhHGw5NUjnmSdWKHeWAohi69S7ywbUXEGLy5DCCsemO2rFsat4e6wGmyoBIEW9X+7EJQTdl8ZpQpWPT15IFuQIJqdOxn0L7rnRZs/3jkFXXfnO023+ESMwlFflKIWcMw4gquSlu0dnlGW4ZQ36jufGYcT/lEw+dpVDgrJDUv2b64jHgc/WLTezG2Nd5XS3TfOhrfpgsiGWtsP5In5WgjWF7o7am1moyJsKTPgnrLCCGvIFAU8mGZNWA71H1zkOyAoUIxes2i5Unu+dh1B9eohnIfCE4+NBQtiSWunQk3CEQYFs/ciDpJGdj0jWTiH0t5XcOThx1Qjc3TuGc/xhsZRkZrlaamYjn7l0fX8uZOCzOZdHqXX2L/0x53OXDp9zH4GJzJM+J9WOxF4cfXBOiOrvwlIzr+9oFyr8irbHCjl2zzlLJ/xZyvWFiqROU8llGzubx3tonhck03KSmQfnWwVAoHlu9yv+3axhngn3NdQKCOR2Lp/J6DqZt0XAexXyU/0EW6f7OrlhoawJoDRMaJysIxfosVANBABx6nE1K9sGPIcH+Ixlq6VzfZZKBx4InDZYTlOi+jxYlc0zarEDAprMdeHxMp0RS7tQk+/fkzgArO/4DgGPJ7N79GOQLBnuuJ41+7CqeNEGYsNt5+7ua76U2zupx12o5TOSO51nzmEOgxNS3QRZz4PnLiOKJpPMY6TsyOANOfbHT+4jOE2bJDECMuP+4CmdKo8P0xmxXQbVoIjAhOvUHj9sZ/OlzlZWsK5A6nMDlITuP6XTtKsH5GqbzRoJyWgfgBqujLNWP9Iu0Bze47h8Rvb9IflzBVlQHbCs7zG1dQLRTSCHoJrLp9WLqZDXTSy5BXbOTfI9nhjpQ2b5wJNlCp7FmTAwT0N76jHytitjq7fXD5MlEXuzoOwlI+tbWXVtsl8FkAbJyS3jLPYB0Y4o7IjjcSBZzekP4tCZ4YxKkcHjUC0/Vdo5D3fxY00CRZZ5WSrBhM/RxLqXJY89DYnrXRAwy1CJRwDkjoNgjn8SbkfcV+JMkxy+nurKOPd3GDwXIXTXkS3F339IsauYJf4RSm5Zre/ah5KCidDReYYSDNh+BeuxIi4mHHdU5zH830geFMI3LSeHEOyOJK/HXkg+W05vY+RLylYjeNbnPgtUyUEdF4W/FThnuqvsMNVxr1qVe4zawWmesT3+BC5tsC9PBol1Tqj1PEbuDipicoIznnzJIgHsjJdIHtxiX0vLOV5D5WSPIGL4Qd4g4Mqo/LDSZi2fz+YnCOurXIxJnfSQoiFqSDkiB1GpM0tqsYxjhKdUF/iOlFpXxiWCPK67ElQ4hqh13tTWkWUWiOOM2nY/8ldv4+0bDPy3k5rzpX4I4XTg94cJ8o2fqaqSirnHpiZhuSI1e/6mSqoHxOWHlcgGg7OV5TFiL3h+lrYEaynut72/Ssd7hHQKF4M9jlVOuT+GPuP30SM1Tl/dfyvrFAF0JNnzuwTCgNzpkbntWU91bLb9MhQEBz+D/yL2fSWNBvflDYniv5NAFrEqBJOLyHv7ZGSzr34FlMsOmCfSq3DWz7lfqHCZ+Hf3Ha4PNOO5co0xhX2QfnW3KT6onoQsVogzW/4OD9YRPidfHKnlnfj+wp7edecJNuRhG1PgnwZ7qXC4pa0jJeCYYoMV7xVBZzCPwP4lAv6OyNoFW5rKz/WU6tWz0+3556o3ITxnuGwFKdeRw2dIR5kowcsNYfQ3Eq/vFWS+vH6nIPSnCdNegW5YfZfiENLLHFR8AVVMKVTi/m/wsYNxzHsKXTYHlX5EdnDF7VY1B8o0GWd+uer6IGUihYAVV/2Q4kZ3ru/TNTHkEwXuhYLruMj2JWJqXfzZzLultn+Kcu+Irz75D6wGJc7MPbqOew0Beg2v++nYjRRjaEqdOQsE/Yzr6XVfv1+8dVARk11s19Hf+bJvZ/z/K+j11uG2dtWHYEOuc/Fc4e76G91hnh9f567I99on/nkCveXz3tSJpuD8MWcjxaGindoiTFMossYIYZF/15zcz+DuK+b7JCH804RuZ42ikdnLchN/T6cnh+MaQ9+G0JsjuNm69uFIoc/HaBxbo4avUD0mMLNB9o5BK/0oicBzP8UPvbDV6jDe/eVXOUj83OQvNadb5yaHNq+58ro8F686fQz5gBWfOUwadIhVnVNtB9Zn+NK436lzk29+kAe+xS+OFHO6udehk7f9NesPbeGdPTQ28a4/r2PClX+s9tnsBSqiKPEWtxmDUs+wFTBqTJqOvaKZ69qg0+Z2MWwLvi3WqHVz0fCpjT8QhS88Kh9hG7jVg7zpxJucsMKx3UYUtEbWavBx8Wi0Y69cr5jHf0FWkIUw2FN7vHPsltO92ccWq5406Pq6kZPohhvv9ioavZlnXP5VfICqfiT54qO1L0Q53XQENj5ivp4WQ53s8hUdWVX7bfend22su37fJ3b+xgpm/T+pKUDevFvNnRyJ2upFJ995G+u+uc938EzXDij0WJsjTr/UpTiKy/7UHMuJyEeIqieh8vylc73ZmPS1M17zkvhLHZAD3ZqO8VPN2KF/uObgVpAgKxB+SMT2nG4RuLvv81GVfQxi419cWxmJickqeh3rfFxaQULyfoEz6j3rBMLPFje/QAb9TCv1pzoXb4I0Byf+Za4kHGRIXydjakAJk7ZoWQOj8YAqa6JLEKouQ0pw122PHm/vdXliuiIqd7QJExdb/l7MTiOKPACo75HK7wPbjmE2d3N1tLdty02HVR6wzxQm33LMbzvkj42v75g74HXU6P4aErjtLhCyoVVStBRgVRBt1DIvkUF2RvyP2w6DbHV4TcmIBf4AgUQytq41sv4goW7yV2qqdvyjg6ZtYIct6WOKtaFxc59hqO9FafHj1oSf/LDhfEQKT79IgQUFctKQFaQUkOG8eShrJwTdNZ/3+9L/7cdz8Py8YQ+foGYPCPplBLy6f6TpyZpweC4oxrFzGW96hL6sSRbwmyypqR0x05pU5SVUkgRnlFtnpfl5S2e+0RLxiV7Z+u5rmu5z7RAW91TizPuooplbqBgCJLUUQdty0X4w8lfRie9QvTfU4QD+1OluoCXYOr9QS3vBFrvQh+1NT2RnzQ58uDl7gRWtR60hmKwR1CUByfMassCqXz8Ci6qPvbEVQ+jTiWTaGkHvwj0RjxyDvY5T082HM/1e5q+e43qHWZBd0/DgjtpuwnVCyMnUBBV7LUsOYKfannfW3+Pg+eDjntHS51uRa6TcKdpazLtiVmqNbV7c6ft96//bJdCBvt/oKZc7zj3hJ99u8+ncwW1vb8ltHKbxr88PZxHlv6d3r7PlrYyqeUTr0b1j+F1EUEHqKlZ05256EG+o8V7v4JWyO4TgwKBiliQzJ6N63PqJAQyvdKXd80/2/XbeW8w+hvVv0Qqfeaf/PcNfQh1/fY6RFOkTZ7CDCwq0Torxl/am9uKmeHeoWy392IuJxVhsaEuhPVt2bmnUvGOTA1zTyOVdOrIng4SFwGd8zur8o+1pA3zwL9Bfut3/UaxeyzH4p3pnCoJy1a/KH4lqygi6k2Fl/ieq8j8P/X6mu/RT1Xj36x2o9adeeCifd7vvtAuBASP5Zva1LrPmfVZ1iX62u8z2dxgERwoX987Dt1O3Y/aNZLHLsDefbQ1gW9Mp21H/xr0EWgeytCbH4aroVMAWEzKMgGD3uOvIeGh/uzvg76uOtff3yWrIJ9vcB9+s/jM6VjezUNHEZ1ohYAW1H+8HnTli3MtymnxfTvPHS6y4pxFXsIvIZjVGxTxb5jpSAqmXCNo8gBxY09DfJoIlOfhV6vlBEsjH3Ix9t89c1FhK2axqLe/35fTHvpn85W99yEn6Z/46HfMnaY/+x19vUh6nEHivaIT7+z/Wj/3Rl9WzX/77//1PAAAA//9rUQ6t") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/internal/spec/cloudbeat.yml b/internal/spec/cloudbeat.yml index 9cfda42344d..b22ed7c328d 100644 --- a/internal/spec/cloudbeat.yml +++ b/internal/spec/cloudbeat.yml @@ -7,8 +7,6 @@ args: [ ] restart_on_output_change: true artifact: cloudbeat -action_input_types: - - cloudbeat rules: - fix_stream: {} @@ -19,11 +17,12 @@ rules: on_conflict: insert_after type: logs - - filter_values: - selector: inputs + +# All Cloudbeat input types begin with 'cloudbeat'. + - filter_values_with_regexp: key: type - values: - - cloudbeat + re: '^cloudbeat.*' + selector: inputs - inject_agent_info: {} From 121a4ad3ff915d6c9700919e1a1e7f188b03f3e1 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 30 Jun 2022 10:22:02 -0400 Subject: [PATCH 29/69] Disable flaky test download test (#641) --- internal/pkg/artifact/download/http/downloader_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/artifact/download/http/downloader_test.go b/internal/pkg/artifact/download/http/downloader_test.go index 6fa2777c02f..26164df3a49 100644 --- a/internal/pkg/artifact/download/http/downloader_test.go +++ b/internal/pkg/artifact/download/http/downloader_test.go @@ -25,6 +25,7 @@ import ( ) func TestDownloadBodyError(t *testing.T) { + t.Skip("Skipping flaky test: https://github.com/elastic/elastic-agent/issues/640") // This tests the scenario where the download encounters a network error // part way through the download, while copying the response body. From 319ea0a35ebca6f32176b680fbd49fb50e48f552 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 1 Jul 2022 01:33:14 -0400 Subject: [PATCH 30/69] [Automation] Update elastic stack version to 8.4.0-3d206b5d for testing (#656) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 94af4334d03..3af879be603 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-86cc80f3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3d206b5d-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-86cc80f3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-3d206b5d-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From fab95b874d8b64344728038055f35e6b61999273 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 4 Jul 2022 01:35:23 -0400 Subject: [PATCH 31/69] [Automation] Update elastic stack version to 8.4.0-3ad82aa8 for testing (#661) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3af879be603..184c29a188d 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3d206b5d-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3ad82aa8-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-3d206b5d-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-3ad82aa8-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From acc91fcaca2b6bc3f74aa61fc52154641f0d8716 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Jul 2022 09:48:09 +0100 Subject: [PATCH 32/69] jjbb: exclude allowed branches, tags and PRs (#658) cosmetic change in the description and boolean based --- .ci/jobs/elastic-agent-mbp.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/jobs/elastic-agent-mbp.yml b/.ci/jobs/elastic-agent-mbp.yml index f3772fd3855..8947d15880a 100644 --- a/.ci/jobs/elastic-agent-mbp.yml +++ b/.ci/jobs/elastic-agent-mbp.yml @@ -2,7 +2,7 @@ - job: name: "elastic-agent/elastic-agent-mbp" display-name: elastic-agent - description: "POC to isolate elastic agent from beats" + description: "Elastic agent" project-type: multibranch script-path: .ci/Jenkinsfile scm: @@ -12,6 +12,7 @@ discover-pr-forks-trust: permission discover-pr-origin: merge-current discover-tags: true + head-filter-regex: '(main|7\.17|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' notification-context: 'fleet-ci' repo: elastic-agent repo-owner: elastic @@ -39,4 +40,4 @@ timeout: 100 timeout: '15' use-author: true - wipe-workspace: 'True' + wipe-workspace: true From f8ca007e224bc9caa1434cd293bac1e2efddf38b Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Mon, 4 Jul 2022 11:49:44 +0200 Subject: [PATCH 33/69] Update elastic-agent-project-board.yml (#649) --- .github/workflows/elastic-agent-project-board.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elastic-agent-project-board.yml b/.github/workflows/elastic-agent-project-board.yml index 1b296620b09..e6add0d093c 100644 --- a/.github/workflows/elastic-agent-project-board.yml +++ b/.github/workflows/elastic-agent-project-board.yml @@ -14,7 +14,7 @@ jobs: with: headers: '{"GraphQL-Features": "projects_next_graphql"}' query: | - mutation add_to_project($projectid:String!,$contentid:String!) { + mutation add_to_project($projectid:[ID!]!,$contentid:ID!) { updateIssue(input: {id:$contentid, projectIds:$projectid}) { clientMutationId } From 42b1a9654ed82c8e2ecf49a961a270e1440c4b53 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Jul 2022 12:24:58 +0100 Subject: [PATCH 34/69] ci: fix labels that clashes with the Orka workers (#659) --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 10c6c1a4347..9454f8d2cc6 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -79,7 +79,7 @@ pipeline { axes { axis { name 'PLATFORM' - values 'ubuntu-20.04 && immutable', 'aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable', 'darwin && orka && x86_64' + values 'ubuntu-20.04 && immutable', 'aws && aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable', 'darwin && orka && x86_64' } } stages { @@ -164,7 +164,7 @@ pipeline { } } environment { - ARCH = "${PLATFORM.equals('aarch64') ? 'arm64' : 'amd64'}" + ARCH = "${PLATFORM.contains('aarch64') ? 'arm64' : 'amd64'}" DEV = true EXTERNAL = true } From f873f3683bb3cfe3985bf2399ed1373e0a0570aa Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 5 Jul 2022 01:33:54 -0400 Subject: [PATCH 35/69] [Automation] Update elastic stack version to 8.4.0-03bd6f3f for testing (#668) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 184c29a188d..3f14fe74810 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3ad82aa8-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-03bd6f3f-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-3ad82aa8-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-03bd6f3f-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From bbd93ff75ccb5680626a45946e28787ef590ca63 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 6 Jul 2022 01:39:21 -0400 Subject: [PATCH 36/69] [Automation] Update elastic stack version to 8.4.0-533f1e30 for testing (#675) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3f14fe74810..9c988fba8ed 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-03bd6f3f-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-533f1e30-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-03bd6f3f-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-533f1e30-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 6c2d91aca140d6ed6a91a19c426a8676036e40d1 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Wed, 6 Jul 2022 15:38:07 -0400 Subject: [PATCH 37/69] Osquerybeat: Fix osquerybeat is not running with logstash output (#674) --- internal/pkg/agent/program/supported.go | 2 +- internal/spec/osquerybeat.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/pkg/agent/program/supported.go b/internal/pkg/agent/program/supported.go index 50140f27617..fa35f4a4fe3 100644 --- a/internal/pkg/agent/program/supported.go +++ b/internal/pkg/agent/program/supported.go @@ -27,7 +27,7 @@ func init() { // internal/spec/metricbeat.yml // internal/spec/osquerybeat.yml // internal/spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzce1mToziX9v33M/r2m5lmSWc3E/FeGLLY7CTLuBIJ3SHJCdgCu9PGNp6Y/z4hsRgwuVR29ztvzEVGVGGh5egsz3nO4b9+SfPD6jWP2K/73Yr8Gu2yf9+vXo+r1/8oM/bLf/6CM/OAfmzjRaB788BjJEeMxLs1Bot7xzJPeClfEHQVBJ1ZCF0pAigJ1dHfcnLZxuC0jR3DOfhLZ+8Y7iEEkwQpwQGBiTTPgiIE7h6BhUZtV0ZLZ2+k09hJZdNJT7GT0TVUdUYyj+F8obn2QX/+Jv/wAxf4gfviS5q9uGzPjw+65sQ7amTBHbG0klrBBioyo7a7C9XHe8fczxxjmoZQP8xhfabU2RtMmpE82CP4eM/XnS/1NVb1CVT9I1TOO6IuxHPHmMaOxSQEpHvHQnsEAql9bvvHp1Tf4VyXqf04E8+MaYyVyUuoaAXKzrtKPpMjVqf894NjyQl52LZjiWVK0cM2RtmZIbi4Pu/srXk2X+olAvKRZsFLpASTp3jb/lb96a8Ibvh9rEMluBBZS4jFxNgvzWO7rJIpK9CpO0aKSRYcsIoYVA5s9eN6nuZPzJvq/L4LOt2Kd1DG7qDqSSQLEvxjG69UqZYJ2mHbZ4RpSgjOcu/ctsewFayppZVjsq7XkVZQZ9d3UILtgJFLb18HoaeLdi97agXl9ez6BYEzC1X/SPIbud+sW82nydTW5ep8V9l07vLgWKyIsmBNTW2LgLlB0L08pfpvL4udGllB8ZTqewQmObXirWsf6nU8bbac/n/nYRqHYLJxrCQh0oGtlvFmpdRr2tLeMSjDlnmhFlsTJUhI5m3d8hS7qsuQxS5ueeJ7yCPFzCLlWz43pjm2tJyofkKUOJ8ttv/45d+GXqGg6QGvosPQKcBgE0E/RcCUjMw7oIdtHPaemSXqGNwcPKZzYXjXMfMs2IfQkyLweAgB5YZdrIC8d9JT9c7zs3iHKIFE4bSIwPnQdRQoM/dEeU7n055juSDoy8SYSAjIJ2yZElpOGM7MFFvB5jvgyuCx4RoYmIXYAwgKanTGQ7+k4GaNPVZoHoFJPs/OjGbB/jvwWZgH+XBepHgvxArKkCv+g7RZSWbgf2PPi02gB6Zm/5Dow9P62+nRloZrJBT4O2q5LISLonIgLJ9n8hHZwhkUCEySkCvXg5yF4HxBS2cGF7uE5P4OZeaa8jvJgoTaj8fe3eQuE5ee+y/cOWE1kIgdSFDxtiGY5Px+hRNebDtOcdxgjfQxXoDz3vlmyshikljPqA3EqAyRlHqKAN3hVFcj6G8dg1ZnM37P5/F2xOC9F6owKTK1EgHKVva0NsDGEfkTYj1Xxme77Cmd5jCr1p+Xd69u7ZyIEuwR8CSsOvfc8PgZyWk7c0pdR5Z/cQy6w5nPVg/bWBhCKf+GLFMKA+1CbZeFQCrE/6HHqC0dVoqfUMt8IapfImAe5tkkwSC4EMtcIyjlPAA4VpCEShxHYHKicFFwuUVg8gffB1RYgazgrnEy1GYnLmuxdndftq/iUk+Q5b+gjDEsnuklVjxGVI87JO6Ejjh7jlGmlY4VKFUQFfu7ILioHYVWoKWeYdXhc29C6Cft/S0n4v/ctubGzW/i3uaGnuBsEQ9lInwA9E4h8Fh1f61Tq8fIL8R2jyLoKFrrVG8DISuIEpTU1BKU+4y8LZcHrEwUBF2puUcgS0VzD0Q+UP6cWr/Xcvcv/HehszCRCLcFU9tHUNwB9wuXZk9NwG3eq/VsR63g8JTqzXma5xK+dMaKs3cCuK0n1Iq7ujYWvNfOt+r3So+Di2OhI0l1vWPv3G7zCNzFwg6MaV7Z7+LolnexDxEjOZMiHoi5nnDZpboUWSaXz6VdxxbykRBYxCF8jKmVMMeq/cdSL0Ig8/trghqXU0GUc0Kt4A3764MUcU+LTrC7scW4mJWbvPuumPdhG7sP32Yd8CD20gMeFiqxIvVt3bjeAQbahoIz68g0c6xvcWuvhr4jpX4hVrCOANoJGShaQbOg5H6plpXQGVdp7MbnwRi7AuDJ1C3v2j1iYL6iQEtI7iZ9gNPqSmtPvfNav9871nX++rcDlxUGmoICTfi07js1iBnO95l32rj1s+u0tr/YfgwKLQHyhDz5PUdAK9p7MF1GYLAj2XMc8rhgeUecoR0qdQmX+horMnNsf0syTUZKXOnkNzPn/ssxkgKX+h4rXoINPYvAmZFyU/zEWaq4Ymol1w2ooB22gtpO3wKVHaC3vvEDNYCubaaWDc3MPQXBpSMTDiaFzqNp95lY/yUCHIvQMoJ+B3hLMVHZheuOEW/Xzjdzg+zHmNr06FjmBplaFoFgz591fSkpBdBs/Hrc4KYIjPr8FhBCpZLxPGObuTHyPPeKEOqXyNKk8d+5z/F2WLkbxLTqdxHbst/vHfugVfL2jsRu7kjPSKYdbgB7lTB1cc6Mzz30Byh3jxxv9LFj38dWdtZio1mNV3qg/ooH3Pbu5sv++q2915io1T2u81C8y6r7qmOeobdAvvEnlZ+tgbnBdX6SRYDKJDOFrQxi5iAR2M6okjC85vEvKJDqb2eG/1v/7M05/1RiIJIBkvF5H0Uy8FOJAWHbgo4lBsr5SMEgMeg9GyQG/1Igf0c/D44HQLt7xngnHFCdcQpHsGrBkVQZSVpnktmOOx/uJDYIxp1MVwS+e8c6H5HKFbo/vgE/UO0Aw/X2HdZhHGw5NUjnmSdWKHeWAohi69S7ywbUXEGLy5DCCsemO2rFsat4e6wGmyoBIEW9X+7EJQTdl8ZpQpWPT15IFuQIJqdOxn0L7rnRZs/3jkFXXfnO023+ESMwlFflKIWcMw4gquSlu0dnlGW4ZQ36jufGYcT/lEw+dpVDgrJDUv2b64jHgc/WLTezG2Nd5XS3TfOhrfpgsiGWtsP5In5WgjWF7o7am1moyJsKTPgnrLCCGvIFAU8mGZNWA71H1zkOyAoUIxes2i5Unu+dh1B9eohnIfCE4+NBQtiSWunQk3CEQYFs/ciDpJGdj0jWTiH0t5XcOThx1Qjc3TuGc/xhsZRkZrlaamYjn7l0fX8uZOCzOZdHqXX2L/0x53OXDp9zH4GJzJM+J9WOxF4cfXBOiOrvwlIzr+9oFyr8irbHCjl2zzlLJ/xZyvWFiqROU8llGzubx3tonhck03KSmQfnWwVAoHlu9yv+3axhngn3NdQKCOR2Lp/J6DqZt0XAexXyU/0EW6f7OrlhoawJoDRMaJysIxfosVANBABx6nE1K9sGPIcH+Ixlq6VzfZZKBx4InDZYTlOi+jxYlc0zarEDAprMdeHxMp0RS7tQk+/fkzgArO/4DgGPJ7N79GOQLBnuuJ41+7CqeNEGYsNt5+7ua76U2zupx12o5TOSO51nzmEOgxNS3QRZz4PnLiOKJpPMY6TsyOANOfbHT+4jOE2bJDECMuP+4CmdKo8P0xmxXQbVoIjAhOvUHj9sZ/OlzlZWsK5A6nMDlITuP6XTtKsH5GqbzRoJyWgfgBqujLNWP9Iu0Bze47h8Rvb9IflzBVlQHbCs7zG1dQLRTSCHoJrLp9WLqZDXTSy5BXbOTfI9nhjpQ2b5wJNlCp7FmTAwT0N76jHytitjq7fXD5MlEXuzoOwlI+tbWXVtsl8FkAbJyS3jLPYB0Y4o7IjjcSBZzekP4tCZ4YxKkcHjUC0/Vdo5D3fxY00CRZZ5WSrBhM/RxLqXJY89DYnrXRAwy1CJRwDkjoNgjn8SbkfcV+JMkxy+nurKOPd3GDwXIXTXkS3F339IsauYJf4RSm5Zre/ah5KCidDReYYSDNh+BeuxIi4mHHdU5zH830geFMI3LSeHEOyOJK/HXkg+W05vY+RLylYjeNbnPgtUyUEdF4W/FThnuqvsMNVxr1qVe4zawWmesT3+BC5tsC9PBol1Tqj1PEbuDipicoIznnzJIgHsjJdIHtxiX0vLOV5D5WSPIGL4Qd4g4Mqo/LDSZi2fz+YnCOurXIxJnfSQoiFqSDkiB1GpM0tqsYxjhKdUF/iOlFpXxiWCPK67ElQ4hqh13tTWkWUWiOOM2nY/8ldv4+0bDPy3k5rzpX4I4XTg94cJ8o2fqaqSirnHpiZhuSI1e/6mSqoHxOWHlcgGg7OV5TFiL3h+lrYEaynut72/Ssd7hHQKF4M9jlVOuT+GPuP30SM1Tl/dfyvrFAF0JNnzuwTCgNzpkbntWU91bLb9MhQEBz+D/yL2fSWNBvflDYniv5NAFrEqBJOLyHv7ZGSzr34FlMsOmCfSq3DWz7lfqHCZ+Hf3Ha4PNOO5co0xhX2QfnW3KT6onoQsVogzW/4OD9YRPidfHKnlnfj+wp7edecJNuRhG1PgnwZ7qXC4pa0jJeCYYoMV7xVBZzCPwP4lAv6OyNoFW5rKz/WU6tWz0+3556o3ITxnuGwFKdeRw2dIR5kowcsNYfQ3Eq/vFWS+vH6nIPSnCdNegW5YfZfiENLLHFR8AVVMKVTi/m/wsYNxzHsKXTYHlX5EdnDF7VY1B8o0GWd+uer6IGUihYAVV/2Q4kZ3ru/TNTHkEwXuhYLruMj2JWJqXfzZzLultn+Kcu+Irz75D6wGJc7MPbqOew0Beg2v++nYjRRjaEqdOQsE/Yzr6XVfv1+8dVARk11s19Hf+bJvZ/z/K+j11uG2dtWHYEOuc/Fc4e76G91hnh9f567I99on/nkCveXz3tSJpuD8MWcjxaGindoiTFMossYIYZF/15zcz+DuK+b7JCH804RuZ42ikdnLchN/T6cnh+MaQ9+G0JsjuNm69uFIoc/HaBxbo4avUD0mMLNB9o5BK/0oicBzP8UPvbDV6jDe/eVXOUj83OQvNadb5yaHNq+58ro8F686fQz5gBWfOUwadIhVnVNtB9Zn+NK436lzk29+kAe+xS+OFHO6udehk7f9NesPbeGdPTQ28a4/r2PClX+s9tnsBSqiKPEWtxmDUs+wFTBqTJqOvaKZ69qg0+Z2MWwLvi3WqHVz0fCpjT8QhS88Kh9hG7jVg7zpxJucsMKx3UYUtEbWavBx8Wi0Y69cr5jHf0FWkIUw2FN7vHPsltO92ccWq5406Pq6kZPohhvv9ioavZlnXP5VfICqfiT54qO1L0Q53XQENj5ivp4WQ53s8hUdWVX7bfend22su37fJ3b+xgpm/T+pKUDevFvNnRyJ2upFJ995G+u+uc938EzXDij0WJsjTr/UpTiKy/7UHMuJyEeIqieh8vylc73ZmPS1M17zkvhLHZAD3ZqO8VPN2KF/uObgVpAgKxB+SMT2nG4RuLvv81GVfQxi419cWxmJickqeh3rfFxaQULyfoEz6j3rBMLPFje/QAb9TCv1pzoXb4I0Byf+Za4kHGRIXydjakAJk7ZoWQOj8YAqa6JLEKouQ0pw122PHm/vdXliuiIqd7QJExdb/l7MTiOKPACo75HK7wPbjmE2d3N1tLdty02HVR6wzxQm33LMbzvkj42v75g74HXU6P4aErjtLhCyoVVStBRgVRBt1DIvkUF2RvyP2w6DbHV4TcmIBf4AgUQytq41sv4goW7yV2qqdvyjg6ZtYIct6WOKtaFxc59hqO9FafHj1oSf/LDhfEQKT79IgQUFctKQFaQUkOG8eShrJwTdNZ/3+9L/7cdz8Py8YQ+foGYPCPplBLy6f6TpyZpweC4oxrFzGW96hL6sSRbwmyypqR0x05pU5SVUkgRnlFtnpfl5S2e+0RLxiV7Z+u5rmu5z7RAW91TizPuooplbqBgCJLUUQdty0X4w8lfRie9QvTfU4QD+1OluoCXYOr9QS3vBFrvQh+1NT2RnzQ58uDl7gRWtR60hmKwR1CUByfMassCqXz8Ci6qPvbEVQ+jTiWTaGkHvwj0RjxyDvY5T082HM/1e5q+e43qHWZBd0/DgjtpuwnVCyMnUBBV7LUsOYKfannfW3+Pg+eDjntHS51uRa6TcKdpazLtiVmqNbV7c6ft96//bJdCBvt/oKZc7zj3hJ99u8+ncwW1vb8ltHKbxr88PZxHlv6d3r7PlrYyqeUTr0b1j+F1EUEHqKlZ05256EG+o8V7v4JWyO4TgwKBiliQzJ6N63PqJAQyvdKXd80/2/XbeW8w+hvVv0Qqfeaf/PcNfQh1/fY6RFOkTZ7CDCwq0Torxl/am9uKmeHeoWy392IuJxVhsaEuhPVt2bmnUvGOTA1zTyOVdOrIng4SFwGd8zur8o+1pA3zwL9Bfut3/UaxeyzH4p3pnCoJy1a/KH4lqygi6k2Fl/ieq8j8P/X6mu/RT1Xj36x2o9adeeCifd7vvtAuBASP5Zva1LrPmfVZ1iX62u8z2dxgERwoX987Dt1O3Y/aNZLHLsDefbQ1gW9Mp21H/xr0EWgeytCbH4aroVMAWEzKMgGD3uOvIeGh/uzvg76uOtff3yWrIJ9vcB9+s/jM6VjezUNHEZ1ohYAW1H+8HnTli3MtymnxfTvPHS6y4pxFXsIvIZjVGxTxb5jpSAqmXCNo8gBxY09DfJoIlOfhV6vlBEsjH3Ix9t89c1FhK2axqLe/35fTHvpn85W99yEn6Z/46HfMnaY/+x19vUh6nEHivaIT7+z/Wj/3Rl9WzX/77//1PAAAA//9rUQ6t") + unpacked := packer.MustUnpack("eJzce0mTqziX9v77GbX9ursY0llFR7wLQyaTnWQZXyOhHZKcYFtgV9rYxh393zskBgMmx7r19hu9yIh7sdBwdIbnPOfwX7+sssPyNYvYr/vdkvwa7dJ/3y9fj8vX/yhS9st//oJT84B+bONZoHvTwGMkQ4zEuzUGs3vHMk94Ll8QdBUEnUkIXSkCKAnVwd8yctnG4LSNHcM5+HNn7xjuIQSjBCnBAYGRNE2DPATuHoGZRm1XRnNnb6zGsbOSTWd1ip2UrqGqM5J6DGczzbUP+uJR/uEHLvAD98WXNHt22Z6fHnTNiXfUSIM7YmkFtYINVGRGbXcXqk/3jrmfOMZ4FUL9MIXVmVbO3mDShGTBHsGne77udK6vsaqPoOofoXLeEXUmnjvGOHYsJiEg3TsW2iMQSM1z2z8+r/QdznSZ2k8T8cwYx1gZvYSKlqP0vCvlMzpidcx/PziWnJCHbTOWWKYUPWxjlJ4ZgrPr89be6mfTuV4gIB9pGrxESjB6jrfNb+Wf/orght/HOlSCC5G1hFhMjP3WPLbLSpmyHJ3aY6SYpMEBq4hB5cCWP67nqf/EvCud33dOx1vxDkrZHVQ9iaRBgn9s46UqVTJBO2z7jDBNCcFZ7pzb9hi2gjW1tGJI1tU60hLq7PoOSrAdMHLp7Osg9HTW7GVPraC4nl2/IHBmoeofSXYj95t1y/k0mdq6XJ7vKpvWXR4ci+VRGqypqW0RMDcIupfnlf7by2ynRlaQP6/0PQKjjFrx1rUP1TqeNpmP/7/zMI5DMNo4VpIQ6cCW83izVKo1bWnvGJRhy7xQi62JEiQk9bZucYpd1WXIYhe3OPE9ZJFippHymE2NcYYtLSOqnxAlziaz7T9++be+V8jp6oCX0aHvFGCwiaC/QsCUjNQ7oIdtHHaemQVqGdwUPK2mwvCuY6ZpsA+hJ0Xg6RACyg07XwJ576xO5TuLhXiHKIFE4TiPwPnQdhQoNfdEWaym445juSDoy8QYSQjIJ2yZEpqPGE7NFbaCzR+AK4PH+mtgYOZiDyDIqdEaD/2Cgps19lihWQRG2TQ9M5oG+z+Az8IsyPrzIsV7IVZQhFzxH6TNUjID/5EtZptAD0zN/iHRh+f14+nJlvprJBT4O2q5LISzvHQgLJum8hHZwhnkCIySkCvXg5yG4HxBc2cCZ7uEZP4Opeaa8jtJg4TaT8fO3WQuE5ee+S/cOWE1kIgdSFDxtiEYZfx+hROebVtOcdhgjdVTPAPnvfNoyshikljPqAzEKA2RFPoKAbrDK12NoL91DFqezfg9m8bbAYP3XqjCpMjUCgQoW9rjygBrR+SPiLUojc922fNqnMG0XH9a3L26lXMiSrBHwJOw6txzw+NnJKftxCl0HVn+xTHoDqc+Wz5sY2EIhfwbskwpDLQLtV0WAikX/4ceo7Z0WCp+Qi3zhah+gYB5mKajBIPgQixzjaCU8QDgWEESKnEcgdGJwlnO5RaB0Z98H1BhObKCu9rJUJuduKzF2u192b6KCz1Blv+CUsaweKYXWPEYUT3ukLgTOuJ0EaNUKxwrUMogKvZ3QXBWOQotR3M9xarD596E0E+a+5uPxP+5bU2Nm9/EvU0NPcHpLO7LRPgA6J1C4LHy/hqnVo2RX4jtHkXQUbTGqd4GQpYTJSioqSUo8xl5Wy4PWBkpCLpSfY9AlvL6Hoh8oPw5tX6v5O5f+O9CZ2EiEW4LpraPoLgD7hcu9Z7qgFu/V+nZjlrB4Xml1+epn0v40horzt4K4LaeUCtu69pQ8F47j+XvpR4HF8dCR7LS9Za9c7vNInAXCzswxllpv7OjW9zFPkSMZEyKeCDmesJlt9KlyDK5fC7NOraQj4TALA7hU0ythDlW5T/meh4Cmd9fHdS4nHKinBNqBW/YXxekiHuatYLdjS3G+aTYZO13xbwP29h9eJy0wIPYSwd4WKjAitS1deN6BxhoGwrOrCXT1LEe48ZeDX1HCv1CrGAdAbQTMlC0nKZBwf1SJSuhM65S243PgzF2BcCTqVvcNXvEwHxFgZaQzE26AKfRlcaeOue1fr93rOv81W8HLisMNAUFmvBp7XcqENOf7zPvNHHrq+s0tj/bfgwKLQHyhDz5PUdAy5t7MF1GYLAj6SIOeVywvCNO0Q4VuoQLfY0VmTm2vyWpJiMlLnXy0cy4/3KMJMeFvseKl2BDTyNwZqTY5F84SxlXTK3gugEVtMNWUNnpW6CyBfTWN36gAtCVzVSyoam5pyC4tGTCwaTQeTRuPxPrv0SAYxFaRNBvAW8pJiq7cN0x4u3aeTQ3yH6KqU2PjmVukKmlEQj2/Fnbl5JCAM3ar8c1borAoM9vACFUShlPU7aZGgPPMy8PoX6JLE0a/p37HG+HlbteTCt/F7Et/f3esQ9aKW/vSOz6jvSUpNrhBrCXCVMb50z43H1/gDL3yPFGFzt2fWxpZw02mlR4pQPqr3jAbe5uOu+u39h7hYka3eM6D8W7rLyvKuYZegPka39S+tkKmBtc50dpBKhMUlPYSi9m9hKB7YQqCcNrHv+CHKn+dmL4v3XPXp/zLyUGIhkgKZ/3SSQDX0oMCNvmdCgxUM5HCnqJQedZLzH4lwL5O/p5cNwD2u0zxjvhgKqMUziCZQOOpNJIVlUmme648+FOYoNg3Mp0ReC7d6zzEalcobvja/AD1RYwXG/fYR2GwZZTgXSeeWKFcmcpgCi2Tp27rEHNFbS4DCksd2y6o1Ycu4q3x2qwKRMAklf75U5cQtB9qZ0mVPn45IWkQYZgcmpl3Lfgnhtturh3DLpsy3e62mYfMQJ9eZWOUsg55QCiTF7ae3QGWYZb1qDreG4cRvxPyeRjVzkkKD0k5b+5jngc+GzdYjO5MdZlRnfbVda3VR+MNsTSdjibxQslWFPo7qi9mYSKvCnBhH/CCsupIV8Q8GSSMmnZ03t0neOArEAxMsGq7UJlce88hOrzQzwJgSccHw8SwpbUUoeehSMMcmTrRx4kjfR8RLJ2CqG/LeXOwYmrRuDu3jGc4w+LrUhqFsu5ZtbymUrX96dCBj6bcnkUWmv/0p9TPnfh8Dn3ERjJPOlzVtqR2LOjD84JUf1dWGjm9R3tQoVf0fZYIcf2OSerEX+24vpCRVKnqeSyjZ3N0z00zzOSahlJzYPzWAIQaJ6b/Yp/12uYZ8J9DbUCArmdy2cyuE7qbRHwXoX8VD/B1um+Sm5YKGsCKPUTGidtyQV6LFQDAUCcalzFyjYBz+EBPmXpcu5cn62kAw8EThMsxyui+jxYFfUzarEDAprMdeHpMp4QS7tQk+/fkzgArO74DgGPJ7N79KOXLBnusJ7V+7DKeNEEYsNt5m7vazqXmzupxl2o5TOSOa1nzmEKgxNS3QRZi95zlxFFk0nqMVK0ZPCGHLvjR/cRHK/qJDECMuP+4Hk1Vp4exhNiuwyqQR6BEdepPX7YTqZznS2tYF2C1EUNlITuP6/Gq7YekKtt1mskJKVdAGq4Mk4b/Vi1gWb/HoflM7DvD8mfK8iCao9lfY+prRKIdgLZB9VcPo1ejIW8bmLJLbBzbpLv4cRI7zPLB54sU7AQZ8LAPPXtqcPI266Mrc5eP0yWROxNg6KTjKxvZdW2yW4VQOolJ7eMs9gHRDuisCOOh4FkOaffi0NnhlMqRQaPQ5X8VGnnPNzFTxUJFFnmZa4EIz5HHete5jz21CSud0HALEIlHgCQOw6COf5JuB1xX4lTTXL4eqor48zfYbDIQ+iuI1uK//ghxa5iFvhHKLlFub5rHwoKRkJHpylKMGD7JazGiriYcNxRnsfwfyNZkAvfNB8dQrA7kqwaeyHZZD6+jZEvK7YcwLM+91mgTA6quCj8rcA5411physdd6pVmceoHZymKdvjT+DSGvvyZJBY54RaiyFyt1cRkxOc8uRLFglga7xEsuAW+1paxvEaKkZ7BBHDD/IGAVdGxYeVNmu+OJufIKyvcjFGVdJD8pqoIcWAHESlziyoxVKOEZ5XusB3pNDaMi4Q5HHdlaDCMUSl86a2jiwzRxxnVLb7kb96G2/fYOC/ndSczvVDCMc9v99PkG/8TFmVVMw9NjUJyyWp2fE3ZVLdIy4/rETWGJwtLY8Re8bzs1VDsBbifpv7K3W8Q0iv4Ky3x6HKKffH0Gf8Pjqkxum7+29kvUIAHUm6eJdA6JE7HTK3Oeupis22X4SC4OBn8F/Evq+kUe++vD5R/HcSyCJWhWB0EXlvl4ys99WtgHLZAfNEOhXO6jn3CyUuE/9uv8P1gaY8V64wprAP0q3u1sUH1ZOQxXJxZsvf4d46wudksyO1vBPfX9jRu/Y8wYY8bGMK/FNvLyUOt7R1pAQcU2yw4r0i6PTmEdi/QMDfEVm7YEtT+bmeV3r57HR7/qnqjQjPGS5bQcq15PAZ0lEmSvByQxj9jcTrewWZb6/fKgj9ZcK0U6DrV9+lOIT0MgUlX0AVUwqVuPsbfGphHPOeQpdNQakfkR1ccbtVzoFSTcapXyzbPkgZSSFg+VU/pLjWnev7dE0M+USBe6HgOi6yfYmYWht/1vNuqe2fosw74qtP/hOrQYFTc4+u415DgF7D635adiPFGJpSa84cQT/lenrd1+8Xbx2UxGQb27X0dzrv2hn//xJ6nXW4rV31IdiQ61w8V7i7/kZ3mOfH17lL8r3yiX+dQG/4vDd1oi44f8zZSHGoaKemCFMXiqwhQljk3xUn9xXcfcV8nySEv0zottbIa5m9zDfxH6vxyeG4xtC3IfSmCG62rn04UujzMRrH1qjmK1SPCcxskL1j0FI/CiLw3Jf4oRe2XB6Gu7/8MgeJF3X+UnG6VW5yaPKaK6/Lc/Gy08eQD1jxmcOkXodY2TnVdGB9hi+Nu506N/nmB3ngW/ziQDGnnXsdWnnbz1m/bwvv7KG2iXf9eRUTrvxjuc96L1ARRYm3uM0YFHqKrYBRY1R37OX1XNcGnSa3i2FT8G2wRqWbs5pPrf2BKHzhQfkI28CNHmR1J97ohBWO7TaioDWwVo2P8yejGXvlesU8/guygjSEwZ7aw51jt5zuzT62WPWkXtfXjZxEN9xwt1de68005fIv4wNU9SPJZh+tfSHK6aYjsPYR0/U47+tkm69oyarcb7M/vW1j7fW7PrH1N1Qw6/5JdQHy5t1y7uRI1EYvWvnO21j3zX2+g2fadkChx5occfytLsVBXPaX5piPRD5CVD0JlcW3zvVmY9L3znjNS+JvdUD2dGs8xE/VY/v+4ZqDW0GCrED4IRHbM7pF4O6+y0eV9tGLjT+5tjIQE5Nl9DrU+Ti3goRk3QJn1HnWCoSfLW5+gwz6Siv1pzoXb4I0Byf+ZaokHGRI3ydjKkAJk6ZoWQGj4YAqa6JLEKouQ0pw126PHm7vdXliuiQqd7QJExdb/J5PTgOK3AOo75HK7wPblmHWd3N1tLdty3WHVRawzxQm33LMbzvkj42v65hb4HXQ6H4OCdx0FwjZ0DIpmguwKog2apmXyCA7I/7HbYdBujy8rsiABf4AgURStq40svogoWryVyqqdvijg7ptYIct6WOKtaZxM59hqO9FafHj1oQvfthwPiKFp18kx4ICOWnIClYUkP68WShrJwTdNZ/3j7n/249FsFhs2MMnqNkDgn4RAa/qH6l7skYcnguKcehcxpseoStrkgb8JgtqakfMtDpVeQmVJMEp5dZZan7W0JlvtER8ole2uvuKpvtcO4TFPZU48z4qaeYGKoYASQ1F0LRcNB+M/Cw68R2q94Y67MGfKt0NtARb5xdqaS/YYhf6sL3piWyt2YIPN2fPsaJ1qDUEkzWCuiQgeVZBFlj260dgVvax17ZiCH06kVRbI+hduCfikaO312Fquv5wptvL/N1zXO8wDdJrGh7cUdtNuE4IOZmaoGKvZcke7FSb8066e+w9733cM1j6fCtyDZQ7RVuLeZdPCq22zYs7fr9v/X+7BNrT9xs95XLHmSf85NttPq07uO3tLbiNw1X86+LhLKL8H6u718n8VkblPKL16N4x/DYiKCF1GSvac9c9iDfUeKd38ErZHUJwYFAxC5Kao0E9bvxED4aXutLs+Yt9v633ZpOPYf1btMJn3ul+z/BTqOPvzzGQIn3iDHZwQYHWSjF+am9qJ26Kd/u61dCPnZiYD8WGphTasWXnlkbNWjbZwzW1XN6lIzsySFgIfMbnLM8/2J7Wwwf/Av2l2/2f+fK1GIJ/qnemICiW3ar8kaimjKA76lfmv1CV/zr0+0p36aeq8e73O1CrT71wXz7vdt9pFwIDRrLN5HtdZvX7rOwS/Wx3me3vMAiOFM7unYfHU7tj9o1ksc2w159t9WBb3SnbUv/avQRaC7I0JsfhquhUwBYTMoyAYPe460h5aH+7O+Dvq4419/fJasgn29x736z+0ztWP/j29LZDqNvxw5PKRHOMR80xyOX5IcwGk8ZdRDbLIdpmYZnrSAmkTtJo82BzYHXzf5M0FuTgl2nqBwkjH3Mz9t2edCGVQjbLusz7PTzdsW8mitlbH32S7pm/T938RYqk+6HYm/TIKQTeKxrgCf+P9W5/9BX25Jf//n//EwAA//82Bxyl") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/internal/spec/osquerybeat.yml b/internal/spec/osquerybeat.yml index 36e60901a34..6a7f3be562e 100644 --- a/internal/spec/osquerybeat.yml +++ b/internal/spec/osquerybeat.yml @@ -35,5 +35,6 @@ rules: - inputs - output -when: length(${inputs}) > 0 and hasKey(${output}, 'elasticsearch') +when: length(${inputs}) > 0 and hasKey(${output}, 'elasticsearch', 'redis', + 'kafka', 'logstash') constraints: ${runtime.arch} != '386' From a18cfadcd38fef27be0307ae5307b562ceed44fc Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 7 Jul 2022 01:41:51 -0400 Subject: [PATCH 38/69] [Automation] Update elastic stack version to 8.4.0-d0a4da44 for testing (#684) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 9c988fba8ed..f2f3b3b330f 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-533f1e30-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-d0a4da44-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-533f1e30-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-d0a4da44-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From fe7881952826fa19c54e6eb32a3aa4ae0925e58e Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 11 Jul 2022 01:37:29 -0400 Subject: [PATCH 39/69] [Automation] Update elastic stack version to 8.4.0-dd98ded4 for testing (#703) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f2f3b3b330f..bc05ed04fa0 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-d0a4da44-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-dd98ded4-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-d0a4da44-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-dd98ded4-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From d1d56a9d46751b9fcbbf2b77cf3eb0349032fc6a Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 12 Jul 2022 01:49:59 -0400 Subject: [PATCH 40/69] [Automation] Update elastic stack version to 8.4.0-164d9a10 for testing (#705) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index bc05ed04fa0..218ea14d077 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-dd98ded4-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-164d9a10-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-dd98ded4-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-164d9a10-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From bda217dd48e1cc6ca1d2b338ec67dd495d7aa769 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:49:39 -0700 Subject: [PATCH 41/69] Add missing license headers (#711) --- internal/pkg/core/status/handler.go | 4 ++++ internal/pkg/testutils/status_reporter.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/internal/pkg/core/status/handler.go b/internal/pkg/core/status/handler.go index 2e7476901c5..e82f73fb216 100644 --- a/internal/pkg/core/status/handler.go +++ b/internal/pkg/core/status/handler.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package status import ( diff --git a/internal/pkg/testutils/status_reporter.go b/internal/pkg/testutils/status_reporter.go index a045e50304a..45448aa53b2 100644 --- a/internal/pkg/testutils/status_reporter.go +++ b/internal/pkg/testutils/status_reporter.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package testutils import ( From be332390064f924c00b5668785aea81b22af6e70 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 13 Jul 2022 01:48:41 -0400 Subject: [PATCH 42/69] [Automation] Update elastic stack version to 8.4.0-00048b66 for testing (#713) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 218ea14d077..0d7fb81c508 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-164d9a10-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-00048b66-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-164d9a10-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-00048b66-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 28721ebe888b377c36b43c7dbf94978e3d16ba9c Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:29:18 -0700 Subject: [PATCH 43/69] Allow - in eql variable names (#710) * fix to allow dashes in variable names in EQL expressions extend eql to allow the '-' char to appear in variable names, i.e., ${data.some-var} and additional test cases to eql, the transpiler, and the k8s provider to verify this works. Note that the bug was caused by the EQL limitation, the otehr test cases were added when attempting to find it. * Regenerate grammer with antlr 4.7.1, add CHANGELOG * Fix linter issue * Fix typo --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/transpiler/vars_test.go | 11 +- .../providers/kubernetes/node_test.go | 14 +- .../providers/kubernetes/pod_test.go | 34 ++-- .../providers/kubernetes/service_test.go | 12 +- internal/pkg/eql/Eql.g4 | 2 +- internal/pkg/eql/eql_test.go | 21 ++- internal/pkg/eql/parser/EqlLexer.interp | 2 +- internal/pkg/eql/parser/eql_lexer.go | 155 +++++++++--------- 9 files changed, 143 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 602b0293cc6..c9e71a035b7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,6 +107,7 @@ - Collects stdout and stderr of applications run as a process and logs them. {issue}[88] - Remove VerificationMode option to empty string. Default value is `full`. {issue}[184] - diagnostics collect file mod times are set. {pull}570[570] +- Allow the - char to appear as part of variable names in eql expressions. {issue}709[709] {pull}710[710] ==== New features diff --git a/internal/pkg/agent/transpiler/vars_test.go b/internal/pkg/agent/transpiler/vars_test.go index 5dd6d41ec72..8171a4b7a9f 100644 --- a/internal/pkg/agent/transpiler/vars_test.go +++ b/internal/pkg/agent/transpiler/vars_test.go @@ -17,8 +17,9 @@ import ( func TestVars_Replace(t *testing.T) { vars := mustMakeVars(map[string]interface{}{ "un-der_score": map[string]interface{}{ - "key1": "data1", - "key2": "data2", + "key1": "data1", + "key2": "data2", + "with-dash": "dash-value", "list": []string{ "array1", "array2", @@ -44,6 +45,12 @@ func TestVars_Replace(t *testing.T) { false, false, }, + { + "${un-der_score.with-dash}", + NewStrVal("dash-value"), + false, + false, + }, { "${un-der_score.missing}", NewStrVal(""), diff --git a/internal/pkg/composable/providers/kubernetes/node_test.go b/internal/pkg/composable/providers/kubernetes/node_test.go index 7d8abfcea4e..547702573c4 100644 --- a/internal/pkg/composable/providers/kubernetes/node_test.go +++ b/internal/pkg/composable/providers/kubernetes/node_test.go @@ -26,7 +26,8 @@ func TestGenerateNodeData(t *testing.T) { Name: "testnode", UID: types.UID(uid), Labels: map[string]string{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, Annotations: map[string]string{ "baz": "ban", @@ -54,7 +55,8 @@ func TestGenerateNodeData(t *testing.T) { "baz": "ban", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, } @@ -64,7 +66,10 @@ func TestGenerateNodeData(t *testing.T) { "name": "devcluster", "url": "8.8.8.8:9090"}, }, "kubernetes": mapstr.M{ - "labels": mapstr.M{"foo": "bar"}, + "labels": mapstr.M{ + "foo": "bar", + "with-dash": "dash-value", + }, "annotations": mapstr.M{"baz": "ban"}, "node": mapstr.M{ "ip": "node1", @@ -123,7 +128,8 @@ func (n *nodeMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOp "ip": "node1", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{ "baz": "ban", diff --git a/internal/pkg/composable/providers/kubernetes/pod_test.go b/internal/pkg/composable/providers/kubernetes/pod_test.go index 45fd78ac76c..feeba193472 100644 --- a/internal/pkg/composable/providers/kubernetes/pod_test.go +++ b/internal/pkg/composable/providers/kubernetes/pod_test.go @@ -27,7 +27,8 @@ func TestGeneratePodData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, Annotations: map[string]string{ "app": "production", @@ -59,7 +60,8 @@ func TestGeneratePodData(t *testing.T) { "nsa": "nsb", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{ "app": "production", @@ -74,7 +76,8 @@ func TestGeneratePodData(t *testing.T) { }, "kubernetes": mapstr.M{ "namespace": "testns", "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{"app": "production"}, "pod": mapstr.M{ @@ -119,7 +122,8 @@ func TestGenerateContainerPodData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, Annotations: map[string]string{ "app": "production", @@ -175,7 +179,8 @@ func TestGenerateContainerPodData(t *testing.T) { "app": "production", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, } @@ -191,7 +196,9 @@ func TestGenerateContainerPodData(t *testing.T) { }, "kubernetes": mapstr.M{ "namespace": "testns", "annotations": mapstr.M{"app": "production"}, - "labels": mapstr.M{"foo": "bar"}, + "labels": mapstr.M{"foo": "bar", + "with-dash": "dash-value", + }, "pod": mapstr.M{ "ip": "127.0.0.5", "name": "testpod", @@ -232,7 +239,8 @@ func TestEphemeralContainers(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, Annotations: map[string]string{ "app": "production", @@ -274,7 +282,8 @@ func TestEphemeralContainers(t *testing.T) { "ip": pod.Status.PodIP, }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "container": mapstr.M{ "id": "asdfghdeadbeef", @@ -300,8 +309,10 @@ func TestEphemeralContainers(t *testing.T) { "name": "devcluster", "url": "8.8.8.8:9090"}, }, "kubernetes": mapstr.M{ - "namespace": "testns", - "labels": mapstr.M{"foo": "bar"}, + "namespace": "testns", + "labels": mapstr.M{"foo": "bar", + "with-dash": "dash-value", + }, "annotations": mapstr.M{"app": "production"}, "pod": mapstr.M{ "ip": "127.0.0.5", @@ -383,7 +394,8 @@ func (p *podMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOpt "ip": k8sPod.Status.PodIP, }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{ "app": "production", diff --git a/internal/pkg/composable/providers/kubernetes/service_test.go b/internal/pkg/composable/providers/kubernetes/service_test.go index 47d420fb233..0fbed196908 100644 --- a/internal/pkg/composable/providers/kubernetes/service_test.go +++ b/internal/pkg/composable/providers/kubernetes/service_test.go @@ -25,7 +25,8 @@ func TestGenerateServiceData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, Annotations: map[string]string{ "baz": "ban", @@ -64,7 +65,8 @@ func TestGenerateServiceData(t *testing.T) { "baz": "ban", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, } @@ -80,7 +82,8 @@ func TestGenerateServiceData(t *testing.T) { "ip": "1.2.3.4", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{ "baz": "ban", @@ -139,7 +142,8 @@ func (s *svcMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOpt "ip": "1.2.3.4", }, "labels": mapstr.M{ - "foo": "bar", + "foo": "bar", + "with-dash": "dash-value", }, "annotations": mapstr.M{ "baz": "ban", diff --git a/internal/pkg/eql/Eql.g4 b/internal/pkg/eql/Eql.g4 index d46e2571812..b6731d41cef 100644 --- a/internal/pkg/eql/Eql.g4 +++ b/internal/pkg/eql/Eql.g4 @@ -22,7 +22,7 @@ NUMBER: [\-]? [0-9]+; WHITESPACE: [ \r\n\t]+ -> skip; NOT: 'NOT' | 'not'; NAME: [a-zA-Z_] [a-zA-Z0-9_]*; -VNAME: [a-zA-Z0-9_.]+('.'[a-zA-Z0-9_]+)*; +VNAME: [a-zA-Z0-9_.-]+('.'[a-zA-Z0-9_-]+)*; STEXT: '\'' ~[\r\n']* '\''; DTEXT: '"' ~[\r\n"]* '"'; LPAR: '('; diff --git a/internal/pkg/eql/eql_test.go b/internal/pkg/eql/eql_test.go index eab34f69026..77bf5cb37f0 100644 --- a/internal/pkg/eql/eql_test.go +++ b/internal/pkg/eql/eql_test.go @@ -42,6 +42,8 @@ func TestEql(t *testing.T) { {expression: "${env.MISSING|host.MISSING|true} == true", result: true}, {expression: "${env.MISSING|host.MISSING|false} == false", result: true}, {expression: "${'constant'} == 'constant'", result: true}, + {expression: "${data.with-dash} == 'dash-value'", result: true}, + {expression: "${'dash-value'} == 'dash-value'", result: true}, // boolean {expression: "true", result: true}, @@ -306,9 +308,10 @@ func TestEql(t *testing.T) { store := &testVarStore{ vars: map[string]interface{}{ - "env.HOSTNAME": "my-hostname", - "host.name": "host-name", - "data.array": []interface{}{"array1", "array2", "array3"}, + "env.HOSTNAME": "my-hostname", + "host.name": "host-name", + "data.array": []interface{}{"array1", "array2", "array3"}, + "data.with-dash": "dash-value", "data.dict": map[string]interface{}{ "key1": "dict1", "key2": "dict2", @@ -327,7 +330,7 @@ func TestEql(t *testing.T) { } t.Run(title, func(t *testing.T) { if showDebug == "1" { - debug(test.expression) + debug(t, test.expression) } r, err := Eval(test.expression, store) @@ -343,17 +346,17 @@ func TestEql(t *testing.T) { } } -func debug(expression string) { +func debug(t *testing.T, expression string) { raw := antlr.NewInputStream(expression) lexer := parser.NewEqlLexer(raw) for { - t := lexer.NextToken() - if t.GetTokenType() == antlr.TokenEOF { + token := lexer.NextToken() + if token.GetTokenType() == antlr.TokenEOF { break } - fmt.Printf("%s (%q)\n", - lexer.SymbolicNames[t.GetTokenType()], t.GetText()) + t.Logf("%s (%q)\n", + lexer.SymbolicNames[token.GetTokenType()], token.GetText()) } } diff --git a/internal/pkg/eql/parser/EqlLexer.interp b/internal/pkg/eql/parser/EqlLexer.interp index 2131aba8177..3432105b62f 100644 --- a/internal/pkg/eql/parser/EqlLexer.interp +++ b/internal/pkg/eql/parser/EqlLexer.interp @@ -113,4 +113,4 @@ mode names: DEFAULT_MODE atn: -[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 35, 230, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 12, 3, 12, 3, 13, 3, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 5, 16, 108, 10, 16, 3, 17, 3, 17, 3, 17, 3, 17, 5, 17, 114, 10, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 5, 18, 124, 10, 18, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 5, 19, 136, 10, 19, 3, 20, 5, 20, 139, 10, 20, 3, 20, 6, 20, 142, 10, 20, 13, 20, 14, 20, 143, 3, 20, 3, 20, 6, 20, 148, 10, 20, 13, 20, 14, 20, 149, 3, 21, 5, 21, 153, 10, 21, 3, 21, 6, 21, 156, 10, 21, 13, 21, 14, 21, 157, 3, 22, 6, 22, 161, 10, 22, 13, 22, 14, 22, 162, 3, 22, 3, 22, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 173, 10, 23, 3, 24, 3, 24, 7, 24, 177, 10, 24, 12, 24, 14, 24, 180, 11, 24, 3, 25, 6, 25, 183, 10, 25, 13, 25, 14, 25, 184, 3, 25, 3, 25, 6, 25, 189, 10, 25, 13, 25, 14, 25, 190, 7, 25, 193, 10, 25, 12, 25, 14, 25, 196, 11, 25, 3, 26, 3, 26, 7, 26, 200, 10, 26, 12, 26, 14, 26, 203, 11, 26, 3, 26, 3, 26, 3, 27, 3, 27, 7, 27, 209, 10, 27, 12, 27, 14, 27, 212, 11, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 30, 3, 30, 3, 31, 3, 31, 3, 32, 3, 32, 3, 33, 3, 33, 3, 34, 3, 34, 3, 34, 2, 2, 35, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 10, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 48, 48, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 6, 2, 2, 188, 187, 3, 2, 2, 2, 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 8, 2, 2, 199, 198, 3, 2, 2, 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 9, 2, 2, 208, 207, 3, 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, 3, 8, 2, 2] \ No newline at end of file +[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 35, 230, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 12, 3, 12, 3, 13, 3, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 5, 16, 108, 10, 16, 3, 17, 3, 17, 3, 17, 3, 17, 5, 17, 114, 10, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 5, 18, 124, 10, 18, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 5, 19, 136, 10, 19, 3, 20, 5, 20, 139, 10, 20, 3, 20, 6, 20, 142, 10, 20, 13, 20, 14, 20, 143, 3, 20, 3, 20, 6, 20, 148, 10, 20, 13, 20, 14, 20, 149, 3, 21, 5, 21, 153, 10, 21, 3, 21, 6, 21, 156, 10, 21, 13, 21, 14, 21, 157, 3, 22, 6, 22, 161, 10, 22, 13, 22, 14, 22, 162, 3, 22, 3, 22, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 173, 10, 23, 3, 24, 3, 24, 7, 24, 177, 10, 24, 12, 24, 14, 24, 180, 11, 24, 3, 25, 6, 25, 183, 10, 25, 13, 25, 14, 25, 184, 3, 25, 3, 25, 6, 25, 189, 10, 25, 13, 25, 14, 25, 190, 7, 25, 193, 10, 25, 12, 25, 14, 25, 196, 11, 25, 3, 26, 3, 26, 7, 26, 200, 10, 26, 12, 26, 14, 26, 203, 11, 26, 3, 26, 3, 26, 3, 27, 3, 27, 7, 27, 209, 10, 27, 12, 27, 14, 27, 212, 11, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 30, 3, 30, 3, 31, 3, 31, 3, 32, 3, 32, 3, 33, 3, 33, 3, 34, 3, 34, 3, 34, 2, 2, 35, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 11, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 48, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 47, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, 3, 8, 2, 2] \ No newline at end of file diff --git a/internal/pkg/eql/parser/eql_lexer.go b/internal/pkg/eql/parser/eql_lexer.go index da1bf4d112e..d817439a858 100644 --- a/internal/pkg/eql/parser/eql_lexer.go +++ b/internal/pkg/eql/parser/eql_lexer.go @@ -46,84 +46,85 @@ var serializedLexerAtn = []uint16{ 34, 2, 2, 35, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, - 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 10, 3, 2, 47, + 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 11, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, - 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 48, 48, 50, 59, 67, 92, - 97, 97, 99, 124, 5, 2, 12, 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, - 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, - 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, - 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, - 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, - 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, - 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, - 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, - 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, - 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, - 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, - 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, - 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, - 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, - 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, - 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, - 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, - 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, - 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, - 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, - 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, - 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, - 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, - 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, - 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, - 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, - 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, - 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, - 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, - 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, - 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, - 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, - 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, - 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, - 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, - 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, - 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, - 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, 2, 2, 128, 129, 7, 117, 2, - 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, - 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, - 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, - 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, - 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, - 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, - 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, - 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, - 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, - 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, - 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, - 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, - 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, - 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, 167, 7, 80, 2, 2, 167, 168, - 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, 7, 112, 2, 2, 170, 171, 7, - 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, 3, 2, 2, 2, 172, 169, 3, 2, - 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, 2, 2, 175, 177, 9, 6, 2, 2, - 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, 178, 176, 3, 2, 2, 2, 178, - 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 181, 183, 9, - 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 182, 3, 2, 2, - 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, 186, 188, 7, 48, 2, 2, 187, - 189, 9, 6, 2, 2, 188, 187, 3, 2, 2, 2, 189, 190, 3, 2, 2, 2, 190, 188, - 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, 3, 2, 2, 2, 192, 186, 3, 2, - 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, - 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 197, 201, 7, 41, 2, 2, 198, - 200, 10, 8, 2, 2, 199, 198, 3, 2, 2, 2, 200, 203, 3, 2, 2, 2, 201, 199, - 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, 3, 2, 2, 2, 203, 201, 3, 2, - 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, - 207, 209, 10, 9, 2, 2, 208, 207, 3, 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, - 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 213, 3, 2, 2, 2, 212, 210, - 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, 3, 2, 2, 2, 215, 216, 7, 42, - 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, 2, 2, 218, 58, 3, 2, 2, 2, - 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, 221, 222, 7, 95, 2, 2, 222, - 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, 64, 3, 2, 2, 2, 225, 226, - 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, 7, 38, 2, 2, 228, 229, 7, - 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, 123, 135, 138, 143, 149, - 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, 3, 8, 2, 2, + 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 48, 50, 59, 67, 92, + 97, 97, 99, 124, 7, 2, 47, 47, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, + 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, + 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, + 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, + 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, + 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, + 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, + 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, + 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, + 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, + 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, + 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, + 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, + 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, + 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, + 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, + 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, + 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, + 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, + 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, + 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, + 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, + 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, + 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, + 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, + 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, + 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, + 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, + 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, + 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, + 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, + 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, + 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, + 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, + 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, + 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, + 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, + 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, + 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, + 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, + 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, + 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, + 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, + 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, + 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, + 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, + 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, + 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, + 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, + 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, + 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, + 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, + 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, + 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, + 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, + 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, + 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, + 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, + 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, + 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, + 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, + 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, + 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, + 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, + 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, + 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, + 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, + 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, + 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, + 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, + 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, + 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, + 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, + 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, + 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, + 210, 3, 8, 2, 2, } var lexerDeserializer = antlr.NewATNDeserializer(nil) From 0560b466d838d07b2565536cd96e2cfa98165e89 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:41:20 -0700 Subject: [PATCH 44/69] Fix transpiler to allow : in dynamic variables. (#680) Fix transpiler regex to allow ':' characters in dynamic variables so that users can input "${dynamic.lookup|'fallback.here'}". Co-authored-by: Aleksandr Maus --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/transpiler/vars.go | 2 +- internal/pkg/agent/transpiler/vars_test.go | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c9e71a035b7..06b85fa7e96 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,6 +107,7 @@ - Collects stdout and stderr of applications run as a process and logs them. {issue}[88] - Remove VerificationMode option to empty string. Default value is `full`. {issue}[184] - diagnostics collect file mod times are set. {pull}570[570] +- Allow ':' characters in dynamic variables {issue}624[624] {pull}680[680] - Allow the - char to appear as part of variable names in eql expressions. {issue}709[709] {pull}710[710] ==== New features diff --git a/internal/pkg/agent/transpiler/vars.go b/internal/pkg/agent/transpiler/vars.go index 8daacf606fe..a9f96b15ee8 100644 --- a/internal/pkg/agent/transpiler/vars.go +++ b/internal/pkg/agent/transpiler/vars.go @@ -14,7 +14,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/core/composable" ) -var varsRegex = regexp.MustCompile(`\${([\p{L}\d\s\\\-_|.'"]*)}`) +var varsRegex = regexp.MustCompile(`\${([\p{L}\d\s\\\-_|.'":]*)}`) // ErrNoMatch is return when the replace didn't fail, just that no vars match to perform the replace. var ErrNoMatch = fmt.Errorf("no matching vars") diff --git a/internal/pkg/agent/transpiler/vars_test.go b/internal/pkg/agent/transpiler/vars_test.go index 8171a4b7a9f..142ab132109 100644 --- a/internal/pkg/agent/transpiler/vars_test.go +++ b/internal/pkg/agent/transpiler/vars_test.go @@ -81,12 +81,24 @@ func TestVars_Replace(t *testing.T) { false, false, }, + { + `${"with:colon"}`, + NewStrVal("with:colon"), + false, + false, + }, { `${"direct"}`, NewStrVal("direct"), false, false, }, + { + `${un-der_score.missing|'with:colon'}`, + NewStrVal("with:colon"), + false, + false, + }, { `${un-der_score.}`, NewStrVal(""), From 4dcc16b82ea10bbafde1bd88658d9380e1194e3d Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Thu, 14 Jul 2022 11:25:15 -0400 Subject: [PATCH 45/69] Fix for the filebeat spec file picking up packetbeat inputs (#700) * Reproduce filebeat picking up packetbeat inputs * Filebeat: filter inputs as first input transform. Move input filtering to be the first input transformation that occurs in the filebeat spec file. Fixes https://github.com/elastic/elastic-agent/issues/427. * Update changelog. --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/program/supported.go | 2 +- .../testdata/single_config-packetbeat.yml | 7 ++ .../agent/program/testdata/single_config.yml | 7 ++ internal/spec/filebeat.yml | 66 +++++++++++-------- 5 files changed, 54 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 06b85fa7e96..33aa1756f0d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -193,3 +193,4 @@ - Add `@metadata.input_id` and `@metadata.stream_id` when applying the inject stream processor {pull}527[527] - Add liveness endpoint, allow fleet-gateway component to report degraded state, add update time and messages to status output. {issue}390[390] {pull}569[569] - Redact sensitive information on diagnostics collect command. {issue}[241] {pull}[566] +- Fix incorrectly creating a filebeat redis input when a policy contains a packetbeat redis input. {issue}[427] {pull}[700] diff --git a/internal/pkg/agent/program/supported.go b/internal/pkg/agent/program/supported.go index fa35f4a4fe3..985ebb03709 100644 --- a/internal/pkg/agent/program/supported.go +++ b/internal/pkg/agent/program/supported.go @@ -27,7 +27,7 @@ func init() { // internal/spec/metricbeat.yml // internal/spec/osquerybeat.yml // internal/spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzce0mTqziX9v77GbX9ursY0llFR7wLQyaTnWQZXyOhHZKcYFtgV9rYxh393zskBgMmx7r19hu9yIh7sdBwdIbnPOfwX7+sssPyNYvYr/vdkvwa7dJ/3y9fj8vX/yhS9st//oJT84B+bONZoHvTwGMkQ4zEuzUGs3vHMk94Ll8QdBUEnUkIXSkCKAnVwd8yctnG4LSNHcM5+HNn7xjuIQSjBCnBAYGRNE2DPATuHoGZRm1XRnNnb6zGsbOSTWd1ip2UrqGqM5J6DGczzbUP+uJR/uEHLvAD98WXNHt22Z6fHnTNiXfUSIM7YmkFtYINVGRGbXcXqk/3jrmfOMZ4FUL9MIXVmVbO3mDShGTBHsGne77udK6vsaqPoOofoXLeEXUmnjvGOHYsJiEg3TsW2iMQSM1z2z8+r/QdznSZ2k8T8cwYx1gZvYSKlqP0vCvlMzpidcx/PziWnJCHbTOWWKYUPWxjlJ4ZgrPr89be6mfTuV4gIB9pGrxESjB6jrfNb+Wf/orght/HOlSCC5G1hFhMjP3WPLbLSpmyHJ3aY6SYpMEBq4hB5cCWP67nqf/EvCud33dOx1vxDkrZHVQ9iaRBgn9s46UqVTJBO2z7jDBNCcFZ7pzb9hi2gjW1tGJI1tU60hLq7PoOSrAdMHLp7Osg9HTW7GVPraC4nl2/IHBmoeofSXYj95t1y/k0mdq6XJ7vKpvWXR4ci+VRGqypqW0RMDcIupfnlf7by2ynRlaQP6/0PQKjjFrx1rUP1TqeNpmP/7/zMI5DMNo4VpIQ6cCW83izVKo1bWnvGJRhy7xQi62JEiQk9bZucYpd1WXIYhe3OPE9ZJFippHymE2NcYYtLSOqnxAlziaz7T9++be+V8jp6oCX0aHvFGCwiaC/QsCUjNQ7oIdtHHaemQVqGdwUPK2mwvCuY6ZpsA+hJ0Xg6RACyg07XwJ576xO5TuLhXiHKIFE4TiPwPnQdhQoNfdEWaym445juSDoy8QYSQjIJ2yZEpqPGE7NFbaCzR+AK4PH+mtgYOZiDyDIqdEaD/2Cgps19lihWQRG2TQ9M5oG+z+Az8IsyPrzIsV7IVZQhFzxH6TNUjID/5EtZptAD0zN/iHRh+f14+nJlvprJBT4O2q5LISzvHQgLJum8hHZwhnkCIySkCvXg5yG4HxBc2cCZ7uEZP4Opeaa8jtJg4TaT8fO3WQuE5ee+S/cOWE1kIgdSFDxtiEYZfx+hROebVtOcdhgjdVTPAPnvfNoyshikljPqAzEKA2RFPoKAbrDK12NoL91DFqezfg9m8bbAYP3XqjCpMjUCgQoW9rjygBrR+SPiLUojc922fNqnMG0XH9a3L26lXMiSrBHwJOw6txzw+NnJKftxCl0HVn+xTHoDqc+Wz5sY2EIhfwbskwpDLQLtV0WAikX/4ceo7Z0WCp+Qi3zhah+gYB5mKajBIPgQixzjaCU8QDgWEESKnEcgdGJwlnO5RaB0Z98H1BhObKCu9rJUJuduKzF2u192b6KCz1Blv+CUsaweKYXWPEYUT3ukLgTOuJ0EaNUKxwrUMogKvZ3QXBWOQotR3M9xarD596E0E+a+5uPxP+5bU2Nm9/EvU0NPcHpLO7LRPgA6J1C4LHy/hqnVo2RX4jtHkXQUbTGqd4GQpYTJSioqSUo8xl5Wy4PWBkpCLpSfY9AlvL6Hoh8oPw5tX6v5O5f+O9CZ2EiEW4LpraPoLgD7hcu9Z7qgFu/V+nZjlrB4Xml1+epn0v40horzt4K4LaeUCtu69pQ8F47j+XvpR4HF8dCR7LS9Za9c7vNInAXCzswxllpv7OjW9zFPkSMZEyKeCDmesJlt9KlyDK5fC7NOraQj4TALA7hU0ythDlW5T/meh4Cmd9fHdS4nHKinBNqBW/YXxekiHuatYLdjS3G+aTYZO13xbwP29h9eJy0wIPYSwd4WKjAitS1deN6BxhoGwrOrCXT1LEe48ZeDX1HCv1CrGAdAbQTMlC0nKZBwf1SJSuhM65S243PgzF2BcCTqVvcNXvEwHxFgZaQzE26AKfRlcaeOue1fr93rOv81W8HLisMNAUFmvBp7XcqENOf7zPvNHHrq+s0tj/bfgwKLQHyhDz5PUdAy5t7MF1GYLAj6SIOeVywvCNO0Q4VuoQLfY0VmTm2vyWpJiMlLnXy0cy4/3KMJMeFvseKl2BDTyNwZqTY5F84SxlXTK3gugEVtMNWUNnpW6CyBfTWN36gAtCVzVSyoam5pyC4tGTCwaTQeTRuPxPrv0SAYxFaRNBvAW8pJiq7cN0x4u3aeTQ3yH6KqU2PjmVukKmlEQj2/Fnbl5JCAM3ar8c1borAoM9vACFUShlPU7aZGgPPMy8PoX6JLE0a/p37HG+HlbteTCt/F7Et/f3esQ9aKW/vSOz6jvSUpNrhBrCXCVMb50z43H1/gDL3yPFGFzt2fWxpZw02mlR4pQPqr3jAbe5uOu+u39h7hYka3eM6D8W7rLyvKuYZegPka39S+tkKmBtc50dpBKhMUlPYSi9m9hKB7YQqCcNrHv+CHKn+dmL4v3XPXp/zLyUGIhkgKZ/3SSQDX0oMCNvmdCgxUM5HCnqJQedZLzH4lwL5O/p5cNwD2u0zxjvhgKqMUziCZQOOpNJIVlUmme648+FOYoNg3Mp0ReC7d6zzEalcobvja/AD1RYwXG/fYR2GwZZTgXSeeWKFcmcpgCi2Tp27rEHNFbS4DCksd2y6o1Ycu4q3x2qwKRMAklf75U5cQtB9qZ0mVPn45IWkQYZgcmpl3Lfgnhtturh3DLpsy3e62mYfMQJ9eZWOUsg55QCiTF7ae3QGWYZb1qDreG4cRvxPyeRjVzkkKD0k5b+5jngc+GzdYjO5MdZlRnfbVda3VR+MNsTSdjibxQslWFPo7qi9mYSKvCnBhH/CCsupIV8Q8GSSMmnZ03t0neOArEAxMsGq7UJlce88hOrzQzwJgSccHw8SwpbUUoeehSMMcmTrRx4kjfR8RLJ2CqG/LeXOwYmrRuDu3jGc4w+LrUhqFsu5ZtbymUrX96dCBj6bcnkUWmv/0p9TPnfh8Dn3ERjJPOlzVtqR2LOjD84JUf1dWGjm9R3tQoVf0fZYIcf2OSerEX+24vpCRVKnqeSyjZ3N0z00zzOSahlJzYPzWAIQaJ6b/Yp/12uYZ8J9DbUCArmdy2cyuE7qbRHwXoX8VD/B1um+Sm5YKGsCKPUTGidtyQV6LFQDAUCcalzFyjYBz+EBPmXpcu5cn62kAw8EThMsxyui+jxYFfUzarEDAprMdeHpMp4QS7tQk+/fkzgArO74DgGPJ7N79KOXLBnusJ7V+7DKeNEEYsNt5m7vazqXmzupxl2o5TOSOa1nzmEKgxNS3QRZi95zlxFFk0nqMVK0ZPCGHLvjR/cRHK/qJDECMuP+4Hk1Vp4exhNiuwyqQR6BEdepPX7YTqZznS2tYF2C1EUNlITuP6/Gq7YekKtt1mskJKVdAGq4Mk4b/Vi1gWb/HoflM7DvD8mfK8iCao9lfY+prRKIdgLZB9VcPo1ejIW8bmLJLbBzbpLv4cRI7zPLB54sU7AQZ8LAPPXtqcPI266Mrc5eP0yWROxNg6KTjKxvZdW2yW4VQOolJ7eMs9gHRDuisCOOh4FkOaffi0NnhlMqRQaPQ5X8VGnnPNzFTxUJFFnmZa4EIz5HHete5jz21CSud0HALEIlHgCQOw6COf5JuB1xX4lTTXL4eqor48zfYbDIQ+iuI1uK//ghxa5iFvhHKLlFub5rHwoKRkJHpylKMGD7JazGiriYcNxRnsfwfyNZkAvfNB8dQrA7kqwaeyHZZD6+jZEvK7YcwLM+91mgTA6quCj8rcA5411physdd6pVmceoHZymKdvjT+DSGvvyZJBY54RaiyFyt1cRkxOc8uRLFglga7xEsuAW+1paxvEaKkZ7BBHDD/IGAVdGxYeVNmu+OJufIKyvcjFGVdJD8pqoIcWAHESlziyoxVKOEZ5XusB3pNDaMi4Q5HHdlaDCMUSl86a2jiwzRxxnVLb7kb96G2/fYOC/ndSczvVDCMc9v99PkG/8TFmVVMw9NjUJyyWp2fE3ZVLdIy4/rETWGJwtLY8Re8bzs1VDsBbifpv7K3W8Q0iv4Ky3x6HKKffH0Gf8Pjqkxum7+29kvUIAHUm6eJdA6JE7HTK3Oeupis22X4SC4OBn8F/Evq+kUe++vD5R/HcSyCJWhWB0EXlvl4ys99WtgHLZAfNEOhXO6jn3CyUuE/9uv8P1gaY8V64wprAP0q3u1sUH1ZOQxXJxZsvf4d46wudksyO1vBPfX9jRu/Y8wYY8bGMK/FNvLyUOt7R1pAQcU2yw4r0i6PTmEdi/QMDfEVm7YEtT+bmeV3r57HR7/qnqjQjPGS5bQcq15PAZ0lEmSvByQxj9jcTrewWZb6/fKgj9ZcK0U6DrV9+lOIT0MgUlX0AVUwqVuPsbfGphHPOeQpdNQakfkR1ccbtVzoFSTcapXyzbPkgZSSFg+VU/pLjWnev7dE0M+USBe6HgOi6yfYmYWht/1vNuqe2fosw74qtP/hOrQYFTc4+u415DgF7D635adiPFGJpSa84cQT/lenrd1+8Xbx2UxGQb27X0dzrv2hn//xJ6nXW4rV31IdiQ61w8V7i7/kZ3mOfH17lL8r3yiX+dQG/4vDd1oi44f8zZSHGoaKemCFMXiqwhQljk3xUn9xXcfcV8nySEv0zottbIa5m9zDfxH6vxyeG4xtC3IfSmCG62rn04UujzMRrH1qjmK1SPCcxskL1j0FI/CiLw3Jf4oRe2XB6Gu7/8MgeJF3X+UnG6VW5yaPKaK6/Lc/Gy08eQD1jxmcOkXodY2TnVdGB9hi+Nu506N/nmB3ngW/ziQDGnnXsdWnnbz1m/bwvv7KG2iXf9eRUTrvxjuc96L1ARRYm3uM0YFHqKrYBRY1R37OX1XNcGnSa3i2FT8G2wRqWbs5pPrf2BKHzhQfkI28CNHmR1J97ohBWO7TaioDWwVo2P8yejGXvlesU8/guygjSEwZ7aw51jt5zuzT62WPWkXtfXjZxEN9xwt1de68005fIv4wNU9SPJZh+tfSHK6aYjsPYR0/U47+tkm69oyarcb7M/vW1j7fW7PrH1N1Qw6/5JdQHy5t1y7uRI1EYvWvnO21j3zX2+g2fadkChx5occfytLsVBXPaX5piPRD5CVD0JlcW3zvVmY9L3znjNS+JvdUD2dGs8xE/VY/v+4ZqDW0GCrED4IRHbM7pF4O6+y0eV9tGLjT+5tjIQE5Nl9DrU+Ti3goRk3QJn1HnWCoSfLW5+gwz6Siv1pzoXb4I0Byf+ZaokHGRI3ydjKkAJk6ZoWQGj4YAqa6JLEKouQ0pw126PHm7vdXliuiQqd7QJExdb/J5PTgOK3AOo75HK7wPblmHWd3N1tLdty3WHVRawzxQm33LMbzvkj42v65hb4HXQ6H4OCdx0FwjZ0DIpmguwKog2apmXyCA7I/7HbYdBujy8rsiABf4AgURStq40svogoWryVyqqdvijg7ptYIct6WOKtaZxM59hqO9FafHj1oQvfthwPiKFp18kx4ICOWnIClYUkP68WShrJwTdNZ/3j7n/249FsFhs2MMnqNkDgn4RAa/qH6l7skYcnguKcehcxpseoStrkgb8JgtqakfMtDpVeQmVJMEp5dZZan7W0JlvtER8ole2uvuKpvtcO4TFPZU48z4qaeYGKoYASQ1F0LRcNB+M/Cw68R2q94Y67MGfKt0NtARb5xdqaS/YYhf6sL3piWyt2YIPN2fPsaJ1qDUEkzWCuiQgeVZBFlj260dgVvax17ZiCH06kVRbI+hduCfikaO312Fquv5wptvL/N1zXO8wDdJrGh7cUdtNuE4IOZmaoGKvZcke7FSb8066e+w9733cM1j6fCtyDZQ7RVuLeZdPCq22zYs7fr9v/X+7BNrT9xs95XLHmSf85NttPq07uO3tLbiNw1X86+LhLKL8H6u718n8VkblPKL16N4x/DYiKCF1GSvac9c9iDfUeKd38ErZHUJwYFAxC5Kao0E9bvxED4aXutLs+Yt9v633ZpOPYf1btMJn3ul+z/BTqOPvzzGQIn3iDHZwQYHWSjF+am9qJ26Kd/u61dCPnZiYD8WGphTasWXnlkbNWjbZwzW1XN6lIzsySFgIfMbnLM8/2J7Wwwf/Av2l2/2f+fK1GIJ/qnemICiW3ar8kaimjKA76lfmv1CV/zr0+0p36aeq8e73O1CrT71wXz7vdt9pFwIDRrLN5HtdZvX7rOwS/Wx3me3vMAiOFM7unYfHU7tj9o1ksc2w159t9WBb3SnbUv/avQRaC7I0JsfhquhUwBYTMoyAYPe460h5aH+7O+Dvq4419/fJasgn29x736z+0ztWP/j29LZDqNvxw5PKRHOMR80xyOX5IcwGk8ZdRDbLIdpmYZnrSAmkTtJo82BzYHXzf5M0FuTgl2nqBwkjH3Mz9t2edCGVQjbLusz7PTzdsW8mitlbH32S7pm/T938RYqk+6HYm/TIKQTeKxrgCf+P9W5/9BX25Jf//n//EwAA//82Bxyl") + unpacked := packer.MustUnpack("eJzce1uTqkiX9v33M/r2m5nmUFZvJuK9EKpA0KJa3JJJ3pGZFqgJ2iWoODH/fSKTg4DUsXv6fWMuKmLvFPKwch2e9azFf/2yTrPVaxqyXw/7Ffk13Cf/fli9Hlev/1Ek7Jf//AUnZoZ+7qK5r7sz32UkRYxE+w0G83vbMk94IV8QdBQE7WkAHSkEKA7Uwd9SctlF4LSLbMPOvIV9sA0nC8AoRoqfITCSZomfB8A5IDDX6MSR0cI+GOtxZK9l016fIjuhG6jqjCQuw+lccyaZvnyUf3q+AzzfefEkbTK/7M5PD7pmR3tqJP4dsbSCWv4WKjKjE2cfqE/3tnmY2sZ4HUA9m8HqTGv7YDBpSlL/gODTPV93ttA3WNVHUPWOUDnviToX47YxjmyLSQhI97aFDgj4UjM+8Y7Pa32PU12mk6epGDPGEVZGL4Gi5Sg570v5jI5YHfPfM9uSY/Kwa54llimFD7sIJWeG4Pw63tpbPTZb6AUC8pEm/kuo+KPnaNf8Vv7prwhu+X1sAsW/EFmLicXEs9+aZ+KwUqYsR6f2M1JEEj/DKmJQydjq5/U89Z+Yd63z+87peCfeQQm7g6orkcSP8c9dtFKlSiZojyceI0xTAnCWO+eeuAxb/oZaWjEk62odaQV1dn0HxXjiM3Lp7CsTejpv9nKgll9cz65fEDizQPWOJL2R+8265XyaTCe6XJ7vKpvWXWa2xfIw8TfU1HYImFsEncvzWv/tZb5XQ8vPn9f6AYFRSq1o50yyah1Xmy7G/99+GEcBGG1tK46JlLHVItqulGrNiXSwDcqwZV6oxTZE8WOSuDunOEWO6jBksYtTnPge0lAxk1B5TGfGOMWWlhLVi4kSpdP57h+//FvfK+R0neFVmPWdAvS3IfTWCJiSkbgZethFQWfMLFDL4GbgaT0Thnd9Zpb4hwC6UgiesgBQbtj5CsgHe30q31kuxTtE8SUKx3kIzlnbUaDEPBBluZ6NO47lgqAnE2MkISCfsGVKaDFiODHX2PK3vwOuDC7rr4GBmYs9AD+nRut56BUU3KxxwApNQzBKZ8mZ0cQ//A48FqR+2p8XKe4Lsfwi4Ir/IG1Xkul7j2w53/q6b2qTnxJ9eN48np4mUn+NmAJvTy2HBXCelw6EpbNEPqKJcAY5AqM44Mr1ICcBOF/Qwp7C+T4mqbdHibmh/E4SP6aTp2PnblKHiUtPvRfunLDqS2TiS1BxdwEYpfx+hROe71pOcdhgjfVTNAfng/1oyshikljPqAzEKA2RFPoaAbrHa10NobezDVqezfiRzqLdgMG7L1RhUmhqBQKUrSbjygBrR+SNiLUsjW/isOf1OIVJuf6suHt1KudEFP+AgCth1b7nhsfPSE67qV3oOrK8i23QPU48tnrYRcIQCvk3ZJlS4GsXOnFYAKRc/B+6jE6kbKV4MbXMF6J6BQJmNktGMQb+hVjmBkEp5QHAtvw4UKIoBKMThfOcyy0Eoz/4PqDCcmT5d7WToRN24rIWa7f3NfFUXOgxsrwXlDCGxZheYMVlRHW5Q+JO6IiTZYQSrbAtXymDqNjfBcF55Si0HC30BKs2n3sbQC9u7m8xEv/ntjUzbn4T9zYz9Bgn86gvE+EDoHsKgMvK+2ucWvWM/EImzlEEHUVrnOptIGQ5UfyCmlqMUo+Rt+XygJWRgqAj1fcIZCmv74HIGeXj1PpRyd278N+FzsJYItwWTO0QQnEH3C9c6j3VAbd+r9KzPbX87Hmt1+epxyV8aT0rzt4K4BM9plbU1rWh4L2xH8vfSz32L7aFjmSt6y1753abhuAuEnZgjNPSfudHp7iLPIgYSZkU8kDM9YTLbq1LoWVy+VyadSZCPhIC8yiATxG1YmZblf9Y6HkAZH5/dVDjcsqJco6p5b9hf12QIu5p3gp2N7YY5dNim7bfFfM+7CLn4XHaAg9iLx3gYaECK1LX1o3rHWCgbSk4s5ZME9t6jBp7NfQ9KfQLsfxNCNBeyEDRcpr4BfdLlayEzjhKbTceD8bYEQBPpk5x1+wRA/MV+VpMUifuApxGVxp76pzX+nFvW9f5q98yLisMNAX5mvBp7XcqENOf7zPvNHHrq+s0tj/ffQwKLQHyhDz5PYdAy5t7MB1GoL8nyTIKeFyw3CNO0B4VuoQLfYMVmdkTb0cSTUZKVOrko5ly/2UbcY4L/YAVN8aGnoTgzEixzb9wljKumFrBdQMqaI8tv7LTt0BlC+htbvxABaArm6lkQxPzQIF/acmEg0mh82jcHhPrv4SAYxFahNBrAW8pIiq7cN0xot3GfjS3aPIU0Qk92pa5RaaWhMA/8LG2LyWFAJq1X49q3BSCQZ/fAEKolDKeJWw7MwbGUzcPoH4JLU0a/p37HHePlbteTCt/F7Et+XFvTzKtlLd7JJP6jvSEJFp2A9jLhKmNc6Z87r4/QKlz5Hijix27Pra0swYbTSu80gH1VzzgNHc3W3TXb+y9wkSN7nGdh+JdVt5XFfMMvQHytT8p/WwFzA2u86MkBFQmiSlspRcze4nAbkqVmOENj39+jlRvNzW837pnr8/5pxIDkQyQhM/7JJKBLyUGhO1yOpQYKOcjBb3EoDPWSwz+pUD+nn4eHPeAdvuM0V44oCrjFI5g1YAjqTSSdZVJJnvufLiT2CIYtTJdEfjubet8RCpX6O7zNfiBagsYbnbvsA7DYMuuQDrPPLFCubMUQBRbp85d1qDmClochhSW2xO6p1YUOYp7wKq/LRMAklf75U5cQtB5qZ0mVPnz8QtJ/BTB+NTKuG/BPTfaZHlvG3TVlu9svUs/YgT68iodpZBzwgFEmby092gPsgy3rEHX8dw4jOhvyeQjR8lilGRx+W+uIy4HPjun2E5vjHWV0v1unfZt1QOjLbG0PU7n0VLxNxQ6ezrZTgNF3pZgwjthheXUkC8IuDJJmLTq6T26zpEhy1eMVLBq+0BZ3tsPgfr8EE0D4ArHx4OEsCW11KFn4Qj9HE30Iw+SRnI+Ilk7BdDblXLn4MRRQ3B3bxv28afF1iQxi9VCM2v5zKTr+zMhA4/NuDwKrbV/6Y8Zn7uw+ZyHEIxknvTZa+1IJvOjB84xUb19UGjm9R3tQoVf0Q5YIcf2OafrER9bc32hIqnTVHLZRfb26R6a5zlJtJQkZmY/lgAEmudmv+Lf9RrmmXBfQy2fQG7n8pkMrpO4OwTcVyE/1YuxdbqvkhsWyJoASv2Exk5acoEuC1RfABC7eq5iZZuAZ/MAn7BktbCvY2sp44HAboLleE1Ujweroh6jFssQ0GSuC0+X8ZRY2oWafP+uxAFgdcd3CLg8mT2gn71kyXCG9azeh1XGiyYQG04zd3tfs4Xc3En13IVaHiOp3Rqzsxn0T0h1YmQte+MOI4omk8RlpGjJ4A05dp8f3YdwvK6TxBDIjPuD5/VYeXoYT8nEYVD18xCMuE4d8MNuOlvobGX5mxKkLmugJHT/eT1et/WAXG2zXiMmCe0CUMORcdLox7oNNPv3OCyfgX1/SP5cQRZUeyzre0xtlUC0E8g+qObyafRiLOR1E0tugZ19k3wPJ0Z6n1nOeLJMwVKcCQPz1LenDiM/cWRsdfb6YbIkYm/iF51kZHMrq7ZNdqsAUi85uWWcxT4g2hOFHXE0DCTLOb1eHDoznFApNHgcquSnSnv74S56qkig0DIvC8Uf8TnqWPey4LGnJnHdCwJmESjRAIDccxDM8U/M7Yj7Spxoks3XUx0Zp94eg2UeQGcTTqTo959S5ChmgX8GklOU6zuTrKBgJHR0lqAYA3ZYwepZERdjjjvK8xjebyT1c+GbFqMsAPsjSatnLySdLsa3MfJlzVYDeNbjPguUyUEVF4W/FThnvC/tcK3jTrUqdRmd+KdZwg74E7i0xr48GSTWOabWcojc7VXE5BgnPPmSRQLYel4iqX+LfS0t5XgNFaMDgojhB3mLgCOj4sNKm7VYns1PENZXuRijKukheU3UkGJADqJSZxbUYgnHCM9rXeA7UmhtGRcI8rjuSFDhGKLSeVPbhJaZI44zKtv9yF+9jbdvMPDfQWo6FUle+7U9TkiEE58JIsD6wW3uRCw/F1g70Qq00GOcbiOsejsE7Zpk5vlIjFM3waqTBdDbc1ub8STVdBla6Ds68U7ksjvOFIG95WDN70c6oivuafDmDFQJv8IuRPUZKX5ITxuS/y/7+piqT1mF8dUAeptw3P2NXJ4aPxjAvUySZSb0OPV2FFzjdTVHglUet53R1Ve6R5x63P80dzJb6FusuK8IXt8v55wfBVa5kmGMJn6FCb09jm7GpaCs6HbmoMA7XdcvsWCVA16fVdzTDOpykLpycJ1X3BlUWti4mSeW6ET/gyjalRiz9kcKnTwA522LsBD5wfX/52NdtZ0t9BLTWtomVPyi80yhFQh4eyJrF2xpKtfX3u+XFXT5+1eyhPuWrrwyAr3WnkeMWuiA1dY7lyfFBSVp1ib40JXAzuyJKyGLtQhAd0SUzjk6cu/Ie+LnIYyuvyksF2f5HmlfEVrshePOwa6AmoSaVD4HtqvD+hEzTfhEbDGRN+FEO5J2pdjy7wLFP/HfQOE0hYfSJzd8SBkvOoSgQ/t7pJb2gi12oQ+DRPWJJNoGQfeCVefy3f1zLIuET65801xgtILHIQyWnyH+6pz4WmSpcgXuz7iPFmcwNbHvq6/p3Zfa22/dndE/R6874y2c1yOA2zxOqUelrz3wPLOD8+p93RYssgDqpw6/UI7zWFPmbiLukG7XRF3Uq/KDtv3Wf7Wf4TmZyJG79lKv0/ZRfH+bjt615uH287zWZTQZ9/Yi7L72lfe25R0DJWtxSBXnZGnprOIooOoesEr5ue57dt06PzkSlXE/kz6v9Ru/8nZXSc0V+Rfka0cKPR4HP/+e1S3kfrtYshgJopqoehwoy2+v3yrc/umcqsGB4zdjrcCHPYxTdubwvD1Bx5qIrrug+kR8z0ZEQQ61cPIXyPcWzmvGbu6Ty56o3pEkZX52LT7ELAA8h32qCg+D3Fsbl/7zO2pe2GqVDXfaeWW+Fy3rXLHiz6s8MGtyyCuHvp4t5PLuDDnDisdsJvW68coutabb7TPcdNTtirrRww9y7re43IHCWTvPzVo58l+zvtXEzQ/3UDe8vOsHKhu7cr3lPuu9QEUUgN7ikSNQ6Am2fEaNUd0dmddzXZuhmjw6gk1xveaFrrZRcdd18VIUGfGgfERhCTd6kNZdj6MTVnjOshXFw4G1av+SPxnNs1deXczjvSDLTwLoH+hkuEvvlj+/2ccOq67U67C7kZPoPBzurMtrvZklXP5+gRPzAFX9SNL5R2tfiHK66b6s/dtsM877OtnGDC1Zlftt9tfHDa0coM37tP6G/GP3T6qLvTfvVnnBkaiNXmQBbDVtvIFx39znO3GwbQcUuqzJx8ff6ggdbD74U3MMxuSvnevNJrDvnbGNb7/TbdrTrfEQF3gTO2/qi5YfI8sXfkhwtyndIY4ZO9xfaR8vi230+3p8si0zR8ZfXccaqE/Fq/B1qMt0YfkxSbvF5LAz1gqEny0kf4N4+0rb+qe6RG+CNAdO3mXGAQ10pO8TXxWognEDLiswNxxQZU10ZEIOaBT/rt2KPtxK7fBkZUVU7mhjJi62+JFPTwOK3OumeY/Af78L521wO0Dkd0HuJ4rAbznmtx3yx8bXdcytzpFBo/trCPemk0PIhr4GAL0GC8KNUJCaPMEIDbI3on/cgtJklb2uyYAF/gS+RBK2qTSy+vij+qBCqWjx4Q886haNPbakj+nsmjJPPYahLtLrT7SBfPEjkvMRKXSPE5JjkV6fNGT5awpIf940kLUTgs6Gz/v7wvvt59JfLrfs4RM0eIagV4TArXp16v63EYfnomw0dC7jTY/QlTVJfH6TBTW1I2Z16uu9BEoc44Ry6yw1P21KVG+0n3yiL7m6e3LaTT/demJxTyXOfAhLSr+BigFAUkO1Nu0tzcc57/Z+DaWc9VjTB1zuswV9+lDW7ffz3tA7gaKdVr4WY+v8Fo0m1m6t2YIPN2fPsaKd2l4CwXiDoC4JSJ42FJWIRmFF7zW2Ygh96tB1PHL09iphuext7pYdaxqs0zf+3XNc7zDxk6oH7m+l6j4duQbKDaKFyLzLp4VW2+bFGb//jcA/u9z8FRry7Zaqd2nJgts4XEe/Lh/OIsr/vr57nS5uZVTOI9q87m3DayOCElKXsaI9d037dtFDv0+zVS4IQMagYhYkMUeDetz4iR4ML3Wl2fMXe6xb732FPuzTCn8r5fidvvC/hrbs0a1/cR9wJ26Kd/u6VX8T0MUf+VBsaMrOHVu2b3u+03aJpxtra7m82wv8dTqyhw/+BXp5d4c/8tVrMQT/VPdMgV+suh0QR6KaMoLOqN8F8YUOiK9Dv6908n6q88H5frdv9Vkd7svn3U5H7UKgz0i6nX6vo69+n5UduZ/t5Jt4ewz8I4Xze/vh8dTuTn4jWWxXBupP5Hqwre5Kbql/7V58rQVZGpN7p4JoJjy0v92J8Sc+3eh8KthnkblpV/c370Lc4W7hT1c1et8H/+3dwR9UJW67sbrdVTypjDXbeNRsg1yeH4J0MGnch2S7GqJtlpa5CRVf6iSNEx5sMlZ/aNEkjQXJvDJN/SBh5M/cPPtu/7+QSiGbZY/0+/1S3WffTBTTtz6wJd0zf5+6+ZMUSfejvDfpkVMA3Fc0wBP+H+uT/6g+N/3lv//f/wQAAP//vp+ZfQ==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/internal/pkg/agent/program/testdata/single_config-packetbeat.yml b/internal/pkg/agent/program/testdata/single_config-packetbeat.yml index f800d0bd2a0..3d62d2c49c6 100644 --- a/internal/pkg/agent/program/testdata/single_config-packetbeat.yml +++ b/internal/pkg/agent/program/testdata/single_config-packetbeat.yml @@ -23,6 +23,13 @@ inputs: data_stream: dataset: packet.icmp type: logs + - id: packet-network_traffic.redis-387bdc6a-0acb-4ef2-9552-c21e524a2d21 + type: redis + data_stream: + dataset: network_traffic.redis + type: logs + ports: + - 6379 output: elasticsearch: hosts: diff --git a/internal/pkg/agent/program/testdata/single_config.yml b/internal/pkg/agent/program/testdata/single_config.yml index 16a03f9a77d..b2cd1f87466 100644 --- a/internal/pkg/agent/program/testdata/single_config.yml +++ b/internal/pkg/agent/program/testdata/single_config.yml @@ -104,6 +104,13 @@ inputs: data_stream: dataset: packet.icmp type: logs + - id: packet-network_traffic.redis-387bdc6a-0acb-4ef2-9552-c21e524a2d21 + type: redis + data_stream: + dataset: network_traffic.redis + type: logs + ports: + - 6379 - id: endpoint-id type: endpoint name: endpoint-1 diff --git a/internal/spec/filebeat.yml b/internal/spec/filebeat.yml index 10f8ee4493b..e4dbd2a9892 100644 --- a/internal/spec/filebeat.yml +++ b/internal/spec/filebeat.yml @@ -19,6 +19,44 @@ rules: on_conflict: insert_after type: logs +# Input filtering needs to happen before any other input transformations. +# See https://github.com/elastic/elastic-agent/issues/427. +- filter_values: + selector: inputs + key: type + values: + - aws-cloudwatch + - aws-s3 + - azure-eventhub + - cloudfoundry + - container + - docker + - event/file + - event/stdin + - event/tcp + - event/udp + - filestream + - gcp-pubsub + - http_endpoint + - httpjson + - journald + - kafka + - log + - log/docker + - log/redis_slowlog + - log/syslog + - logfile + - mqtt + - netflow + - o365audit + - redis + - stdin + - syslog + - tcp + - udp + - unix + - winlog + - map: path: inputs rules: @@ -63,34 +101,6 @@ rules: - remove_key: key: data_stream.dataset -- filter_values: - selector: inputs - key: type - values: - - aws-cloudwatch - - aws-s3 - - azure-eventhub - - cloudfoundry - - container - - docker - - gcp-pubsub - - http_endpoint - - httpjson - - journald - - kafka - - log - - mqtt - - netflow - - o365audit - - redis - - stdin - - syslog - - tcp - - udp - - unix - - winlog - - filestream - - filter_values: selector: inputs key: enabled From f4d2b48fc2c07f45f98631af541ee9802e6959c3 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 15 Jul 2022 01:43:11 -0400 Subject: [PATCH 46/69] [Automation] Update elastic stack version to 8.4.0-3cd57abb for testing (#724) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0d7fb81c508..f05494199a3 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-00048b66-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3cd57abb-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-00048b66-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-3cd57abb-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From a5351f961d535b0bc39f9ea283fa7058c6fd6f5e Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 18 Jul 2022 01:45:03 -0400 Subject: [PATCH 47/69] [Automation] Update elastic stack version to 8.4.0-a324b98b for testing (#727) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f05494199a3..f75cba71333 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3cd57abb-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-a324b98b-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-3cd57abb-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-a324b98b-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From c6cdfc0a499f821d269ca35575151453408ba962 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 18 Jul 2022 15:43:13 +0100 Subject: [PATCH 48/69] ci: run on MacOS12 (#696) --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9454f8d2cc6..34a72100707 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -79,7 +79,7 @@ pipeline { axes { axis { name 'PLATFORM' - values 'ubuntu-20.04 && immutable', 'aws && aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable', 'darwin && orka && x86_64' + values 'ubuntu-20.04 && immutable', 'aws && aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable', 'macos12 && x86_64' } } stages { @@ -277,7 +277,7 @@ def isCodeCoverageEnabled() { def withPackageEnv(platform, Closure body) { if (isUnix()) { - if (platform.contains('macosx')) { + if (isDarwin()) { withPackageDarwinEnv() { body() } From 21c48120f7e9e722a83153b2d07169c930117e8b Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 19 Jul 2022 01:43:20 -0400 Subject: [PATCH 49/69] [Automation] Update elastic stack version to 8.4.0-31315ca3 for testing (#732) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index f75cba71333..43b309fb98e 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-a324b98b-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-31315ca3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-a324b98b-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-31315ca3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 78019a9c9615592c2e957a97af0079129132fcfd Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 19 Jul 2022 10:56:10 +0200 Subject: [PATCH 50/69] fix typo on package command (#734) This commit fixes the typo in the package command on the README.md. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 25aff95042e..df27ac6f99a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ In Linux operating systems that you can not run docker as a root user you need t Running Elastic Agent in a docker container is a common use case. To build the Elastic Agent and create a docker image run the following command: ``` -DEV=true SNAPSHOT=true PLATFORMS=linux/amd64 TYPES=docker mage package +DEV=true SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=docker mage package ``` If you are in the 7.13 branch, this will create the `docker.elastic.co/beats/elastic-agent:7.13.0-SNAPSHOT` image in your local environment. Now you can use this to for example test this container with the stack in elastic-package: @@ -45,7 +45,7 @@ for the standard variant. 1. Build elastic-agent: ```bash -DEV=true PLATFORMS=linux/amd64 TYPES=docker mage package +DEV=true PLATFORMS=linux/amd64 PACKAGES=docker mage package ``` Use environmental variables `GOHOSTOS` and `GOHOSTARCH` to specify PLATFORMS variable accordingly. eg. From e34e0e71d0c106381f9120b5d05032f6358a64cc Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 19 Jul 2022 11:53:00 -0700 Subject: [PATCH 51/69] Allow / to be used in variable names (#718) * Allow the / character to be used in variable names. Allow / to be used in variable names from dynamic providers and eql expressions. Ensure that k8s providers can provide variables with slashes in their names. * run antlr4 * Fix tests --- CHANGELOG.next.asciidoc | 1 + internal/pkg/agent/transpiler/vars.go | 2 +- internal/pkg/agent/transpiler/vars_test.go | 7 + .../providers/kubernetes/node_test.go | 20 ++- .../providers/kubernetes/pod_test.go | 52 +++--- .../providers/kubernetes/service_test.go | 20 ++- internal/pkg/eql/Eql.g4 | 2 +- internal/pkg/eql/eql_test.go | 10 +- internal/pkg/eql/parser/EqlLexer.interp | 2 +- internal/pkg/eql/parser/eql_lexer.go | 154 +++++++++--------- 10 files changed, 150 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33aa1756f0d..6ee9674253e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -109,6 +109,7 @@ - diagnostics collect file mod times are set. {pull}570[570] - Allow ':' characters in dynamic variables {issue}624[624] {pull}680[680] - Allow the - char to appear as part of variable names in eql expressions. {issue}709[709] {pull}710[710] +- Allow the / char in variable names in eql and transpiler. {issue}715[715] {pull}718[718] ==== New features diff --git a/internal/pkg/agent/transpiler/vars.go b/internal/pkg/agent/transpiler/vars.go index a9f96b15ee8..e8f06a6928b 100644 --- a/internal/pkg/agent/transpiler/vars.go +++ b/internal/pkg/agent/transpiler/vars.go @@ -14,7 +14,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/core/composable" ) -var varsRegex = regexp.MustCompile(`\${([\p{L}\d\s\\\-_|.'":]*)}`) +var varsRegex = regexp.MustCompile(`\${([\p{L}\d\s\\\-_|.'":\/]*)}`) // ErrNoMatch is return when the replace didn't fail, just that no vars match to perform the replace. var ErrNoMatch = fmt.Errorf("no matching vars") diff --git a/internal/pkg/agent/transpiler/vars_test.go b/internal/pkg/agent/transpiler/vars_test.go index 142ab132109..56e27694a33 100644 --- a/internal/pkg/agent/transpiler/vars_test.go +++ b/internal/pkg/agent/transpiler/vars_test.go @@ -24,6 +24,7 @@ func TestVars_Replace(t *testing.T) { "array1", "array2", }, + "with/slash": "some/path", "dict": map[string]interface{}{ "key1": "value1", "key2": "value2", @@ -168,6 +169,12 @@ func TestVars_Replace(t *testing.T) { false, false, }, + { + `${un-der_score.with/slash}`, + NewStrVal(`some/path`), + false, + false, + }, { `list inside string ${un-der_score.list} causes no match`, NewList([]Node{ diff --git a/internal/pkg/composable/providers/kubernetes/node_test.go b/internal/pkg/composable/providers/kubernetes/node_test.go index 547702573c4..ab19e7d2ce2 100644 --- a/internal/pkg/composable/providers/kubernetes/node_test.go +++ b/internal/pkg/composable/providers/kubernetes/node_test.go @@ -26,8 +26,9 @@ func TestGenerateNodeData(t *testing.T) { Name: "testnode", UID: types.UID(uid), Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, Annotations: map[string]string{ "baz": "ban", @@ -55,8 +56,9 @@ func TestGenerateNodeData(t *testing.T) { "baz": "ban", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, } @@ -67,8 +69,9 @@ func TestGenerateNodeData(t *testing.T) { "url": "8.8.8.8:9090"}, }, "kubernetes": mapstr.M{ "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{"baz": "ban"}, "node": mapstr.M{ @@ -128,8 +131,9 @@ func (n *nodeMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOp "ip": "node1", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{ "baz": "ban", diff --git a/internal/pkg/composable/providers/kubernetes/pod_test.go b/internal/pkg/composable/providers/kubernetes/pod_test.go index feeba193472..95361fd2ce0 100644 --- a/internal/pkg/composable/providers/kubernetes/pod_test.go +++ b/internal/pkg/composable/providers/kubernetes/pod_test.go @@ -27,8 +27,9 @@ func TestGeneratePodData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, Annotations: map[string]string{ "app": "production", @@ -60,8 +61,9 @@ func TestGeneratePodData(t *testing.T) { "nsa": "nsb", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{ "app": "production", @@ -76,8 +78,9 @@ func TestGeneratePodData(t *testing.T) { }, "kubernetes": mapstr.M{ "namespace": "testns", "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{"app": "production"}, "pod": mapstr.M{ @@ -122,8 +125,9 @@ func TestGenerateContainerPodData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, Annotations: map[string]string{ "app": "production", @@ -179,8 +183,9 @@ func TestGenerateContainerPodData(t *testing.T) { "app": "production", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, } @@ -196,8 +201,10 @@ func TestGenerateContainerPodData(t *testing.T) { }, "kubernetes": mapstr.M{ "namespace": "testns", "annotations": mapstr.M{"app": "production"}, - "labels": mapstr.M{"foo": "bar", - "with-dash": "dash-value", + "labels": mapstr.M{ + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "pod": mapstr.M{ "ip": "127.0.0.5", @@ -239,8 +246,9 @@ func TestEphemeralContainers(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, Annotations: map[string]string{ "app": "production", @@ -282,8 +290,9 @@ func TestEphemeralContainers(t *testing.T) { "ip": pod.Status.PodIP, }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "container": mapstr.M{ "id": "asdfghdeadbeef", @@ -310,8 +319,10 @@ func TestEphemeralContainers(t *testing.T) { "url": "8.8.8.8:9090"}, }, "kubernetes": mapstr.M{ "namespace": "testns", - "labels": mapstr.M{"foo": "bar", - "with-dash": "dash-value", + "labels": mapstr.M{ + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{"app": "production"}, "pod": mapstr.M{ @@ -394,8 +405,9 @@ func (p *podMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOpt "ip": k8sPod.Status.PodIP, }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{ "app": "production", diff --git a/internal/pkg/composable/providers/kubernetes/service_test.go b/internal/pkg/composable/providers/kubernetes/service_test.go index 0fbed196908..69e945ee1cd 100644 --- a/internal/pkg/composable/providers/kubernetes/service_test.go +++ b/internal/pkg/composable/providers/kubernetes/service_test.go @@ -25,8 +25,9 @@ func TestGenerateServiceData(t *testing.T) { UID: types.UID(uid), Namespace: "testns", Labels: map[string]string{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, Annotations: map[string]string{ "baz": "ban", @@ -65,8 +66,9 @@ func TestGenerateServiceData(t *testing.T) { "baz": "ban", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, } @@ -82,8 +84,9 @@ func TestGenerateServiceData(t *testing.T) { "ip": "1.2.3.4", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{ "baz": "ban", @@ -142,8 +145,9 @@ func (s *svcMeta) GenerateK8s(obj kubernetes.Resource, opts ...metadata.FieldOpt "ip": "1.2.3.4", }, "labels": mapstr.M{ - "foo": "bar", - "with-dash": "dash-value", + "foo": "bar", + "with-dash": "dash-value", + "with/slash": "some/path", }, "annotations": mapstr.M{ "baz": "ban", diff --git a/internal/pkg/eql/Eql.g4 b/internal/pkg/eql/Eql.g4 index b6731d41cef..bb7b5a88770 100644 --- a/internal/pkg/eql/Eql.g4 +++ b/internal/pkg/eql/Eql.g4 @@ -22,7 +22,7 @@ NUMBER: [\-]? [0-9]+; WHITESPACE: [ \r\n\t]+ -> skip; NOT: 'NOT' | 'not'; NAME: [a-zA-Z_] [a-zA-Z0-9_]*; -VNAME: [a-zA-Z0-9_.-]+('.'[a-zA-Z0-9_-]+)*; +VNAME: [a-zA-Z0-9_.\-/]+('.'[a-zA-Z0-9_\-/]+)*; STEXT: '\'' ~[\r\n']* '\''; DTEXT: '"' ~[\r\n"]* '"'; LPAR: '('; diff --git a/internal/pkg/eql/eql_test.go b/internal/pkg/eql/eql_test.go index 77bf5cb37f0..54f7741f88d 100644 --- a/internal/pkg/eql/eql_test.go +++ b/internal/pkg/eql/eql_test.go @@ -44,6 +44,7 @@ func TestEql(t *testing.T) { {expression: "${'constant'} == 'constant'", result: true}, {expression: "${data.with-dash} == 'dash-value'", result: true}, {expression: "${'dash-value'} == 'dash-value'", result: true}, + {expression: "${data.with/slash} == 'some/path'", result: true}, // boolean {expression: "true", result: true}, @@ -308,10 +309,11 @@ func TestEql(t *testing.T) { store := &testVarStore{ vars: map[string]interface{}{ - "env.HOSTNAME": "my-hostname", - "host.name": "host-name", - "data.array": []interface{}{"array1", "array2", "array3"}, - "data.with-dash": "dash-value", + "env.HOSTNAME": "my-hostname", + "host.name": "host-name", + "data.array": []interface{}{"array1", "array2", "array3"}, + "data.with-dash": "dash-value", + "data.with/slash": "some/path", "data.dict": map[string]interface{}{ "key1": "dict1", "key2": "dict2", diff --git a/internal/pkg/eql/parser/EqlLexer.interp b/internal/pkg/eql/parser/EqlLexer.interp index 3432105b62f..66413a00c42 100644 --- a/internal/pkg/eql/parser/EqlLexer.interp +++ b/internal/pkg/eql/parser/EqlLexer.interp @@ -113,4 +113,4 @@ mode names: DEFAULT_MODE atn: -[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 35, 230, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 12, 3, 12, 3, 13, 3, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 5, 16, 108, 10, 16, 3, 17, 3, 17, 3, 17, 3, 17, 5, 17, 114, 10, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 5, 18, 124, 10, 18, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 5, 19, 136, 10, 19, 3, 20, 5, 20, 139, 10, 20, 3, 20, 6, 20, 142, 10, 20, 13, 20, 14, 20, 143, 3, 20, 3, 20, 6, 20, 148, 10, 20, 13, 20, 14, 20, 149, 3, 21, 5, 21, 153, 10, 21, 3, 21, 6, 21, 156, 10, 21, 13, 21, 14, 21, 157, 3, 22, 6, 22, 161, 10, 22, 13, 22, 14, 22, 162, 3, 22, 3, 22, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 173, 10, 23, 3, 24, 3, 24, 7, 24, 177, 10, 24, 12, 24, 14, 24, 180, 11, 24, 3, 25, 6, 25, 183, 10, 25, 13, 25, 14, 25, 184, 3, 25, 3, 25, 6, 25, 189, 10, 25, 13, 25, 14, 25, 190, 7, 25, 193, 10, 25, 12, 25, 14, 25, 196, 11, 25, 3, 26, 3, 26, 7, 26, 200, 10, 26, 12, 26, 14, 26, 203, 11, 26, 3, 26, 3, 26, 3, 27, 3, 27, 7, 27, 209, 10, 27, 12, 27, 14, 27, 212, 11, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 30, 3, 30, 3, 31, 3, 31, 3, 32, 3, 32, 3, 33, 3, 33, 3, 34, 3, 34, 3, 34, 2, 2, 35, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 11, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 48, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 47, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, 3, 8, 2, 2] \ No newline at end of file +[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 35, 230, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 12, 3, 12, 3, 13, 3, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 5, 16, 108, 10, 16, 3, 17, 3, 17, 3, 17, 3, 17, 5, 17, 114, 10, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 5, 18, 124, 10, 18, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 5, 19, 136, 10, 19, 3, 20, 5, 20, 139, 10, 20, 3, 20, 6, 20, 142, 10, 20, 13, 20, 14, 20, 143, 3, 20, 3, 20, 6, 20, 148, 10, 20, 13, 20, 14, 20, 149, 3, 21, 5, 21, 153, 10, 21, 3, 21, 6, 21, 156, 10, 21, 13, 21, 14, 21, 157, 3, 22, 6, 22, 161, 10, 22, 13, 22, 14, 22, 162, 3, 22, 3, 22, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 173, 10, 23, 3, 24, 3, 24, 7, 24, 177, 10, 24, 12, 24, 14, 24, 180, 11, 24, 3, 25, 6, 25, 183, 10, 25, 13, 25, 14, 25, 184, 3, 25, 3, 25, 6, 25, 189, 10, 25, 13, 25, 14, 25, 190, 7, 25, 193, 10, 25, 12, 25, 14, 25, 196, 11, 25, 3, 26, 3, 26, 7, 26, 200, 10, 26, 12, 26, 14, 26, 203, 11, 26, 3, 26, 3, 26, 3, 27, 3, 27, 7, 27, 209, 10, 27, 12, 27, 14, 27, 212, 11, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 30, 3, 30, 3, 31, 3, 31, 3, 32, 3, 32, 3, 33, 3, 33, 3, 34, 3, 34, 3, 34, 2, 2, 35, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 11, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 6, 2, 47, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 47, 49, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, 3, 8, 2, 2] \ No newline at end of file diff --git a/internal/pkg/eql/parser/eql_lexer.go b/internal/pkg/eql/parser/eql_lexer.go index d817439a858..b8eb1eeed6d 100644 --- a/internal/pkg/eql/parser/eql_lexer.go +++ b/internal/pkg/eql/parser/eql_lexer.go @@ -48,83 +48,83 @@ var serializedLexerAtn = []uint16{ 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 3, 2, 11, 3, 2, 47, 47, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, 34, 5, 2, 67, 92, 97, 97, 99, - 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 48, 50, 59, 67, 92, - 97, 97, 99, 124, 7, 2, 47, 47, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, - 12, 15, 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, - 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, - 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, - 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, - 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, - 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, - 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, - 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, - 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, - 2, 2, 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, - 2, 2, 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, - 83, 3, 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, - 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, - 3, 2, 2, 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, - 2, 37, 135, 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, - 3, 2, 2, 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, - 2, 51, 197, 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, - 3, 2, 2, 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, - 2, 65, 225, 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, - 4, 3, 2, 2, 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, - 2, 74, 8, 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, - 3, 2, 2, 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, - 2, 81, 82, 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, - 3, 2, 2, 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, - 2, 88, 89, 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, - 7, 45, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, - 2, 95, 96, 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, - 3, 2, 2, 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, - 2, 2, 102, 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, - 2, 2, 105, 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, - 2, 107, 104, 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, - 114, 7, 116, 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, - 3, 2, 2, 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, - 2, 2, 116, 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, - 2, 2, 119, 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, - 2, 122, 124, 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, - 36, 3, 2, 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, - 7, 110, 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, - 7, 72, 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, - 85, 2, 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, - 2, 2, 136, 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, - 138, 139, 3, 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, - 140, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, - 3, 2, 2, 2, 144, 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, - 2, 2, 147, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, - 149, 150, 3, 2, 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, - 3, 2, 2, 2, 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, - 2, 2, 155, 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, - 157, 158, 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, - 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, - 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, - 166, 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, - 170, 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, - 166, 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, - 5, 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, - 2, 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, - 178, 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, - 3, 2, 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, - 2, 2, 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, - 189, 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, - 193, 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, - 3, 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, - 2, 2, 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, - 2, 200, 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, - 204, 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, - 3, 2, 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, - 2, 2, 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, - 2, 211, 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, - 54, 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, - 43, 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, - 2, 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, - 224, 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, - 228, 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, - 113, 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, - 210, 3, 8, 2, 2, + 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 6, 2, 47, 59, 67, 92, 97, 97, + 99, 124, 7, 2, 47, 47, 49, 59, 67, 92, 97, 97, 99, 124, 5, 2, 12, 12, 15, + 15, 41, 41, 5, 2, 12, 12, 15, 15, 36, 36, 2, 246, 2, 3, 3, 2, 2, 2, 2, + 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, + 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, + 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, + 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, + 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, + 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, + 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, + 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, + 2, 67, 3, 2, 2, 2, 3, 69, 3, 2, 2, 2, 5, 71, 3, 2, 2, 2, 7, 73, 3, 2, 2, + 2, 9, 75, 3, 2, 2, 2, 11, 78, 3, 2, 2, 2, 13, 81, 3, 2, 2, 2, 15, 83, 3, + 2, 2, 2, 17, 85, 3, 2, 2, 2, 19, 88, 3, 2, 2, 2, 21, 91, 3, 2, 2, 2, 23, + 93, 3, 2, 2, 2, 25, 95, 3, 2, 2, 2, 27, 97, 3, 2, 2, 2, 29, 99, 3, 2, 2, + 2, 31, 107, 3, 2, 2, 2, 33, 113, 3, 2, 2, 2, 35, 123, 3, 2, 2, 2, 37, 135, + 3, 2, 2, 2, 39, 138, 3, 2, 2, 2, 41, 152, 3, 2, 2, 2, 43, 160, 3, 2, 2, + 2, 45, 172, 3, 2, 2, 2, 47, 174, 3, 2, 2, 2, 49, 182, 3, 2, 2, 2, 51, 197, + 3, 2, 2, 2, 53, 206, 3, 2, 2, 2, 55, 215, 3, 2, 2, 2, 57, 217, 3, 2, 2, + 2, 59, 219, 3, 2, 2, 2, 61, 221, 3, 2, 2, 2, 63, 223, 3, 2, 2, 2, 65, 225, + 3, 2, 2, 2, 67, 227, 3, 2, 2, 2, 69, 70, 7, 126, 2, 2, 70, 4, 3, 2, 2, + 2, 71, 72, 7, 46, 2, 2, 72, 6, 3, 2, 2, 2, 73, 74, 7, 60, 2, 2, 74, 8, + 3, 2, 2, 2, 75, 76, 7, 63, 2, 2, 76, 77, 7, 63, 2, 2, 77, 10, 3, 2, 2, + 2, 78, 79, 7, 35, 2, 2, 79, 80, 7, 63, 2, 2, 80, 12, 3, 2, 2, 2, 81, 82, + 7, 64, 2, 2, 82, 14, 3, 2, 2, 2, 83, 84, 7, 62, 2, 2, 84, 16, 3, 2, 2, + 2, 85, 86, 7, 64, 2, 2, 86, 87, 7, 63, 2, 2, 87, 18, 3, 2, 2, 2, 88, 89, + 7, 62, 2, 2, 89, 90, 7, 63, 2, 2, 90, 20, 3, 2, 2, 2, 91, 92, 7, 45, 2, + 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 47, 2, 2, 94, 24, 3, 2, 2, 2, 95, 96, + 7, 44, 2, 2, 96, 26, 3, 2, 2, 2, 97, 98, 7, 49, 2, 2, 98, 28, 3, 2, 2, + 2, 99, 100, 7, 39, 2, 2, 100, 30, 3, 2, 2, 2, 101, 102, 7, 99, 2, 2, 102, + 103, 7, 112, 2, 2, 103, 108, 7, 102, 2, 2, 104, 105, 7, 67, 2, 2, 105, + 106, 7, 80, 2, 2, 106, 108, 7, 70, 2, 2, 107, 101, 3, 2, 2, 2, 107, 104, + 3, 2, 2, 2, 108, 32, 3, 2, 2, 2, 109, 110, 7, 113, 2, 2, 110, 114, 7, 116, + 2, 2, 111, 112, 7, 81, 2, 2, 112, 114, 7, 84, 2, 2, 113, 109, 3, 2, 2, + 2, 113, 111, 3, 2, 2, 2, 114, 34, 3, 2, 2, 2, 115, 116, 7, 118, 2, 2, 116, + 117, 7, 116, 2, 2, 117, 118, 7, 119, 2, 2, 118, 124, 7, 103, 2, 2, 119, + 120, 7, 86, 2, 2, 120, 121, 7, 84, 2, 2, 121, 122, 7, 87, 2, 2, 122, 124, + 7, 71, 2, 2, 123, 115, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 36, 3, 2, + 2, 2, 125, 126, 7, 104, 2, 2, 126, 127, 7, 99, 2, 2, 127, 128, 7, 110, + 2, 2, 128, 129, 7, 117, 2, 2, 129, 136, 7, 103, 2, 2, 130, 131, 7, 72, + 2, 2, 131, 132, 7, 67, 2, 2, 132, 133, 7, 78, 2, 2, 133, 134, 7, 85, 2, + 2, 134, 136, 7, 71, 2, 2, 135, 125, 3, 2, 2, 2, 135, 130, 3, 2, 2, 2, 136, + 38, 3, 2, 2, 2, 137, 139, 9, 2, 2, 2, 138, 137, 3, 2, 2, 2, 138, 139, 3, + 2, 2, 2, 139, 141, 3, 2, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, + 2, 142, 143, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, + 145, 3, 2, 2, 2, 145, 147, 7, 48, 2, 2, 146, 148, 9, 3, 2, 2, 147, 146, + 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, + 2, 2, 150, 40, 3, 2, 2, 2, 151, 153, 9, 2, 2, 2, 152, 151, 3, 2, 2, 2, + 152, 153, 3, 2, 2, 2, 153, 155, 3, 2, 2, 2, 154, 156, 9, 3, 2, 2, 155, + 154, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 155, 3, 2, 2, 2, 157, 158, + 3, 2, 2, 2, 158, 42, 3, 2, 2, 2, 159, 161, 9, 4, 2, 2, 160, 159, 3, 2, + 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, + 163, 164, 3, 2, 2, 2, 164, 165, 8, 22, 2, 2, 165, 44, 3, 2, 2, 2, 166, + 167, 7, 80, 2, 2, 167, 168, 7, 81, 2, 2, 168, 173, 7, 86, 2, 2, 169, 170, + 7, 112, 2, 2, 170, 171, 7, 113, 2, 2, 171, 173, 7, 118, 2, 2, 172, 166, + 3, 2, 2, 2, 172, 169, 3, 2, 2, 2, 173, 46, 3, 2, 2, 2, 174, 178, 9, 5, + 2, 2, 175, 177, 9, 6, 2, 2, 176, 175, 3, 2, 2, 2, 177, 180, 3, 2, 2, 2, + 178, 176, 3, 2, 2, 2, 178, 179, 3, 2, 2, 2, 179, 48, 3, 2, 2, 2, 180, 178, + 3, 2, 2, 2, 181, 183, 9, 7, 2, 2, 182, 181, 3, 2, 2, 2, 183, 184, 3, 2, + 2, 2, 184, 182, 3, 2, 2, 2, 184, 185, 3, 2, 2, 2, 185, 194, 3, 2, 2, 2, + 186, 188, 7, 48, 2, 2, 187, 189, 9, 8, 2, 2, 188, 187, 3, 2, 2, 2, 189, + 190, 3, 2, 2, 2, 190, 188, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 193, + 3, 2, 2, 2, 192, 186, 3, 2, 2, 2, 193, 196, 3, 2, 2, 2, 194, 192, 3, 2, + 2, 2, 194, 195, 3, 2, 2, 2, 195, 50, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, + 197, 201, 7, 41, 2, 2, 198, 200, 10, 9, 2, 2, 199, 198, 3, 2, 2, 2, 200, + 203, 3, 2, 2, 2, 201, 199, 3, 2, 2, 2, 201, 202, 3, 2, 2, 2, 202, 204, + 3, 2, 2, 2, 203, 201, 3, 2, 2, 2, 204, 205, 7, 41, 2, 2, 205, 52, 3, 2, + 2, 2, 206, 210, 7, 36, 2, 2, 207, 209, 10, 10, 2, 2, 208, 207, 3, 2, 2, + 2, 209, 212, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, + 213, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, 213, 214, 7, 36, 2, 2, 214, 54, + 3, 2, 2, 2, 215, 216, 7, 42, 2, 2, 216, 56, 3, 2, 2, 2, 217, 218, 7, 43, + 2, 2, 218, 58, 3, 2, 2, 2, 219, 220, 7, 93, 2, 2, 220, 60, 3, 2, 2, 2, + 221, 222, 7, 95, 2, 2, 222, 62, 3, 2, 2, 2, 223, 224, 7, 125, 2, 2, 224, + 64, 3, 2, 2, 2, 225, 226, 7, 127, 2, 2, 226, 66, 3, 2, 2, 2, 227, 228, + 7, 38, 2, 2, 228, 229, 7, 125, 2, 2, 229, 68, 3, 2, 2, 2, 20, 2, 107, 113, + 123, 135, 138, 143, 149, 152, 157, 162, 172, 178, 184, 190, 194, 201, 210, + 3, 8, 2, 2, } var lexerDeserializer = antlr.NewATNDeserializer(nil) From 8ef98f102abf196903ff08f394797cd7888f5ac2 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 19 Jul 2022 16:04:38 -0400 Subject: [PATCH 52/69] Fix Elastic Agent non-fleet broken upgrade between 8.3.x releases (#701) * Fix Elastic Agent non-fleet broken upgrade between 8.3.x releases * Migrates vault directory on linux and windows to the top directory of the agent, so it can be shared without needing the upgrade handler call, like for example with side-by-side install/upgrade from .rpm/.deb * Extended vault to allow read-only open, useful when the vault at particular location needs to be only read not created. * Correct the typo in the log messages * Update lint flagged function comment with 'unused', was flagged with 'deadcode' on the previous run * Address code review feedback * Add missing import for linux utz * Change vault path from Top() to Config(), this a better location, next to fleet.enc based on the install/upgrade testing with .rpm/.deb installs * Fix the missing state migration for .rpm/.deb upgrade. The post install script now performs the migration and creates the symlink after that. * Fix typo in the postinstall script * Update the vault migration code, add the agent configuration match check with the agent secret --- dev-tools/packaging/packages.yml | 10 +- .../templates/linux/postinstall.sh.tmpl | 38 ++ .../agent/application/paths/paths_linux.go | 2 +- .../agent/application/paths/paths_windows.go | 2 +- .../pkg/agent/application/secret/secret.go | 41 +- .../pkg/agent/application/upgrade/upgrade.go | 37 -- internal/pkg/agent/cmd/run.go | 17 + .../pkg/agent/migration/migrate_secret.go | 163 ++++++++ .../agent/migration/migrate_secret_test.go | 386 ++++++++++++++++++ internal/pkg/agent/vault/seed.go | 27 ++ internal/pkg/agent/vault/seed_test.go | 43 +- internal/pkg/agent/vault/vault_darwin.go | 2 +- internal/pkg/agent/vault/vault_linux.go | 24 +- internal/pkg/agent/vault/vault_options.go | 28 ++ internal/pkg/agent/vault/vault_windows.go | 29 +- 15 files changed, 772 insertions(+), 77 deletions(-) create mode 100644 dev-tools/packaging/templates/linux/postinstall.sh.tmpl create mode 100644 internal/pkg/agent/migration/migrate_secret.go create mode 100644 internal/pkg/agent/migration/migrate_secret_test.go create mode 100644 internal/pkg/agent/vault/vault_options.go diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index acc89420081..bd5e9d1722c 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -20,11 +20,8 @@ shared: # Deb/RPM spec for community beats. - &deb_rpm_agent_spec <<: *common - post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/systemd-daemon-reload.sh' + post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/postinstall.sh.tmpl' files: - /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: - source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - mode: 0755 /usr/share/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 @@ -1083,11 +1080,6 @@ specs: spec: <<: *deb_rpm_agent_spec <<: *elastic_license_for_deb_rpm - files: - /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: - source: /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}} - symlink: true - mode: 0755 - os: linux arch: amd64 diff --git a/dev-tools/packaging/templates/linux/postinstall.sh.tmpl b/dev-tools/packaging/templates/linux/postinstall.sh.tmpl new file mode 100644 index 00000000000..083ebb91060 --- /dev/null +++ b/dev-tools/packaging/templates/linux/postinstall.sh.tmpl @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +symlink="/usr/share/elastic-agent/bin/elastic-agent" +old_agent_dir="$( dirname "$(readlink -f -- "$symlink")" )" + +commit_hash="{{ commit_short }}" + +yml_path="$old_agent_dir/state.yml" +enc_path="$old_agent_dir/state.enc" + +new_agent_dir="$( dirname "$old_agent_dir")/elastic-agent-$commit_hash" + +if ! [[ "$old_agent_dir" -ef "$new_agent_dir" ]]; then + echo "migrate state from $old_agent_dir to $new_agent_dir" + + if test -f "$yml_path"; then + echo "found "$yml_path", copy to "$new_agent_dir"." + cp "$yml_path" "$new_agent_dir" + fi + + if test -f "$enc_path"; then + echo "found "$enc_path", copy to "$new_agent_dir"." + cp "$enc_path" "$new_agent_dir" + fi + + if test -f "$symlink"; then + echo "found symlink $symlink, unlink" + unlink "$symlink" + fi + + echo "create symlink "$symlink" to "$new_agent_dir/elastic-agent"" + ln -s "$new_agent_dir/elastic-agent" "$symlink" +fi + +systemctl daemon-reload 2> /dev/null +exit 0 diff --git a/internal/pkg/agent/application/paths/paths_linux.go b/internal/pkg/agent/application/paths/paths_linux.go index 22faeb5f75a..37cc57c33af 100644 --- a/internal/pkg/agent/application/paths/paths_linux.go +++ b/internal/pkg/agent/application/paths/paths_linux.go @@ -14,5 +14,5 @@ const defaultAgentVaultPath = "vault" // AgentVaultPath is the directory that contains all the files for the value func AgentVaultPath() string { - return filepath.Join(Home(), defaultAgentVaultPath) + return filepath.Join(Config(), defaultAgentVaultPath) } diff --git a/internal/pkg/agent/application/paths/paths_windows.go b/internal/pkg/agent/application/paths/paths_windows.go index 2fc6fd008a0..0b81aa2061b 100644 --- a/internal/pkg/agent/application/paths/paths_windows.go +++ b/internal/pkg/agent/application/paths/paths_windows.go @@ -42,5 +42,5 @@ func ArePathsEqual(expected, actual string) bool { // AgentVaultPath is the directory that contains all the files for the value func AgentVaultPath() string { - return filepath.Join(Home(), defaultAgentVaultPath) + return filepath.Join(Config(), defaultAgentVaultPath) } diff --git a/internal/pkg/agent/application/secret/secret.go b/internal/pkg/agent/application/secret/secret.go index edce9eda174..cf690bf24e8 100644 --- a/internal/pkg/agent/application/secret/secret.go +++ b/internal/pkg/agent/application/secret/secret.go @@ -6,6 +6,7 @@ package secret import ( "encoding/json" + "fmt" "runtime" "sync" "time" @@ -52,7 +53,7 @@ func Create(key string, opts ...OptionFunc) error { options := applyOptions(opts...) v, err := vault.New(options.vaultPath) if err != nil { - return err + return fmt.Errorf("could not create new vault: %w", err) } defer v.Close() @@ -80,12 +81,7 @@ func Create(key string, opts ...OptionFunc) error { CreatedOn: time.Now().UTC(), } - b, err := json.Marshal(secret) - if err != nil { - return err - } - - return v.Set(key, b) + return set(v, key, secret) } // GetAgentSecret read the agent secret from the vault @@ -93,10 +89,17 @@ func GetAgentSecret(opts ...OptionFunc) (secret Secret, err error) { return Get(agentSecretKey, opts...) } +// SetAgentSecret saves the agent secret from the vault +// This is needed for migration from 8.3.0-8.3.2 to higher versions +func SetAgentSecret(secret Secret, opts ...OptionFunc) error { + return Set(agentSecretKey, secret, opts...) +} + // Get reads the secret key from the vault func Get(key string, opts ...OptionFunc) (secret Secret, err error) { options := applyOptions(opts...) - v, err := vault.New(options.vaultPath) + // open vault readonly, will not create the vault directory or the seed it was not created before + v, err := vault.New(options.vaultPath, vault.WithReadonly(true)) if err != nil { return secret, err } @@ -111,12 +114,32 @@ func Get(key string, opts ...OptionFunc) (secret Secret, err error) { return secret, err } +// Set saves the secret key to the vault +func Set(key string, secret Secret, opts ...OptionFunc) error { + options := applyOptions(opts...) + v, err := vault.New(options.vaultPath) + if err != nil { + return fmt.Errorf("could not create new vault: %w", err) + } + defer v.Close() + return set(v, key, secret) +} + +func set(v *vault.Vault, key string, secret Secret) error { + b, err := json.Marshal(secret) + if err != nil { + return fmt.Errorf("could not marshal secret: %w", err) + } + + return v.Set(key, b) +} + // Remove removes the secret key from the vault func Remove(key string, opts ...OptionFunc) error { options := applyOptions(opts...) v, err := vault.New(options.vaultPath) if err != nil { - return err + return fmt.Errorf("could not create new vault: %w", err) } defer v.Close() diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 9d67165d0eb..1d370cb5301 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "os" "path/filepath" - "runtime" "strings" "github.com/otiai10/copy" @@ -33,7 +32,6 @@ const ( agentName = "elastic-agent" hashLen = 6 agentCommitFile = ".elastic-agent.active.commit" - darwin = "darwin" ) var ( @@ -161,11 +159,6 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree return nil, nil } - // Copy vault directory for linux/windows only - if err := copyVault(newHash); err != nil { - return nil, errors.New(err, "failed to copy vault") - } - if err := copyActionStore(newHash); err != nil { return nil, errors.New(err, "failed to copy action store") } @@ -300,36 +293,6 @@ func copyActionStore(newHash string) error { return nil } -func getVaultPath(newHash string) string { - vaultPath := paths.AgentVaultPath() - if runtime.GOOS == darwin { - return vaultPath - } - newHome := filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, newHash)) - return filepath.Join(newHome, filepath.Base(vaultPath)) -} - -// Copies the vault files for windows and linux -func copyVault(newHash string) error { - // No vault files to copy on darwin - if runtime.GOOS == darwin { - return nil - } - - vaultPath := paths.AgentVaultPath() - newVaultPath := getVaultPath(newHash) - - err := copyDir(vaultPath, newVaultPath) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - return nil -} - // shutdownCallback returns a callback function to be executing during shutdown once all processes are closed. // this goes through runtime directory of agent and copies all the state files created by processes to new versioned // home directory with updated process name to match new version. diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index ad508af9086..732831d87d8 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -37,6 +37,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/control/server" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + "github.com/elastic/elastic-agent/internal/pkg/agent/migration" "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/elastic/elastic-agent/internal/pkg/config" @@ -121,6 +122,22 @@ func run(override cfgOverrider) error { createAgentID = false } + // This is specific for the agent upgrade from 8.3.0 - 8.3.2 to 8.x and above on Linux and Windows platforms. + // Addresses the issue: https://github.com/elastic/elastic-agent/issues/682 + // The vault directory was located in the hash versioned "Home" directory of the agent. + // This moves the vault directory two levels up into the "Config" directory next to fleet.enc file + // in order to be able to "upgrade" the agent from deb/rpm that is not invoking the upgrade handle and + // doesn't perform the migration of the state or vault. + // If the agent secret doesn't exist, then search for the newest agent secret in the agent data directories + // and migrate it into the new vault location. + err = migration.MigrateAgentSecret(logger) + logger.Debug("migration of agent secret completed, err: %v", err) + if err != nil { + err = errors.New(err, "failed to perfrom the agent secret migration") + logger.Error(err) + return err + } + // Ensure we have the agent secret created. // The secret is not created here if it exists already from the previous enrollment. // This is needed for compatibility with agent running in standalone mode, diff --git a/internal/pkg/agent/migration/migrate_secret.go b/internal/pkg/agent/migration/migrate_secret.go new file mode 100644 index 00000000000..08cfc3e5eb1 --- /dev/null +++ b/internal/pkg/agent/migration/migrate_secret.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package migration + +import ( + "errors" + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" + "github.com/elastic/elastic-agent/internal/pkg/agent/storage" + "github.com/elastic/elastic-agent/internal/pkg/fileutil" +) + +const ( + darwin = "darwin" +) + +// MigrateAgentSecret migrates agent secret if the secret doesn't exists agent upgrade from 8.3.0 - 8.3.2 to 8.x and above on Linux and Windows platforms. +func MigrateAgentSecret(log *logp.Logger) error { + // Nothing to migrate for darwin + if runtime.GOOS == darwin { + return nil + } + + // Check if the secret already exists + log.Debug("migrate agent secret, check if secret already exists") + _, err := secret.GetAgentSecret() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // The secret doesn't exists, perform migration below + log.Debug("agent secret doesn't exists, perform migration") + } else { + err = fmt.Errorf("failed read the agent secret: %w", err) + log.Error(err) + return err + } + } else { + // The secret already exists, nothing to migrate + log.Debug("secret already exists nothing to migrate") + return nil + } + + // Check if the secret was copied by the fleet upgrade handler to the legacy location + log.Debug("check if secret was copied over by 8.3.0-8.3.2 version of the agent") + sec, err := getAgentSecretFromHomePath(paths.Home()) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // The secret is not found in this instance of the vault, continue with migration + log.Debug("agent secret copied from 8.3.0-8.3.2 doesn't exists, continue with migration") + } else { + err = fmt.Errorf("failed agent 8.3.0-8.3.2 secret check: %w", err) + log.Error(err) + return err + } + } else { + // The secret is found, save in the new agent vault + log.Debug("agent secret from 8.3.0-8.3.2 is found, migrate to the new vault") + return secret.SetAgentSecret(sec) + } + + // Scan other agent data directories, find the latest agent secret + log.Debug("search for possible latest agent 8.3.0-8.3.2 secret") + dataDir := paths.Data() + + sec, err = findPreviousAgentSecret(dataDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // The secret is not found + log.Debug("no previous agent 8.3.0-8.3.2 secrets found, nothing to migrate") + return nil + } + err = fmt.Errorf("search for possible latest agent 8.3.0-8.3.2 secret failed: %w", err) + log.Error(err) + return err + } + log.Debug("found previous agent 8.3.0-8.3.2 secret, migrate to the new vault") + return secret.SetAgentSecret(sec) +} + +func findPreviousAgentSecret(dataDir string) (secret.Secret, error) { + found := false + var sec secret.Secret + fileSystem := os.DirFS(dataDir) + _ = fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if strings.HasPrefix(d.Name(), "elastic-agent-") { + vaultPath := getLegacyVaultPathFromPath(filepath.Join(dataDir, path)) + s, err := secret.GetAgentSecret(secret.WithVaultPath(vaultPath)) + if err != nil { + // Ignore if fs.ErrNotExist error, keep scanning + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + + // Check that the configuration can be decrypted with the found agent secret + exists, _ := fileutil.FileExists(paths.AgentConfigFile()) + if exists { + store := storage.NewEncryptedDiskStore(paths.AgentConfigFile(), storage.WithVaultPath(vaultPath)) + r, err := store.Load() + if err != nil { + //nolint:nilerr // ignore the error keep scanning + return nil + } + + defer r.Close() + _, err = ioutil.ReadAll(r) + if err != nil { + //nolint:nilerr // ignore the error keep scanning + return nil + } + + sec = s + found = true + return io.EOF + } + } else if d.Name() != "." { + return fs.SkipDir + } + } + return nil + }) + if !found { + return sec, fs.ErrNotExist + } + return sec, nil +} + +func getAgentSecretFromHomePath(homePath string) (sec secret.Secret, err error) { + vaultPath := getLegacyVaultPathFromPath(homePath) + fi, err := os.Stat(vaultPath) + if err != nil { + return + } + + if !fi.IsDir() { + return sec, fs.ErrNotExist + } + return secret.GetAgentSecret(secret.WithVaultPath(vaultPath)) +} + +func getLegacyVaultPath() string { + return getLegacyVaultPathFromPath(paths.Home()) +} + +func getLegacyVaultPathFromPath(path string) string { + return filepath.Join(path, "vault") +} diff --git a/internal/pkg/agent/migration/migrate_secret_test.go b/internal/pkg/agent/migration/migrate_secret_test.go new file mode 100644 index 00000000000..562549c6db8 --- /dev/null +++ b/internal/pkg/agent/migration/migrate_secret_test.go @@ -0,0 +1,386 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux || windows +// +build linux windows + +package migration + +import ( + "errors" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" + "github.com/elastic/elastic-agent/internal/pkg/agent/storage" + "github.com/elastic/elastic-agent/internal/pkg/agent/vault" + "github.com/gofrs/uuid" + "github.com/google/go-cmp/cmp" +) + +func TestFindAgentSecretFromHomePath(t *testing.T) { + + tests := []struct { + name string + setupFn func(homePath string) error + wantErr error + }{ + { + name: "no data dir", + wantErr: fs.ErrNotExist, + }, + { + name: "no vault dir", + setupFn: func(homePath string) error { + return os.MkdirAll(homePath, 0750) + }, + wantErr: fs.ErrNotExist, + }, + { + name: "vault file instead of directory", + setupFn: func(homePath string) error { + err := os.MkdirAll(homePath, 0750) + if err != nil { + return err + } + return ioutil.WriteFile(getLegacyVaultPathFromPath(homePath), []byte{}, 0600) + }, + wantErr: fs.ErrNotExist, + }, + { + name: "empty vault directory", + setupFn: func(homePath string) error { + return os.MkdirAll(getLegacyVaultPathFromPath(homePath), 0750) + }, + wantErr: fs.ErrNotExist, + }, + { + name: "empty vault", + setupFn: func(homePath string) error { + v, err := vault.New(getLegacyVaultPathFromPath(homePath)) + if err != nil { + return err + } + defer v.Close() + return nil + }, + wantErr: fs.ErrNotExist, + }, + { + name: "vault dir with no seed", + setupFn: func(homePath string) error { + vaultPath := getLegacyVaultPathFromPath(homePath) + v, err := vault.New(vaultPath) + if err != nil { + return err + } + defer v.Close() + return os.Remove(filepath.Join(vaultPath, ".seed")) + }, + wantErr: fs.ErrNotExist, + }, + { + name: "vault with secret and misplaced seed vault", + setupFn: func(homePath string) error { + vaultPath := getLegacyVaultPathFromPath(homePath) + err := secret.CreateAgentSecret(secret.WithVaultPath(vaultPath)) + if err != nil { + return err + } + return os.Remove(filepath.Join(vaultPath, ".seed")) + }, + wantErr: fs.ErrNotExist, + }, + { + name: "vault with valid secret", + setupFn: func(homePath string) error { + vaultPath := getLegacyVaultPathFromPath(homePath) + err := secret.CreateAgentSecret(secret.WithVaultPath(vaultPath)) + if err != nil { + return err + } + return generateTestConfig(vaultPath) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + top := t.TempDir() + paths.SetTop(top) + homePath := paths.Home() + + if tc.setupFn != nil { + if err := tc.setupFn(homePath); err != nil { + t.Fatal(err) + } + } + + sec, err := getAgentSecretFromHomePath(homePath) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("want err: %v, got err: %v", tc.wantErr, err) + } + + foundSec, err := findPreviousAgentSecret(filepath.Dir(homePath)) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("want err: %v, got err: %v", tc.wantErr, err) + } + diff := cmp.Diff(sec, foundSec) + if diff != "" { + t.Fatal(diff) + } + + }) + } +} + +type configType int + +const ( + NoConfig configType = iota + MatchingConfig + NonMatchingConfig +) + +func TestFindNewestAgentSecret(t *testing.T) { + + tests := []struct { + name string + cfgType configType + wantErr error + }{ + { + name: "missing config", + cfgType: NoConfig, + wantErr: fs.ErrNotExist, + }, + { + name: "matching config", + cfgType: MatchingConfig, + }, + { + name: "non-matching config", + cfgType: NonMatchingConfig, + wantErr: fs.ErrNotExist, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + top := t.TempDir() + paths.SetTop(top) + paths.SetConfig(top) + dataDir := paths.Data() + + wantSecret, err := generateTestSecrets(dataDir, 3, tc.cfgType) + if err != nil { + t.Fatal(err) + } + sec, err := findPreviousAgentSecret(dataDir) + + if !errors.Is(err, tc.wantErr) { + t.Fatalf("want err: %v, got err: %v", tc.wantErr, err) + } + diff := cmp.Diff(sec, wantSecret) + if diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestMigrateAgentSecret(t *testing.T) { + top := t.TempDir() + paths.SetTop(top) + paths.SetConfig(top) + dataDir := paths.Data() + + // No vault home path + homePath := generateTestHomePath(dataDir) + if err := os.MkdirAll(homePath, 0750); err != nil { + t.Fatal(err) + } + + // Empty vault home path + homePath = generateTestHomePath(dataDir) + vaultPath := getLegacyVaultPathFromPath(homePath) + if err := os.MkdirAll(vaultPath, 0750); err != nil { + t.Fatal(err) + } + + // Vault with missing seed + homePath = generateTestHomePath(dataDir) + vaultPath = getLegacyVaultPathFromPath(homePath) + v, err := vault.New(vaultPath) + if err != nil { + t.Fatal(err) + } + defer v.Close() + + if err = os.Remove(filepath.Join(vaultPath, ".seed")); err != nil { + t.Fatal(err) + } + + // Generate few valid secrets to scan for + wantSecret, err := generateTestSecrets(dataDir, 5, MatchingConfig) + if err != nil { + t.Fatal(err) + } + + // Expect no agent secret found + _, err = secret.GetAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("expected err: %v", fs.ErrNotExist) + } + + // Perform migration + log := logp.NewLogger("test_agent_secret") + err = MigrateAgentSecret(log) + if err != nil { + t.Fatal(err) + } + + // Expect the agent secret is migrated now + sec, err := secret.GetAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if err != nil { + t.Fatal(err) + } + + // Compare the migrated secret with the expected newest one + diff := cmp.Diff(sec, wantSecret) + if diff != "" { + t.Fatal(diff) + } +} + +func TestMigrateAgentSecretAlreadyExists(t *testing.T) { + top := t.TempDir() + paths.SetTop(top) + err := secret.CreateAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if err != nil { + t.Fatal(err) + } + + // Expect agent secret created + wantSecret, err := secret.GetAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if err != nil { + t.Fatal(err) + } + + // Perform migration + log := logp.NewLogger("test_agent_secret") + err = MigrateAgentSecret(log) + if err != nil { + t.Fatal(err) + } + + sec, err := secret.GetAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if err != nil { + t.Fatal(err) + } + + // Compare, should be the same secret + diff := cmp.Diff(sec, wantSecret) + if diff != "" { + t.Fatal(diff) + } +} + +func TestMigrateAgentSecretFromLegacyLocation(t *testing.T) { + top := t.TempDir() + paths.SetTop(top) + paths.SetConfig(top) + vaultPath := getLegacyVaultPath() + err := secret.CreateAgentSecret(secret.WithVaultPath(vaultPath)) + if err != nil { + t.Fatal(err) + } + + // Expect agent secret created + wantSecret, err := secret.GetAgentSecret(secret.WithVaultPath(vaultPath)) + if err != nil { + t.Fatal(err) + } + + // Perform migration + log := logp.NewLogger("test_agent_secret") + err = MigrateAgentSecret(log) + if err != nil { + t.Fatal(err) + } + + sec, err := secret.GetAgentSecret(secret.WithVaultPath(paths.AgentVaultPath())) + if err != nil { + t.Fatal(err) + } + + // Compare, should be the same secret + diff := cmp.Diff(sec, wantSecret) + if diff != "" { + t.Fatal(diff) + } +} + +func generateTestHomePath(dataDir string) string { + suffix := uuid.Must(uuid.NewV4()).String()[:6] + return filepath.Join(dataDir, "elastic-agent-"+suffix) +} + +func generateTestConfig(vaultPath string) error { + fleetEncConfigFile := paths.AgentConfigFile() + store := storage.NewEncryptedDiskStore(fleetEncConfigFile, storage.WithVaultPath(vaultPath)) + return store.Save(strings.NewReader("foo")) +} + +func generateTestSecrets(dataDir string, count int, cfgType configType) (wantSecret secret.Secret, err error) { + now := time.Now() + + // Generate multiple home paths + //homePaths := make([]string, count) + for i := 0; i < count; i++ { + homePath := generateTestHomePath(dataDir) + k, err := vault.NewKey(vault.AES256) + if err != nil { + return wantSecret, err + } + + sec := secret.Secret{ + Value: k, + CreatedOn: now.Add(-time.Duration(i+1) * time.Minute), + } + + vaultPath := getLegacyVaultPathFromPath(homePath) + err = secret.SetAgentSecret(sec, secret.WithVaultPath(vaultPath)) + if err != nil { + return wantSecret, err + } + + switch cfgType { + case NoConfig: + case MatchingConfig, NonMatchingConfig: + if i == 0 { + wantSecret = sec + // Create matching encrypted config file, the content of the file doesn't matter for this test + err = generateTestConfig(vaultPath) + if err != nil { + return wantSecret, err + } + } + } + // Delete + if cfgType == NonMatchingConfig && i == 0 { + _ = os.RemoveAll(vaultPath) + wantSecret = secret.Secret{} + } + } + + return wantSecret, nil +} diff --git a/internal/pkg/agent/vault/seed.go b/internal/pkg/agent/vault/seed.go index 698bd0f0135..773c42e7465 100644 --- a/internal/pkg/agent/vault/seed.go +++ b/internal/pkg/agent/vault/seed.go @@ -9,6 +9,8 @@ package vault import ( "errors" + "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -29,6 +31,24 @@ func getSeed(path string) ([]byte, error) { mxSeed.Lock() defer mxSeed.Unlock() + b, err := ioutil.ReadFile(fp) + if err != nil { + return nil, fmt.Errorf("could not read seed file: %w", err) + } + + // return fs.ErrNotExists if invalid length of bytes returned + if len(b) != int(AES256) { + return nil, fmt.Errorf("invalid seed length, expected: %v, got: %v: %w", int(AES256), len(b), fs.ErrNotExist) + } + return b, nil +} + +func createSeedIfNotExists(path string) ([]byte, error) { + fp := filepath.Join(path, seedFile) + + mxSeed.Lock() + defer mxSeed.Unlock() + b, err := ioutil.ReadFile(fp) if err != nil { if !errors.Is(err, os.ErrNotExist) { @@ -52,3 +72,10 @@ func getSeed(path string) ([]byte, error) { return seed, nil } + +func getOrCreateSeed(path string, readonly bool) ([]byte, error) { + if readonly { + return getSeed(path) + } + return createSeedIfNotExists(path) +} diff --git a/internal/pkg/agent/vault/seed_test.go b/internal/pkg/agent/vault/seed_test.go index bb9197ea614..d10be29634f 100644 --- a/internal/pkg/agent/vault/seed_test.go +++ b/internal/pkg/agent/vault/seed_test.go @@ -10,12 +10,14 @@ package vault import ( "context" "encoding/hex" + "io/fs" "path/filepath" "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -24,12 +26,45 @@ func TestGetSeed(t *testing.T) { fp := filepath.Join(dir, seedFile) + require.NoFileExists(t, fp) + + // seed is not yet created + _, err := getSeed(dir) + + // should be not found + require.ErrorIs(t, err, fs.ErrNotExist) + + b, err := createSeedIfNotExists(dir) + assert.NoError(t, err) + + require.FileExists(t, fp) + + diff := cmp.Diff(int(AES256), len(b)) + if diff != "" { + t.Error(diff) + } + + // try get seed + gotSeed, err := getSeed(dir) + assert.NoError(t, err) + + diff = cmp.Diff(b, gotSeed) + if diff != "" { + t.Error(diff) + } +} + +func TestCreateSeedIfNotExists(t *testing.T) { + dir := t.TempDir() + + fp := filepath.Join(dir, seedFile) + assert.NoFileExists(t, fp) - b, err := getSeed(dir) + b, err := createSeedIfNotExists(dir) assert.NoError(t, err) - assert.FileExists(t, fp) + require.FileExists(t, fp) diff := cmp.Diff(int(AES256), len(b)) if diff != "" { @@ -37,7 +72,7 @@ func TestGetSeed(t *testing.T) { } } -func TestGetSeedRace(t *testing.T) { +func TestCreateSeedIfNotExistsRace(t *testing.T) { var err error dir := t.TempDir() @@ -51,7 +86,7 @@ func TestGetSeedRace(t *testing.T) { for i := 0; i < count; i++ { g.Go(func(idx int) func() error { return func() error { - seed, err := getSeed(dir) + seed, err := createSeedIfNotExists(dir) mx.Lock() res[idx] = seed mx.Unlock() diff --git a/internal/pkg/agent/vault/vault_darwin.go b/internal/pkg/agent/vault/vault_darwin.go index 4119b27a586..bfcb636da6f 100644 --- a/internal/pkg/agent/vault/vault_darwin.go +++ b/internal/pkg/agent/vault/vault_darwin.go @@ -38,7 +38,7 @@ type Vault struct { // New initializes the vault store // Call Close when done to release the resouces -func New(name string) (*Vault, error) { +func New(name string, opts ...OptionFunc) (*Vault, error) { var keychain C.SecKeychainRef err := statusToError(C.OpenKeychain(keychain)) if err != nil { diff --git a/internal/pkg/agent/vault/vault_linux.go b/internal/pkg/agent/vault/vault_linux.go index a3737d5c625..23d0a0e859f 100644 --- a/internal/pkg/agent/vault/vault_linux.go +++ b/internal/pkg/agent/vault/vault_linux.go @@ -11,6 +11,7 @@ import ( "crypto/rand" "crypto/sha256" "errors" + "fmt" "io/fs" "io/ioutil" "os" @@ -29,8 +30,9 @@ type Vault struct { mx sync.Mutex } -// Open initializes the vault store -func New(path string) (*Vault, error) { +// New creates the vault store +func New(path string, opts ...OptionFunc) (v *Vault, err error) { + options := applyOptions(opts...) dir := filepath.Dir(path) // If there is no specific path then get the executable directory @@ -43,12 +45,22 @@ func New(path string) (*Vault, error) { path = filepath.Join(dir, path) } - err := os.MkdirAll(path, 0750) - if err != nil { - return nil, err + if options.readonly { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fs.ErrNotExist + } + } else { + err := os.MkdirAll(path, 0750) + if err != nil { + return nil, fmt.Errorf("failed to create vault path: %v, err: %w", path, err) + } } - key, err := getSeed(path) + key, err := getOrCreateSeed(path, options.readonly) if err != nil { return nil, err } diff --git a/internal/pkg/agent/vault/vault_options.go b/internal/pkg/agent/vault/vault_options.go new file mode 100644 index 00000000000..2673ae6aa53 --- /dev/null +++ b/internal/pkg/agent/vault/vault_options.go @@ -0,0 +1,28 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package vault + +type Options struct { + readonly bool +} + +type OptionFunc func(o *Options) + +func WithReadonly(readonly bool) OptionFunc { + return func(o *Options) { + o.readonly = readonly + } +} + +//nolint:unused // not used on darwin +func applyOptions(opts ...OptionFunc) Options { + var options Options + + for _, opt := range opts { + opt(&options) + } + + return options +} diff --git a/internal/pkg/agent/vault/vault_windows.go b/internal/pkg/agent/vault/vault_windows.go index 7468fe16814..c39769cc8da 100644 --- a/internal/pkg/agent/vault/vault_windows.go +++ b/internal/pkg/agent/vault/vault_windows.go @@ -27,7 +27,8 @@ type Vault struct { } // Open initializes the vault store -func New(path string) (*Vault, error) { +func New(path string, opts ...OptionFunc) (v *Vault, err error) { + options := applyOptions(opts...) dir := filepath.Dir(path) // If there is no specific path then get the executable directory @@ -40,16 +41,26 @@ func New(path string) (*Vault, error) { path = filepath.Join(dir, path) } - err := os.MkdirAll(path, 0750) - if err != nil { - return nil, err - } - err = systemAdministratorsOnly(path, false) - if err != nil { - return nil, err + if options.readonly { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fs.ErrNotExist + } + } else { + err := os.MkdirAll(path, 0750) + if err != nil { + return nil, err + } + err = systemAdministratorsOnly(path, false) + if err != nil { + return nil, err + } } - entropy, err := getSeed(path) + entropy, err := getOrCreateSeed(path, options.readonly) if err != nil { return nil, err } From 394033dbd9cc2105be37fc1bb971d7ba3449fc3d Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 20 Jul 2022 02:12:00 -0400 Subject: [PATCH 53/69] [Automation] Update elastic stack version to 8.4.0-31269fd2 for testing (#746) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 43b309fb98e..9e2fd843b08 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-31315ca3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-31269fd2-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-31315ca3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-31269fd2-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 97e6597a24c0f16e530fdbf86c82953374f28549 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 20 Jul 2022 11:29:07 +0200 Subject: [PATCH 54/69] wrap errors and fix some docs typo and convention (#743) --- internal/pkg/agent/application/info/agent_id.go | 16 ++++++++-------- .../pkg/agent/application/info/agent_metadata.go | 7 ++++--- .../pkg/agent/storage/encrypted_disk_store.go | 3 ++- internal/pkg/agent/vault/vault_darwin.go | 6 ++++-- internal/pkg/agent/vault/vault_linux.go | 4 ++-- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/pkg/agent/application/info/agent_id.go b/internal/pkg/agent/application/info/agent_id.go index e376a0fbfb4..2d5fbfc76e1 100644 --- a/internal/pkg/agent/application/info/agent_id.go +++ b/internal/pkg/agent/application/info/agent_id.go @@ -71,7 +71,7 @@ func getInfoFromStore(s ioStore, logLevel string) (*persistentAgentInfo, error) agentConfigFile := paths.AgentConfigFile() reader, err := s.Load() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load from ioStore: %w", err) } // reader is closed by this function @@ -195,20 +195,20 @@ func loadAgentInfo(forceUpdate bool, logLevel string, createAgentID bool) (*pers agentConfigFile := paths.AgentConfigFile() diskStore := storage.NewEncryptedDiskStore(agentConfigFile) - agentinfo, err := getInfoFromStore(diskStore, logLevel) + agentInfo, err := getInfoFromStore(diskStore, logLevel) if err != nil { - return nil, err + return nil, fmt.Errorf("could not get agent info from store: %w", err) } - if agentinfo != nil && !forceUpdate && (agentinfo.ID != "" || !createAgentID) { - return agentinfo, nil + if agentInfo != nil && !forceUpdate && (agentInfo.ID != "" || !createAgentID) { + return agentInfo, nil } - if err := updateID(agentinfo, diskStore); err != nil { - return nil, err + if err := updateID(agentInfo, diskStore); err != nil { + return nil, fmt.Errorf("could not update agent ID on disk store: %w", err) } - return agentinfo, nil + return agentInfo, nil } func updateID(agentInfo *persistentAgentInfo, s ioStore) error { diff --git a/internal/pkg/agent/application/info/agent_metadata.go b/internal/pkg/agent/application/info/agent_metadata.go index a532487a446..49afeca9dc7 100644 --- a/internal/pkg/agent/application/info/agent_metadata.go +++ b/internal/pkg/agent/application/info/agent_metadata.go @@ -10,10 +10,11 @@ import ( "runtime" "strings" - "github.com/elastic/elastic-agent/internal/pkg/agent/errors" - "github.com/elastic/elastic-agent/internal/pkg/release" "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" + + "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + "github.com/elastic/elastic-agent/internal/pkg/release" ) // ECSMeta is a collection of agent related metadata in ECS compliant object form. @@ -123,7 +124,7 @@ const ( func Metadata() (*ECSMeta, error) { agentInfo, err := NewAgentInfo(false) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create new agent info: %w", err) } meta, err := agentInfo.ECSMetadata() diff --git a/internal/pkg/agent/storage/encrypted_disk_store.go b/internal/pkg/agent/storage/encrypted_disk_store.go index 7fe2f70339a..e3ea3c3f6cf 100644 --- a/internal/pkg/agent/storage/encrypted_disk_store.go +++ b/internal/pkg/agent/storage/encrypted_disk_store.go @@ -15,6 +15,7 @@ import ( "github.com/hectane/go-acl" "github.com/elastic/elastic-agent-libs/file" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" @@ -78,7 +79,7 @@ func (d *EncryptedDiskStore) ensureKey() error { if d.key == nil { key, err := secret.GetAgentSecret(secret.WithVaultPath(d.vaultPath)) if err != nil { - return err + return fmt.Errorf("could not get agent key: %w", err) } d.key = key.Value } diff --git a/internal/pkg/agent/vault/vault_darwin.go b/internal/pkg/agent/vault/vault_darwin.go index bfcb636da6f..5f63a496179 100644 --- a/internal/pkg/agent/vault/vault_darwin.go +++ b/internal/pkg/agent/vault/vault_darwin.go @@ -37,13 +37,15 @@ type Vault struct { } // New initializes the vault store -// Call Close when done to release the resouces +// Call Close when done to release the resources func New(name string, opts ...OptionFunc) (*Vault, error) { var keychain C.SecKeychainRef + err := statusToError(C.OpenKeychain(keychain)) if err != nil { - return nil, err + return nil, fmt.Errorf("could not open keychain: %w", err) } + return &Vault{ name: name, keychain: keychain, diff --git a/internal/pkg/agent/vault/vault_linux.go b/internal/pkg/agent/vault/vault_linux.go index 23d0a0e859f..51f6a3fa651 100644 --- a/internal/pkg/agent/vault/vault_linux.go +++ b/internal/pkg/agent/vault/vault_linux.go @@ -39,7 +39,7 @@ func New(path string, opts ...OptionFunc) (v *Vault, err error) { if dir == "." { exefp, err := os.Executable() if err != nil { - return nil, err + return nil, fmt.Errorf("could not get executable path: %w", err) } dir = filepath.Dir(exefp) path = filepath.Join(dir, path) @@ -62,7 +62,7 @@ func New(path string, opts ...OptionFunc) (v *Vault, err error) { key, err := getOrCreateSeed(path, options.readonly) if err != nil { - return nil, err + return nil, fmt.Errorf("could not get seed to create new valt: %w", err) } return &Vault{ From b0e9623f33c84f1e19515a4306e3b9d37a560a85 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 20 Jul 2022 11:56:52 +0100 Subject: [PATCH 55/69] automate the ironbank docker context generation (#679) --- .ci/Jenkinsfile | 2 +- .gitignore | 1 - dev-tools/packaging/files/ironbank/LICENSE | 280 ++++++++++++++++++ .../files/ironbank/config/docker-entrypoint | 11 + .../templates/ironbank/Dockerfile.tmpl | 90 ++++++ .../templates/ironbank/README.md.tmpl | 43 +++ .../ironbank/hardening_manifest.yaml.tmpl | 68 +++++ magefile.go | 108 +++++++ 8 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 dev-tools/packaging/files/ironbank/LICENSE create mode 100644 dev-tools/packaging/files/ironbank/config/docker-entrypoint create mode 100644 dev-tools/packaging/templates/ironbank/Dockerfile.tmpl create mode 100644 dev-tools/packaging/templates/ironbank/README.md.tmpl create mode 100644 dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 34a72100707..9d24c6ea810 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -175,7 +175,7 @@ pipeline { withMageEnv(){ dir("${BASE_DIR}"){ withPackageEnv("${PLATFORM}") { - cmd(label: 'Go package', script: 'mage package') + cmd(label: 'Go package', script: 'mage package ironbank') uploadPackagesToGoogleBucket( credentialsId: env.JOB_GCS_EXT_CREDENTIALS, repo: env.REPO, diff --git a/.gitignore b/.gitignore index 89eaa67db73..939765242f5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,3 @@ internal/pkg/agent/transpiler/tests/exec-1.0-darwin-x86_64/exec # VSCode /.vscode - diff --git a/dev-tools/packaging/files/ironbank/LICENSE b/dev-tools/packaging/files/ironbank/LICENSE new file mode 100644 index 00000000000..ef2739c152e --- /dev/null +++ b/dev-tools/packaging/files/ironbank/LICENSE @@ -0,0 +1,280 @@ +ELASTIC LICENSE AGREEMENT + +PLEASE READ CAREFULLY THIS ELASTIC LICENSE AGREEMENT (THIS "AGREEMENT"), WHICH +CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS ALL OF YOUR USE OF ALL OF +THE ELASTIC SOFTWARE WITH WHICH THIS AGREEMENT IS INCLUDED ("ELASTIC SOFTWARE") +THAT IS PROVIDED IN OBJECT CODE FORMAT, AND, IN ACCORDANCE WITH SECTION 2 BELOW, +CERTAIN OF THE ELASTIC SOFTWARE THAT IS PROVIDED IN SOURCE CODE FORMAT. BY +INSTALLING OR USING ANY OF THE ELASTIC SOFTWARE GOVERNED BY THIS AGREEMENT, YOU +ARE ASSENTING TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE +WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE THE ELASTIC SOFTWARE +GOVERNED BY THIS AGREEMENT. IF YOU ARE INSTALLING OR USING THE SOFTWARE ON +BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL +AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF +SUCH ENTITY. + +Posted Date: April 20, 2018 + +This Agreement is entered into by and between Elasticsearch BV ("Elastic") and +You, or the legal entity on behalf of whom You are acting (as applicable, +"You"). + +1. OBJECT CODE END USER LICENSES, RESTRICTIONS AND THIRD PARTY OPEN SOURCE +SOFTWARE + + 1.1 Object Code End User License. Subject to the terms and conditions of + Section 1.2 of this Agreement, Elastic hereby grants to You, AT NO CHARGE and + for so long as you are not in breach of any provision of this Agreement, a + License to the Basic Features and Functions of the Elastic Software. + + 1.2 Reservation of Rights; Restrictions. As between Elastic and You, Elastic + and its licensors own all right, title and interest in and to the Elastic + Software, and except as expressly set forth in Sections 1.1, and 2.1 of this + Agreement, no other license to the Elastic Software is granted to You under + this Agreement, by implication, estoppel or otherwise. You agree not to: (i) + reverse engineer or decompile, decrypt, disassemble or otherwise reduce any + Elastic Software provided to You in Object Code, or any portion thereof, to + Source Code, except and only to the extent any such restriction is prohibited + by applicable law, (ii) except as expressly permitted in this Agreement, + prepare derivative works from, modify, copy or use the Elastic Software Object + Code or the Commercial Software Source Code in any manner; (iii) except as + expressly permitted in Section 1.1 above, transfer, sell, rent, lease, + distribute, sublicense, loan or otherwise transfer, Elastic Software Object + Code, in whole or in part, to any third party; (iv) use Elastic Software + Object Code for providing time-sharing services, any software-as-a-service, + service bureau services or as part of an application services provider or + other service offering (collectively, "SaaS Offering") where obtaining access + to the Elastic Software or the features and functions of the Elastic Software + is a primary reason or substantial motivation for users of the SaaS Offering + to access and/or use the SaaS Offering ("Prohibited SaaS Offering"); (v) + circumvent the limitations on use of Elastic Software provided to You in + Object Code format that are imposed or preserved by any License Key, or (vi) + alter or remove any Marks and Notices in the Elastic Software. If You have any + question as to whether a specific SaaS Offering constitutes a Prohibited SaaS + Offering, or are interested in obtaining Elastic's permission to engage in + commercial or non-commercial distribution of the Elastic Software, please + contact elastic_license@elastic.co. + + 1.3 Third Party Open Source Software. The Commercial Software may contain or + be provided with third party open source libraries, components, utilities and + other open source software (collectively, "Open Source Software"), which Open + Source Software may have applicable license terms as identified on a website + designated by Elastic. Notwithstanding anything to the contrary herein, use of + the Open Source Software shall be subject to the license terms and conditions + applicable to such Open Source Software, to the extent required by the + applicable licensor (which terms shall not restrict the license rights granted + to You hereunder, but may contain additional rights). To the extent any + condition of this Agreement conflicts with any license to the Open Source + Software, the Open Source Software license will govern with respect to such + Open Source Software only. Elastic may also separately provide you with + certain open source software that is licensed by Elastic. Your use of such + Elastic open source software will not be governed by this Agreement, but by + the applicable open source license terms. + +2. COMMERCIAL SOFTWARE SOURCE CODE + + 2.1 Limited License. Subject to the terms and conditions of Section 2.2 of + this Agreement, Elastic hereby grants to You, AT NO CHARGE and for so long as + you are not in breach of any provision of this Agreement, a limited, + non-exclusive, non-transferable, fully paid up royalty free right and license + to the Commercial Software in Source Code format, without the right to grant + or authorize sublicenses, to prepare Derivative Works of the Commercial + Software, provided You (i) do not hack the licensing mechanism, or otherwise + circumvent the intended limitations on the use of Elastic Software to enable + features other than Basic Features and Functions or those features You are + entitled to as part of a Subscription, and (ii) use the resulting object code + only for reasonable testing purposes. + + 2.2 Restrictions. Nothing in Section 2.1 grants You the right to (i) use the + Commercial Software Source Code other than in accordance with Section 2.1 + above, (ii) use a Derivative Work of the Commercial Software outside of a + Non-production Environment, in any production capacity, on a temporary or + permanent basis, or (iii) transfer, sell, rent, lease, distribute, sublicense, + loan or otherwise make available the Commercial Software Source Code, in whole + or in part, to any third party. Notwithstanding the foregoing, You may + maintain a copy of the repository in which the Source Code of the Commercial + Software resides and that copy may be publicly accessible, provided that you + include this Agreement with Your copy of the repository. + +3. TERMINATION + + 3.1 Termination. This Agreement will automatically terminate, whether or not + You receive notice of such Termination from Elastic, if You breach any of its + provisions. + + 3.2 Post Termination. Upon any termination of this Agreement, for any reason, + You shall promptly cease the use of the Elastic Software in Object Code format + and cease use of the Commercial Software in Source Code format. For the + avoidance of doubt, termination of this Agreement will not affect Your right + to use Elastic Software, in either Object Code or Source Code formats, made + available under the Apache License Version 2.0. + + 3.3 Survival. Sections 1.2, 2.2. 3.3, 4 and 5 shall survive any termination or + expiration of this Agreement. + +4. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY + + 4.1 Disclaimer of Warranties. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE + LAW, THE ELASTIC SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, + AND ELASTIC AND ITS LICENSORS MAKE NO WARRANTIES WHETHER EXPRESSED, IMPLIED OR + STATUTORY REGARDING OR RELATING TO THE ELASTIC SOFTWARE. TO THE MAXIMUM EXTENT + PERMITTED UNDER APPLICABLE LAW, ELASTIC AND ITS LICENSORS SPECIFICALLY + DISCLAIM ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE ELASTIC SOFTWARE, AND WITH + RESPECT TO THE USE OF THE FOREGOING. FURTHER, ELASTIC DOES NOT WARRANT RESULTS + OF USE OR THAT THE ELASTIC SOFTWARE WILL BE ERROR FREE OR THAT THE USE OF THE + ELASTIC SOFTWARE WILL BE UNINTERRUPTED. + + 4.2 Limitation of Liability. IN NO EVENT SHALL ELASTIC OR ITS LICENSORS BE + LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, + INCLUDING, WITHOUT LIMITATION, FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS + INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY + SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, IN CONNECTION WITH + OR ARISING OUT OF THE USE OR INABILITY TO USE THE ELASTIC SOFTWARE, OR THE + PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A + BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF ELASTIC + HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +5. MISCELLANEOUS + + This Agreement completely and exclusively states the entire agreement of the + parties regarding the subject matter herein, and it supersedes, and its terms + govern, all prior proposals, agreements, or other communications between the + parties, oral or written, regarding such subject matter. This Agreement may be + modified by Elastic from time to time, and any such modifications will be + effective upon the "Posted Date" set forth at the top of the modified + Agreement. If any provision hereof is held unenforceable, this Agreement will + continue without said provision and be interpreted to reflect the original + intent of the parties. This Agreement and any non-contractual obligation + arising out of or in connection with it, is governed exclusively by Dutch law. + This Agreement shall not be governed by the 1980 UN Convention on Contracts + for the International Sale of Goods. All disputes arising out of or in + connection with this Agreement, including its existence and validity, shall be + resolved by the courts with jurisdiction in Amsterdam, The Netherlands, except + where mandatory law provides for the courts at another location in The + Netherlands to have jurisdiction. The parties hereby irrevocably waive any and + all claims and defenses either might otherwise have in any such action or + proceeding in any of such courts based upon any alleged lack of personal + jurisdiction, improper venue, forum non conveniens or any similar claim or + defense. A breach or threatened breach, by You of Section 2 may cause + irreparable harm for which damages at law may not provide adequate relief, and + therefore Elastic shall be entitled to seek injunctive relief without being + required to post a bond. You may not assign this Agreement (including by + operation of law in connection with a merger or acquisition), in whole or in + part to any third party without the prior written consent of Elastic, which + may be withheld or granted by Elastic in its sole and absolute discretion. + Any assignment in violation of the preceding sentence is void. Notices to + Elastic may also be sent to legal@elastic.co. + +6. DEFINITIONS + + The following terms have the meanings ascribed: + + 6.1 "Affiliate" means, with respect to a party, any entity that controls, is + controlled by, or which is under common control with, such party, where + "control" means ownership of at least fifty percent (50%) of the outstanding + voting shares of the entity, or the contractual right to establish policy for, + and manage the operations of, the entity. + + 6.2 "Basic Features and Functions" means those features and functions of the + Elastic Software that are eligible for use under a Basic license, as set forth + at https://www.elastic.co/subscriptions, as may be modified by Elastic from + time to time. + + 6.3 "Commercial Software" means the Elastic Software Source Code in any file + containing a header stating the contents are subject to the Elastic License or + which is contained in the repository folder labeled "x-pack", unless a LICENSE + file present in the directory subtree declares a different license. + + 6.4 "Derivative Work of the Commercial Software" means, for purposes of this + Agreement, any modification(s) or enhancement(s) to the Commercial Software, + which represent, as a whole, an original work of authorship. + + 6.5 "License" means a limited, non-exclusive, non-transferable, fully paid up, + royalty free, right and license, without the right to grant or authorize + sublicenses, solely for Your internal business operations to (i) install and + use the applicable Features and Functions of the Elastic Software in Object + Code, and (ii) permit Contractors and Your Affiliates to use the Elastic + software as set forth in (i) above, provided that such use by Contractors must + be solely for Your benefit and/or the benefit of Your Affiliates, and You + shall be responsible for all acts and omissions of such Contractors and + Affiliates in connection with their use of the Elastic software that are + contrary to the terms and conditions of this Agreement. + + 6.6 "License Key" means a sequence of bytes, including but not limited to a + JSON blob, that is used to enable certain features and functions of the + Elastic Software. + + 6.7 "Marks and Notices" means all Elastic trademarks, trade names, logos and + notices present on the Documentation as originally provided by Elastic. + + 6.8 "Non-production Environment" means an environment for development, testing + or quality assurance, where software is not used for production purposes. + + 6.9 "Object Code" means any form resulting from mechanical transformation or + translation of Source Code form, including but not limited to compiled object + code, generated documentation, and conversions to other media types. + + 6.10 "Source Code" means the preferred form of computer software for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + 6.11 "Subscription" means the right to receive Support Services and a License + to the Commercial Software. + + +GOVERNMENT END USER ADDENDUM TO THE ELASTIC LICENSE AGREEMENT + + This ADDENDUM TO THE ELASTIC LICENSE AGREEMENT (this "Addendum") applies +only to U.S. Federal Government, State Government, and Local Government +entities ("Government End Users") of the Elastic Software. This Addendum is +subject to, and hereby incorporated into, the Elastic License Agreement, +which is being entered into as of even date herewith, by Elastic and You (the +"Agreement"). This Addendum sets forth additional terms and conditions +related to Your use of the Elastic Software. Capitalized terms not defined in +this Addendum have the meaning set forth in the Agreement. + + 1. LIMITED LICENSE TO DISTRIBUTE (DSOP ONLY). Subject to the terms and +conditions of the Agreement (including this Addendum), Elastic grants the +Department of Defense Enterprise DevSecOps Initiative (DSOP) a royalty-free, +non-exclusive, non-transferable, limited license to reproduce and distribute +the Elastic Software solely through a software distribution repository +controlled and managed by DSOP, provided that DSOP: (i) distributes the +Elastic Software complete and unmodified, inclusive of the Agreement +(including this Addendum) and (ii) does not remove or alter any proprietary +legends or notices contained in the Elastic Software. + + 2. CHOICE OF LAW. The choice of law and venue provisions set forth shall +prevail over those set forth in Section 5 of the Agreement. + + "For U.S. Federal Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by U.S. Federal law. To the extent permitted by + federal law, the laws of the State of Delaware (excluding Delaware choice + of law rules) will apply in the absence of applicable federal law. + + For State and Local Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by the laws of the state in which you are located + without reference to conflict of laws. Furthermore, the Parties agree that + the Uniform Computer Information Transactions Act or any version thereof, + adopted by any state in any form ('UCITA'), shall not apply to this + Agreement and, to the extent that UCITA is applicable, the Parties agree to + opt out of the applicability of UCITA pursuant to the opt-out provision(s) + contained therein." + + 3. ELASTIC LICENSE MODIFICATION. Section 5 of the Agreement is hereby +amended to replace + + "This Agreement may be modified by Elastic from time to time, and any + such modifications will be effective upon the "Posted Date" set forth at + the top of the modified Agreement." + + with: + + "This Agreement may be modified by Elastic from time to time; provided, + however, that any such modifications shall apply only to Elastic Software + that is installed after the "Posted Date" set forth at the top of the + modified Agreement." + +V100820.0 diff --git a/dev-tools/packaging/files/ironbank/config/docker-entrypoint b/dev-tools/packaging/files/ironbank/config/docker-entrypoint new file mode 100644 index 00000000000..7ebe21745f4 --- /dev/null +++ b/dev-tools/packaging/files/ironbank/config/docker-entrypoint @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +# For information on the possible environment variables that can be passed into the container. Run the following +# command for information on the options that are available. +# +# `./elastic-agent container --help` +# + +elastic-agent container "$@" diff --git a/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl b/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl new file mode 100644 index 00000000000..04c4dfde930 --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl @@ -0,0 +1,90 @@ +################################################################################ +# Build stage 0 +# Extract Elastic Agent and make various file manipulations. +################################################################################ +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 +ARG BASE_TAG=8.6 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as prep_files + +ARG ELASTIC_STACK={{ beat_version }} +ARG ELASTIC_PRODUCT=elastic-agent +ARG OS_AND_ARCH=linux-x86_64 + +RUN mkdir /usr/share/${ELASTIC_PRODUCT} +WORKDIR /usr/share/${ELASTIC_PRODUCT} +COPY --chown=1000:0 ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz . +RUN tar --strip-components=1 -zxf ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz \ + && rm ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz + +# Support arbitrary user ids +# Ensure that group permissions are the same as user permissions. +# This will help when relying on GID-0 to run Kibana, rather than UID-1000. +# OpenShift does this, for example. +# REF: https://docs.okd.io/latest/openshift_images/create-images.html +RUN chmod -R g=u /usr/share/${ELASTIC_PRODUCT} + +# Create auxiliary folders and assigning default permissions. +RUN mkdir -p /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs && \ + chown -R root:root /usr/share/${ELASTIC_PRODUCT} && \ + find /usr/share/${ELASTIC_PRODUCT} -type d -exec chmod 0750 {} \; && \ + find /usr/share/${ELASTIC_PRODUCT} -type f -exec chmod 0640 {} \; && \ + chmod 0750 /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} && \ + chmod 0770 /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs + +################################################################################ +# Build stage 1 +# Copy prepared files from the previous stage and complete the image. +################################################################################ +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +ARG ELASTIC_PRODUCT=elastic-agent + +COPY LICENSE /licenses/elastic-${ELASTIC_PRODUCT} + +# Add a dumb init process +COPY tinit /tinit +RUN chmod +x /tinit + +# Bring in product from the initial stage. +COPY --from=prep_files --chown=1000:0 /usr/share/${ELASTIC_PRODUCT} /usr/share/${ELASTIC_PRODUCT} +WORKDIR /usr/share/${ELASTIC_PRODUCT} +RUN ln -s /usr/share/${ELASTIC_PRODUCT} /opt/${ELASTIC_PRODUCT} + +ENV ELASTIC_CONTAINER="true" +RUN ln -s /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} /usr/bin/${ELASTIC_PRODUCT} + +# Support arbitrary user ids +# Ensure gid 0 write permissions for OpenShift. +RUN chmod -R g+w /usr/share/${ELASTIC_PRODUCT} + +# config file ("${ELASTIC_PRODUCT}.yml") can only be writable by the root and group root +# it is needed on some configurations where the container needs to run as root +RUN chown root:root /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml \ + && chmod go-w /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml + +# Remove the suid bit everywhere to mitigate "Stack Clash" +RUN find / -xdev -perm -4000 -exec chmod u-s {} + + +# Provide a non-root user to run the process. +RUN groupadd --gid 1000 ${ELASTIC_PRODUCT} && useradd --uid 1000 --gid 1000 --groups 0 --home-dir /usr/share/${ELASTIC_PRODUCT} --no-create-home ${ELASTIC_PRODUCT} + +# Elastic Agent permissions +RUN find /usr/share//elastic-agent/data -type d -exec chmod 0770 {} \; && \ + find /usr/share//elastic-agent/data -type f -exec chmod 0660 {} \; && \ + chmod +x /usr/share//elastic-agent/data/elastic-agent-*/elastic-agent + +COPY jq /usr/local/bin +RUN chown root:root /usr/local/bin/jq && chmod 0755 /usr/local/bin/jq + +COPY config/docker-entrypoint /usr/local/bin/docker-entrypoint +RUN chmod 755 /usr/local/bin/docker-entrypoint + +USER ${ELASTIC_PRODUCT} +ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} + +ENTRYPOINT ["/tinit", "--", "/usr/local/bin/docker-entrypoint"] +CMD [""] + +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD test -w '/tmp/elastic-agent/elastic-agent.sock' diff --git a/dev-tools/packaging/templates/ironbank/README.md.tmpl b/dev-tools/packaging/templates/ironbank/README.md.tmpl new file mode 100644 index 00000000000..271fdb8c0d7 --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/README.md.tmpl @@ -0,0 +1,43 @@ +# elastic-agent + +**elastic-agent** is a single, unified way to add monitoring for logs, metrics, and other types of data to each host. A single agent makes it easier and faster to deploy monitoring across your infrastructure. The agent’s single, unified configuration makes it easier to add integrations for new data sources. + +For more information about elastic-agent, please visit +https://www.elastic.co/guide/en/ingest-management/7.17/index.html. + +--- + +**NOTE** + +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. + +--- + +### Installation instructions + +Please follow the documentation on [Quick start](https://www.elastic.co/guide/en/fleet/{{ .MajorMinor }}/fleet-elastic-agent-quick-start.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/elastic-agent/issues) +- [PRs](https://github.com/elastic/elastic-agent/pulls) + +### DoD Restrictions + +### Where to get help + +- [elastic-agent Discuss Forums](https://discuss.elastic.co/tags/c/elastic-stack/beats/28/elastic-agent) +- [elastic-agent Documentation](https://www.elastic.co/guide/en/ingest-management/current/index.html) + +### Still need help? + +You can learn more about the Elastic Community and also understand how to get more help +visiting [Elastic Community](https://www.elastic.co/community). + +This software is governed by the [Elastic +License](https://github.com/elastic/beats/blob/{{ .MajorMinor }}/licenses/ELASTIC-LICENSE.txt), +and includes the full set of [free +features](https://www.elastic.co/subscriptions). + +View the detailed release notes +[here](https://www.elastic.co/guide/en/beats/libbeat/current/release-notes-{{ beat_version }}.html). diff --git a/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl b/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl new file mode 100644 index 00000000000..3c753caa0fb --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl @@ -0,0 +1,68 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "elastic/beats/elastic-agent" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dsop.io +tags: +- "{{ beat_version }}" +- "latest" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/ubi/ubi8" + BASE_TAG: "8.6" + ELASTIC_STACK: "{{ beat_version }}" + ELASTIC_PRODUCT: "elastic-agent" + +# Docker image labels +labels: + org.opencontainers.image.title: "elastic-agent" + ## Human-readable description of the software packaged in the image + org.opencontainers.image.description: "elastic-agent is a single, unified way to add monitoring for logs, metrics, and other types of data to each host" + ## License(s) under which contained software is distributed + org.opencontainers.image.licenses: "Elastic License" + ## URL to find more information on the image + org.opencontainers.image.url: "https://www.elastic.co/products/beats/elastic-agent" + ## Name of the distributing entity, organization or individual + org.opencontainers.image.vendor: "Elastic" + org.opencontainers.image.version: "{{ beat_version }}" + ## Keywords to help with search (ex. "cicd,gitops,golang") + mil.dso.ironbank.image.keywords: "log,metrics,monitoring,observabilty,o11y,oblt,beats,elastic,elasticsearch,golang" + ## This value can be "opensource" or "commercial" + mil.dso.ironbank.image.type: "commercial" + ## Product the image belongs to for grouping multiple images + mil.dso.ironbank.product.name: "beats" + +# List of resources to make available to the offline build context +resources: + - filename: "elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz" + url: "/elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz" + validation: + type: "sha512" + value: "" + - filename: tinit + url: https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 + validation: + type: sha256 + value: 93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c + - filename: jq + url: https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + validation: + type: sha256 + value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 + +# List of project maintainers +maintainers: + - email: "nassim.kammah@elastic.co" + name: "Nassim Kammah" + username: "nassim.kammah" + - email: "ivan.fernandez@elastic.co" + name: "Ivan Fernandez Calvo" + username: "ivan.fernandez" + - email: "victor.martinez@elastic.co" + name: "Victor Martinez" + username: "victor.martinez" diff --git a/magefile.go b/magefile.go index aba39385159..a215f1639d7 100644 --- a/magefile.go +++ b/magefile.go @@ -817,3 +817,111 @@ func injectBuildVars(m map[string]string) { m[k] = v } } + +// Package packages elastic-agent for the IronBank distribution, relying on the +// binaries having already been built. +// +// Use SNAPSHOT=true to build snapshots. +func Ironbank() error { + if runtime.GOARCH != "amd64" { + fmt.Printf(">> IronBank images are only supported for amd64 arch (%s is not supported)\n", runtime.GOARCH) + return nil + } + if err := prepareIronbankBuild(); err != nil { + return errors.Wrap(err, "failed to prepare the IronBank context") + } + if err := saveIronbank(); err != nil { + return errors.Wrap(err, "failed to save artifacts for IronBank") + } + return nil +} + +func saveIronbank() error { + fmt.Println(">> saveIronbank: save the IronBank container context.") + + ironbank := getIronbankContextName() + buildDir := filepath.Join("build", ironbank) + if _, err := os.Stat(buildDir); os.IsNotExist(err) { + return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err) + } + + distributionsDir := "build/distributions" + if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { + err := os.MkdirAll(distributionsDir, 0750) + if err != nil { + return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) + } + } + + // change dir to the buildDir location where the ironbank folder exists + // this will generate a tar.gz without some nested folders. + wd, _ := os.Getwd() + os.Chdir(buildDir) + defer os.Chdir(wd) + + // move the folder to the parent folder, there are two parent folder since + // buildDir contains a two folders dir. + tarGzFile := filepath.Join("..", "..", distributionsDir, ironbank+".tar.gz") + + // Save the build context as tar.gz artifact + err := devtools.Tar("./", tarGzFile) + if err != nil { + return fmt.Errorf("cannot compress the tar.gz file: %+v", err) + } + + return errors.Wrap(devtools.CreateSHA512File(tarGzFile), "failed to create .sha512 file") +} + +func getIronbankContextName() string { + version, _ := devtools.BeatQualifiedVersion() + defaultBinaryName := "{{.Name}}-ironbank-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}" + outputDir, _ := devtools.Expand(defaultBinaryName+"-docker-build-context", map[string]interface{}{ + "Name": "elastic-agent", + "Version": version, + }) + return outputDir +} + +func prepareIronbankBuild() error { + fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.") + buildDir := filepath.Join("build", getIronbankContextName()) + templatesDir := filepath.Join("dev-tools", "packaging", "templates", "ironbank") + + data := map[string]interface{}{ + "MajorMinor": majorMinor(), + } + + err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { + if !info.IsDir() { + target := strings.TrimSuffix( + filepath.Join(buildDir, filepath.Base(path)), + ".tmpl", + ) + + err := devtools.ExpandFile(path, target, data) + if err != nil { + return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) + } + } + return nil + }) + + if err != nil { + return fmt.Errorf("cannot create templates for the IronBank: %+v", err) + } + + // copy files + sourcePath := filepath.Join("dev-tools", "packaging", "files", "ironbank") + if err := devtools.Copy(sourcePath, buildDir); err != nil { + return fmt.Errorf("cannot create files for the IronBank: %+v", err) + } + return nil +} + +func majorMinor() string { + if v, _ := devtools.BeatQualifiedVersion(); v != "" { + parts := strings.SplitN(v, ".", 3) + return parts[0] + "." + parts[1] + } + return "" +} From f26b0eb53fa5b08c23a3145156cd4a245ad04f32 Mon Sep 17 00:00:00 2001 From: Pierre HILBERT Date: Wed, 20 Jul 2022 21:01:21 +0200 Subject: [PATCH 56/69] Update README.md Adding M1 variable to export to be able to build AMD images --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index df27ac6f99a..2c0dbe31f69 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ Prerequisites: - [Docker](https://docs.docker.com/get-docker/) - [X-pack](https://github.com/elastic/beats/tree/main/x-pack) to pre-exist in the parent folder of the local Git repository checkout +If you are on a Mac with M1 chip, don't forget to export some docker variable to be able to build for AMD +``` +export DOCKER_BUILDKIT=0 +export COMPOSE_DOCKER_CLI_BUILD=0 +export DOCKER_DEFAULT_PLATFORM=linux/amd64 +``` + In Linux operating systems that you can not run docker as a root user you need to follow [linux-postinstall steps](https://docs.docker.com/engine/install/linux-postinstall/) ### Testing docker container From d84b957de85441e42932b5dee0bd93b8d8e2076d Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Thu, 21 Jul 2022 15:15:11 +0200 Subject: [PATCH 57/69] fix flaky (#730) --- .../pkg/artifact/download/http/downloader_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/pkg/artifact/download/http/downloader_test.go b/internal/pkg/artifact/download/http/downloader_test.go index 26164df3a49..aac16a60f5d 100644 --- a/internal/pkg/artifact/download/http/downloader_test.go +++ b/internal/pkg/artifact/download/http/downloader_test.go @@ -25,7 +25,6 @@ import ( ) func TestDownloadBodyError(t *testing.T) { - t.Skip("Skipping flaky test: https://github.com/elastic/elastic-agent/issues/640") // This tests the scenario where the download encounters a network error // part way through the download, while copying the response body. @@ -65,9 +64,9 @@ func TestDownloadBodyError(t *testing.T) { } require.GreaterOrEqual(t, len(log.info), 1, "download error not logged at info level") - assert.Equal(t, log.info[len(log.info)-1].record, "download from %s failed at %s @ %sps: %s") + assert.True(t, containsMessage(log.info, "download from %s failed at %s @ %sps: %s")) require.GreaterOrEqual(t, len(log.warn), 1, "download error not logged at warn level") - assert.Equal(t, log.warn[len(log.warn)-1].record, "download from %s failed at %s @ %sps: %s") + assert.True(t, containsMessage(log.warn, "download from %s failed at %s @ %sps: %s")) } func TestDownloadLogProgressWithLength(t *testing.T) { @@ -208,3 +207,12 @@ func (f *recordLogger) Warnf(record string, args ...interface{}) { defer f.lock.Unlock() f.warn = append(f.warn, logMessage{record, args}) } + +func containsMessage(logs []logMessage, msg string) bool { + for _, item := range logs { + if item.record == msg { + return true + } + } + return false +} From be3c48e98171548f61f7aa4de16e48d1a288cff7 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 22 Jul 2022 15:20:14 +0200 Subject: [PATCH 58/69] Add filestream ID on standalone kubernetes manifest (#742) This commit add unique IDs for the filestream inputs used by the Kubernetes integration in the Elastic-Agent standalone Kubernetes configuration/manifest file. --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml | 2 ++ .../elastic-agent-standalone-daemonset-configmap.yaml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6ee9674253e..75d574a23b5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -110,6 +110,7 @@ - Allow ':' characters in dynamic variables {issue}624[624] {pull}680[680] - Allow the - char to appear as part of variable names in eql expressions. {issue}709[709] {pull}710[710] - Allow the / char in variable names in eql and transpiler. {issue}715[715] {pull}718[718] +- Fix data duplication for standalone agent on Kubernetes using the default manifest {issue-beats}31512[31512] {pull}742[742] ==== New features diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index ab360f19bcb..0c084c4a81b 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -227,6 +227,7 @@ data: fields: ecs.version: 1.12.0 - name: container-log + id: container-log-${kubernetes.pod.name}-${kubernetes.container.id} type: filestream use_output: default meta: @@ -252,6 +253,7 @@ data: paths: - /var/log/containers/*${kubernetes.container.id}.log - name: audit-log + id: audit-log type: filestream use_output: default meta: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 6894f32bbe4..0581585e2fb 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -227,6 +227,7 @@ data: fields: ecs.version: 1.12.0 - name: container-log + id: container-log-${kubernetes.pod.name}-${kubernetes.container.id} type: filestream use_output: default meta: @@ -252,6 +253,7 @@ data: paths: - /var/log/containers/*${kubernetes.container.id}.log - name: audit-log + id: audit-log type: filestream use_output: default meta: From 83c4fa2cad2575103407df3e2e8ae0e8207e1dbc Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Fri, 22 Jul 2022 13:33:59 -0700 Subject: [PATCH 59/69] Alter github action to run on different OSs (#769) Alter the linter action to run on different OSs instead of on linux with the $GOOS env var. --- .github/workflows/golangci-lint.yml | 14 ++------------ internal/pkg/agent/application/secret/secret.go | 1 + 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 79a22cbabc5..8079fe1c673 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -14,18 +14,10 @@ jobs: golangci: strategy: matrix: - include: - - GOOS: windows - - GOOS: linux - - GOOS: darwin + os: [ ubuntu-latest, macos-latest, windows-latest ] name: lint - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - - name: Echo details - env: - GOOS: ${{ matrix.GOOS }} - run: echo Go GOOS=$GOOS - - uses: actions/checkout@v2 # Uses Go version from the repository. @@ -38,8 +30,6 @@ jobs: go-version: "${{ steps.goversion.outputs.version }}" - name: golangci-lint - env: - GOOS: ${{ matrix.GOOS }} uses: golangci/golangci-lint-action@v2 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version diff --git a/internal/pkg/agent/application/secret/secret.go b/internal/pkg/agent/application/secret/secret.go index cf690bf24e8..bd2ee546454 100644 --- a/internal/pkg/agent/application/secret/secret.go +++ b/internal/pkg/agent/application/secret/secret.go @@ -2,6 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// Package secret manages application secrets. package secret import ( From c05ce638b80ff5b6c76c1242ba4e5506bc903ebe Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 25 Jul 2022 01:35:11 -0400 Subject: [PATCH 60/69] [Automation] Update elastic stack version to 8.4.0-d058e92f for testing (#771) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 9e2fd843b08..b8b6792b912 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-31269fd2-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-d058e92f-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-31269fd2-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-d058e92f-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 623fe82552a34196e51a9ce1749bf4e37cb23ec5 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 25 Jul 2022 10:40:25 +0200 Subject: [PATCH 61/69] elastic-agent manifests: add comments; add cloudnative team as a codeowner for the k8s manifests (#708) * managed elastic-agent: add comments; add cloudnative team as a codeowner for the k8s manifests Signed-off-by: Tetiana Kravchenko * add comments to the standalone elastic-agent, similar to the documentation we have https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-standalone.html Signed-off-by: Tetiana Kravchenko * Apply suggestions from code review Co-authored-by: Michael Katsoulis Co-authored-by: Andrew Gizas * remove comment for FLEET_ENROLLMENT_TOKEN; use Needed everywhere instead of Required Signed-off-by: Tetiana Kravchenko * rephrase regarding accessing kube-state-metrics when used third party tools, like kube-rbac-proxy Signed-off-by: Tetiana Kravchenko * run make check Signed-off-by: Tetiana Kravchenko * keep manifests in sync to pass ci check Signed-off-by: Tetiana Kravchenko * add info on where to find FLEET_URL and FLEET_ENROLLMENT_TOKEN Signed-off-by: Tetiana Kravchenko * add links to elastic-agent documentation Signed-off-by: Tetiana Kravchenko * update comment on FLEET_ENROLLMENT_TOKEN Signed-off-by: Tetiana Kravchenko Co-authored-by: Michael Katsoulis Co-authored-by: Andrew Gizas --- .github/CODEOWNERS | 4 +- .../elastic-agent-managed-kubernetes.yaml | 30 ++++- .../elastic-agent-managed-daemonset.yaml | 23 +++- .../elastic-agent-managed-role.yaml | 7 +- .../elastic-agent-standalone-kubernetes.yaml | 107 +++++++++++++++++- ...-agent-standalone-daemonset-configmap.yaml | 89 ++++++++++++++- .../elastic-agent-standalone-daemonset.yaml | 11 ++ .../elastic-agent-standalone-role.yaml | 7 +- 8 files changed, 253 insertions(+), 25 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bc147bf0680..d8bc0072d7b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,4 @@ -# Team responsable for Fleet Server +# Team responsible for Fleet Server * @elastic/elastic-agent-control-plane + +/deploy/kubernetes @elastic/obs-cloudnative-monitoring diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml index 4771cf37727..1e2403f47a2 100644 --- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml @@ -1,3 +1,4 @@ +# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html apiVersion: apps/v1 kind: DaemonSet metadata: @@ -14,34 +15,41 @@ spec: labels: app: elastic-agent spec: + # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. + # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule serviceAccountName: elastic-agent hostNetwork: true + # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host. # Sharing the host process ID namespace gives visibility of all processes running on the same host. - # This enables the Elastic Security integration to observe all process exec events on the host. hostPID: true dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent image: docker.elastic.co/beats/elastic-agent:8.3.0 env: + # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode - name: FLEET_ENROLL value: "1" - # Set to true in case of insecure or unverified HTTP + # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS - name: FLEET_INSECURE value: "true" - # The ip:port pair of fleet server + # Fleet Server URL to enroll the Elastic Agent into + # FLEET_URL can be found in Kibana, go to Management > Fleet > Settings - name: FLEET_URL value: "https://fleet-server:8220" - # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed + # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens) + # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed - name: FLEET_ENROLLMENT_TOKEN value: "" - name: KIBANA_HOST value: "http://kibana:5601" + # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet - name: KIBANA_FLEET_USERNAME value: "elastic" + # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet - name: KIBANA_FLEET_PASSWORD value: "changeme" - name: NODE_NAME @@ -104,21 +112,28 @@ spec: - name: varlog hostPath: path: /var/log + # Needed for cloudbeat - name: etc-kubernetes hostPath: path: /etc/kubernetes + # Needed for cloudbeat - name: var-lib hostPath: path: /var/lib + # Needed for cloudbeat - name: passwd hostPath: path: /etc/passwd + # Needed for cloudbeat - name: group hostPath: path: /etc/group + # Needed for cloudbeat - name: etcsysmd hostPath: path: /etc/systemd + # Mount /etc/machine-id from the host to determine host ID + # Needed for Elastic Security integration - name: etc-mid hostPath: path: /etc/machine-id @@ -180,6 +195,7 @@ rules: - pods - services - configmaps + # Needed for cloudbeat - serviceaccounts - persistentvolumes - persistentvolumeclaims @@ -211,11 +227,12 @@ rules: - jobs - cronjobs verbs: [ "get", "list", "watch" ] - # required for apiserver + # Needed for apiserver - nonResourceURLs: - "/metrics" verbs: - get + # Needed for cloudbeat - apiGroups: ["rbac.authorization.k8s.io"] resources: - clusterrolebindings @@ -223,6 +240,7 @@ rules: - rolebindings - roles verbs: ["get", "list", "watch"] + # Needed for cloudbeat - apiGroups: ["policy"] resources: - podsecuritypolicies @@ -232,7 +250,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: elastic-agent - # should be the namespace where elastic-agent is running + # Should be the namespace where elastic-agent is running namespace: kube-system labels: k8s-app: elastic-agent diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml index 231b976fe71..c3c679efa36 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml @@ -1,3 +1,4 @@ +# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html apiVersion: apps/v1 kind: DaemonSet metadata: @@ -14,34 +15,41 @@ spec: labels: app: elastic-agent spec: + # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. + # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule serviceAccountName: elastic-agent hostNetwork: true + # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host. # Sharing the host process ID namespace gives visibility of all processes running on the same host. - # This enables the Elastic Security integration to observe all process exec events on the host. hostPID: true dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent image: docker.elastic.co/beats/elastic-agent:%VERSION% env: + # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode - name: FLEET_ENROLL value: "1" - # Set to true in case of insecure or unverified HTTP + # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS - name: FLEET_INSECURE value: "true" - # The ip:port pair of fleet server + # Fleet Server URL to enroll the Elastic Agent into + # FLEET_URL can be found in Kibana, go to Management > Fleet > Settings - name: FLEET_URL value: "https://fleet-server:8220" - # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed + # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens) + # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed - name: FLEET_ENROLLMENT_TOKEN value: "" - name: KIBANA_HOST value: "http://kibana:5601" + # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet - name: KIBANA_FLEET_USERNAME value: "elastic" + # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet - name: KIBANA_FLEET_PASSWORD value: "changeme" - name: NODE_NAME @@ -104,21 +112,28 @@ spec: - name: varlog hostPath: path: /var/log + # Needed for cloudbeat - name: etc-kubernetes hostPath: path: /etc/kubernetes + # Needed for cloudbeat - name: var-lib hostPath: path: /var/lib + # Needed for cloudbeat - name: passwd hostPath: path: /etc/passwd + # Needed for cloudbeat - name: group hostPath: path: /etc/group + # Needed for cloudbeat - name: etcsysmd hostPath: path: /etc/systemd + # Mount /etc/machine-id from the host to determine host ID + # Needed for Elastic Security integration - name: etc-mid hostPath: path: /etc/machine-id diff --git a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-role.yaml b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-role.yaml index 0ef5b850782..0d961215f4e 100644 --- a/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-role.yaml +++ b/deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-role.yaml @@ -13,6 +13,7 @@ rules: - pods - services - configmaps + # Needed for cloudbeat - serviceaccounts - persistentvolumes - persistentvolumeclaims @@ -44,11 +45,12 @@ rules: - jobs - cronjobs verbs: [ "get", "list", "watch" ] - # required for apiserver + # Needed for apiserver - nonResourceURLs: - "/metrics" verbs: - get + # Needed for cloudbeat - apiGroups: ["rbac.authorization.k8s.io"] resources: - clusterrolebindings @@ -56,6 +58,7 @@ rules: - rolebindings - roles verbs: ["get", "list", "watch"] + # Needed for cloudbeat - apiGroups: ["policy"] resources: - podsecuritypolicies @@ -65,7 +68,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: elastic-agent - # should be the namespace where elastic-agent is running + # Should be the namespace where elastic-agent is running namespace: kube-system labels: k8s-app: elastic-agent diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 0c084c4a81b..0984f0dc8ac 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -1,3 +1,4 @@ +# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-standalone.html apiVersion: v1 kind: ConfigMap metadata: @@ -63,7 +64,9 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - # If `https` is used to access `kube-state-metrics`, then to all `kubernetes.state_*` datasets should be added: + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt @@ -76,6 +79,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_daemonset type: metrics @@ -85,6 +94,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_deployment type: metrics @@ -94,6 +109,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_job type: metrics @@ -103,6 +124,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_node type: metrics @@ -112,6 +139,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_persistentvolume type: metrics @@ -121,6 +154,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_persistentvolumeclaim type: metrics @@ -130,6 +169,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_pod type: metrics @@ -139,6 +184,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_replicaset type: metrics @@ -148,6 +199,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_resourcequota type: metrics @@ -157,6 +214,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_service type: metrics @@ -166,6 +229,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_statefulset type: metrics @@ -175,6 +244,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_storageclass type: metrics @@ -184,6 +259,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - name: system-logs type: logfile use_output: default @@ -417,7 +498,7 @@ data: period: 10s ssl.verification_mode: none condition: ${kubernetes.labels.component} == 'kube-controller-manager' - # Openshift: + # On Openshift condition should be adjusted: # condition: ${kubernetes.labels.app} == 'kube-controller-manager' - data_stream: dataset: kubernetes.scheduler @@ -430,7 +511,7 @@ data: period: 10s ssl.verification_mode: none condition: ${kubernetes.labels.component} == 'kube-scheduler' - # Openshift: + # On Openshift condition should be adjusted: # condition: ${kubernetes.labels.app} == 'openshift-kube-scheduler' - data_stream: dataset: kubernetes.proxy @@ -439,7 +520,7 @@ data: - proxy hosts: - 'localhost:10249' - # Openshift: + # On Openshift port should be adjusted: # - 'localhost:29101' period: 10s - data_stream: @@ -559,6 +640,8 @@ spec: labels: app: elastic-agent-standalone spec: + # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. + # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -573,10 +656,14 @@ spec: "-e", ] env: + # The basic authentication username used to connect to Elasticsearch + # This user needs the privileges required to publish events to Elasticsearch. - name: ES_USERNAME value: "elastic" + # The basic authentication password used to connect to Elasticsearch - name: ES_PASSWORD value: "" + # The Elasticsearch host to communicate with - name: ES_HOST value: "" - name: NODE_NAME @@ -644,18 +731,23 @@ spec: - name: varlog hostPath: path: /var/log + # Needed for cloudbeat - name: etc-kubernetes hostPath: path: /etc/kubernetes + # Needed for cloudbeat - name: var-lib hostPath: path: /var/lib + # Needed for cloudbeat - name: passwd hostPath: path: /etc/passwd + # Needed for cloudbeat - name: group hostPath: path: /etc/group + # Needed for cloudbeat - name: etcsysmd hostPath: path: /etc/systemd @@ -716,6 +808,7 @@ rules: - pods - services - configmaps + # Needed for cloudbeat - serviceaccounts - persistentvolumes - persistentvolumeclaims @@ -747,11 +840,12 @@ rules: - nodes/stats verbs: - get - # required for apiserver + # Needed for apiserver - nonResourceURLs: - "/metrics" verbs: - get + # Needed for cloudbeat - apiGroups: ["rbac.authorization.k8s.io"] resources: - clusterrolebindings @@ -759,6 +853,7 @@ rules: - rolebindings - roles verbs: ["get", "list", "watch"] + # Needed for cloudbeat - apiGroups: ["policy"] resources: - podsecuritypolicies @@ -768,7 +863,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: elastic-agent-standalone - # should be the namespace where elastic-agent is running + # Should be the namespace where elastic-agent is running namespace: kube-system labels: k8s-app: elastic-agent-standalone diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 0581585e2fb..7048bf22adb 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -1,3 +1,4 @@ +# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-standalone.html apiVersion: v1 kind: ConfigMap metadata: @@ -63,7 +64,9 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - # If `https` is used to access `kube-state-metrics`, then to all `kubernetes.state_*` datasets should be added: + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt @@ -76,6 +79,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_daemonset type: metrics @@ -85,6 +94,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_deployment type: metrics @@ -94,6 +109,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_job type: metrics @@ -103,6 +124,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_node type: metrics @@ -112,6 +139,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_persistentvolume type: metrics @@ -121,6 +154,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_persistentvolumeclaim type: metrics @@ -130,6 +169,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_pod type: metrics @@ -139,6 +184,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_replicaset type: metrics @@ -148,6 +199,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_resourcequota type: metrics @@ -157,6 +214,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_service type: metrics @@ -166,6 +229,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_statefulset type: metrics @@ -175,6 +244,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - data_stream: dataset: kubernetes.state_storageclass type: metrics @@ -184,6 +259,12 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s + # Openshift: + # if to access `kube-state-metrics` are used third party tools, like kube-rbac-proxy or similar, that perform RBAC authorization + # and/or tls termination, then configuration below should be considered: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - name: system-logs type: logfile use_output: default @@ -417,7 +498,7 @@ data: period: 10s ssl.verification_mode: none condition: ${kubernetes.labels.component} == 'kube-controller-manager' - # Openshift: + # On Openshift condition should be adjusted: # condition: ${kubernetes.labels.app} == 'kube-controller-manager' - data_stream: dataset: kubernetes.scheduler @@ -430,7 +511,7 @@ data: period: 10s ssl.verification_mode: none condition: ${kubernetes.labels.component} == 'kube-scheduler' - # Openshift: + # On Openshift condition should be adjusted: # condition: ${kubernetes.labels.app} == 'openshift-kube-scheduler' - data_stream: dataset: kubernetes.proxy @@ -439,7 +520,7 @@ data: - proxy hosts: - 'localhost:10249' - # Openshift: + # On Openshift port should be adjusted: # - 'localhost:29101' period: 10s - data_stream: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index 2a0f23107f1..0bf131ec8ea 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -14,6 +14,8 @@ spec: labels: app: elastic-agent-standalone spec: + # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. + # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -28,10 +30,14 @@ spec: "-e", ] env: + # The basic authentication username used to connect to Elasticsearch + # This user needs the privileges required to publish events to Elasticsearch. - name: ES_USERNAME value: "elastic" + # The basic authentication password used to connect to Elasticsearch - name: ES_PASSWORD value: "" + # The Elasticsearch host to communicate with - name: ES_HOST value: "" - name: NODE_NAME @@ -99,18 +105,23 @@ spec: - name: varlog hostPath: path: /var/log + # Needed for cloudbeat - name: etc-kubernetes hostPath: path: /etc/kubernetes + # Needed for cloudbeat - name: var-lib hostPath: path: /var/lib + # Needed for cloudbeat - name: passwd hostPath: path: /etc/passwd + # Needed for cloudbeat - name: group hostPath: path: /etc/group + # Needed for cloudbeat - name: etcsysmd hostPath: path: /etc/systemd diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml index b253f0520fe..8a644f3aadf 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml @@ -13,6 +13,7 @@ rules: - pods - services - configmaps + # Needed for cloudbeat - serviceaccounts - persistentvolumes - persistentvolumeclaims @@ -44,11 +45,12 @@ rules: - nodes/stats verbs: - get - # required for apiserver + # Needed for apiserver - nonResourceURLs: - "/metrics" verbs: - get + # Needed for cloudbeat - apiGroups: ["rbac.authorization.k8s.io"] resources: - clusterrolebindings @@ -56,6 +58,7 @@ rules: - rolebindings - roles verbs: ["get", "list", "watch"] + # Needed for cloudbeat - apiGroups: ["policy"] resources: - podsecuritypolicies @@ -65,7 +68,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: elastic-agent-standalone - # should be the namespace where elastic-agent is running + # Should be the namespace where elastic-agent is running namespace: kube-system labels: k8s-app: elastic-agent-standalone From 56f22161e677f5524241588264ce9a38abb21c97 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 26 Jul 2022 14:17:59 +0200 Subject: [PATCH 62/69] [Elastic-Agent] Added source uri reloading (#686) --- .../pkg/agent/application/managed_mode.go | 2 + .../handlers/handler_action_policy_change.go | 2 +- .../handler_action_policy_change_test.go | 8 +-- internal/pkg/artifact/config.go | 53 ++++++++++++++++++- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/internal/pkg/agent/application/managed_mode.go b/internal/pkg/agent/application/managed_mode.go index d334ae0198c..3f98e78fd62 100644 --- a/internal/pkg/agent/application/managed_mode.go +++ b/internal/pkg/agent/application/managed_mode.go @@ -31,6 +31,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/operation" "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/agent/storage/store" + "github.com/elastic/elastic-agent/internal/pkg/artifact" "github.com/elastic/elastic-agent/internal/pkg/capabilities" "github.com/elastic/elastic-agent/internal/pkg/composable" "github.com/elastic/elastic-agent/internal/pkg/config" @@ -157,6 +158,7 @@ func newManaged( }, caps, monitor, + artifact.NewReloader(cfg.Settings.DownloadConfig, log), ) if err != nil { return nil, err diff --git a/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go b/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go index 3775d12b352..ad75299e420 100644 --- a/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go +++ b/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go @@ -151,7 +151,7 @@ func (h *PolicyChange) handleFleetServerHosts(ctx context.Context, c *config.Con errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts)) } // discard body for proper cancellation and connection reuse - io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() reader, err := fleetToReader(h.agentInfo, h.config) diff --git a/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go b/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go index d887e755154..e2d480ee6fe 100644 --- a/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go +++ b/internal/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go @@ -44,7 +44,7 @@ func TestPolicyChange(t *testing.T) { conf := map[string]interface{}{"hello": "world"} action := &fleetapi.ActionPolicyChange{ - ActionID: "abc123", + ActionID: "TestPolicyChange-abc1", ActionType: "POLICY_CHANGE", Policy: conf, } @@ -69,7 +69,7 @@ func TestPolicyChange(t *testing.T) { conf := map[string]interface{}{"hello": "world"} action := &fleetapi.ActionPolicyChange{ - ActionID: "abc123", + ActionID: "TestPolicyChange-abc2", ActionType: "POLICY_CHANGE", Policy: conf, } @@ -100,7 +100,7 @@ func TestPolicyAcked(t *testing.T) { emitter := &mockEmitter{err: mockErr} config := map[string]interface{}{"hello": "world"} - actionID := "abc123" + actionID := "TestPolicyAcked-abc1" action := &fleetapi.ActionPolicyChange{ ActionID: actionID, ActionType: "POLICY_CHANGE", @@ -129,7 +129,7 @@ func TestPolicyAcked(t *testing.T) { emitter := &mockEmitter{} config := map[string]interface{}{"hello": "world"} - actionID := "abc123" + actionID := "TestPolicyAcked-abc2" action := &fleetapi.ActionPolicyChange{ ActionID: actionID, ActionType: "POLICY_CHANGE", diff --git a/internal/pkg/artifact/config.go b/internal/pkg/artifact/config.go index c190c02d239..fa57ca06870 100644 --- a/internal/pkg/artifact/config.go +++ b/internal/pkg/artifact/config.go @@ -11,12 +11,17 @@ import ( "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + "github.com/elastic/elastic-agent/internal/pkg/config" + "github.com/elastic/elastic-agent/pkg/core/logger" ) const ( darwin = "darwin" linux = "linux" windows = "windows" + + defaultSourceURI = "https://artifacts.elastic.co/downloads/" ) // Config is a configuration used for verifier and downloader @@ -46,6 +51,52 @@ type Config struct { httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"` // Note: use anonymous struct for json inline } +type Reloader struct { + log *logger.Logger + cfg *Config +} + +func NewReloader(cfg *Config, log *logger.Logger) *Reloader { + return &Reloader{ + cfg: cfg, + log: log, + } +} + +func (r *Reloader) Reload(rawConfig *config.Config) error { + type reloadConfig struct { + // SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ + SourceURI string `json:"agent.download.sourceURI" config:"agent.download.sourceURI"` + + // FleetSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ coming from fleet which uses + // different naming. + FleetSourceURI string `json:"agent.download.source_uri" config:"agent.download.source_uri"` + } + cfg := &reloadConfig{} + if err := rawConfig.Unpack(&cfg); err != nil { + return errors.New(err, "failed to unpack config during reload") + } + + var newSourceURI string + if cfg.FleetSourceURI != "" { + // fleet configuration takes precedence + newSourceURI = cfg.FleetSourceURI + } else if cfg.SourceURI != "" { + newSourceURI = cfg.SourceURI + } + + if newSourceURI != "" { + r.log.Infof("Source URI changed from %q to %q", r.cfg.SourceURI, newSourceURI) + r.cfg.SourceURI = newSourceURI + } else { + // source uri unset, reset to default + r.log.Infof("Source URI reset from %q to %q", r.cfg.SourceURI, defaultSourceURI) + r.cfg.SourceURI = defaultSourceURI + } + + return nil +} + // DefaultConfig creates a config with pre-set default values. func DefaultConfig() *Config { transport := httpcommon.DefaultHTTPTransportSettings() @@ -56,7 +107,7 @@ func DefaultConfig() *Config { transport.Timeout = 10 * time.Minute return &Config{ - SourceURI: "https://artifacts.elastic.co/downloads/", + SourceURI: defaultSourceURI, TargetDirectory: paths.Downloads(), InstallPath: paths.Install(), HTTPTransportSettings: transport, From 3e877825b91cdde39f2ea94eabdd6275176b7b82 Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 26 Jul 2022 13:17:34 -0700 Subject: [PATCH 63/69] Update will cleanup unneeded artifacts. (#752) * Update will cleanup unneeded artifacts. The update process will cleanup unneeded artifacts. When an update starts all artifacts that do not have the current version number in it's name will be removed. If artifact retrieval fails, downloaded artifacts are removed. On a successful upgrade, all contents of the downloads dir will be removed. * Clean up linter warnings * Wrap errors * cleanup tests * Fix passed version * Use os.RemoveAll --- CHANGELOG.next.asciidoc | 1 + .../pkg/agent/application/upgrade/cleanup.go | 36 +++++++++++++++ .../agent/application/upgrade/cleanup_test.go | 44 +++++++++++++++++++ .../pkg/agent/application/upgrade/upgrade.go | 20 +++++++++ 4 files changed, 101 insertions(+) create mode 100644 internal/pkg/agent/application/upgrade/cleanup.go create mode 100644 internal/pkg/agent/application/upgrade/cleanup_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 75d574a23b5..2361baf73f5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -111,6 +111,7 @@ - Allow the - char to appear as part of variable names in eql expressions. {issue}709[709] {pull}710[710] - Allow the / char in variable names in eql and transpiler. {issue}715[715] {pull}718[718] - Fix data duplication for standalone agent on Kubernetes using the default manifest {issue-beats}31512[31512] {pull}742[742] +- Agent updates will clean up unneeded artifacts. {issue}693[693] {issue}694[694] {pull}752[752] ==== New features diff --git a/internal/pkg/agent/application/upgrade/cleanup.go b/internal/pkg/agent/application/upgrade/cleanup.go new file mode 100644 index 00000000000..5e0618dfe78 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/cleanup.go @@ -0,0 +1,36 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package upgrade + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/go-multierror" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" +) + +// preUpgradeCleanup will remove files that do not have the passed version number from the downloads directory. +func preUpgradeCleanup(version string) error { + files, err := os.ReadDir(paths.Downloads()) + if err != nil { + return fmt.Errorf("unable to read directory %q: %w", paths.Downloads(), err) + } + var rErr error + for _, file := range files { + if file.IsDir() { + continue + } + if !strings.Contains(file.Name(), version) { + if err := os.Remove(filepath.Join(paths.Downloads(), file.Name())); err != nil { + rErr = multierror.Append(rErr, fmt.Errorf("unable to remove file %q: %w", filepath.Join(paths.Downloads(), file.Name()), err)) + } + } + } + return rErr +} diff --git a/internal/pkg/agent/application/upgrade/cleanup_test.go b/internal/pkg/agent/application/upgrade/cleanup_test.go new file mode 100644 index 00000000000..736a9c42b3d --- /dev/null +++ b/internal/pkg/agent/application/upgrade/cleanup_test.go @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package upgrade + +import ( + "os" + "path/filepath" + "testing" + + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" + + "github.com/stretchr/testify/require" +) + +func setupDir(t *testing.T) { + t.Helper() + dir := t.TempDir() + paths.SetDownloads(dir) + + err := os.WriteFile(filepath.Join(dir, "test-8.3.0-file"), []byte("hello, world!"), 0600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "test-8.4.0-file"), []byte("hello, world!"), 0600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "test-8.5.0-file"), []byte("hello, world!"), 0600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "test-hash-file"), []byte("hello, world!"), 0600) + require.NoError(t, err) +} + +func TestPreUpgradeCleanup(t *testing.T) { + setupDir(t) + err := preUpgradeCleanup("8.4.0") + require.NoError(t, err) + + files, err := os.ReadDir(paths.Downloads()) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "test-8.4.0-file", files[0].Name()) + p, err := os.ReadFile(filepath.Join(paths.Downloads(), files[0].Name())) + require.NoError(t, err) + require.Equal(t, []byte("hello, world!"), p) +} diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 1d370cb5301..ce811036176 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -126,6 +126,11 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree "running under control of the systems supervisor") } + err = preUpgradeCleanup(u.agentInfo.Version()) + if err != nil { + u.log.Errorf("Unable to clean downloads dir %q before update: %v", paths.Downloads(), err) + } + if u.caps != nil { if _, err := u.caps.Apply(a); errors.Is(err, capabilities.ErrBlocked) { return nil, nil @@ -137,6 +142,11 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree sourceURI := u.sourceURI(a.SourceURI()) archivePath, err := u.downloadArtifact(ctx, a.Version(), sourceURI) if err != nil { + // Run the same preUpgradeCleanup task to get rid of any newly downloaded files + // This may have an issue if users are upgrading to the same version number. + if dErr := preUpgradeCleanup(u.agentInfo.Version()); dErr != nil { + u.log.Errorf("Unable to remove file after verification failure: %v", dErr) + } return nil, err } @@ -180,10 +190,20 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree cb := shutdownCallback(u.log, paths.Home(), release.Version(), a.Version(), release.TrimCommit(newHash)) if reexecNow { + err = os.RemoveAll(paths.Downloads()) + if err != nil { + u.log.Errorf("Unable to clean downloads dir %q after update: %v", paths.Downloads(), err) + } u.reexec.ReExec(cb) return nil, nil } + // Clean everything from the downloads dir + err = os.RemoveAll(paths.Downloads()) + if err != nil { + u.log.Errorf("Unable to clean downloads dir %q after update: %v", paths.Downloads(), err) + } + return cb, nil } From 87f3798e1481387c766ddad037139dbf51e75ba4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 27 Jul 2022 09:21:29 +0100 Subject: [PATCH 64/69] ci: propagate e2e-testing errors (#695) --- .ci/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9d24c6ea810..78078f79358 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -256,7 +256,9 @@ pipeline { elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT", gitHubCheckName: "e2e-tests", gitHubCheckRepo: env.REPO, - gitHubCheckSha1: env.GIT_BASE_COMMIT) + gitHubCheckSha1: env.GIT_BASE_COMMIT, + propagate: true, + wait: true) } } } From e87810336ea3f244feb76ee6130b365a67270b24 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 27 Jul 2022 22:50:09 +0930 Subject: [PATCH 65/69] [Release] add-backport-next (#784) --- .mergify.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 00b49806e47..3fe46362854 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -207,3 +207,16 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" + - name: backport patches to 8.4 branch + conditions: + - merged + - label=backport-v8.4.0 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.4" + labels: + - "backport" + title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" From 3f322dee74a93117b50c5b4343fccfec56e883bd Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 27 Jul 2022 10:02:12 -0400 Subject: [PATCH 66/69] Update go.sum. --- go.sum | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go.sum b/go.sum index d12450c533f..df4668bfc70 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,7 @@ github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/containerd v1.5.10 h1:3cQ2uRVCkJVcx5VombsE7105Gl9Wrl7ORAO3+4+ogf4= github.com/containerd/containerd v1.5.10/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= +github.com/containerd/containerd v1.5.13/go.mod h1:3AlCrzKROjIuP3JALsY14n8YtntaUDBu7vek+rPN5Vc= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -388,6 +389,8 @@ github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= github.com/elastic/elastic-agent-libs v0.2.3 h1:GY8M0fxOs/GBY2nIB+JOB91aoD72S87iEcm2qVGFUqI= github.com/elastic/elastic-agent-libs v0.2.3/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= +github.com/elastic/elastic-agent-libs v0.2.6 h1:DpcUcCVYZ7lNtHLUlyT1u/GtGAh49wpL15DTH7+8O5o= +github.com/elastic/elastic-agent-libs v0.2.6/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-system-metrics v0.3.0 h1:W8L0E8lWJmdguH+oIR7OzuFgopvw8ucZAE9w6iqVlpE= github.com/elastic/elastic-agent-system-metrics v0.3.0/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o= github.com/elastic/elastic-package v0.32.1/go.mod h1:l1fEnF52XRBL6a5h6uAemtdViz2bjtjUtgdQcuRhEAY= @@ -1263,6 +1266,7 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1890,6 +1894,7 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v1.7.0 h1:RwpqwwFKBAa2h+F6pMEGpE707Edld0etUD3GhqqhDNc= From 7cf50637d978f905d6fc87cb96aaf2736692ae14 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 27 Jul 2022 14:31:38 -0400 Subject: [PATCH 67/69] Fix upgrade. --- internal/pkg/agent/application/application.go | 2 +- internal/pkg/agent/application/upgrade/upgrade.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index d5bc6a182d8..e42be3d443f 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -70,7 +70,7 @@ func New( return nil, fmt.Errorf("failed to load configuration: %w", err) } - upgrader := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig) + upgrader := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, agentInfo) runtime, err := runtime.NewManager(log, cfg.Settings.GRPC.String(), tracer) if err != nil { diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 169414bb897..fed53376966 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -30,7 +30,6 @@ const ( agentName = "elastic-agent" hashLen = 6 agentCommitFile = ".elastic-agent.active.commit" - darwin = "darwin" ) var ( @@ -50,6 +49,7 @@ var ( type Upgrader struct { log *logger.Logger settings *artifact.Config + agentInfo *info.AgentInfo upgradeable bool } @@ -61,10 +61,11 @@ func IsUpgradeable() bool { } // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(log *logger.Logger, settings *artifact.Config) *Upgrader { +func NewUpgrader(log *logger.Logger, settings *artifact.Config, agentInfo *info.AgentInfo) *Upgrader { return &Upgrader{ log: log, settings: settings, + agentInfo: agentInfo, upgradeable: IsUpgradeable(), } } @@ -74,8 +75,7 @@ func (u *Upgrader) Upgradeable() bool { return u.upgradeable } -// Upgrade upgrades running agent, function returns shutdown callback if some needs to be executed for cases when -// reexec is called by caller. +// Upgrade upgrades running agent, function returns shutdown callback that must be called by reexec. func (u *Upgrader) Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade) (_ reexec.ShutdownCallbackFn, err error) { span, ctx := apm.StartSpan(ctx, "upgrade", "app.internal") defer span.End() From 366dd67ef4c9950f0296a226ca7964e5cb1649e2 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 27 Jul 2022 14:55:35 -0400 Subject: [PATCH 68/69] Fix the upgrade artifact reload. --- internal/pkg/agent/application/application.go | 2 +- .../application/coordinator/coordinator.go | 13 ++++- .../application/coordinator}/handler.go | 26 +++++---- .../application/upgrade/artifact/config.go | 54 ++----------------- .../pkg/agent/application/upgrade/upgrade.go | 36 +++++++++++++ 5 files changed, 66 insertions(+), 65 deletions(-) rename internal/pkg/{core/status => agent/application/coordinator}/handler.go (59%) diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index e42be3d443f..bf0d0fd6444 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -121,7 +121,7 @@ func New( return nil, errors.New(err, "failed to initialize composable controller") } - coord := coordinator.New(log, specs, reexec, upgrader, runtime, configMgr, composable, caps, compModifiers...) + coord := coordinator.New(log, agentInfo, specs, reexec, upgrader, runtime, configMgr, composable, caps, compModifiers...) if managed != nil { // the coordinator requires the config manager as well as in managed-mode the config manager requires the // coordinator, so it must be set here once the coordinator is created diff --git a/internal/pkg/agent/application/coordinator/coordinator.go b/internal/pkg/agent/application/coordinator/coordinator.go index dac48400179..4a6a9f6414b 100644 --- a/internal/pkg/agent/application/coordinator/coordinator.go +++ b/internal/pkg/agent/application/coordinator/coordinator.go @@ -42,6 +42,9 @@ type UpgradeManager interface { // Upgradeable returns true if can be upgraded. Upgradeable() bool + // Reload reloads the configuration for the upgrade manager. + Reload(rawConfig *config.Config) error + // Upgrade upgrades running agent. Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade) (_ reexec.ShutdownCallbackFn, err error) } @@ -129,7 +132,8 @@ type StateFetcher interface { // // All configuration changes, update variables, and upgrade actions are managed and controlled by the coordinator. type Coordinator struct { - logger *logger.Logger + logger *logger.Logger + agentInfo *info.AgentInfo specs component.RuntimeSpecs @@ -150,9 +154,10 @@ type Coordinator struct { } // New creates a new coordinator. -func New(logger *logger.Logger, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, modifiers ...ComponentsModifier) *Coordinator { +func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, modifiers ...ComponentsModifier) *Coordinator { return &Coordinator{ logger: logger, + agentInfo: agentInfo, specs: specs, reexecMgr: reexecMgr, upgradeMgr: upgradeMgr, @@ -429,6 +434,10 @@ func (c *Coordinator) processConfig(ctx context.Context, cfg *config.Config) (er } } + if err := c.upgradeMgr.Reload(cfg); err != nil { + return fmt.Errorf("failed to reload upgrade manager configuration: %w", err) + } + c.state.config = cfg c.state.ast = rawAst diff --git a/internal/pkg/core/status/handler.go b/internal/pkg/agent/application/coordinator/handler.go similarity index 59% rename from internal/pkg/core/status/handler.go rename to internal/pkg/agent/application/coordinator/handler.go index e82f73fb216..22130d1a776 100644 --- a/internal/pkg/core/status/handler.go +++ b/internal/pkg/agent/application/coordinator/handler.go @@ -2,12 +2,14 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package status +package coordinator import ( "encoding/json" "net/http" "time" + + "github.com/elastic/elastic-agent/internal/pkg/agent/control/client" ) // LivenessResponse is the response body for the liveness endpoint. @@ -18,19 +20,21 @@ type LivenessResponse struct { UpdateTime time.Time `json:"update_timestamp"` } -// ServeHTTP is an HTTP Handler for the status controller. -// Respose code is 200 for a healthy agent, and 503 otherwise. +// ServeHTTP is an HTTP Handler for the coordinatorr. +// Response code is 200 for a healthy agent, and 503 otherwise. // Response body is a JSON object that contains the agent ID, status, message, and the last status update time. -func (r *controller) ServeHTTP(wr http.ResponseWriter, req *http.Request) { - s := r.Status() +func (c *Coordinator) ServeHTTP(wr http.ResponseWriter, req *http.Request) { + s := c.State() lr := LivenessResponse{ - ID: r.agentID, - Status: s.Status.String(), - Message: s.Message, - UpdateTime: s.UpdateTime, + ID: c.agentInfo.AgentID(), + Status: s.State.String(), + Message: s.Message, + + // TODO(blakerouse): Coordinator should be changed to store the last timestamp that the state has changed. + UpdateTime: time.Now().UTC(), } status := http.StatusOK - if s.Status != Healthy { + if s.State != client.Healthy { status = http.StatusServiceUnavailable } @@ -38,6 +42,6 @@ func (r *controller) ServeHTTP(wr http.ResponseWriter, req *http.Request) { wr.WriteHeader(status) enc := json.NewEncoder(wr) if err := enc.Encode(lr); err != nil { - r.log.Errorf("Unable to encode liveness response: %v", err) + c.logger.Errorf("Unable to encode liveness response: %v", err) } } diff --git a/internal/pkg/agent/application/upgrade/artifact/config.go b/internal/pkg/agent/application/upgrade/artifact/config.go index fa57ca06870..6db38fa612c 100644 --- a/internal/pkg/agent/application/upgrade/artifact/config.go +++ b/internal/pkg/agent/application/upgrade/artifact/config.go @@ -11,9 +11,6 @@ import ( "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" - "github.com/elastic/elastic-agent/internal/pkg/agent/errors" - "github.com/elastic/elastic-agent/internal/pkg/config" - "github.com/elastic/elastic-agent/pkg/core/logger" ) const ( @@ -21,7 +18,8 @@ const ( linux = "linux" windows = "windows" - defaultSourceURI = "https://artifacts.elastic.co/downloads/" + // DefaultSourceURI is the default source URI for downloading artifacts. + DefaultSourceURI = "https://artifacts.elastic.co/downloads/" ) // Config is a configuration used for verifier and downloader @@ -51,52 +49,6 @@ type Config struct { httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"` // Note: use anonymous struct for json inline } -type Reloader struct { - log *logger.Logger - cfg *Config -} - -func NewReloader(cfg *Config, log *logger.Logger) *Reloader { - return &Reloader{ - cfg: cfg, - log: log, - } -} - -func (r *Reloader) Reload(rawConfig *config.Config) error { - type reloadConfig struct { - // SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ - SourceURI string `json:"agent.download.sourceURI" config:"agent.download.sourceURI"` - - // FleetSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ coming from fleet which uses - // different naming. - FleetSourceURI string `json:"agent.download.source_uri" config:"agent.download.source_uri"` - } - cfg := &reloadConfig{} - if err := rawConfig.Unpack(&cfg); err != nil { - return errors.New(err, "failed to unpack config during reload") - } - - var newSourceURI string - if cfg.FleetSourceURI != "" { - // fleet configuration takes precedence - newSourceURI = cfg.FleetSourceURI - } else if cfg.SourceURI != "" { - newSourceURI = cfg.SourceURI - } - - if newSourceURI != "" { - r.log.Infof("Source URI changed from %q to %q", r.cfg.SourceURI, newSourceURI) - r.cfg.SourceURI = newSourceURI - } else { - // source uri unset, reset to default - r.log.Infof("Source URI reset from %q to %q", r.cfg.SourceURI, defaultSourceURI) - r.cfg.SourceURI = defaultSourceURI - } - - return nil -} - // DefaultConfig creates a config with pre-set default values. func DefaultConfig() *Config { transport := httpcommon.DefaultHTTPTransportSettings() @@ -107,7 +59,7 @@ func DefaultConfig() *Config { transport.Timeout = 10 * time.Minute return &Config{ - SourceURI: defaultSourceURI, + SourceURI: DefaultSourceURI, TargetDirectory: paths.Downloads(), InstallPath: paths.Install(), HTTPTransportSettings: transport, diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index fed53376966..edc70c3f5c0 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -12,6 +12,8 @@ import ( "path/filepath" "strings" + "github.com/elastic/elastic-agent/internal/pkg/config" + "github.com/otiai10/copy" "go.elastic.co/apm" @@ -70,6 +72,40 @@ func NewUpgrader(log *logger.Logger, settings *artifact.Config, agentInfo *info. } } +// Reload reloads the artifact configuration for the upgrader. +func (u *Upgrader) Reload(rawConfig *config.Config) error { + type reloadConfig struct { + // SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ + SourceURI string `json:"agent.download.sourceURI" config:"agent.download.sourceURI"` + + // FleetSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ coming from fleet which uses + // different naming. + FleetSourceURI string `json:"agent.download.source_uri" config:"agent.download.source_uri"` + } + cfg := &reloadConfig{} + if err := rawConfig.Unpack(&cfg); err != nil { + return errors.New(err, "failed to unpack config during reload") + } + + var newSourceURI string + if cfg.FleetSourceURI != "" { + // fleet configuration takes precedence + newSourceURI = cfg.FleetSourceURI + } else if cfg.SourceURI != "" { + newSourceURI = cfg.SourceURI + } + + if newSourceURI != "" { + u.log.Infof("Source URI changed from %q to %q", u.settings.SourceURI, newSourceURI) + u.settings.SourceURI = newSourceURI + } else { + // source uri unset, reset to default + u.log.Infof("Source URI reset from %q to %q", u.settings.SourceURI, artifact.DefaultSourceURI) + u.settings.SourceURI = artifact.DefaultSourceURI + } + return nil +} + // Upgradeable returns true if the Elastic Agent can be upgraded. func (u *Upgrader) Upgradeable() bool { return u.upgradeable From 692b71ef1378d22b7f1b2718d845905223e07805 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 27 Jul 2022 15:01:09 -0400 Subject: [PATCH 69/69] Fix lint in coordinator. --- internal/pkg/agent/application/coordinator/coordinator.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/pkg/agent/application/coordinator/coordinator.go b/internal/pkg/agent/application/coordinator/coordinator.go index 4a6a9f6414b..906b9af2d64 100644 --- a/internal/pkg/agent/application/coordinator/coordinator.go +++ b/internal/pkg/agent/application/coordinator/coordinator.go @@ -514,10 +514,9 @@ type coordinatorState struct { message string overrideState *coordinatorOverrideState - config *config.Config - ast *transpiler.AST - vars []*transpiler.Vars - components []component.Component + config *config.Config + ast *transpiler.AST + vars []*transpiler.Vars } type coordinatorOverrideState struct {