From a5acefb77e1bc28909121ce11e14d0b64f053fda Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 6 Sep 2022 09:44:36 -0700 Subject: [PATCH 1/9] Update Metricbeat, Filebeat, libbeat with elastic-agent V2 support (#32673) * basic framework * continued tinkering * move away from ast code, use a struct * get metricbeat working, starting on filebeat * add notice update * add basic config register * move over processors to individual beats * remove comments * start to integrate V2 client changes * finishing touches * lint * cleanup merge * remove V1 controller * stil tinkering with linter * still fixing linter * plz linter * fmt x-pack files * notice update * fix output test * refactor stop functions, refactor tests, some misc cleanup * fix client version string * add devguide * linter * expand filebeat test * cleanup test * fix docs, add tests, debuggin * add signal handler * fix mutex issue in register * Fix osquerybeat configuration for V2 * clean up component registration * spelling * remove workaround for filebeat types * try to fix filebeat tests * add nil checks, fix test, fix unit stop * continue tinkering with nil type checks * add test for missing config datastreams, clean up nil handling * change nil protections, use getter methods * fix config access in output code Co-authored-by: Aleksandr Maus --- NOTICE.txt | 20 +- filebeat/beater/filebeat.go | 3 +- go.mod | 10 +- go.sum | 18 +- heartbeat/beater/heartbeat.go | 4 +- libbeat/cfgfile/cfgfile.go | 48 ++- libbeat/cmd/instance/beat.go | 4 +- libbeat/common/reload/reload.go | 41 +- metricbeat/beater/metricbeat.go | 2 +- packetbeat/beater/packetbeat.go | 2 +- x-pack/filebeat/cmd/agent.go | 28 ++ x-pack/filebeat/cmd/root.go | 2 + x-pack/libbeat/management/blacklist.go | 3 +- x-pack/libbeat/management/config.go | 2 +- x-pack/libbeat/management/devguide.asciidoc | 116 ++++++ x-pack/libbeat/management/generate.go | 270 ++++++++++++ x-pack/libbeat/management/generate_test.go | 165 ++++++++ x-pack/libbeat/management/manager.go | 389 ------------------ x-pack/libbeat/management/managerV2.go | 377 +++++++++++++++++ x-pack/libbeat/management/manager_test.go | 75 ---- x-pack/libbeat/management/plugin.go | 10 +- .../tests/fbtest/filebeat_v2_test.go | 115 ++++++ .../management/tests/fbtest/testdata/messages | 62 +++ .../management/tests/fbtest/testdata/secure | 30 ++ x-pack/libbeat/management/tests/init.go | 88 ++++ .../tests/mbtest/metricbeat_v2_test.go | 124 ++++++ .../libbeat/management/tests/mock_server.go | 159 +++++++ .../libbeat/management/tests/output_read.go | 94 +++++ x-pack/metricbeat/cmd/agent.go | 37 ++ x-pack/metricbeat/cmd/root.go | 2 + x-pack/osquerybeat/cmd/root.go | 33 ++ x-pack/osquerybeat/internal/config/watcher.go | 2 +- 32 files changed, 1807 insertions(+), 528 deletions(-) create mode 100644 x-pack/filebeat/cmd/agent.go create mode 100644 x-pack/libbeat/management/devguide.asciidoc create mode 100644 x-pack/libbeat/management/generate.go create mode 100644 x-pack/libbeat/management/generate_test.go delete mode 100644 x-pack/libbeat/management/manager.go create mode 100644 x-pack/libbeat/management/managerV2.go delete mode 100644 x-pack/libbeat/management/manager_test.go create mode 100644 x-pack/libbeat/management/tests/fbtest/filebeat_v2_test.go create mode 100644 x-pack/libbeat/management/tests/fbtest/testdata/messages create mode 100644 x-pack/libbeat/management/tests/fbtest/testdata/secure create mode 100644 x-pack/libbeat/management/tests/init.go create mode 100644 x-pack/libbeat/management/tests/mbtest/metricbeat_v2_test.go create mode 100644 x-pack/libbeat/management/tests/mock_server.go create mode 100644 x-pack/libbeat/management/tests/output_read.go create mode 100644 x-pack/metricbeat/cmd/agent.go diff --git a/NOTICE.txt b/NOTICE.txt index 6f6ddb63cd3..c167bc1b9d8 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9625,11 +9625,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.0.0-20210727140539-f0905d9377f6 +Version: v7.0.0-20220804181728-b0328d2fe484 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20210727140539-f0905d9377f6/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20220804181728-b0328d2fe484/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -21297,11 +21297,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.0.0-20220225172249-27dd8689420f +Version: v0.0.0-20220425223048-2871e0cb64e4 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20220225172249-27dd8689420f/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20220425223048-2871e0cb64e4/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -21593,11 +21593,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : google.golang.org/genproto -Version: v0.0.0-20220329172620-7be39ac1afc7 +Version: v0.0.0-20220426171045-31bebdecfb46 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0-20220329172620-7be39ac1afc7/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0-20220426171045-31bebdecfb46/LICENSE: Apache License @@ -21805,11 +21805,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0- -------------------------------------------------------------------------------- Dependency : google.golang.org/grpc -Version: v1.45.0 +Version: v1.46.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.45.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.46.0/LICENSE: Apache License @@ -32500,11 +32500,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/envoyproxy/go-control-plane -Version: v0.10.1 +Version: v0.10.2-0.20220325020618-49ff273808a1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.10.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.10.2-0.20220325020618-49ff273808a1/LICENSE: Apache License Version 2.0, January 2004 diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 868e8ea4a4d..53c82d5ee7e 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -353,10 +353,9 @@ func (fb *Filebeat) Run(b *beat.Beat) error { // Register reloadable list of inputs and modules inputs := cfgfile.NewRunnerList(management.DebugK, inputLoader, fb.pipeline) - reload.Register.MustRegisterList("filebeat.inputs", inputs) + reload.RegisterV2.MustRegisterInput(inputs) modules := cfgfile.NewRunnerList(management.DebugK, moduleLoader, fb.pipeline) - reload.Register.MustRegisterList("filebeat.modules", modules) var adiscover *autodiscover.Autodiscover if fb.config.Autodiscover != nil { diff --git a/go.mod b/go.mod index 0aed1eee6c7..5f8037df8ae 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 + github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.3.2 github.com/elastic/go-licenser v0.4.0 @@ -168,7 +168,7 @@ require ( golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.5.1 - golang.org/x/net v0.0.0-20220225172249-27dd8689420f + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 @@ -176,8 +176,8 @@ require ( golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac golang.org/x/tools v0.1.9 google.golang.org/api v0.62.0 - google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7 - google.golang.org/grpc v1.45.0 + google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 + google.golang.org/grpc v1.46.0 google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect @@ -256,7 +256,7 @@ require ( github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect - github.com/envoyproxy/go-control-plane v0.10.1 // indirect + github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fearful-symmetry/gomsr v0.0.1 // indirect diff --git a/go.sum b/go.sum index 1dfcc55e0c7..287efdad0c6 100644 --- a/go.sum +++ b/go.sum @@ -596,8 +596,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.7/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= @@ -666,8 +666,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1 h1:cgDRLG7bs59Zd+apAWuzLQL95obVYAymNJek76W3mgw= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1:xvqufLtNVwAhN8NMyWklVgxnWohi+wtMGQMhtxexlm0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2 h1:JiO+kJTpmYGjEodY7O1Zk8oZcNz1+f30UtwtXoFUPzE= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= @@ -1999,8 +2000,9 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2170,6 +2172,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2419,8 +2422,8 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7 h1:HOL66YCI20JvN2hVk6o2YIp9i/3RvzVUz82PqNr7fXw= -google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 h1:G1IeWbjrqEq9ChWxEuRPJu6laA67+XgTFHVSAvepr38= +google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2456,8 +2459,9 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index 5e0f1ff9b6e..05914a9c4a2 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -182,10 +182,8 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) (stop func(), err error) { // RunCentralMgmtMonitors loads any central management configured configs. func (bt *Heartbeat) RunCentralMgmtMonitors(b *beat.Beat) { - mons := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher) - reload.Register.MustRegisterList(b.Info.Beat+".monitors", mons) inputs := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher) - reload.Register.MustRegisterList("inputs", inputs) + reload.RegisterV2.MustRegisterInput(inputs) } // RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present. diff --git a/libbeat/cfgfile/cfgfile.go b/libbeat/cfgfile/cfgfile.go index d97107aaffd..ca19af8cb9f 100644 --- a/libbeat/cfgfile/cfgfile.go +++ b/libbeat/cfgfile/cfgfile.go @@ -23,6 +23,7 @@ import ( "path/filepath" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/fleetmode" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -101,13 +102,13 @@ func HandleFlags() error { home, err := filepath.Abs(filepath.Dir(os.Args[0])) if err != nil { if *homePath == "" { - return fmt.Errorf("The absolute path to %s could not be obtained. %v", + return fmt.Errorf("The absolute path to %s could not be obtained. %w", os.Args[0], err) } home = *homePath } - defaults.SetString("path.home", -1, home) + _ = defaults.SetString("path.home", -1, home) if len(overwrites.GetFields()) > 0 { common.PrintConfigDebugf(overwrites, "CLI setting overwrites (-E flag):") @@ -133,30 +134,36 @@ func Read(out interface{}, path string) error { // Load reads the configuration from a YAML file structure. If path is empty // this method reads from the configuration file specified by the '-c' command // line flag. +// This function cares about the underlying fleet setting, and if beats is running with +// the management.enabled flag, Load() will bypass reading a config file, and merely merge any overrides. func Load(path string, beatOverrides []ConditionalOverride) (*config.C, error) { var c *config.C var err error cfgpath := GetPathConfig() - if path == "" { - list := []string{} - for _, cfg := range configfiles.List() { - if !filepath.IsAbs(cfg) { - list = append(list, filepath.Join(cfgpath, cfg)) - } else { - list = append(list, cfg) + if !fleetmode.Enabled() { + if path == "" { + list := []string{} + for _, cfg := range configfiles.List() { + if !filepath.IsAbs(cfg) { + list = append(list, filepath.Join(cfgpath, cfg)) + } else { + list = append(list, cfg) + } + } + c, err = common.LoadFiles(list...) + } else { + if !filepath.IsAbs(path) { + path = filepath.Join(cfgpath, path) } + c, err = common.LoadFile(path) } - c, err = common.LoadFiles(list...) - } else { - if !filepath.IsAbs(path) { - path = filepath.Join(cfgpath, path) + if err != nil { + return nil, err } - c, err = common.LoadFile(path) - } - if err != nil { - return nil, err + } else { + c = config.NewConfig() } if beatOverrides != nil { @@ -183,6 +190,9 @@ func Load(path string, beatOverrides []ConditionalOverride) (*config.C, error) { c, overwrites, ) + if err != nil { + return nil, err + } } common.PrintConfigDebugf(c, "Complete configuration loaded:") @@ -194,13 +204,13 @@ func LoadList(file string) ([]*config.C, error) { logp.Debug("cfgfile", "Load config from file: %s", file) rawConfig, err := common.LoadFile(file) if err != nil { - return nil, fmt.Errorf("invalid config: %s", err) + return nil, fmt.Errorf("invalid config: %w", err) } var c []*config.C err = rawConfig.Unpack(&c) if err != nil { - return nil, fmt.Errorf("error reading configuration from file %s: %s", file, err) + return nil, fmt.Errorf("error reading configuration from file %s: %w", file, err) } return c, nil diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 30a960c3288..5272bb3251e 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -350,7 +350,7 @@ func (b *Beat) createBeater(bt beat.Creator) (beat.Beater, error) { return nil, fmt.Errorf("error initializing publisher: %w", err) } - reload.Register.MustRegister("output", b.makeOutputReloader(publisher.OutputReloader())) + reload.RegisterV2.MustRegisterOutput(b.makeOutputReloader(publisher.OutputReloader())) // TODO: some beats race on shutdown with publisher.Stop -> do not call Stop yet, // but refine publisher to disconnect clients on stop automatically @@ -710,7 +710,7 @@ func (b *Beat) configure(settings Settings) error { logp.Info("Beat ID: %v", b.Info.ID) // initialize config manager - b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID) + b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID) if err != nil { return err } diff --git a/libbeat/common/reload/reload.go b/libbeat/common/reload/reload.go index e1838619245..c3a57bc027a 100644 --- a/libbeat/common/reload/reload.go +++ b/libbeat/common/reload/reload.go @@ -25,8 +25,14 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -// Register holds a registry of reloadable objects -var Register = NewRegistry() +// RegisterV2 is the special registry used for the V2 controller +var RegisterV2 = NewRegistry() + +// InputRegName is the registation name for V2 inputs +const InputRegName = "input" + +// OutputRegName is the registation name for V2 Outputs +const OutputRegName = "output" // ConfigWithMeta holds a pair of config.C and optional metadata for it type ConfigWithMeta struct { @@ -106,13 +112,38 @@ func (r *Registry) MustRegister(name string, obj Reloadable) { } } -// MustRegisterList declares a reloadable object list -func (r *Registry) MustRegisterList(name string, list ReloadableList) { - if err := r.RegisterList(name, list); err != nil { +// MustRegisterOutput is a V2-specific registration function +// That declares a reloadable output +func (r *Registry) MustRegisterOutput(obj Reloadable) { + if err := r.Register(OutputRegName, obj); err != nil { panic(err) } } +// MustRegisterInput is a V2-specific registration function +// that declares a reloadable object list for a beat input +func (r *Registry) MustRegisterInput(list ReloadableList) { + if err := r.RegisterList(InputRegName, list); err != nil { + panic(err) + } +} + +// GetInputList is a V2-specific function +// That returns the reloadable list created for an input +func (r *Registry) GetInputList() ReloadableList { + r.RLock() + defer r.RUnlock() + return r.confsLists[InputRegName] +} + +// GetReloadableOutput is a V2-specific function +// That returns the reloader for the registered output +func (r *Registry) GetReloadableOutput() Reloadable { + r.RLock() + defer r.RUnlock() + return r.confs[OutputRegName] +} + // GetRegisteredNames returns the list of names registered func (r *Registry) GetRegisteredNames() []string { r.RLock() diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 8b204a3cad3..9e2c101a253 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -218,7 +218,7 @@ func (bt *Metricbeat) Run(b *beat.Beat) error { // Centrally managed modules factory := module.NewFactory(b.Info, bt.moduleOptions...) modules := cfgfile.NewRunnerList(management.DebugK, factory, b.Publisher) - reload.Register.MustRegisterList(b.Info.Beat+".modules", modules) + reload.RegisterV2.MustRegisterInput(modules) wg.Add(1) go func() { defer wg.Done() diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index a0dde1e28a6..dc8b1999c99 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -151,7 +151,7 @@ func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error { // the runner by starting the beat's manager. It returns on the first fatal error. func (pb *packetbeat) runManaged(b *beat.Beat, factory *processorFactory) error { runner := newReloader(management.DebugK, factory, b.Publisher) - reload.Register.MustRegisterList("inputs", runner) + reload.RegisterV2.MustRegisterInput(runner) logp.Debug("main", "Waiting for the runner to finish") // Start the manager after all the hooks are registered and terminates when diff --git a/x-pack/filebeat/cmd/agent.go b/x-pack/filebeat/cmd/agent.go new file mode 100644 index 00000000000..d5cb432ce90 --- /dev/null +++ b/x-pack/filebeat/cmd/agent.go @@ -0,0 +1,28 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" +) + +func filebeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + modules, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + // format for the reloadable list needed bythe cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating config for reloader: %w", err) + } + return configList, nil +} diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index 18bdc321b30..c6f55a02379 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -7,6 +7,7 @@ package cmd import ( fbcmd "github.com/elastic/beats/v7/filebeat/cmd" cmd "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/x-pack/libbeat/management" // Register the includes. _ "github.com/elastic/beats/v7/x-pack/filebeat/include" @@ -18,6 +19,7 @@ const Name = fbcmd.Name // Filebeat build the beat root command for executing filebeat and it's subcommands. func Filebeat() *cmd.BeatsRootCmd { + management.ConfigTransform.SetTransform(filebeatCfg) settings := fbcmd.FilebeatSettings() settings.ElasticLicensed = true command := fbcmd.Filebeat(inputs.Init, settings) diff --git a/x-pack/libbeat/management/blacklist.go b/x-pack/libbeat/management/blacklist.go index 75d7425e89c..ca5b07d5a5e 100644 --- a/x-pack/libbeat/management/blacklist.go +++ b/x-pack/libbeat/management/blacklist.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/match" conf "github.com/elastic/elastic-agent-libs/config" @@ -52,7 +51,7 @@ func NewConfigBlacklist(cfg ConfigBlacklistSettings) (*ConfigBlacklist, error) { for field, pattern := range cfg.Patterns { exp, err := match.Compile(pattern) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Given expression is not a valid regexp: %s", pattern)) + return nil, fmt.Errorf("given expression is not a valid regexp: %s", pattern) } list.patterns[field] = exp diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index a4500941b86..ae19bd08cc4 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -29,7 +29,7 @@ type ConfigBlocksWithType struct { // ConfigBlocks holds a list of type + configs objects type ConfigBlocks []ConfigBlocksWithType -func defaultConfig() *Config { +func DefaultConfig() *Config { return &Config{ Blacklist: ConfigBlacklistSettings{ Patterns: map[string]string{ diff --git a/x-pack/libbeat/management/devguide.asciidoc b/x-pack/libbeat/management/devguide.asciidoc new file mode 100644 index 00000000000..a2553a6e5df --- /dev/null +++ b/x-pack/libbeat/management/devguide.asciidoc @@ -0,0 +1,116 @@ +# Writing V2 support for a beat + +## What is this document? + +This guide is meant to be a high-level guide for adapting an individual beat to the newer V2 control protocol used by elastic-agent; +this includes component registration, config transformation, and tests. + + +## Introduction: The V2 controller + +The V2 remote management protocol is fundamentally different from V1; whereas V1 required the registration of a number of callbacks from a client to a server, +such as `OnConfig()` and `OnStop()`, V2, is more parallelized and flexible, +but as a trade-off requires more work on behalf of a client to manage the flow of commands from the server. + +From the perspective of an individual beat however, little has changed, as this newer V2 server will still look for a `ReloadableList` callback registered by an +individual beat. However, the V2 server also offloads the work of transforming the "agent-native" config generated by fleet onto clients. +Previously, this config transformation was performed by an AST layer in elastic-agent, +as configured by YML spec files that lived in `elastic-agent/internal/spec`. In V2, this transformation must be performed by an individual beat. + + +## Component Registration + +There are two components, one of which is optional, that must be registered with the management controller for a beat to run under the V2 controller: + +### The Reloader + +The reloader component is not unique to V2, and remains largely unchanged from it's V1 state. Currently, all beats have a line like this (the following example is from metricbeat), in their early startup state: + +``` +reload.RegisterV2.MustRegisterInput(modules) +``` + +The `MustRegisterInput` method takes a single argument: component that satisfies a `ReloadableList` interface. +For most beats, this component will be a wrapper around an array of individual modules/inputs that maps to each individual input in the upstream fleet config. +In this example agent YAML config, each item under the `streams` key will become an individual input config in the array sent to the `ReloadableList` interface: +``` + - id: logfile-system-default-system + name: system-1 + revision: 1 + type: logfile + use_output: default + meta: + package: + name: system + version: 1.17.0 + data_stream: + namespace: default + streams: + - id: logfile-system.auth-default-system + [...] + - id: logfile-system.syslog-default-system + [...] +``` + +### Config Transformation + +Unlike with the V1 agent controller, with the V2 controller, individual beats will be required to make any transformations needed to format their config. +In V1, config transformation was performed by the agent and configured by a YAML file. In V2, this transformation process is fundamentally different, +as beats no longer receive the entire agent config, but individual per-input configs, +and so the logic of the AST transformations present in `elastic-agent/internal/spec` cannot be moved one-to-one over to the V2 client in beats. + +In order for a beat to perform its own config transformations, it must register a callback function, as such: +``` +import "github.com/elastic/beats/v7/x-pack/libbeat/management" +... +management.ConfigTransform.SetTransform(metricbeatCfg) +``` + +The `SetTransform` method takes a function with the following signature: +``` +func(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) +``` + +This registration will usually happen early in the startup process, and for licensing reasons, must happen in `x-pack` code. +For the sake of consistency, both filebeat and metricbeat register this callback in `x-pack/[beatname]/cmd`. If a beat does not register a callback, +the V2 client will perform a basic config transformation that adds metadata `processor` fields and breaks apart the individual inputs under the `streams` key. + +This function callback takes an `AgentInfo` struct that can be used for creating metadata `add_field` processors, as well as a `UnitExpectedConfig` structure that +represents the individual input config demonstrated in the previous section. The return value, `[]*reload.ConfigWithMeta`, should represent the final config that +will be passed to the reloader interface. + +The V2 client library provides a number of convenience functions. `CreateInputsFromStreams` will generate an array of input configs from the `streams` value, +complete with metadata fields. `CreateReloadConfigFromInputs` will turn a list of input configs in the form of `[]map[string]interface{}` into a list of reloader +configs that can be returned and sent to the registered reloader. + + +## Tests + +The V2 client controller contains a test suite under `x-pack/libbeat/management/tests`. These tests are broken down to one beat per folder, +as the tests themselves will try to initialize a number of global variables, requiring the CI process to break them apart to individual executables. +To add a new beat to the test, you must fetch the `BeatsRootCmd` object that starts a beat executible, and pass it to a wrapper that will +modify the beats environment for use in a test wrapper: +``` +import fbroot "github.com/elastic/beats/v7/x-pack/filebeat/cmd" +... +filebeatCmd := fbroot.Filebeat() +tests.InitBeatsForTest(t, filebeatCmd) +``` + +The test suite uses a mock `elastic-agent` server, requiring only a `UnitExpectedConfig{}` with the beat's desired config. After the mock server has been configured, +the `BeatsRootCmd` object can be started as it would in any normal non-test setup: +``` + // Setup the mock server, return the tmpfile where the beat will output metrics, and the server handler + outPath, server := tests.SetupTestEnv(t, expectedBeatsConfig, serverRuntimeInSeconds) + defer server.Srv.Stop() + + // start the beat. This is a blocking command, and beats will shut down after `serverRuntimeInSeconds`. + err := filebeatCmd.Execute() + require.NoError(t, err) + + // Read the reported metrics send to the `file` output by the beat + events := tests.ReadEvents(t, outPath) + t.Logf("Got %d events", len(events)) +``` + +For more in-depth examples, examine the `fbtest` and `mbtest` directories. \ No newline at end of file diff --git a/x-pack/libbeat/management/generate.go b/x-pack/libbeat/management/generate.go new file mode 100644 index 00000000000..957b8477b6d --- /dev/null +++ b/x-pack/libbeat/management/generate.go @@ -0,0 +1,270 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package management + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var DefaultNamespaceName = "default" +var DefaultDatasetName = "generic" + +// =========== +// Config Transformation Registry +// =========== + +var ConfigTransform = TransformRegister{} + +// TransformRegister is a hack that allows an individual beat to set a transform function +// so the V2 controller can perform beat-specific config transformations. +// This is mostly done this way so we can avoid mixing up code with different licenses, +// as this is entirely xpack/Elastic License code, and the normal beat init process happens in libbeat. +// This is fairly simple, as only one beat will ever register a callback. +type TransformRegister struct { + transformFunc func(*proto.UnitExpectedConfig, *client.AgentInfo) ([]*reload.ConfigWithMeta, error) +} + +// SetTransform sets a transform function callback +func (r *TransformRegister) SetTransform(transform func(*proto.UnitExpectedConfig, *client.AgentInfo) ([]*reload.ConfigWithMeta, error)) { + r.transformFunc = transform +} + +// SetTransform sets a transform function callback +func (r *TransformRegister) Transform(cfg *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + // If no transform is registered, fallback to a basic setup + if r.transformFunc == nil { + streamList, err := CreateInputsFromStreams(cfg, "log", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from fallback function: %w", err) + } + // format for the reloadable list needed bythe cm.Reload() method + configList, err := CreateReloadConfigFromInputs(streamList) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + return configList, nil + } + + return r.transformFunc(cfg, agentInfo) +} + +// =========== +// Stream and Input processors +// =========== + +// CreateInputsFromStreams breaks down the raw Expected config into an array of individual inputs/modules from the Streams values +// that can later be formatted into the reloader's ConfigWithMetaData and sent to an indvidual beat/ +// This also performs the basic task of inserting module-level add_field processors into the inputs/modules. +func CreateInputsFromStreams(raw *proto.UnitExpectedConfig, inputType string, agentInfo *client.AgentInfo) ([]map[string]interface{}, error) { + inputs := make([]map[string]interface{}, len(raw.Streams)) + + for iter, stream := range raw.GetStreams() { + streamSource := raw.GetStreams()[iter].GetSource().AsMap() + + streamSource = injectIndexStream(raw, inputType, stream, streamSource) + streamSource, err := injectStreamProcessors(raw, inputType, stream, streamSource) + if err != nil { + return nil, fmt.Errorf("Error injecting stream processors: %w", err) + } + streamSource, err = injectAgentInfoRule(streamSource, agentInfo) + if err != nil { + return nil, fmt.Errorf("Error injecting agent processors: %w", err) + } + inputs[iter] = streamSource + } + + return inputs, nil +} + +// CreateReloadConfigFromInputs turns a raw input/module list into the ConfigWithMeta type used by the reloader interface +func CreateReloadConfigFromInputs(raw []map[string]interface{}) ([]*reload.ConfigWithMeta, error) { + // format for the reloadable list needed bythe cm.Reload() method + configList := make([]*reload.ConfigWithMeta, len(raw)) + + for iter := range raw { + uconfig, err := conf.NewConfigFrom(raw[iter]) + if err != nil { + return nil, fmt.Errorf("error in conversion to conf.C: %w", err) + } + configList[iter] = &reload.ConfigWithMeta{Config: uconfig} + } + return configList, nil +} + +// Emulates the InjectAgentInfoRule and InjectHeadersRule ast rules +func injectAgentInfoRule(inputs map[string]interface{}, agentInfo *client.AgentInfo) (map[string]interface{}, error) { + // upstream API can sometimes return a nil agent info + if agentInfo == nil { + return inputs, nil + } + var processors []interface{} + + processors = append(processors, generateAddFieldsProcessor( + mapstr.M{"id": agentInfo.ID, "snapshot": agentInfo.Snapshot, "version": agentInfo.Version}, + "elastic_agent")) + processors = append(processors, generateAddFieldsProcessor( + mapstr.M{"id": agentInfo.ID}, + "agent")) + + currentProcs, ok := inputs["processors"] + if !ok { + inputs["processors"] = processors + } else { + currentProcsList, ok := currentProcs.([]interface{}) + if !ok { + return nil, fmt.Errorf("error creating list of existing processors, got: %#v", currentProcs) + } + inputs["processors"] = append(processors, currentProcsList...) + + } + + return inputs, nil +} + +// injectIndexStream is an emulation of the InjectIndexProcessor AST code +func injectIndexStream(expected *proto.UnitExpectedConfig, inputType string, streamExpected *proto.Stream, stream map[string]interface{}) map[string]interface{} { + streamType := expected.GetDataStream().GetType() + if streamType == "" { + streamType = inputType + } + + dataset := DefaultDatasetName + if testDataset := streamExpected.GetDataStream().GetDataset(); testDataset != "" { + dataset = testDataset + } + + namespace := DefaultNamespaceName + if testNamespace := expected.GetDataStream().GetNamespace(); testNamespace != "" { + namespace = testNamespace + } + + index := fmt.Sprintf("%s-%s-%s", streamType, dataset, namespace) + stream["index"] = index + return stream +} + +//injectStreamProcessors is an emulation of the InjectStreamProcessorRule AST code +func injectStreamProcessors(expected *proto.UnitExpectedConfig, inputType string, streamExpected *proto.Stream, stream map[string]interface{}) (map[string]interface{}, error) { + //1. start by "repairing" config to add any missing fields + // logic from datastreamTypeFromInputNode + procInputType := inputType + if testInputType := expected.GetDataStream().GetType(); testInputType != "" { + procInputType = testInputType + } + + procInputNamespace := DefaultNamespaceName + if testInputNamespace := expected.GetDataStream().GetNamespace(); testInputNamespace != "" { + procInputNamespace = testInputNamespace + } + + var processors = []interface{}{} + + // the AST injects input_id at the input level and not the stream level, + // for reasons I can't understand, as it just ends up shuffling it around + // to individual metricsets anyway, at least on metricbeat + if expected.GetId() != "" { + inputId := generateAddFieldsProcessor(mapstr.M{"input_id": expected.Id}, "@metadata") + processors = append(processors, inputId) + } + + procInputDataset := DefaultDatasetName + if testStreamDataset := streamExpected.GetDataStream().GetDataset(); testStreamDataset != "" { + procInputDataset = testStreamDataset + } + + //2. Actually add the processors + // namespace + datastream := generateAddFieldsProcessor(mapstr.M{"dataset": procInputDataset, + "namespace": procInputNamespace, "type": procInputType}, "data_stream") + processors = append(processors, datastream) + + // dataset + event := generateAddFieldsProcessor(mapstr.M{"dataset": procInputDataset}, "event") + processors = append(processors, event) + + // source stream + streamID := streamExpected.GetId() + sourceStream := generateAddFieldsProcessor(mapstr.M{"stream_id": streamID}, "@metadata") + processors = append(processors, sourceStream) + + // figure out if we have any existing processors + currentProcs, ok := stream["processors"] + if !ok { + stream["processors"] = processors + } else { + currentProcsList, ok := currentProcs.([]interface{}) + if !ok { + return nil, fmt.Errorf("error creating list of existing processors, got: %#v", currentProcs) + } + stream["processors"] = append(processors, currentProcsList...) + + } + + return stream, nil +} + +// =========== +// Config Processors +// =========== + +func generateAddFieldsProcessor(fields mapstr.M, target string) mapstr.M { + return mapstr.M{ + "add_fields": mapstr.M{ + "fields": fields, + "target": target, + }, + } +} + +// This generates an opaque config blob used by all the beats +// This has to handle both universal config changes and changes specific to the beats +// This is a replacement for the AST code that lived in V1 +func generateBeatConfig(unitRaw *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + // We aren't guaranteed a DataStream field from the config + if unitRaw.GetDataStream() == nil { + unitRaw.DataStream = &proto.DataStream{ + Namespace: DefaultNamespaceName, + Dataset: DefaultDatasetName, + } + } else { + if unitRaw.GetDataStream().GetNamespace() == "" { + unitRaw.DataStream.Namespace = DefaultNamespaceName + } + if unitRaw.GetDataStream().GetDataset() == "" { + unitRaw.DataStream.Dataset = DefaultDatasetName + } + } + + // Generate the config that's unique to a beat + metaConfig, err := ConfigTransform.Transform(unitRaw, agentInfo) + if err != nil { + return nil, fmt.Errorf("error transforming config for beats: %w", err) + } + return metaConfig, nil +} + +// generate the output config, including shuffling around the `type` key +// In V1, this was done by the groupByOutputs function buried in the AST init +func groupByOutputs(outCfg *proto.UnitExpectedConfig) (*reload.ConfigWithMeta, error) { + // We still need to emulate the InjectHeadersRule AST code, + // I don't think we can get the `Headers()` data reported by the AgentInfo() + sourceMap := outCfg.GetSource().AsMap() + outputType := outCfg.GetType() + formattedOut := mapstr.M{ + outputType: sourceMap, + } + uconfig, err := conf.NewConfigFrom(formattedOut) + if err != nil { + return nil, fmt.Errorf("error creating reloader config for output: %w", err) + } + + return &reload.ConfigWithMeta{Config: uconfig}, nil +} diff --git a/x-pack/libbeat/management/generate_test.go b/x-pack/libbeat/management/generate_test.go new file mode 100644 index 00000000000..b81931342a0 --- /dev/null +++ b/x-pack/libbeat/management/generate_test.go @@ -0,0 +1,165 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package management + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestBareConfig(t *testing.T) { + // config with datastreams, metadata, etc, removed + rawExpected := proto.UnitExpectedConfig{ + Id: "system/metrics-system-default-system", + Type: "system/metrics", + Name: "system-1", + Streams: []*proto.Stream{ + { + Id: "system/metrics-system.filesystem-default-system", + Source: requireNewStruct(t, map[string]interface{}{ + "metricsets": []interface{}{"filesystem"}, + "period": "1m", + }), + }, + }, + } + + // First test: this doesn't panic on nil pointer dereference + reloadCfg, err := generateBeatConfig(&rawExpected, &client.AgentInfo{ID: "beat-ID", Version: "8.0.0", Snapshot: true}) + require.NoError(t, err, "error in generateBeatConfig") + cfgMap := mapstr.M{} + err = reloadCfg[0].Config.Unpack(&cfgMap) + require.NoError(t, err, "error in unpack for config %#v", reloadCfg[0].Config) + + // Actual checks + processorFields := map[string]interface{}{ + "add_fields.fields.stream_id": "system/metrics-system.filesystem-default-system", + "add_fields.fields.dataset": "generic", + "add_fields.fields.namespace": "default", + "add_fields.fields.type": "log", + "add_fields.fields.input_id": "system/metrics-system-default-system", + "add_fields.fields.id": "beat-ID", + } + findFieldsInProcessors(t, processorFields, cfgMap) +} + +func TestMBGenerate(t *testing.T) { + sourceStream := requireNewStruct(t, map[string]interface{}{ + "metricsets": []interface{}{"filesystem"}, + "period": "1m", + "processors": []interface{}{ + map[string]interface{}{ + "drop_event.when.regexp": map[string]interface{}{ + "system.filesystem.mount_point": "^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)", + }, + }, + }, + }) + + rawExpected := proto.UnitExpectedConfig{ + DataStream: &proto.DataStream{ + Namespace: "default", + }, + Id: "system/metrics-system-default-system", + Type: "system/metrics", + Name: "system-1", + Revision: 1, + Meta: &proto.Meta{ + Package: &proto.Package{ + Name: "system", + Version: "1.17.0", + }, + }, + Streams: []*proto.Stream{ + { + Id: "system/metrics-system.filesystem-default-system", + DataStream: &proto.DataStream{ + Dataset: "system.filesystem", + Type: "metrics", + }, + Source: sourceStream, + }, + }, + } + + reloadCfg, err := generateBeatConfig(&rawExpected, &client.AgentInfo{ID: "beat-ID", Version: "8.0.0", Snapshot: true}) + require.NoError(t, err, "error in generateBeatConfig") + cfgMap := mapstr.M{} + err = reloadCfg[0].Config.Unpack(&cfgMap) + require.NoError(t, err, "error in unpack for config %#v", reloadCfg[0].Config) + + configFields := map[string]interface{}{ + "drop_event": nil, + "add_fields.fields.stream_id": "system/metrics-system.filesystem-default-system", + "add_fields.fields.dataset": "system.filesystem", + "add_fields.fields.input_id": "system/metrics-system-default-system", + "add_fields.fields.id": "beat-ID", + } + findFieldsInProcessors(t, configFields, cfgMap) + +} + +func TestOutputGen(t *testing.T) { + testExpected := proto.UnitExpectedConfig{ + Type: "elasticsearch", + Source: requireNewStruct(t, map[string]interface{}{ + "hosts": []interface{}{"localhost:9200"}, + "username": "elastic", + "password": "changeme", + }), + } + + cfg, err := groupByOutputs(&testExpected) + require.NoError(t, err) + testStruct := mapstr.M{} + err = cfg.Config.Unpack(&testStruct) + require.NoError(t, err) + innerCfg, exists := testStruct["elasticsearch"] + assert.True(t, exists, "elasticsearch key does not exist") + _, pwExists := innerCfg.(map[string]interface{})["password"] + assert.True(t, pwExists, "password config not found") + +} + +func requireNewStruct(t *testing.T, v map[string]interface{}) *structpb.Struct { + str, err := structpb.NewStruct(v) + if err != nil { + require.NoError(t, err) + } + return str +} + +func findFieldsInProcessors(t *testing.T, configFields map[string]interface{}, cfgMap mapstr.M) { + for key, val := range configFields { + gotKey := false + gotVal := false + errStr := "" + for _, proc := range cfgMap["processors"].([]interface{}) { + processor := mapstr.M(proc.(map[string]interface{})) + found, ok := processor.GetValue(key) + if ok == nil { + gotKey = true + if val == nil { + gotVal = true + } else { + if val == found { + gotVal = true + } else { + errStr = found.(string) + } + } + } + } + assert.True(t, gotKey, "did not find key for %s", key) + assert.True(t, gotVal, "got incorrect key for %s, expected %s, got %s", key, val, errStr) + } +} diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go deleted file mode 100644 index 11b9a294d02..00000000000 --- a/x-pack/libbeat/management/manager.go +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "context" - "fmt" - "os" - "sort" - "sync" - - "github.com/gofrs/uuid" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/common/reload" - lbmanagement "github.com/elastic/beats/v7/libbeat/management" - "github.com/elastic/elastic-agent-libs/logp" -) - -var notReportedErrors = []error{ - context.Canceled, -} - -// Manager handles internal config updates. By retrieving -// new configs from Kibana and applying them to the Beat. -type Manager struct { - config *Config - logger *logp.Logger - beatUUID uuid.UUID - registry *reload.Registry - blacklist *ConfigBlacklist - client client.Client - lock sync.Mutex - status lbmanagement.Status - msg string - payload map[string]interface{} - - stopFunc func() - isRunning bool -} - -// NewFleetManager returns a X-Pack Beats Fleet Management manager. -func NewFleetManager(config *conf.C, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { - c := defaultConfig() - if config.Enabled() { - if err := config.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "parsing fleet management settings") - } - } - return NewFleetManagerWithConfig(c, registry, beatUUID) -} - -// NewFleetManagerWithConfig returns a X-Pack Beats Fleet Management manager. -func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { - log := logp.NewLogger(lbmanagement.DebugK) - - m := &Manager{ - config: c, - logger: log.Named("fleet"), - beatUUID: beatUUID, - registry: registry, - } - - var err error - var blacklist *ConfigBlacklist - var eac client.Client - if c.Enabled { - // Initialize configs blacklist - blacklist, err = NewConfigBlacklist(c.Blacklist) - if err != nil { - return nil, errors.Wrap(err, "wrong settings for configurations blacklist") - } - - // Initialize the client - eac, err = client.NewFromReader(os.Stdin, m) - if err != nil { - return nil, errors.Wrap(err, "failed to create elastic-agent-client") - } - } - - m.blacklist = blacklist - m.client = eac - return m, nil -} - -// Enabled returns true if config management is enabled. -func (cm *Manager) Enabled() bool { - return cm.config.Enabled -} - -// SetStopCallback sets the callback to run when the manager want to shutdown the beats gracefully. -func (cm *Manager) SetStopCallback(stopFunc func()) { - cm.lock.Lock() - defer cm.lock.Unlock() - cm.stopFunc = stopFunc -} - -// Start the config manager. -func (cm *Manager) Start() error { - cm.lock.Lock() - defer cm.lock.Unlock() - - if !cm.Enabled() { - return nil - } - - cfgwarn.Beta("Fleet management is enabled") - cm.logger.Info("Starting fleet management service") - - cm.isRunning = true - err := cm.client.Start(context.Background()) - if err != nil { - cm.logger.Errorf("failed to start elastic-agent-client: %s", err) - return err - } - cm.logger.Info("Ready to receive configuration") - return nil -} - -// Stop stops the current Manager and close the connection to Elastic Agent. -func (cm *Manager) Stop() { - cm.lock.Lock() - defer cm.lock.Unlock() - - if !cm.Enabled() { - return - } - - cm.logger.Info("Stopping fleet management service") - cm.isRunning = false - cm.client.Stop() -} - -// CheckRawConfig check settings are correct to start the beat. This method -// checks there are no collision between the existing configuration and what -// fleet management can configure. -// -// NOTE: This is currently not implemented for fleet. -func (cm *Manager) CheckRawConfig(cfg *conf.C) error { - // TODO implement this method - return nil -} - -// UpdateStatus updates the manager with the current status for the beat. -func (cm *Manager) UpdateStatus(status lbmanagement.Status, msg string) { - cm.lock.Lock() - defer cm.lock.Unlock() - - if cm.status != status || cm.msg != msg { - cm.status = status - cm.msg = msg - cm.client.Status(statusToProtoStatus(status), msg, nil) - cm.logger.Infof("Status change to %s: %s", status, msg) - } -} - -// updateStatusWithError updates the manager with the current status for the beat with error. -func (cm *Manager) updateStatusWithError(err error) { - if err == nil { - return - } - - for _, e := range notReportedErrors { - if errors.Is(err, e) { - return - } - } - - cm.logger.Error(err) - cm.UpdateStatus(lbmanagement.Failed, err.Error()) -} - -func (cm *Manager) OnConfig(s string) { - cm.UpdateStatus(lbmanagement.Configuring, "Updating configuration") - - var configMap mapstr.M - uconfig, err := conf.NewConfigFrom(s) - if err != nil { - err = errors.Wrap(err, "config blocks unsuccessfully generated") - cm.updateStatusWithError(err) - return - } - - err = uconfig.Unpack(&configMap) - if err != nil { - err = errors.Wrap(err, "config blocks unsuccessfully generated") - cm.updateStatusWithError(err) - return - } - - blocks, err := cm.toConfigBlocks(configMap) - if err != nil { - err = errors.Wrap(err, "failed to parse configuration") - cm.updateStatusWithError(err) - return - } - - if err := cm.apply(blocks); err != nil { - // `cm.apply` already logs the errors; currently allow beat to run degraded - cm.updateStatusWithError(err) - cm.logger.Errorf("failed applying config blocks: %v", err) - return - } - - cm.client.Status(proto.StateObserved_HEALTHY, "Running", cm.payload) -} - -func (cm *Manager) RegisterAction(action client.Action) { - cm.client.RegisterAction(action) -} - -func (cm *Manager) UnregisterAction(action client.Action) { - cm.client.UnregisterAction(action) -} - -func (cm *Manager) SetPayload(payload map[string]interface{}) { - cm.lock.Lock() - cm.payload = payload - cm.lock.Unlock() -} - -func (cm *Manager) OnStop() { - cm.lock.Lock() - defer cm.lock.Unlock() - - if cm.stopFunc != nil { - cm.client.Status(proto.StateObserved_STOPPING, "Stopping", nil) - cm.stopFunc() - } -} - -func (cm *Manager) OnError(err error) { - isStopped := false - cm.lock.Lock() - isStopped = !cm.isRunning - cm.lock.Unlock() - - if isStopped && errors.Is(err, context.Canceled) { - // don't report context cancelled on shutdown - return - } - cm.logger.Errorf("elastic-agent-client got error: %s", err) -} - -func (cm *Manager) apply(blocks ConfigBlocks) error { - missing := map[string]bool{} - for _, name := range cm.registry.GetRegisteredNames() { - missing[name] = true - } - - // Detect unwanted configs from the list - if err := cm.blacklist.Detect(blocks); err != nil { - return err - } - - var errors *multierror.Error - // Reload configs - for _, b := range blocks { - if err := cm.reload(b.Type, b.Blocks); err != nil { - errors = multierror.Append(errors, err) - } - missing[b.Type] = false - } - - // Unset missing configs - for name, isMissing := range missing { - if isMissing { - if err := cm.reload(name, []*ConfigBlock{}); err != nil { - errors = multierror.Append(errors, err) - } - } - } - - return errors.ErrorOrNil() -} - -func (cm *Manager) reload(t string, blocks []*ConfigBlock) error { - cm.logger.Infof("Applying settings for %s", t) - if obj := cm.registry.GetReloadable(t); obj != nil { - // Single object - if len(blocks) > 1 { - err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) - cm.logger.Error(err) - return err - } - - var config *reload.ConfigWithMeta - var err error - if len(blocks) == 1 { - config, err = blocks[0].ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return err - } - } - - if err := obj.Reload(config); err != nil { - cm.logger.Error(err) - return err - } - } else if obj := cm.registry.GetReloadableList(t); obj != nil { - // List - var configs []*reload.ConfigWithMeta - for _, block := range blocks { - config, err := block.ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return err - } - configs = append(configs, config) - } - - if err := obj.Reload(configs); err != nil { - cm.logger.Error(err) - return err - } - } - - return nil -} - -func (cm *Manager) toConfigBlocks(cfg mapstr.M) (ConfigBlocks, error) { - blocks := map[string][]*ConfigBlock{} - - // Extract all registered values beat can respond to - for _, regName := range cm.registry.GetRegisteredNames() { - iBlock, err := cfg.GetValue(regName) - if err != nil { - cm.logger.Warnf("failed to get '%s' from config: %v. Continuing to next one", regName, err) - continue - } - - if mapBlock, ok := iBlock.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) - } else if arrayBlock, ok := iBlock.([]interface{}); ok { - for _, item := range arrayBlock { - if mapBlock, ok := item.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) - } - } - } - } - - // keep the ordering consistent while grouping the items. - keys := make([]string, 0, len(blocks)) - for k := range blocks { - keys = append(keys, k) - } - sort.Strings(keys) - - res := ConfigBlocks{} - for _, t := range keys { - b := blocks[t] - res = append(res, ConfigBlocksWithType{Type: t, Blocks: b}) - } - - return res, nil -} - -func statusToProtoStatus(status lbmanagement.Status) proto.StateObserved_Status { - switch status { - case lbmanagement.Unknown: - // unknown is reported as healthy, as the status is unknown - return proto.StateObserved_HEALTHY - case lbmanagement.Starting: - return proto.StateObserved_STARTING - case lbmanagement.Configuring: - return proto.StateObserved_CONFIGURING - case lbmanagement.Running: - return proto.StateObserved_HEALTHY - case lbmanagement.Degraded: - return proto.StateObserved_DEGRADED - case lbmanagement.Failed: - return proto.StateObserved_FAILED - case lbmanagement.Stopping: - return proto.StateObserved_STOPPING - } - // unknown status, still reported as healthy - return proto.StateObserved_HEALTHY -} diff --git a/x-pack/libbeat/management/managerV2.go b/x-pack/libbeat/management/managerV2.go new file mode 100644 index 00000000000..060c23161ce --- /dev/null +++ b/x-pack/libbeat/management/managerV2.go @@ -0,0 +1,377 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package management + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + + "github.com/gofrs/uuid" + + "github.com/elastic/beats/v7/libbeat/common/reload" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +// BeatV2Manager is the main type for tracing V2-related config updates +type BeatV2Manager struct { + config *Config + registry *reload.Registry + client client.V2 + + logger *logp.Logger + + // Track individual units given to us by the V2 API + unitsMut sync.Mutex + units map[string]*client.Unit + mainUnit string + + // This satisfies the SetPayload() function, and will pass along this value to the UpdateStatus() + // call whenever a config is re-registered + payload map[string]interface{} + + // stop callback must be registered by libbeat, as with the V1 callback + stopFunc func() + stopMut sync.Mutex + beatStop sync.Once + + // sync channel for shutting down the manager after we get a stop from + // either the agent or the beat + stopChan chan struct{} + + isRunning bool +} + +// NewV2AgentManager returns a remote config manager for the agent V2 protocol. +// This is meant to be used by the management plugin system, which will register this as a callback. +func NewV2AgentManager(config *conf.C, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { + c := DefaultConfig() + if config.Enabled() { + if err := config.Unpack(&c); err != nil { + return nil, fmt.Errorf("parsing fleet management settings: %w", err) + } + } + agentClient, _, err := client.NewV2FromReader(os.Stdin, client.VersionInfo{ + Name: "beat-v2-client", + Version: version.GetDefaultVersion(), + Meta: map[string]string{ + "commit": version.Commit(), + "build_time": version.BuildTime().String(), + }, + }) + if err != nil { + return nil, fmt.Errorf("error reading control config from agent: %w", err) + } + + return NewV2AgentManagerWithClient(c, registry, agentClient) +} + +// NewV2AgentManagerWithClient actually creates the manager instance used by the rest of the beats. +func NewV2AgentManagerWithClient(config *Config, registry *reload.Registry, agentClient client.V2) (lbmanagement.Manager, error) { + log := logp.NewLogger(lbmanagement.DebugK) + m := &BeatV2Manager{ + config: config, + logger: log.Named("V2-manager"), + registry: registry, + units: make(map[string]*client.Unit), + stopChan: make(chan struct{}, 1), + } + + if config.Enabled { + m.client = agentClient + } + return m, nil +} + +// ================================ +// Beats central management interface implementation +// ================================ + +// UpdateStatus updates the manager with the current status for the beat. +func (cm *BeatV2Manager) UpdateStatus(status lbmanagement.Status, msg string) { + updateState := client.UnitState(status) + stateUnit, exists := cm.getMainUnit() + cm.logger.Debugf("Updating beat status: %s", msg) + if exists { + _ = stateUnit.UpdateState(updateState, msg, cm.payload) + } else { + cm.logger.Warnf("Cannot update state to %s, no main unit is set. Msg: %s", status, msg) + } +} + +// Enabled returns true if config management is enabled. +func (cm *BeatV2Manager) Enabled() bool { + return cm.config.Enabled +} + +// SetStopCallback sets the callback to run when the manager want to shutdown the beats gracefully. +func (cm *BeatV2Manager) SetStopCallback(stopFunc func()) { + cm.stopMut.Lock() + defer cm.stopMut.Unlock() + cm.stopFunc = stopFunc +} + +// Start the config manager. +func (cm *BeatV2Manager) Start() error { + if !cm.Enabled() { + return fmt.Errorf("V2 Manager is disabled") + } + err := cm.client.Start(context.Background()) + if err != nil { + return fmt.Errorf("error starting connection to client") + } + + go cm.unitListen() + cm.isRunning = true + return nil +} + +// Stop stops the current Manager and close the connection to Elastic Agent. +func (cm *BeatV2Manager) Stop() { + cm.stopChan <- struct{}{} +} + +// CheckRawConfig is currently not implemented for V1. +func (cm *BeatV2Manager) CheckRawConfig(cfg *conf.C) error { + // This does not do anything on V1 or V2, but here we are + return nil +} + +func (cm *BeatV2Manager) RegisterAction(action client.Action) { + cm.unitsMut.Lock() + defer cm.unitsMut.Unlock() + stateUnit, exists := cm.units[cm.mainUnit] + if exists { + _ = stateUnit.UpdateState(client.UnitStateHealthy, fmt.Sprintf("Registering action %s for main unit with ID %s", cm.mainUnit, action.Name()), nil) + cm.units[cm.mainUnit].RegisterAction(action) + } else { + cm.logger.Warnf("Cannot register action %s, no main unit found", action.Name()) + } +} + +func (cm *BeatV2Manager) UnregisterAction(action client.Action) { + cm.unitsMut.Lock() + defer cm.unitsMut.Unlock() + stateUnit, exists := cm.units[cm.mainUnit] + if exists { + _ = stateUnit.UpdateState(client.UnitStateHealthy, fmt.Sprintf("Unregistering action %s for main unit with ID %s", cm.mainUnit, action.Name()), nil) + cm.units[cm.mainUnit].UnregisterAction(action) + } else { + cm.logger.Warnf("Cannot Unregister action %s, no main unit found", action.Name()) + } +} + +func (cm *BeatV2Manager) SetPayload(payload map[string]interface{}) { + cm.payload = payload +} + +// ================================ +// Unit manager +// ================================ + +func (cm *BeatV2Manager) addUnit(unit *client.Unit) { + cm.unitsMut.Lock() + cm.units[unit.ID()] = unit + cm.unitsMut.Unlock() +} + +func (cm *BeatV2Manager) getMainUnit() (*client.Unit, bool) { + cm.unitsMut.Lock() + defer cm.unitsMut.Unlock() + if cm.mainUnit == "" { + return nil, false + } + return cm.units[cm.mainUnit], true +} + +// We need a "main" unit that we can send updates to for the StatusReporter interface +// the purpose of this is to just grab the first input-type unit we get and set it as the "main" unit +func (cm *BeatV2Manager) setMainUnitValue(unit *client.Unit) { + cm.unitsMut.Lock() + defer cm.unitsMut.Unlock() + if cm.mainUnit == "" { + cm.logger.Debugf("Set main input unit to ID %s", unit.ID) + cm.mainUnit = unit.ID() + } +} + +func (cm *BeatV2Manager) deleteUnit(unit *client.Unit) { + cm.unitsMut.Lock() + delete(cm.units, unit.ID()) + cm.unitsMut.Unlock() +} + +// ================================ +// Private V2 implementation +// ================================ + +func (cm *BeatV2Manager) unitListen() { + + // register signal handler + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + + cm.logger.Debugf("Listening for agent unit changes") + for { + select { + // The stopChan channel comes from the Manager interface Stop() method + case <-cm.stopChan: + cm.stopBeat() + case sig := <-sigc: + // we can't duplicate the same logic used by stopChan here. + // A beat will also watch for sigint and shut down, if we call the stopFunc + // callback, either the V2 client or the beat will get a panic, + // as the stopFunc sent by the beats is usually unsafe. + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + cm.logger.Debug("Received sigterm/sigint, stopping") + case syscall.SIGHUP: + cm.logger.Debug("Received sighup, stopping") + } + cm.isRunning = false + unit, mainExists := cm.getMainUnit() + if mainExists { + _ = unit.UpdateState(client.UnitStateStopping, "stopping beat", nil) + } + cm.client.Stop() + return + case change := <-cm.client.UnitChanges(): + switch change.Type { + // Within the context of how we send config to beats, I'm not sure there is a difference between + // A unit add and a unit change, since either way we can't do much more than call the reloader + case client.UnitChangedAdded: + // At this point we also get a log level, however I'm not sure the beats core logger provides a + // clean way to "just" change the log level, without resetting the whole log config + state, _, _ := change.Unit.Expected() + cm.logger.Debugf("Got unit added: %s, type: %s expected state: %s", change.Unit.ID(), change.Unit.Type(), state.String()) + go cm.handleUnitReload(change.Unit) + + case client.UnitChangedModified: + state, _, _ := change.Unit.Expected() + cm.logger.Debugf("Got unit modified: %s, type: %s expected state: %s", change.Unit.ID(), change.Unit.Type(), state.String()) + // I'm assuming that a state STOPPED just tells us to shut down the entire beat, + // as such we don't really care about updating via a particular unit + if state == client.UnitStateStopped { + cm.stopBeat() + } else { + go cm.handleUnitReload(change.Unit) + } + + case client.UnitChangedRemoved: + cm.logger.Debugf("Got unit removed: %s", change.Unit.ID()) + cm.deleteUnit(change.Unit) + } + } + + } +} + +func (cm *BeatV2Manager) stopBeat() { + if !cm.isRunning { + return + } + // will we ever get a Unit removed for anything other than the main beat? + // Individual reloaders don't have a "stop" function, so the most we can do + // is just shut down a beat, I think. + cm.logger.Debugf("Stopping beat") + // stop the "main" beat runtime + unit, mainExists := cm.getMainUnit() + if mainExists { + _ = unit.UpdateState(client.UnitStateStopping, "stopping beat", nil) + } + + cm.isRunning = false + cm.stopMut.Lock() + defer cm.stopMut.Unlock() + if cm.stopFunc != nil { + // I'm not 100% sure the once here is needed, + // but various beats tend to handle this in a not-quite-safe way + cm.beatStop.Do(cm.stopFunc) + } + cm.client.Stop() + + if mainExists { + _ = unit.UpdateState(client.UnitStateStopped, "stopped beat", nil) + } + +} + +func (cm *BeatV2Manager) handleUnitReload(unit *client.Unit) { + cm.addUnit(unit) + unitType := unit.Type() + + if unitType == client.UnitTypeOutput { + cm.handleOutputReload(unit) + } else if unitType == client.UnitTypeInput { + cm.handleInputReload(unit) + } +} + +// Handle the updated config for an output unit +func (cm *BeatV2Manager) handleOutputReload(unit *client.Unit) { + _, _, rawConfig := unit.Expected() + cm.logger.Debugf("Got Output unit config: %s, ID: %s", rawConfig.Type, rawConfig.Id) + + reloadConfig, err := groupByOutputs(rawConfig) + if err != nil { + errString := fmt.Errorf("Failed to generate config for output: %w", err) + _ = unit.UpdateState(client.UnitStateFailed, errString.Error(), nil) + return + } + // Assuming that the output reloadable isn't a list, see createBeater() in cmd/instance/beat.go + output := cm.registry.GetReloadableOutput() + if output == nil { + _ = unit.UpdateState(client.UnitStateFailed, "failed to find beat reloadable type 'output'", nil) + return + } + + _ = unit.UpdateState(client.UnitStateConfiguring, "reloading output component", nil) + err = output.Reload(reloadConfig) + if err != nil { + errString := fmt.Errorf("Failed to reload component: %w", err) + _ = unit.UpdateState(client.UnitStateFailed, errString.Error(), nil) + return + } + _ = unit.UpdateState(client.UnitStateHealthy, "reloaded output component", nil) +} + +// handle the updated config for an input unit +func (cm *BeatV2Manager) handleInputReload(unit *client.Unit) { + _, _, rawConfig := unit.Expected() + cm.setMainUnitValue(unit) + cm.logger.Debugf("Got Input unit config: %s, ID: %s", rawConfig.Type, rawConfig.Id) + + // Find the V2 inputs we need to reload + // The reloader provides list and non-list types, but all the beats register as lists, + // so just go with that for V2 + obj := cm.registry.GetInputList() + if obj == nil { + _ = unit.UpdateState(client.UnitStateFailed, "failed to find beat reloadable type 'input'", nil) + return + } + _ = unit.UpdateState(client.UnitStateConfiguring, "found reloader for 'input'", nil) + + beatCfg, err := generateBeatConfig(rawConfig, cm.client.AgentInfo()) + if err != nil { + errString := fmt.Errorf("Failed to create Unit config: %w", err) + _ = unit.UpdateState(client.UnitStateFailed, errString.Error(), nil) + return + } + + err = obj.Reload(beatCfg) + if err != nil { + errString := fmt.Errorf("Error reloading input: %w", err) + _ = unit.UpdateState(client.UnitStateFailed, errString.Error(), nil) + return + } + _ = unit.UpdateState(client.UnitStateHealthy, "beat reloaded", nil) +} diff --git a/x-pack/libbeat/management/manager_test.go b/x-pack/libbeat/management/manager_test.go deleted file mode 100644 index b05d70fab2d..00000000000 --- a/x-pack/libbeat/management/manager_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - - "github.com/elastic/beats/v7/libbeat/common/reload" - lbmanagement "github.com/elastic/beats/v7/libbeat/management" -) - -func TestConfigBlocks(t *testing.T) { - input := ` -filebeat: - inputs: - - type: log - paths: - - /var/log/hello1.log - - /var/log/hello2.log -output: - elasticsearch: - hosts: - - localhost:9200` - - var cfg mapstr.M - uconfig, err := conf.NewConfigFrom(input) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - err = uconfig.Unpack(&cfg) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - reg := reload.NewRegistry() - reg.Register("output", &dummyReloadable{}) - reg.Register("filebeat.inputs", &dummyReloadable{}) - - cm := &Manager{ - registry: reg, - } - blocks, err := cm.toConfigBlocks(cfg) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - if len(blocks) != 2 { - t.Fatalf("Expected 2 block have %d: %+v", len(blocks), blocks) - } -} - -func TestStatusToProtoStatus(t *testing.T) { - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Unknown)) - assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(lbmanagement.Starting)) - assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(lbmanagement.Configuring)) - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Running)) - assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(lbmanagement.Degraded)) - assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(lbmanagement.Failed)) - assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(lbmanagement.Stopping)) -} - -type dummyReloadable struct{} - -func (dummyReloadable) Reload(config *reload.ConfigWithMeta) error { - return nil -} diff --git a/x-pack/libbeat/management/plugin.go b/x-pack/libbeat/management/plugin.go index 8568f7029f6..95286be4d82 100644 --- a/x-pack/libbeat/management/plugin.go +++ b/x-pack/libbeat/management/plugin.go @@ -11,17 +11,17 @@ import ( ) func init() { - lbmanagement.Register("x-pack-fleet", NewFleetManagerPlugin, feature.Beta) + lbmanagement.Register("x-pack-fleet", NewFleetManagerPluginV2, feature.Beta) } -// NewFleetManagerPlugin creates a plugin function returning factory if configuration matches the criteria -func NewFleetManagerPlugin(config *conf.C) lbmanagement.FactoryFunc { - c := defaultConfig() +// NewFleetManagerPluginV2 registers the V2 callback +func NewFleetManagerPluginV2(config *conf.C) lbmanagement.FactoryFunc { + c := DefaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { return nil } - return NewFleetManager + return NewV2AgentManager } return nil diff --git a/x-pack/libbeat/management/tests/fbtest/filebeat_v2_test.go b/x-pack/libbeat/management/tests/fbtest/filebeat_v2_test.go new file mode 100644 index 00000000000..7d8a73fa5c2 --- /dev/null +++ b/x-pack/libbeat/management/tests/fbtest/filebeat_v2_test.go @@ -0,0 +1,115 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fbtest + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + fbroot "github.com/elastic/beats/v7/x-pack/filebeat/cmd" + // initialize the plugin system before libbeat does, so we can overwrite it properly + _ "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/beats/v7/x-pack/libbeat/management/tests" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" +) + +var expectedFBStreams = &proto.UnitExpectedConfig{ + DataStream: &proto.DataStream{ + Namespace: "default", + }, + Type: "logfile", + Id: "logfile-system-default-system", + Name: "system-1", + Revision: 1, + Meta: &proto.Meta{ + Package: &proto.Package{ + Name: "system", + Version: "1.17.0", + }, + }, +} + +func TestFilebeat(t *testing.T) { + filebeatCmd := fbroot.Filebeat() + tests.InitBeatsForTest(t, filebeatCmd) + var fbStreams = []*proto.Stream{ + { + Id: "logfile-system.syslog-default-system", + DataStream: &proto.DataStream{ + Dataset: "system.syslog", + Type: "logs", + }, + Source: tests.RequireNewStruct(map[string]interface{}{ + "paths": []interface{}{"./testdata/messages"}, + "exclude_files": []interface{}{".gz$"}, + "multiline": map[string]interface{}{ + "pattern": `^\s`, + "match": "after", + }, + }), + }, + { + Id: "logfile-system.auth-default-system", + DataStream: &proto.DataStream{ + Dataset: "system.auth", + Type: "logs", + }, + Source: tests.RequireNewStruct(map[string]interface{}{ + "paths": []interface{}{"./testdata/secure*"}, + "exclude_files": []interface{}{".gz$"}, + "multiline": map[string]interface{}{ + "pattern": `^\s`, + "match": "after", + }, + }), + }, + } + + expectedFBStreams.Streams = fbStreams + outPath, server := tests.SetupTestEnv(t, expectedFBStreams, time.Second*6) + defer server.Srv.Stop() + + defer func() { + err := os.RemoveAll(outPath) + require.NoError(t, err) + }() + + t.Logf("Running beats...") + err := filebeatCmd.Execute() + require.NoError(t, err) + + t.Logf("Reading events...") + events := tests.ReadEvents(t, outPath) + t.Logf("Got %d events", len(events)) + // Look for processors + expectedMetaValuesSyslog := map[string]interface{}{ + // Processors created by + "@metadata.input_id": "logfile-system-default-system", + "@metadata.stream_id": "logfile-system.syslog-default-system", + "agent.id": "test-agent", + "data_stream.dataset": "system.syslog", + "data_stream.namespace": "default", + "data_stream.type": "logs", + } + tests.ValuesExist(t, expectedMetaValuesSyslog, events, tests.ONCE) + + expectedMetaValuesAuth := map[string]interface{}{ + // Processors created by + "@metadata.input_id": "logfile-system-default-system", + "@metadata.stream_id": "logfile-system.auth-default-system", + "agent.id": "test-agent", + "data_stream.dataset": "system.auth", + } + tests.ValuesExist(t, expectedMetaValuesAuth, events, tests.ONCE) + + expectedLogValues := map[string]interface{}{ + "log.file.path": nil, + "message": nil, + } + tests.ValuesExist(t, expectedLogValues, events, tests.ONCE) +} diff --git a/x-pack/libbeat/management/tests/fbtest/testdata/messages b/x-pack/libbeat/management/tests/fbtest/testdata/messages new file mode 100644 index 00000000000..6eacd05dada --- /dev/null +++ b/x-pack/libbeat/management/tests/fbtest/testdata/messages @@ -0,0 +1,62 @@ +Aug 7 00:00:00 test-server systemd[1]: Starting unbound-anchor.service - update of the root trust anchor for DNSSEC validation in unbound... +Aug 7 00:00:00 test-server audit: BPF prog-id=1328 op=LOAD +Aug 7 00:00:00 test-server systemd[1]: Starting logrotate.service - Rotate log files... +Aug 7 00:00:00 test-server systemd[1]: unbound-anchor.service: Deactivated successfully. +Aug 7 00:00:00 test-server systemd[1]: Finished unbound-anchor.service - update of the root trust anchor for DNSSEC validation in unbound. +Aug 7 00:00:00 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=unbound-anchor comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:00:00 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=unbound-anchor comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:00:00 test-server systemd[1]: rsyslog.service: Sent signal SIGHUP to main process 978 (rsyslogd) on client request. +Aug 7 00:00:00 test-server rsyslogd[978]: [origin software="rsyslogd" swVersion="8.2204.0-2.fc36" x-pid="978" x-info="https://www.rsyslog.com"] rsyslogd was HUPed +Aug 7 00:00:00 test-server systemd[1]: logrotate.service: Deactivated successfully. +Aug 7 00:00:00 test-server systemd[1]: Finished logrotate.service - Rotate log files. +Aug 7 00:00:00 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=logrotate comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:00:00 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=logrotate comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:00:00 test-server audit: BPF prog-id=0 op=UNLOAD +Aug 7 00:08:00 test-server systemd[1]: Starting pmie_daily.service - Process PMIE logs... +Aug 7 00:08:00 test-server systemd[1]: Started pmie_daily.service - Process PMIE logs. +Aug 7 00:08:00 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmie_daily comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:08:01 test-server systemd[1]: pmie_daily.service: Deactivated successfully. +Aug 7 00:08:01 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmie_daily comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:09:19 test-server systemd-logind[984]: Session 389 logged out. Waiting for processes to exit. +Aug 7 00:09:19 test-server systemd[1]: session-389.scope: Deactivated successfully. +Aug 7 00:09:19 test-server systemd-logind[984]: Removed session 389. +Aug 7 00:09:21 test-server systemd-logind[984]: New session 390 of user alexk. +Aug 7 00:09:21 test-server systemd[1]: Started session-390.scope - Session 390 of User alexk. +Aug 7 00:09:21 test-server audit: BPF prog-id=1329 op=LOAD +Aug 7 00:09:21 test-server audit: BPF prog-id=1330 op=LOAD +Aug 7 00:09:21 test-server systemd[1]: Starting systemd-hostnamed.service - Hostname Service... +Aug 7 00:09:21 test-server systemd[1]: Started systemd-hostnamed.service - Hostname Service. +Aug 7 00:09:21 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-hostnamed comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:09:51 test-server systemd[1]: systemd-hostnamed.service: Deactivated successfully. +Aug 7 00:09:51 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-hostnamed comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:09:51 test-server audit: BPF prog-id=0 op=UNLOAD +Aug 7 00:09:51 test-server audit: BPF prog-id=0 op=UNLOAD +Aug 7 00:10:00 test-server systemd[1]: Starting pmlogger_daily.service - Process archive logs... +Aug 7 00:10:00 test-server systemd[1]: Started pmlogger_daily.service - Process archive logs. +Aug 7 00:10:00 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_daily comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:10:03 test-server systemd[1]: pmlogger_daily.service: Deactivated successfully. +Aug 7 00:10:03 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_daily comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:10:03 test-server systemd[1]: pmlogger_daily.service: Consumed 1.905s CPU time. +Aug 7 00:11:11 test-server systemd[1]: run-docker-runtime\x2drunc-moby-c2afbe542c67dd805b6017d6e225bf18d830393752e82d9e97351f6b396c0be2-runc.QxRbu3.mount: Deactivated successfully. +Aug 7 00:11:34 test-server systemd[1]: run-docker-runtime\x2drunc-moby-6435295863b92ffce5f9312196b9fe3153cfdeb385b4cea8a06f9532db8ef457-runc.xOrRCg.mount: Deactivated successfully. +Aug 7 00:17:26 test-server systemd[1]: run-docker-runtime\x2drunc-moby-c2afbe542c67dd805b6017d6e225bf18d830393752e82d9e97351f6b396c0be2-runc.QQ0qVl.mount: Deactivated successfully. +Aug 7 00:19:48 test-server systemd[1]: run-docker-runtime\x2drunc-moby-6435295863b92ffce5f9312196b9fe3153cfdeb385b4cea8a06f9532db8ef457-runc.4vtJU0.mount: Deactivated successfully. +Aug 7 00:21:12 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=dnf-makecache comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:21:12 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=dnf-makecache comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:21:12 test-server systemd[1]: dnf-makecache.service: Consumed 16.261s CPU time. +Aug 7 00:22:22 test-server NetworkManager[1141]: [1659856942.7446] dhcp4 (eno1): state changed new lease, address=192.168.1.96 +Aug 7 00:22:52 test-server systemd[1]: run-docker-runtime\x2drunc-moby-6435295863b92ffce5f9312196b9fe3153cfdeb385b4cea8a06f9532db8ef457-runc.8ba3LQ.mount: Deactivated successfully. +Aug 7 00:22:59 test-server systemd[1]: run-docker-runtime\x2drunc-moby-6435295863b92ffce5f9312196b9fe3153cfdeb385b4cea8a06f9532db8ef457-runc.N1risu.mount: Deactivated successfully. +Aug 7 00:23:29 test-server systemd[1]: run-docker-runtime\x2drunc-moby-56c445f9a6c907b6e46555e448111001f8b8ee4d5e44d94380db44702837789d-runc.VXLgpv.mount: Deactivated successfully. +Aug 7 00:24:21 test-server systemd[1]: run-docker-runtime\x2drunc-moby-c2afbe542c67dd805b6017d6e225bf18d830393752e82d9e97351f6b396c0be2-runc.ia9DaI.mount: Deactivated successfully. +Aug 7 00:25:00 test-server systemd[1]: Starting pmlogger_check.service - Check pmlogger instances are running... +Aug 7 00:25:00 test-server systemd[1]: Started pmlogger_check.service - Check pmlogger instances are running. +Aug 7 00:25:00 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_check comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:25:01 test-server systemd[1]: pmlogger_check.service: Deactivated successfully. +Aug 7 00:25:01 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_check comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:25:10 test-server systemd[1]: Starting pmlogger_farm_check.service - Check and migrate non-primary pmlogger farm instances... +Aug 7 00:25:10 test-server systemd[1]: Started pmlogger_farm_check.service - Check and migrate non-primary pmlogger farm instances. +Aug 7 00:25:10 test-server audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_farm_check comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:25:10 test-server systemd[1]: pmlogger_farm_check.service: Deactivated successfully. +Aug 7 00:25:10 test-server audit[1]: SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=pmlogger_farm_check comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' +Aug 7 00:26:30 test-server systemd[1]: run-docker-runtime\x2drunc-moby-c2afbe542c67dd805b6017d6e225bf18d830393752e82d9e97351f6b396c0be2-runc.OqtwmI.mount: Deactivated successfully. \ No newline at end of file diff --git a/x-pack/libbeat/management/tests/fbtest/testdata/secure b/x-pack/libbeat/management/tests/fbtest/testdata/secure new file mode 100644 index 00000000000..8701850320c --- /dev/null +++ b/x-pack/libbeat/management/tests/fbtest/testdata/secure @@ -0,0 +1,30 @@ +Aug 14 00:06:20 shoebill sshd[2305664]: Received disconnect from 192.168.1.141 port 63067:11: disconnected by user +Aug 14 00:06:20 shoebill sshd[2305664]: Disconnected from user user 192.168.1.141 port 63067 +Aug 14 00:06:20 shoebill sshd[2305661]: pam_unix(sshd:session): session closed for user user +Aug 14 00:06:22 shoebill sshd[2355631]: Accepted publickey for user from 192.168.1.141 port 63134 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 00:06:22 shoebill sshd[2355631]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) +Aug 14 00:24:21 shoebill sshd[2355634]: Received disconnect from 192.168.1.141 port 63134:11: disconnected by user +Aug 14 00:24:21 shoebill sshd[2355634]: Disconnected from user user 192.168.1.141 port 63134 +Aug 14 00:24:21 shoebill sshd[2355631]: pam_unix(sshd:session): session closed for user user +Aug 14 00:24:23 shoebill sshd[2406593]: Accepted publickey for user from 192.168.1.141 port 63206 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 00:24:23 shoebill sshd[2406593]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) +Aug 14 00:43:04 shoebill sshd[2406596]: Received disconnect from 192.168.1.141 port 63206:11: disconnected by user +Aug 14 00:43:04 shoebill sshd[2406596]: Disconnected from user user 192.168.1.141 port 63206 +Aug 14 00:43:04 shoebill sshd[2406593]: pam_unix(sshd:session): session closed for user user +Aug 14 00:43:06 shoebill sshd[2459763]: Accepted publickey for user from 192.168.1.141 port 63275 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 00:43:06 shoebill sshd[2459763]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) +Aug 14 01:00:56 shoebill sshd[2459766]: Received disconnect from 192.168.1.141 port 63275:11: disconnected by user +Aug 14 01:00:56 shoebill sshd[2459766]: Disconnected from user user 192.168.1.141 port 63275 +Aug 14 01:00:56 shoebill sshd[2459763]: pam_unix(sshd:session): session closed for user user +Aug 14 01:00:58 shoebill sshd[2510537]: Accepted publickey for user from 192.168.1.141 port 63343 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 01:00:58 shoebill sshd[2510537]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) +Aug 14 01:19:28 shoebill sshd[2510571]: Received disconnect from 192.168.1.141 port 63343:11: disconnected by user +Aug 14 01:19:28 shoebill sshd[2510571]: Disconnected from user user 192.168.1.141 port 63343 +Aug 14 01:19:28 shoebill sshd[2510537]: pam_unix(sshd:session): session closed for user user +Aug 14 01:19:30 shoebill sshd[2562541]: Accepted publickey for user from 192.168.1.141 port 63413 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 01:19:30 shoebill sshd[2562541]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) +Aug 14 01:36:58 shoebill sshd[2562544]: Received disconnect from 192.168.1.141 port 63413:11: disconnected by user +Aug 14 01:36:58 shoebill sshd[2562544]: Disconnected from user user 192.168.1.141 port 63413 +Aug 14 01:36:58 shoebill sshd[2562541]: pam_unix(sshd:session): session closed for user user +Aug 14 01:37:00 shoebill sshd[2612276]: Accepted publickey for user from 192.168.1.141 port 63477 ssh2: RSA SHA256:4PSoxRMl32R2k+79E7+Fufpm2bjX9Q+d2aI3F8GbNyI +Aug 14 01:37:00 shoebill sshd[2612276]: pam_unix(sshd:session): session opened for user user(uid=1000) by (uid=0) \ No newline at end of file diff --git a/x-pack/libbeat/management/tests/init.go b/x-pack/libbeat/management/tests/init.go new file mode 100644 index 00000000000..0344ccc1de1 --- /dev/null +++ b/x-pack/libbeat/management/tests/init.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tests + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/feature" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + conf "github.com/elastic/elastic-agent-libs/config" +) + +var defaultFleetName = "x-pack-fleet" + +// InitBeatsForTest tinkers with a bunch of global variables so beats will start up properly in a test environment +func InitBeatsForTest(t *testing.T, beatRoot *cmd.BeatsRootCmd) { + // this is a tad hacky, but the go test environment will attempt to insert a bunch of CLI args into the executable, + // which beats's CLI library will then choke on + os.Args = os.Args[:1] + + // Set CLI flags needed to run the tests + t.Logf("Setting flags...") + err := beatRoot.PersistentFlags().Set("e", "true") + require.NoError(t, err) + err = beatRoot.PersistentFlags().Set("E", "management.enabled=true") + require.NoError(t, err) + err = beatRoot.PersistentFlags().Set("d", "centralmgmt.V2-manager") + require.NoError(t, err) +} + +// ResetFleetManager re-registers the global fleet handler, if needed, and replace it with the test one. +func ResetFleetManager(handler MockV2Handler) error { + managers, err := feature.GlobalRegistry().LookupAll(lbmanagement.Namespace) + if err != nil { + return fmt.Errorf("error finding management plugin: %w", err) + } + if managers != nil && managers[0].Name() == defaultFleetName { + _ = feature.GlobalRegistry().Unregister(lbmanagement.Namespace, defaultFleetName) + } + lbmanagement.Register("fleet-test", fleetClientFactory(handler), feature.Beta) + return nil +} + +func fleetClientFactory(srv MockV2Handler) lbmanagement.PluginFunc { + return func(config *conf.C) lbmanagement.FactoryFunc { + c := management.DefaultConfig() + if config.Enabled() { + if err := config.Unpack(&c); err != nil { + return nil + } + return func(_ *conf.C, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { + return management.NewV2AgentManagerWithClient(c, registry, srv.Client) + } + } + return nil + } +} + +// SetupTestEnv is a helper to initialize the common files and handlers for metricbeat. +// This returns a string to the tmpdir location +func SetupTestEnv(t *testing.T, config *proto.UnitExpectedConfig, runtime time.Duration) (string, MockV2Handler) { + tmpdir := os.TempDir() + filename := fmt.Sprintf("test-%d", time.Now().Unix()) + outPath := filepath.Join(tmpdir, filename) + t.Logf("writing output to file %s", outPath) + err := os.Mkdir(outPath, 0775) + require.NoError(t, err) + + server := NewMockServer(t, runtime, config, outPath) + t.Logf("Resetting fleet manager...") + err = ResetFleetManager(server) + require.NoError(t, err) + + return outPath, server +} diff --git a/x-pack/libbeat/management/tests/mbtest/metricbeat_v2_test.go b/x-pack/libbeat/management/tests/mbtest/metricbeat_v2_test.go new file mode 100644 index 00000000000..05d206f84d9 --- /dev/null +++ b/x-pack/libbeat/management/tests/mbtest/metricbeat_v2_test.go @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mbtest + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + // initialize the plugin system before libbeat does, so we can overwrite it properly + _ "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/beats/v7/x-pack/libbeat/management/tests" + "github.com/elastic/beats/v7/x-pack/metricbeat/cmd" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" +) + +var expectedMBStreams = &proto.UnitExpectedConfig{ + DataStream: &proto.DataStream{ + Namespace: "default", + }, + Type: "system/metrics", + Id: "system/metrics-system-default-system", + Name: "system-1", + Revision: 1, + Meta: &proto.Meta{ + Package: &proto.Package{ + Name: "system", + Version: "1.17.0", + }, + }, +} + +func TestSingleMetricbeatMetricsetWithProcessors(t *testing.T) { + tests.InitBeatsForTest(t, cmd.RootCmd) + var mbStreams = []*proto.Stream{ + { + Id: "system/metrics-system.cpu-default-system", + DataStream: &proto.DataStream{ + Dataset: "system.cpu", + Type: "metrics", + }, + Source: tests.RequireNewStruct(map[string]interface{}{ + "metricsets": []interface{}{"cpu"}, + "period": "2s", + "processors": []interface{}{ + map[string]interface{}{ + "add_fields": map[string]interface{}{ + "fields": map[string]interface{}{"testfield": true}, + "target": "@metadata", + }, + }, + }, + }), + }, + { + Id: "system/metrics-system.memory-default-system", + DataStream: &proto.DataStream{ + Dataset: "system.memory", + Type: "metrics", + }, + Source: tests.RequireNewStruct(map[string]interface{}{ + "metricsets": []interface{}{"memory"}, + "period": "2s", + }), + }, + } + + expectedMBStreams.Streams = mbStreams + outPath, server := tests.SetupTestEnv(t, expectedMBStreams, time.Second*6) + + defer server.Srv.Stop() + defer func() { + err := os.RemoveAll(outPath) + require.NoError(t, err) + }() + + // After runfor seconds, this should shut down, allowing us to check the output + t.Logf("Running beats...") + err := cmd.RootCmd.Execute() + require.NoError(t, err) + + t.Logf("Reading events...") + events := tests.ReadEvents(t, outPath) + t.Logf("Got %d events", len(events)) + + // Look for processors + expectedCPUMetaValues := map[string]interface{}{ + // Processors created by + "@metadata.input_id": "system/metrics-system-default-system", + "@metadata.stream_id": "system/metrics-system.cpu-default-system", + "agent.id": "test-agent", + "data_stream.dataset": "system.cpu", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + // make sure the V2 shim isn't overwriting any custom processors + "@metadata.testfield": true, + } + tests.ValuesExist(t, expectedCPUMetaValues, events, tests.ONCE) + + expectedMemoryMetaValues := map[string]interface{}{ + "@metadata.stream_id": "system/metrics-system.memory-default-system", + "data_stream.dataset": "system.memory", + } + tests.ValuesExist(t, expectedMemoryMetaValues, events, tests.ONCE) + + // Look for proper CPU/memory config + expectedCPU := map[string]interface{}{ + "system.cpu.cores": nil, + "system.cpu.total": nil, + "system.memory.actual.free": nil, + } + tests.ValuesExist(t, expectedCPU, events, tests.ONCE) + + // If there's a config issue, metricbeat will fallback to default metricsets. Make sure they don't exist. + disabledMetricsets := []string{ + "system.process", + "system.load", + "system.process_summary", + } + tests.ValuesDoNotExist(t, disabledMetricsets, events) +} diff --git a/x-pack/libbeat/management/tests/mock_server.go b/x-pack/libbeat/management/tests/mock_server.go new file mode 100644 index 00000000000..0948704c640 --- /dev/null +++ b/x-pack/libbeat/management/tests/mock_server.go @@ -0,0 +1,159 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tests + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/client/mock" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/structpb" +) + +// MockV2Handler wraps the basic tooling needed to handle a fake V2 controller +type MockV2Handler struct { + Srv mock.StubServerV2 + Client client.V2 +} + +// NewMockServer returns a mocked elastic-agent V2 controller +func NewMockServer(t *testing.T, runtime time.Duration, inputConfig *proto.UnitExpectedConfig, outPath string) MockV2Handler { + unitOneID := mock.NewID() + unitOutID := mock.NewID() + + token := mock.NewID() + //var gotConfig bool + + var mut sync.Mutex + + var logOutputStream = &proto.UnitExpectedConfig{ + DataStream: &proto.DataStream{ + Namespace: "default", + }, + Type: "file", + Revision: 1, + Meta: &proto.Meta{ + Package: &proto.Package{ + Name: "system", + Version: "1.17.0", + }, + }, + Source: RequireNewStruct(map[string]interface{}{ + "type": "file", + "enabled": true, + "path": outPath, + "filename": "beat-out", + "number_of_files": 7, + }), + } + + start := time.Now() + + var stateIndex uint64 = 1 + srv := mock.StubServerV2{ + CheckinV2Impl: func(observed *proto.CheckinObserved) *proto.CheckinExpected { + mut.Lock() + defer mut.Unlock() + if observed.Token == token { + + // initial checkin + if len(observed.Units) == 0 || observed.Units[0].State == proto.State_STARTING { + return sendUnitsWithState(proto.State_HEALTHY, inputConfig, logOutputStream, unitOneID, unitOutID, stateIndex) + } else if checkUnitStateHealthy(observed.Units) { + + if time.Since(start) > runtime { + //remove the units once they've been healthy for a given period of time + return sendUnitsWithState(proto.State_STOPPED, inputConfig, logOutputStream, unitOneID, unitOutID, stateIndex+1) + } + //otherwise, just remove the units + } else if observed.Units[0].State == proto.State_STOPPED { + return &proto.CheckinExpected{ + Units: nil, + } + } else if observed.Units[0].State == proto.State_FAILED { + + return &proto.CheckinExpected{ + Units: nil, + } + } + + } + + return nil + }, + ActionImpl: func(response *proto.ActionResponse) error { + return nil + }, + ActionsChan: make(chan *mock.PerformAction, 100), + } // end of srv declaration + + // The start() needs to happen here, since the client needs the assigned server port + err := srv.Start() + require.NoError(t, err) + + client := client.NewV2(fmt.Sprintf(":%d", srv.Port), token, client.VersionInfo{ + Name: "program", + Version: "v1.0.0", + Meta: map[string]string{ + "key": "value", + }, + }, grpc.WithTransportCredentials(insecure.NewCredentials())) + + return MockV2Handler{Srv: srv, Client: client} +} + +// helper to wrap the CheckinExpected config we need with every refresh of the mock server +func sendUnitsWithState(state proto.State, input, output *proto.UnitExpectedConfig, inId, outId string, stateIndex uint64) *proto.CheckinExpected { + return &proto.CheckinExpected{ + AgentInfo: &proto.CheckinAgentInfo{ + Id: "test-agent", + Version: "8.4.0", + Snapshot: true, + }, + Units: []*proto.UnitExpected{ + { + Id: inId, + Type: proto.UnitType_INPUT, + ConfigStateIdx: stateIndex, + Config: input, + State: state, + LogLevel: proto.UnitLogLevel_DEBUG, + }, + { + Id: outId, + Type: proto.UnitType_OUTPUT, + ConfigStateIdx: stateIndex, + Config: output, + State: state, + }, + }, + } +} + +func checkUnitStateHealthy(units []*proto.UnitObserved) bool { + for _, unit := range units { + if unit.State != proto.State_HEALTHY { + return false + } + } + return true +} + +//RequireNewStruct converts a mapstr to a protobuf struct +func RequireNewStruct(v map[string]interface{}) *structpb.Struct { + str, err := structpb.NewStruct(v) + if err != nil { + panic(err) + } + return str +} diff --git a/x-pack/libbeat/management/tests/output_read.go b/x-pack/libbeat/management/tests/output_read.go new file mode 100644 index 00000000000..e5c7649de6b --- /dev/null +++ b/x-pack/libbeat/management/tests/output_read.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tests + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type findFieldsMode string + +const ALL findFieldsMode = "all" +const ONCE findFieldsMode = "once" + +// ReadEvents reads the ndjson output we get from the beats file output +func ReadEvents(t *testing.T, path string) []mapstr.M { + files, err := filepath.Glob(filepath.Join(path, "*.ndjson")) + require.NoError(t, err) + + events := []mapstr.M{} + for _, file := range files { + rawFile, err := os.ReadFile(file) + require.NoError(t, err) + lines := strings.Split(string(rawFile), "\n") + for _, line := range lines { + var event = mapstr.M{} + // skip newlines that appear at the end of files + if len(line) < 2 { + continue + } + err = json.Unmarshal([]byte(line), &event) + require.NoError(t, err) + events = append(events, event) + } + } + return events +} + +// ValuesExist verifies that the given fields exist in the events. +// the values map takes keys in the form of keys in the events map, which may be in dot form: "system.cpu.cores", etc. +// The value for the map should be the expected value, or a `nil` if you merely want to check for the presence of a field. +// the mode determines if `ValuesExist` must exist in all events, or just one. +func ValuesExist(t *testing.T, values map[string]interface{}, events []mapstr.M, mode findFieldsMode) { + for searchKey, val := range values { + var foundCount = 0 + for eventIter, event := range events { + evt, err := event.GetValue(searchKey) + if errors.Is(err, mapstr.ErrKeyNotFound) { + continue + } + if val == nil { + foundCount++ + } else { + if val == evt { + foundCount++ + } else if val != evt && mode == ALL { + t.Errorf("Key %s was found in event %d, but value was unexpected. Expected %#v, got %#v", searchKey, eventIter, val, evt) + } + } + } + if mode == ALL { + if foundCount != len(events) { + t.Errorf("Expected to find key %s in all %d events, but key was only found %d times.", searchKey, len(events), foundCount) + } + } + if mode == ONCE { + if foundCount == 0 { + t.Errorf("Did not find key %s in any events", searchKey) + } + } + } +} + +// ValuesDoNotExist checks to make sure that the given keys do not exist in any events. +func ValuesDoNotExist(t *testing.T, values []string, events []mapstr.M) { + for _, key := range values { + for eventIter, event := range events { + evt, _ := event.GetValue(key) + if evt != nil { + t.Errorf("key %s with value %#v was found in event %d in the output", key, evt, eventIter) + } + } + } +} diff --git a/x-pack/metricbeat/cmd/agent.go b/x-pack/metricbeat/cmd/agent.go new file mode 100644 index 00000000000..38a7f51e99e --- /dev/null +++ b/x-pack/metricbeat/cmd/agent.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "strings" + + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" +) + +func metricbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + modules, err := management.CreateInputsFromStreams(rawIn, "metrics", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + // Extract the module name from the type, usually in the form system/metric + module := strings.Split(rawIn.Type, "/")[0] + + for iter := range modules { + modules[iter]["module"] = module + } + + // format for the reloadable list needed bythe cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + + return configList, nil +} diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 5f2df6c025e..99fdd9cf49d 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/beater" mbcmd "github.com/elastic/beats/v7/metricbeat/cmd" "github.com/elastic/beats/v7/metricbeat/cmd/test" + "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/elastic-agent-libs/mapstr" // Register the includes. @@ -43,6 +44,7 @@ var withECSVersion = processing.WithFields(mapstr.M{ }) func init() { + management.ConfigTransform.SetTransform(metricbeatCfg) var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) settings := instance.Settings{ diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index 06cb4ab4cb7..3780ef59efc 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -5,11 +5,17 @@ package cmd import ( + "fmt" + cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/common/cli" + "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/publisher/processing" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -35,6 +41,7 @@ var withECSVersion = processing.WithFields(mapstr.M{ var RootCmd = Osquerybeat() func Osquerybeat() *cmd.BeatsRootCmd { + management.ConfigTransform.SetTransform(osquerybeatCfg) settings := instance.Settings{ Name: Name, Processing: processing.MakeDefaultSupport(true, withECSVersion, processing.WithHost, processing.WithAgentMeta()), @@ -63,3 +70,29 @@ func genVerifyCmd(_ instance.Settings) *cobra.Command { }), } } + +func osquerybeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + // Convert to streams, osquerybeat doesn't use streams + streams := make([]*proto.Stream, 1) + streams[0] = &proto.Stream{ + Source: rawIn.GetSource(), + Id: rawIn.GetId(), + DataStream: rawIn.GetDataStream(), + } + rawIn.Streams = streams + + modules, err := management.CreateInputsFromStreams(rawIn, "osquery", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + for iter := range modules { + modules[iter]["type"] = "log" + } + + // format for the reloadable list needed bythe cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating config for reloader: %w", err) + } + return configList, nil +} diff --git a/x-pack/osquerybeat/internal/config/watcher.go b/x-pack/osquerybeat/internal/config/watcher.go index 1e1fbf856e3..4b4aebdcd31 100644 --- a/x-pack/osquerybeat/internal/config/watcher.go +++ b/x-pack/osquerybeat/internal/config/watcher.go @@ -65,7 +65,7 @@ func WatchInputs(ctx context.Context, log *logp.Logger) <-chan []InputConfig { log: log, ch: ch, } - reload.Register.MustRegisterList("inputs", r) + reload.RegisterV2.MustRegisterInput(r) return ch } From 07d43e3c9a4e2404fe5c4ef0156eb4d8ff379665 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:38:10 -0700 Subject: [PATCH 2/9] V2 packetbeat support (#33041) * first attempt at auditbeat support * add license header * initial packetbeat support * fix bad branch * cleanup * typo in comment * clean up, move around files * add new processors to streams --- x-pack/packetbeat/cmd/root.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index 407d24570df..05c878caaab 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -5,8 +5,15 @@ package cmd import ( + "fmt" + "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/libbeat/common/reload" packetbeatCmd "github.com/elastic/beats/v7/packetbeat/cmd" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + conf "github.com/elastic/elastic-agent-libs/config" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" @@ -20,7 +27,32 @@ var Name = packetbeatCmd.Name // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd +// packetbeatCfg is a callback registered via SetTransform that returns a packetbeat Elastic Agent client.Unit +// configuration generated from a raw Elastic Agent config +func packetbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + //grab and properly format the input streams + inputStreams, err := management.CreateInputsFromStreams(rawIn, "metrics", agentInfo) + if err != nil { + return nil, fmt.Errorf("error generating new stream config: %w", err) + } + + // Packetbeat does its own transformations, + // so update the existing config with our new transformations, + // then send to packetbeat + souceMap := rawIn.Source.AsMap() + souceMap["streams"] = inputStreams + + uconfig, err := conf.NewConfigFrom(souceMap) + if err != nil { + return nil, fmt.Errorf("error in conversion to conf.C: %w", err) + } + return []*reload.ConfigWithMeta{{Config: uconfig}}, nil +} + func init() { + // Register packetbeat with central management to perform any needed config + // transformations before agent configs are sent to the beat during reload. + management.ConfigTransform.SetTransform(packetbeatCfg) settings := packetbeatCmd.PacketbeatSettings() settings.ElasticLicensed = true RootCmd = packetbeatCmd.Initialize(settings) From 6eecb84690a50bcbb67668c610987f6a7e9fbac2 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:32:59 -0700 Subject: [PATCH 3/9] First pass at auditbeat support (#33026) * first attempt at auditbeat support * add license header * cleanup * move files around --- x-pack/auditbeat/cmd/root.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 213a0c3c68a..5ccd3a231ad 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -5,8 +5,15 @@ package cmd import ( + "fmt" + "strings" + auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" @@ -19,7 +26,33 @@ var Name = auditbeatcmd.Name // RootCmd to handle beats CLI. var RootCmd *cmd.BeatsRootCmd +// auditbeatCfg is a callback registered with central management to perform any needed config transformations +// before agent configs are sent to a beat +func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + modules, err := management.CreateInputsFromStreams(rawIn, "metrics", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + // Extract the type field that has "audit/auditd", treat this + // as the module config key + module := strings.Split(rawIn.Type, "/")[1] + + for iter := range modules { + modules[iter]["module"] = module + } + + // Format for the reloadable list needed bythe cm.Reload() method. + configList, err := management.CreateReloadConfigFromInputs(modules) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + + return configList, nil +} + func init() { + management.ConfigTransform.SetTransform(auditbeatCfg) settings := auditbeatcmd.AuditbeatSettings() settings.ElasticLicensed = true RootCmd = auditbeatcmd.Initialize(settings) From 8dc24d6aa08c9b1c7c7c7f350417097caf6232c3 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 11 Oct 2022 10:14:44 -0700 Subject: [PATCH 4/9] Add heartbeat support for V2 (#33157) * add v2 config * fix name * fix doc --- x-pack/heartbeat/cmd/root.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index d777f631e48..1fcd69ed6e4 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -5,16 +5,39 @@ package cmd import ( + "fmt" + heartbeatCmd "github.com/elastic/beats/v7/heartbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + "github.com/elastic/beats/v7/x-pack/libbeat/management" ) // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd +// heartbeatCfg is a callback registered via SetTransform that returns a Elastic Agent client.Unit +// configuration generated from a raw Elastic Agent config +func heartbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + //grab and properly format the input streams + inputStreams, err := management.CreateInputsFromStreams(rawIn, "metrics", agentInfo) + if err != nil { + return nil, fmt.Errorf("error generating new stream config: %w", err) + } + + configList, err := management.CreateReloadConfigFromInputs(inputStreams) + if err != nil { + return nil, fmt.Errorf("error creating reloader config: %w", err) + } + return configList, nil +} + func init() { + management.ConfigTransform.SetTransform(heartbeatCfg) settings := heartbeatCmd.HeartbeatSettings() settings.ElasticLicensed = true RootCmd = heartbeatCmd.Initialize(settings) From 7cab21888d35e9271fe736009fe7eca6c115f056 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Tue, 11 Oct 2022 15:05:34 -0700 Subject: [PATCH 5/9] fix go.mod --- go.sum | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/go.sum b/go.sum index 05176c9ebce..9048f0018fa 100644 --- a/go.sum +++ b/go.sum @@ -613,17 +613,10 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -<<<<<<< HEAD -github.com/elastic/elastic-agent-autodiscover v0.3.0 h1:kdpNnIDnVk7gvQxxR6PzZY7aM8LyMTRkwI/p+FNS17s= -github.com/elastic/elastic-agent-autodiscover v0.3.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= -======= github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= ->>>>>>> upstream/main +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk= @@ -2036,11 +2029,9 @@ golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY= @@ -2549,8 +2540,6 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= From 26ef6da081e69c0a7677f7f56bd2a683ecf4446f Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 12 Oct 2022 13:08:13 -0700 Subject: [PATCH 6/9] fix unchecked stream_id --- x-pack/libbeat/management/generate.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/libbeat/management/generate.go b/x-pack/libbeat/management/generate.go index 957b8477b6d..29bcf15d84b 100644 --- a/x-pack/libbeat/management/generate.go +++ b/x-pack/libbeat/management/generate.go @@ -170,8 +170,8 @@ func injectStreamProcessors(expected *proto.UnitExpectedConfig, inputType string // the AST injects input_id at the input level and not the stream level, // for reasons I can't understand, as it just ends up shuffling it around // to individual metricsets anyway, at least on metricbeat - if expected.GetId() != "" { - inputId := generateAddFieldsProcessor(mapstr.M{"input_id": expected.Id}, "@metadata") + if expectedID := expected.GetId(); expectedID != "" { + inputId := generateAddFieldsProcessor(mapstr.M{"input_id": expectedID}, "@metadata") processors = append(processors, inputId) } @@ -191,9 +191,10 @@ func injectStreamProcessors(expected *proto.UnitExpectedConfig, inputType string processors = append(processors, event) // source stream - streamID := streamExpected.GetId() - sourceStream := generateAddFieldsProcessor(mapstr.M{"stream_id": streamID}, "@metadata") - processors = append(processors, sourceStream) + if streamID := streamExpected.GetId(); streamID != "" { + sourceStream := generateAddFieldsProcessor(mapstr.M{"stream_id": streamID}, "@metadata") + processors = append(processors, sourceStream) + } // figure out if we have any existing processors currentProcs, ok := stream["processors"] From c399a9d577554ace7b94720186cb963eb2dd6ea0 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 13 Oct 2022 09:38:26 -0700 Subject: [PATCH 7/9] fix unchecked stream_id (#33335) --- x-pack/libbeat/management/generate.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/libbeat/management/generate.go b/x-pack/libbeat/management/generate.go index 957b8477b6d..29bcf15d84b 100644 --- a/x-pack/libbeat/management/generate.go +++ b/x-pack/libbeat/management/generate.go @@ -170,8 +170,8 @@ func injectStreamProcessors(expected *proto.UnitExpectedConfig, inputType string // the AST injects input_id at the input level and not the stream level, // for reasons I can't understand, as it just ends up shuffling it around // to individual metricsets anyway, at least on metricbeat - if expected.GetId() != "" { - inputId := generateAddFieldsProcessor(mapstr.M{"input_id": expected.Id}, "@metadata") + if expectedID := expected.GetId(); expectedID != "" { + inputId := generateAddFieldsProcessor(mapstr.M{"input_id": expectedID}, "@metadata") processors = append(processors, inputId) } @@ -191,9 +191,10 @@ func injectStreamProcessors(expected *proto.UnitExpectedConfig, inputType string processors = append(processors, event) // source stream - streamID := streamExpected.GetId() - sourceStream := generateAddFieldsProcessor(mapstr.M{"stream_id": streamID}, "@metadata") - processors = append(processors, sourceStream) + if streamID := streamExpected.GetId(); streamID != "" { + sourceStream := generateAddFieldsProcessor(mapstr.M{"stream_id": streamID}, "@metadata") + processors = append(processors, sourceStream) + } // figure out if we have any existing processors currentProcs, ok := stream["processors"] From cae33ef70e165f99cdbae401ebb74dab64d79dfb Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:16:36 -0700 Subject: [PATCH 8/9] Update elastic-agent-libs for output panic fix (#33336) * Fix errors for non-synth capable instances (#33310) Fixes #32694 by making sure we use the lightweight wrapper code always when monitors cannot be initialized. This also fixes an unrelated bug, where errors attached to non-summary events would not be indexed. * [Automation] Update elastic stack version to 8.6.0-5a8d757d for testing (#33323) Co-authored-by: apmmachine * add pid awareness to file locking (#33169) * add pid awareness to file locking * cleanup, logic for handling restarts with the same PID * add zombie-state awareness * fix file naming * add retry for unlock * was confused by unlock code, fix, cleanup * update notice * fix race with file creation, update deps * clean up tests, spelling * hack for cgo * add lic headers * notice * try to fix windows issues * fix typos * small fixes * use exclusive locks * remove feature to start with a specially named pidfile * clean up some error handling, fix test cleanup * forgot changelog * Fix sample config in log rotation docs (#33306) * Add banner to deprecate functionbeat (#33297) * fix unchecked stream_id * packetbeat/protos/dns: clean up package (#33286) * avoid magic numbers * fix hashableDNSTuple size and offsets * avoid use of String and Error methods in formatted print calls * remove redundant conversions * quieten linter * use plugin-owned logp.Logger * update elastic-agent-libs * Revert "fix unchecked stream_id" This reverts commit 26ef6da081e69c0a7677f7f56bd2a683ecf4446f. * [Automation] Update elastic stack version to 8.6.0-40086bc7 for testing (#33339) Co-authored-by: apmmachine Co-authored-by: Andrew Cholakian Co-authored-by: apmmachine <58790750+apmmachine@users.noreply.github.com> Co-authored-by: apmmachine Co-authored-by: Jaime Soriano Pastor Co-authored-by: DeDe Morton Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 2 + NOTICE.txt | 98 ++++++- filebeat/docs/filebeat-log-rotation.asciidoc | 4 +- go.mod | 15 +- go.sum | 25 +- heartbeat/ecserr/ecserr.go | 8 + heartbeat/monitors/monitor.go | 28 +- heartbeat/monitors/wrappers/wrappers.go | 41 ++- heartbeat/monitors/wrappers/wrappers_test.go | 26 +- libbeat/cmd/instance/beat.go | 7 +- libbeat/cmd/instance/locker.go | 76 ----- libbeat/cmd/instance/locker_test.go | 68 ----- libbeat/cmd/instance/locks/lock.go | 262 ++++++++++++++++++ libbeat/cmd/instance/locks/lock_test.go | 155 +++++++++++ .../cmd/instance/locks/process_lookup_cgo.go | 30 ++ .../cmd/instance/locks/process_lookup_stub.go | 31 +++ packetbeat/protos/dns/dns.go | 118 ++++---- packetbeat/protos/dns/dns_tcp.go | 56 ++-- packetbeat/protos/dns/dns_test.go | 6 +- packetbeat/protos/dns/dns_udp.go | 13 +- packetbeat/protos/dns/names.go | 12 +- packetbeat/protos/dns/names_test.go | 3 +- testing/environments/snapshot.yml | 6 +- x-pack/functionbeat/docs/page_header.html | 3 + x-pack/heartbeat/monitors/browser/browser.go | 5 +- 25 files changed, 768 insertions(+), 330 deletions(-) delete mode 100644 libbeat/cmd/instance/locker.go delete mode 100644 libbeat/cmd/instance/locker_test.go create mode 100644 libbeat/cmd/instance/locks/lock.go create mode 100644 libbeat/cmd/instance/locks/lock_test.go create mode 100644 libbeat/cmd/instance/locks/process_lookup_cgo.go create mode 100644 libbeat/cmd/instance/locks/process_lookup_stub.go create mode 100644 x-pack/functionbeat/docs/page_header.html diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8bbb2c71e33..fde2c427394 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -122,6 +122,8 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Affecting all Beats* +- Beats will now attempt to recover if a lockfile has not been removed {pull}[33169] + *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 91c74780814..e0a952bc588 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10100,11 +10100,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.2.11 +Version: v0.2.13 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.11/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.13/LICENSE: Apache License Version 2.0, January 2004 @@ -10414,11 +10414,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.4.4 +Version: v0.4.5-0.20220927192933-25a985b07d51 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.5-0.20220927192933-25a985b07d51/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -17199,11 +17199,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- Dependency : github.com/magefile/mage -Version: v1.13.0 +Version: v1.14.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.14.0/LICENSE: Apache License Version 2.0, January 2004 @@ -19373,11 +19373,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/stretchr/testify -Version: v1.7.1 +Version: v1.8.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.7.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.8.0/LICENSE: MIT License @@ -21378,11 +21378,11 @@ Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1 -------------------------------------------------------------------------------- Dependency : go.uber.org/atomic -Version: v1.9.0 +Version: v1.10.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.9.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.10.0/LICENSE.txt: Copyright (c) 2016 Uber Technologies, Inc. @@ -21436,11 +21436,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/zap -Version: v1.21.0 +Version: v1.23.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.21.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.23.0/LICENSE.txt: Copyright (c) 2016-2017 Uber Technologies, Inc. @@ -42826,6 +42826,76 @@ DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/shirou/gopsutil +Version: v3.21.11+incompatible +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/shirou/gopsutil@v3.21.11+incompatible/LICENSE: + +gopsutil is distributed under BSD license reproduced below. + +Copyright (c) 2014, WAKAYAMA Shirou +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the gopsutil authors nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------- +internal/common/binary.go in the gopsutil is copied and modifid from golang/encoding/binary.go. + + + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + -------------------------------------------------------------------------------- Dependency : github.com/sirupsen/logrus Version: v1.8.1 @@ -42959,11 +43029,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx -Version: v0.2.0 +Version: v0.4.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.4.0/LICENSE: The MIT License diff --git a/filebeat/docs/filebeat-log-rotation.asciidoc b/filebeat/docs/filebeat-log-rotation.asciidoc index 1399f605c6c..69eafd22e7e 100644 --- a/filebeat/docs/filebeat-log-rotation.asciidoc +++ b/filebeat/docs/filebeat-log-rotation.asciidoc @@ -70,8 +70,8 @@ sure it does not miss any events. [source,yaml] ----------------------------------------------------- filebeat.inputs: -- type: log - enabled: false +- type: filestream + id: my-server-filestream-id paths: - /var/log/my-server/my-server.log* ----------------------------------------------------- diff --git a/go.mod b/go.mod index 9037142a481..46e0188e28f 100644 --- a/go.mod +++ b/go.mod @@ -117,7 +117,7 @@ require ( github.com/jonboulle/clockwork v0.2.2 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/lib/pq v1.10.3 - github.com/magefile/mage v1.13.0 + github.com/magefile/mage v1.14.0 github.com/mattn/go-colorable v0.1.12 github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/miekg/dns v1.1.42 @@ -141,7 +141,7 @@ require ( github.com/shopspring/decimal v1.2.0 github.com/spf13/cobra v1.3.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.0 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/ugorji/go/codec v1.1.8 github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 @@ -150,9 +150,9 @@ require ( go.elastic.co/ecszap v1.0.1 go.elastic.co/go-licence-detector v0.5.0 go.etcd.io/bbolt v1.3.6 - go.uber.org/atomic v1.9.0 + go.uber.org/atomic v1.10.0 go.uber.org/multierr v1.8.0 - go.uber.org/zap v1.21.0 + go.uber.org/zap v1.23.0 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.5.1 @@ -192,9 +192,9 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.4.0 - github.com/elastic/elastic-agent-libs v0.2.11 + github.com/elastic/elastic-agent-libs v0.2.13 github.com/elastic/elastic-agent-shipper-client v0.4.0 - github.com/elastic/elastic-agent-system-metrics v0.4.4 + github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 github.com/elastic/go-elasticsearch/v8 v8.2.0 github.com/googleapis/gax-go/v2 v2.5.1 github.com/pierrec/lz4/v4 v4.1.15 @@ -308,8 +308,9 @@ require ( github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sergi/go-diff v1.1.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect diff --git a/go.sum b/go.sum index 9048f0018fa..cc02b7355cb 100644 --- a/go.sum +++ b/go.sum @@ -617,12 +617,13 @@ github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= -github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= +github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA= +github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk= github.com/elastic/elastic-agent-shipper-client v0.4.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q= -github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0= -github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI= +github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51 h1:ZFk7hC6eRPJkJNtOSG+GYbRlsgLjSD8rTj4gQq+7rsA= +github.com/elastic/elastic-agent-system-metrics v0.4.5-0.20220927192933-25a985b07d51/go.mod h1:vTqfhtj83LlPKbusEwrEywZv13nhPExwINB3PkeRQeo= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= @@ -1281,8 +1282,8 @@ github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc8 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.12.1/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= -github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1632,6 +1633,8 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -1692,8 +1695,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -1703,8 +1707,9 @@ github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1849,8 +1854,9 @@ go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -1871,8 +1877,9 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/heartbeat/ecserr/ecserr.go b/heartbeat/ecserr/ecserr.go index 34cee65a96f..9ca594fc0b8 100644 --- a/heartbeat/ecserr/ecserr.go +++ b/heartbeat/ecserr/ecserr.go @@ -120,3 +120,11 @@ func NewCouldNotConnectErr(host, port string, err error) *ECSErr { fmt.Sprintf("Could not connect to '%s:%s' with error: %s", host, port, err), ) } + +func NewNotSyntheticsCapableError() *ECSErr { + return NewECSErr( + TYPE_IO, + "AGENT_NOT_BROWSER_CAPABLE", + "browser monitors cannot be created outside the official elastic docker image", + ) +} diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go index fb759f074d6..17271086d16 100644 --- a/heartbeat/monitors/monitor.go +++ b/heartbeat/monitors/monitor.go @@ -154,15 +154,19 @@ func newMonitorUnsafe( return p.Close() } - // If we've hit an error at this point, still run on schedule, but always return an error. - // This way the error is clearly communicated through to kibana. - // Since the error is not recoverable in these instances, the user will need to reconfigure - // the monitor, which will destroy and recreate it in heartbeat, thus clearing this error. - // - // Note: we do this at this point, and no earlier, because at a minimum we need the - // standard monitor fields (id, name and schedule) to deliver an error to kibana in a way - // that it can render. - if err != nil { + var wrappedJobs []jobs.Job + if err == nil { + wrappedJobs = wrappers.WrapCommon(p.Jobs, m.stdFields, stateLoader) + } else { + // If we've hit an error at this point, still run on schedule, but always return an error. + // This way the error is clearly communicated through to kibana. + // Since the error is not recoverable in these instances, the user will need to reconfigure + // the monitor, which will destroy and recreate it in heartbeat, thus clearing this error. + // + // Note: we do this at this point, and no earlier, because at a minimum we need the + // standard monitor fields (id, name and schedule) to deliver an error to kibana in a way + // that it can render. + // Note, needed to hoist err to this scope, not just to add a prefix fullErr := fmt.Errorf("job could not be initialized: %w", err) // A placeholder job that always returns an error @@ -171,9 +175,13 @@ func newMonitorUnsafe( p.Jobs = []jobs.Job{func(event *beat.Event) ([]jobs.Job, error) { return nil, fullErr }} + + // We need to use the lightweight wrapping for error jobs + // since browser wrapping won't write summaries, but the fake job here is + // effectively a lightweight job + wrappedJobs = wrappers.WrapLightweight(p.Jobs, m.stdFields, monitorstate.NewTracker(stateLoader, false)) } - wrappedJobs := wrappers.WrapCommon(p.Jobs, m.stdFields, stateLoader) m.endpoints = p.Endpoints m.configuredJobs, err = m.makeTasks(config, wrappedJobs) diff --git a/heartbeat/monitors/wrappers/wrappers.go b/heartbeat/monitors/wrappers/wrappers.go index bdc5935ae3b..7bbbe51103c 100644 --- a/heartbeat/monitors/wrappers/wrappers.go +++ b/heartbeat/monitors/wrappers/wrappers.go @@ -60,6 +60,7 @@ func WrapLightweight(js []jobs.Job, stdMonFields stdfields.StdMonitorFields, mst addServiceName(stdMonFields), addMonitorMeta(stdMonFields, len(js) > 1), addMonitorStatus(false), + addMonitorErr, addMonitorDuration, ), func() jobs.JobWrapper { @@ -82,6 +83,7 @@ func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields, mst *mo addServiceName(stdMonFields), addMonitorMeta(stdMonFields, false), addMonitorStatus(true), + addMonitorErr, addMonitorState(stdMonFields, mst), logJourneySummaries, ) @@ -227,28 +229,39 @@ func addMonitorStatus(summaryOnly bool) jobs.JobWrapper { if summaryOnly { hasSummary, _ := event.Fields.HasKey("summary.up") if !hasSummary { - return cont, nil + return cont, err } } - fields := mapstr.M{ + eventext.MergeEventFields(event, mapstr.M{ "monitor": mapstr.M{ "status": look.Status(err), }, + }) + + return cont, err + } + } +} + +func addMonitorErr(origJob jobs.Job) jobs.Job { + return func(event *beat.Event) ([]jobs.Job, error) { + cont, err := origJob(event) + + if err != nil { + var errVal interface{} + var asECS *ecserr.ECSErr + if errors.As(err, &asECS) { + // Override the message of the error in the event it was wrapped + asECS.Message = err.Error() + errVal = asECS + } else { + errVal = look.Reason(err) } - if err != nil { - var asECS *ecserr.ECSErr - if errors.As(err, &asECS) { - // Override the message of the error in the event it was wrapped - asECS.Message = err.Error() - fields["error"] = asECS - } else { - fields["error"] = look.Reason(err) - } - } - eventext.MergeEventFields(event, fields) - return cont, nil + eventext.MergeEventFields(event, mapstr.M{"error": errVal}) } + + return cont, nil } } diff --git a/heartbeat/monitors/wrappers/wrappers_test.go b/heartbeat/monitors/wrappers/wrappers_test.go index 2528514d067..fce80d87e90 100644 --- a/heartbeat/monitors/wrappers/wrappers_test.go +++ b/heartbeat/monitors/wrappers/wrappers_test.go @@ -719,17 +719,27 @@ func TestProjectBrowserJob(t *testing.T) { } func TestECSErrors(t *testing.T) { + // key is test name, value is whether to test a summary event or not + testCases := map[string]bool{ + "on summary event": true, + "on non-summary event": false, + } + ecse := ecserr.NewBadCmdStatusErr(123, "mycommand") - wrappedEcsErr := fmt.Errorf("wrapped: %w", ecse) - expectedEcsErr := ecserr.NewECSErr( + wrappedECSErr := fmt.Errorf("wrapped: %w", ecse) + expectedECSErr := ecserr.NewECSErr( ecse.Type, ecse.Code, - wrappedEcsErr.Error(), + wrappedECSErr.Error(), ) - j := WrapCommon([]jobs.Job{makeProjectBrowserJob(t, "http://example.net", true, wrappedEcsErr, projectMonitorValues)}, testBrowserMonFields, nil) - event := &beat.Event{} - _, err := j[0](event) - require.NoError(t, err) - require.Equal(t, event.Fields["error"], expectedEcsErr) + for name, makeSummaryEvent := range testCases { + t.Run(name, func(t *testing.T) { + j := WrapCommon([]jobs.Job{makeProjectBrowserJob(t, "http://example.net", makeSummaryEvent, wrappedECSErr, projectMonitorValues)}, testBrowserMonFields, nil) + event := &beat.Event{} + _, err := j[0](event) + require.NoError(t, err) + require.Equal(t, expectedECSErr, event.Fields["error"]) + }) + } } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 4e9af610053..28a7f0489f2 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -44,6 +44,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/cloudid" + "github.com/elastic/beats/v7/libbeat/cmd/instance/locks" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/common/seccomp" @@ -385,13 +386,13 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { // Try to acquire exclusive lock on data path to prevent another beat instance // sharing same data path. - bl := newLocker(b) - err := bl.lock() + bl := locks.New(b.Info) + err := bl.Lock() if err != nil { return err } defer func() { - _ = bl.unlock() + _ = bl.Unlock() }() svc.BeforeRun() diff --git a/libbeat/cmd/instance/locker.go b/libbeat/cmd/instance/locker.go deleted file mode 100644 index 0d7a1ff2486..00000000000 --- a/libbeat/cmd/instance/locker.go +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package instance - -import ( - "os" - - "github.com/gofrs/flock" - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-libs/paths" -) - -var ( - // ErrAlreadyLocked is returned when a lock on the data path is attempted but - // unsuccessful because another Beat instance already has the lock on the same - // data path. - ErrAlreadyLocked = errors.New("data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).") -) - -type locker struct { - fl *flock.Flock -} - -func newLocker(b *Beat) *locker { - lockfilePath := paths.Resolve(paths.Data, b.Info.Beat+".lock") - return &locker{ - fl: flock.NewFlock(lockfilePath), - } -} - -// lock attempts to acquire a lock on the data path for the currently-running -// Beat instance. If another Beats instance already has a lock on the same data path -// an ErrAlreadyLocked error is returned. -func (l *locker) lock() error { - isLocked, err := l.fl.TryLock() - if err != nil { - return errors.Wrap(err, "unable to lock data path") - } - - if !isLocked { - return ErrAlreadyLocked - } - - return nil -} - -// unlock attempts to release the lock on a data path previously acquired via Lock(). -func (l *locker) unlock() error { - err := l.fl.Unlock() - if err != nil { - return errors.Wrap(err, "unable to unlock data path") - } - - err = os.Remove(l.fl.Path()) - if err != nil { - return errors.Wrap(err, "unable to unlock data path") - } - - return nil -} diff --git a/libbeat/cmd/instance/locker_test.go b/libbeat/cmd/instance/locker_test.go deleted file mode 100644 index b2b2d2854d1..00000000000 --- a/libbeat/cmd/instance/locker_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !integration -// +build !integration - -package instance - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-libs/paths" -) - -// TestLocker tests that two beats pointing to the same data path cannot -// acquire the same lock. -func TestLocker(t *testing.T) { - // Setup temporary data folder for test + clean it up at end of test - tmpDataDir, err := ioutil.TempDir("", "data") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDataDir) - - origDataPath := paths.Paths.Data - defer func() { - paths.Paths.Data = origDataPath - }() - paths.Paths.Data = tmpDataDir - - // Setup two beats with same name and data path - const beatName = "testbeat" - - b1 := &Beat{} - b1.Info.Beat = beatName - - b2 := &Beat{} - b2.Info.Beat = beatName - - // Try to get a lock for the first beat. Expect it to succeed. - bl1 := newLocker(b1) - err = bl1.lock() - assert.NoError(t, err) - - // Try to get a lock for the second beat. Expect it to fail because the - // first beat already has the lock. - bl2 := newLocker(b2) - err = bl2.lock() - assert.EqualError(t, err, ErrAlreadyLocked.Error()) -} diff --git a/libbeat/cmd/instance/locks/lock.go b/libbeat/cmd/instance/locks/lock.go new file mode 100644 index 00000000000..0da72726182 --- /dev/null +++ b/libbeat/cmd/instance/locks/lock.go @@ -0,0 +1,262 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package locks + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "runtime" + "time" + + "github.com/gofrs/flock" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/paths" + metricproc "github.com/elastic/elastic-agent-system-metrics/metric/system/process" +) + +type Locker struct { + fileLock *flock.Flock + logger *logp.Logger + beatName string + filePath string + beatStart time.Time +} + +type pidfile struct { + PID int `json:"pid"` + WriteTime time.Time `json:"write_time"` +} + +var ( + // ErrAlreadyLocked is returned when a lock on the data path is attempted but + // unsuccessful because another Beat instance already has the lock on the same + // data path. + ErrAlreadyLocked = fmt.Errorf("data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).") + + // ErrLockfileEmpty is returned by readExistingPidfile() when an existing pidfile is found, but the file is empty. + ErrLockfileEmpty = fmt.Errorf("lockfile is empty") +) + +// a little wrapper for the gitpid function to make testing easier. +var pidFetch = os.Getpid + +// New returns a new pid-aware file locker +// all logic, including checking for existing locks, is performed lazily +func New(beatInfo beat.Info) *Locker { + lockfilePath := paths.Resolve(paths.Data, beatInfo.Beat+".lock") + return &Locker{ + fileLock: flock.New(lockfilePath), + logger: logp.L(), + beatName: beatInfo.Beat, + filePath: lockfilePath, + beatStart: beatInfo.StartTime, + } +} + +// Lock attempts to acquire a lock on the data path for the currently-running +// Beat instance. If another Beats instance already has a lock on the same data path +// an ErrAlreadyLocked error is returned. +func (lock *Locker) Lock() error { + new := pidfile{PID: pidFetch(), WriteTime: time.Now()} + encoded, err := json.Marshal(&new) + if err != nil { + return fmt.Errorf("error encoding json for pidfile: %w", err) + } + + // The combination of O_CREATE and O_EXCL will ensure we return an error if we don't + // manage to create the file + fh, openErr := os.OpenFile(lock.filePath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + // Don't trust different OSes to report the errors we expect, just try to recover regardless + if openErr != nil { + err = lock.handleFailedCreate() + if err != nil { + return fmt.Errorf("cannot obtain lockfile: %w", err) + } + // If something fails here, it's probably unrecoverable + fh, err = os.OpenFile(lock.filePath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("cannot re-obtain lockfile %s: %w", lock.filePath, err) + } + } + + // a Process can't write to its own locked file on all platforms, write first + _, err = fh.Write(encoded) + if err != nil { + return fmt.Errorf("error writing pidfile to %s: %w", lock.filePath, err) + } + + // Exclusive lock + isLocked, err := lock.fileLock.TryLock() + if err != nil { + return fmt.Errorf("unable to lock data path: %w", err) + } + // case: lock could not be obtained. + if !isLocked { + // if we're here, things are probably unrecoverable, as we've previously checked for a lockfile. Exit. + return ErrAlreadyLocked + } + + return nil +} + +// Unlock attempts to release the lock on a data path previously acquired via Lock(). +func (lock *Locker) Unlock() error { + err := lock.fileLock.Unlock() + if err != nil { + return fmt.Errorf("unable to unlock data path: %w", err) + } + + err = os.Remove(lock.fileLock.Path()) + if err != nil { + return fmt.Errorf("unable to unlock data path file %s: %w", lock.fileLock.Path(), err) + } + return nil +} + +// ******* private helpers + +// handleFailedCreate will attempt to recover from a failed lock operation in a pid-aware way. +// The point of this is to deal with instances where an improper beat shutdown left us with +// a pre-existing pidfile for a beat process that no longer exists. +func (lock *Locker) handleFailedCreate() error { + // First, try to lock the file as a check to see what state we're in. + // If there's a pre-existing lock that's in effect, we probably can't recover + // Not all OSes will fail on this. + _, err := lock.fileLock.TryLock() + // Case: the file already locked, and in use by another process. + if err != nil { + if runtime.GOOS == "windows" { + // on windows, locks from dead PIDs will be auto-released, but it might take the OS a while. + // However, the time it takes for the operating system to unlock these locks depends upon available system resources. + time.Sleep(time.Second) + _, err := lock.fileLock.TryLock() + if err != nil { + return fmt.Errorf("The lockfile %s is locked after a retry, another beat is probably running", lock.fileLock) + } + } else { + return fmt.Errorf("The lockfile %s is already locked by another beat", lock.fileLock) + } + } + + // if we're here, we've locked the file + // unlock so we can continue + err = lock.fileLock.Unlock() + if err != nil { + return fmt.Errorf("error unlocking a previously found file %s after a temporary lock", lock.filePath) + } + + // read in whatever existing lockfile caused us to fail + pf, err := lock.readExistingPidfile() + // Case: two beats start up simultaneously, there's a chance we could "see" the pidfile before the other process writes to it + // or, the other beat died before it could write the pidfile. + // Sleep, read again. If we still don't have anything, assume the other PID is dead, continue. + if errors.Is(err, ErrLockfileEmpty) { + lock.logger.Debugf("Found other pidfile, but no data. Retrying.") + time.Sleep(time.Millisecond * 500) + pf, err = lock.readExistingPidfile() + if errors.Is(err, ErrLockfileEmpty) { + lock.logger.Debugf("No PID found in other lockfile, continuing") + return lock.recoverLockfile() + } else if err != nil { + return fmt.Errorf("error re-reading existing lockfile: %w", err) + } + } else if err != nil { + return fmt.Errorf("error reading existing lockfile: %w", err) + } + + // Case: the lockfile is locked, but by us. Probably a coding error, + // and probably hard to do + if pf.PID == os.Getpid() { + // the lockfile was written before the beat started, meaning we restarted and somehow got the same pid + // in which case, continue + if lock.beatStart.Before(pf.WriteTime) { + return fmt.Errorf("lockfile for beat has been locked twice by the same PID, potential bug.") + } + lock.logger.Debugf("Beat has started with the same PID, continuing") + return lock.recoverLockfile() + } + + // Check to see if the PID found in the pidfile exists. + existsState, err := findMatchingPID(pf.PID) + // Case: we have a lockfile, but the pid from the pidfile no longer exists + // this was presumably due to the dirty shutdown. + // Try to reset the lockfile and continue. + if errors.Is(err, metricproc.ProcNotExist) { + lock.logger.Debugf("%s shut down without removing previous lockfile, continuing", lock.beatName) + return lock.recoverLockfile() + } else if err != nil { + return fmt.Errorf("error looking up status for pid %d: %w", pf.PID, err) + } else { + // Case: the PID exists, but it's attached to a zombie process + // In this case...we should be okay to restart? + if existsState == metricproc.Zombie { + lock.logger.Debugf("%s shut down without removing previous lockfile and is currently in a zombie state, continuing", lock.beatName) + return lock.recoverLockfile() + } + // Case: we've gotten a lock file for another process that's already running + // This is the "base" lockfile case, which is two beats running from the same directory + // This is where we'll catch this particular case on Linux, due to Linux's advisory-style locks. + return fmt.Errorf("connot start, data directory belongs to process with pid %d", pf.PID) + } +} + +// recoverLockfile attempts to remove the lockfile and continue running +// This should only be called after we're sure it's safe to ignore a pre-existing lockfile +// This will reset the internal lockfile handler when it's successful. +func (lock *Locker) recoverLockfile() error { + // File remove may or not work, depending on os-specific details with lockfiles + err := os.Remove(lock.fileLock.Path()) + if err != nil { + if runtime.GOOS == "windows" { + // retry on windows, the OS can take time to clean up + time.Sleep(time.Second) + err = os.Remove(lock.fileLock.Path()) + if err != nil { + return fmt.Errorf("tried twice to remove lockfile %s on windows: %w", + lock.fileLock.Path(), err) + } + } else { + return fmt.Errorf("lockfile %s cannot be removed: %w", lock.fileLock.Path(), err) + } + + } + lock.fileLock = flock.New(lock.filePath) + return nil +} + +// readExistingPidfile will read the contents of an existing pidfile +// Will return ErrLockfileEmpty if no data is found in the lockfile +func (lock *Locker) readExistingPidfile() (pidfile, error) { + rawPidfile, err := os.ReadFile(lock.filePath) + if err != nil { + return pidfile{}, fmt.Errorf("error reading pidfile from path %s", lock.filePath) + } + if len(rawPidfile) == 0 { + return pidfile{}, ErrLockfileEmpty + } + foundPidFile := pidfile{} + err = json.Unmarshal(rawPidfile, &foundPidFile) + if err != nil { + return pidfile{}, fmt.Errorf("error reading JSON from pid file %s: %w", lock.filePath, err) + } + return foundPidFile, nil +} diff --git a/libbeat/cmd/instance/locks/lock_test.go b/libbeat/cmd/instance/locks/lock_test.go new file mode 100644 index 00000000000..10c63f48567 --- /dev/null +++ b/libbeat/cmd/instance/locks/lock_test.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package locks + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/paths" +) + +func TestMain(m *testing.M) { + err := logp.DevelopmentSetup() + if err != nil { + fmt.Fprintf(os.Stderr, "error creating logger: %s\n", err) + os.Exit(1) + } + tmp, err := os.MkdirTemp("", "pidfile_test") + defer os.RemoveAll(tmp) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating temp directory: %s\n", err) + os.Exit(1) + } + + origDataPath := paths.Paths.Data + defer func() { + paths.Paths.Data = origDataPath + }() + paths.Paths.Data = tmp + + exit := m.Run() + // cleanup tmpdir after run, but let the tests set the exit code + err = os.RemoveAll(tmp) + if err != nil { + fmt.Fprintf(os.Stderr, "Error removing tempdir %s, %s:", tmp, err) + } + + os.Exit(exit) +} + +func TestLockWithDeadPid(t *testing.T) { + // create old lockfile + pidFetch = fakeDeadPid + testBeat := beat.Info{Beat: mustNewUUID(t), StartTime: time.Now()} + locker := New(testBeat) + err := locker.Lock() + require.NoError(t, err) + + // create new locker + pidFetch = os.Getpid + newLocker := New(testBeat) + err = newLocker.Lock() + require.NoError(t, err) +} + +func TestLockWithTwoBeats(t *testing.T) { + testBeat := beat.Info{Beat: mustNewUUID(t), StartTime: time.Now()} + // emulate two beats trying to run from the same data path + locker := New(testBeat) + // use the parent process as another random beat + pidFetch = os.Getppid + err := locker.Lock() + require.NoError(t, err) + + // create new locker for this beat + pidFetch = os.Getpid + newLocker := New(testBeat) + err = newLocker.Lock() + require.Error(t, err) + t.Logf("Got desired error: %s", err) +} + +func TestDoubleLock(t *testing.T) { + testBeat := beat.Info{Beat: mustNewUUID(t), StartTime: time.Now()} + locker := New(testBeat) + err := locker.Lock() + require.NoError(t, err) + + newLocker := New(testBeat) + err = newLocker.Lock() + require.Error(t, err) + t.Logf("Got desired error: %s", err) +} + +func TestUnlock(t *testing.T) { + testBeat := beat.Info{Beat: mustNewUUID(t), StartTime: time.Now()} + locker := New(testBeat) + err := locker.Lock() + require.NoError(t, err) + + err = locker.Unlock() + require.NoError(t, err) +} + +func TestRestartWithSamePID(t *testing.T) { + // create old lockfile + testBeatName := mustNewUUID(t) + testBeat := beat.Info{Beat: testBeatName, StartTime: time.Now().Add(-time.Second * 20)} + locker := New(testBeat) + err := locker.Lock() + require.NoError(t, err) + // create new lockfile with the same PID but a newer time + // create old lockfile + testNewBeat := beat.Info{Name: testBeatName, StartTime: time.Now()} + lockerNew := New(testNewBeat) + err = lockerNew.Lock() + require.NoError(t, err) +} + +func TestEmptyLockfile(t *testing.T) { + testBeat := beat.Info{Beat: mustNewUUID(t), StartTime: time.Now().Add(-time.Second * 1)} + deadLock := New(testBeat) + // Create an empty lockfile + // Might happen in cases where a beat shut down at *just* the right time. + fh, err := os.Create(deadLock.filePath) + require.NoError(t, err) + fh.Close() + + newBeat := New(testBeat) + err = newBeat.Lock() + require.NoError(t, err) + +} + +func mustNewUUID(t *testing.T) string { + uuid, err := uuid.NewV4() + require.NoError(t, err) + return uuid.String() +} + +func fakeDeadPid() int { + return 99999 +} diff --git a/libbeat/cmd/instance/locks/process_lookup_cgo.go b/libbeat/cmd/instance/locks/process_lookup_cgo.go new file mode 100644 index 00000000000..f0f2b253cca --- /dev/null +++ b/libbeat/cmd/instance/locks/process_lookup_cgo.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build (darwin && cgo) || freebsd || linux || windows || aix + +package locks + +import ( + "github.com/elastic/elastic-agent-system-metrics/metric/system/process" + "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" +) + +// findMatchingPID is a small wrapper to deal with cgo compat issues in libbeat's CI +func findMatchingPID(pid int) (process.PidState, error) { + return process.GetPIDState(resolve.NewTestResolver("/"), pid) +} diff --git a/libbeat/cmd/instance/locks/process_lookup_stub.go b/libbeat/cmd/instance/locks/process_lookup_stub.go new file mode 100644 index 00000000000..cd151ebad14 --- /dev/null +++ b/libbeat/cmd/instance/locks/process_lookup_stub.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build (!darwin || !cgo) && !freebsd && !linux && !windows && !aix + +package locks + +import ( + "fmt" + "runtime" + + "github.com/elastic/elastic-agent-system-metrics/metric/system/process" +) + +func findMatchingPID(pid int) (process.PidState, error) { + return process.Dead, fmt.Errorf("findMatchingPID not supported on platform: %s", runtime.GOOS) +} diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go index f531af22ed2..028b9449c29 100644 --- a/packetbeat/protos/dns/dns.go +++ b/packetbeat/protos/dns/dns.go @@ -59,11 +59,9 @@ type dnsPlugin struct { results protos.Reporter // Channel where results are pushed. watcher procs.ProcessesWatcher -} - -var debugf = logp.MakeDebug("dns") -const maxDNSTupleRawSize = 16 + 16 + 2 + 2 + 4 + 1 + logger *logp.Logger +} // Transport protocol. type transport uint8 @@ -92,6 +90,15 @@ func (t transport) String() string { type hashableDNSTuple [maxDNSTupleRawSize]byte +const ( + maxDNSTupleRawSize = 2*(sizeofIP+sizeofPort) + sizeofID + sizeofTransport + + sizeofIP = 16 + sizeofPort = 2 + sizeofID = 2 + sizeofTransport = 1 +) + // DnsMessage contains a single DNS message. type dnsMessage struct { ts time.Time // Time when the message was received. @@ -109,8 +116,8 @@ type dnsTuple struct { transport transport id uint16 - raw hashableDNSTuple // Src_ip:Src_port:Dst_ip:Dst_port:Transport:Id - revRaw hashableDNSTuple // Dst_ip:Dst_port:Src_ip:Src_port:Transport:Id + raw hashableDNSTuple // Src_ip:Src_port:Dst_ip:Dst_port:ID:Transport + revRaw hashableDNSTuple // Dst_ip:Dst_port:Src_ip:Src_port:ID:Transport } func dnsTupleFromIPPort(t *common.IPPortTuple, trans transport, id uint16) dnsTuple { @@ -152,21 +159,21 @@ func (t *dnsTuple) computeHashables() { copy(t.raw[18:34], t.DstIP) copy(t.raw[34:36], []byte{byte(t.DstPort >> 8), byte(t.DstPort)}) copy(t.raw[36:38], []byte{byte(t.id >> 8), byte(t.id)}) - t.raw[39] = byte(t.transport) + t.raw[38] = byte(t.transport) copy(t.revRaw[0:16], t.DstIP) copy(t.revRaw[16:18], []byte{byte(t.DstPort >> 8), byte(t.DstPort)}) copy(t.revRaw[18:34], t.SrcIP) copy(t.revRaw[34:36], []byte{byte(t.SrcPort >> 8), byte(t.SrcPort)}) copy(t.revRaw[36:38], []byte{byte(t.id >> 8), byte(t.id)}) - t.revRaw[39] = byte(t.transport) + t.revRaw[38] = byte(t.transport) } func (t *dnsTuple) String() string { return fmt.Sprintf("DnsTuple src[%s:%d] dst[%s:%d] transport[%s] id[%d]", - t.SrcIP.String(), + t.SrcIP, t.SrcPort, - t.DstIP.String(), + t.DstIP, t.DstPort, t.transport, t.id) @@ -212,13 +219,8 @@ func init() { protos.Register("dns", New) } -func New( - testMode bool, - results protos.Reporter, - watcher procs.ProcessesWatcher, - cfg *conf.C, -) (protos.Plugin, error) { - p := &dnsPlugin{} +func New(testMode bool, results protos.Reporter, watcher procs.ProcessesWatcher, cfg *conf.C) (protos.Plugin, error) { + p := &dnsPlugin{logger: logp.NewLogger("dns")} config := defaultConfig if !testMode { if err := cfg.Unpack(&config); err != nil { @@ -240,7 +242,7 @@ func (dns *dnsPlugin) init(results protos.Reporter, watcher procs.ProcessesWatch func(k common.Key, v common.Value) { trans, ok := v.(*dnsTransaction) if !ok { - logp.Err("Expired value is not a *DnsTransaction.") + dns.logger.Error("Expired value is not a *DnsTransaction.") return } dns.expireTransaction(trans) @@ -253,14 +255,13 @@ func (dns *dnsPlugin) init(results protos.Reporter, watcher procs.ProcessesWatch return nil } -func (dns *dnsPlugin) setFromConfig(config *dnsConfig) error { +func (dns *dnsPlugin) setFromConfig(config *dnsConfig) { dns.ports = config.Ports dns.sendRequest = config.SendRequest dns.sendResponse = config.SendResponse dns.includeAuthorities = config.IncludeAuthorities dns.includeAdditionals = config.IncludeAdditionals dns.transactionTimeout = config.TransactionTimeout - return nil } func newTransaction(ts time.Time, tuple dnsTuple, cmd common.ProcessTuple) *dnsTransaction { @@ -292,14 +293,14 @@ func (dns *dnsPlugin) ConnectionTimeout() time.Duration { } func (dns *dnsPlugin) receivedDNSRequest(tuple *dnsTuple, msg *dnsMessage) { - debugf("Processing query. %s", tuple.String()) + dns.logger.Debugf("Processing query. %s", tuple) trans := dns.deleteTransaction(tuple.hashable()) if trans != nil { // This happens if a client puts multiple requests in flight // with the same ID. trans.notes = append(trans.notes, duplicateQueryMsg.Error()) - debugf("%s %s", duplicateQueryMsg.Error(), tuple.String()) + dns.logger.Debugf("%v %s", duplicateQueryMsg, tuple) dns.publishTransaction(trans) dns.deleteTransaction(trans.tuple.hashable()) } @@ -308,7 +309,7 @@ func (dns *dnsPlugin) receivedDNSRequest(tuple *dnsTuple, msg *dnsMessage) { if tuple.transport == transportUDP && (msg.data.IsEdns0() != nil) && msg.length > maxDNSPacketSize { trans.notes = append(trans.notes, udpPacketTooLarge.Error()) - debugf("%s", udpPacketTooLarge.Error()) + dns.logger.Debugf("%v", udpPacketTooLarge) } dns.transactions.Put(tuple.hashable(), trans) @@ -316,13 +317,13 @@ func (dns *dnsPlugin) receivedDNSRequest(tuple *dnsTuple, msg *dnsMessage) { } func (dns *dnsPlugin) receivedDNSResponse(tuple *dnsTuple, msg *dnsMessage) { - debugf("Processing response. %s", tuple.String()) + dns.logger.Debugf("Processing response. %s", tuple) trans := dns.getTransaction(tuple.revHashable()) if trans == nil { trans = newTransaction(msg.ts, tuple.reverse(), msg.cmdlineTuple.Reverse()) trans.notes = append(trans.notes, orphanedResponse.Error()) - debugf("%s %s", orphanedResponse.Error(), tuple.String()) + dns.logger.Debugf("%v %s", orphanedResponse, tuple) unmatchedResponses.Add(1) } @@ -332,7 +333,7 @@ func (dns *dnsPlugin) receivedDNSResponse(tuple *dnsTuple, msg *dnsMessage) { respIsEdns := msg.data.IsEdns0() != nil if !respIsEdns && msg.length > maxDNSPacketSize { trans.notes = append(trans.notes, udpPacketTooLarge.responseError()) - debugf("%s", udpPacketTooLarge.responseError()) + dns.logger.Debugf("%s", udpPacketTooLarge.responseError()) } request := trans.request @@ -342,10 +343,10 @@ func (dns *dnsPlugin) receivedDNSResponse(tuple *dnsTuple, msg *dnsMessage) { switch { case reqIsEdns && !respIsEdns: trans.notes = append(trans.notes, respEdnsNoSupport.Error()) - debugf("%s %s", respEdnsNoSupport.Error(), tuple.String()) + dns.logger.Debugf("%v %s", respEdnsNoSupport, tuple) case !reqIsEdns && respIsEdns: trans.notes = append(trans.notes, respEdnsUnexpected.Error()) - debugf("%s %s", respEdnsUnexpected.Error(), tuple.String()) + dns.logger.Debugf("%v %s", respEdnsUnexpected, tuple) } } } @@ -359,7 +360,7 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { return } - debugf("Publishing transaction. %s", t.tuple.String()) + dns.logger.Debugf("Publishing transaction. %s", &t.tuple) evt, pbf := pb.NewBeatEvent(t.ts) @@ -388,18 +389,17 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.request.data.Question[0]) fields["resource"] = t.request.data.Question[0].Name } - addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, - dns.includeAdditionals) + addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, dns.includeAdditionals, dns.logger) if t.response.data.Rcode == 0 { fields["status"] = common.OK_STATUS } if dns.sendRequest { - fields["request"] = dnsToString(t.request.data) + fields["request"] = dnsToString(t.request.data, dns.logger) } if dns.sendResponse { - fields["response"] = dnsToString(t.response.data) + fields["response"] = dnsToString(t.response.data, dns.logger) } } else if t.request != nil { pbf.Source.Bytes = int64(t.request.length) @@ -411,11 +411,10 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.request.data.Question[0]) fields["resource"] = t.request.data.Question[0].Name } - addDNSToMapStr(dnsEvent, pbf, t.request.data, dns.includeAuthorities, - dns.includeAdditionals) + addDNSToMapStr(dnsEvent, pbf, t.request.data, dns.includeAuthorities, dns.includeAdditionals, dns.logger) if dns.sendRequest { - fields["request"] = dnsToString(t.request.data) + fields["request"] = dnsToString(t.request.data, dns.logger) } } else if t.response != nil { pbf.Destination.Bytes = int64(t.response.length) @@ -427,10 +426,9 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { fields["query"] = dnsQuestionToString(t.response.data.Question[0]) fields["resource"] = t.response.data.Question[0].Name } - addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, - dns.includeAdditionals) + addDNSToMapStr(dnsEvent, pbf, t.response.data, dns.includeAuthorities, dns.includeAdditionals, dns.logger) if dns.sendResponse { - fields["response"] = dnsToString(t.response.data) + fields["response"] = dnsToString(t.response.data, dns.logger) } } @@ -439,13 +437,13 @@ func (dns *dnsPlugin) publishTransaction(t *dnsTransaction) { func (dns *dnsPlugin) expireTransaction(t *dnsTransaction) { t.notes = append(t.notes, noResponse.Error()) - debugf("%s %s", noResponse.Error(), t.tuple.String()) + dns.logger.Debugf("%v %s", noResponse, &t.tuple) dns.publishTransaction(t) unmatchedRequests.Add(1) } // Adds the DNS message data to the supplied MapStr. -func addDNSToMapStr(m mapstr.M, pbf *pb.Fields, dns *mkdns.Msg, authority bool, additional bool) { +func addDNSToMapStr(m mapstr.M, pbf *pb.Fields, dns *mkdns.Msg, authority bool, additional bool, logger *logp.Logger) { m["id"] = dns.Id m["op_code"] = dnsOpCodeToString(dns.Opcode) @@ -527,7 +525,7 @@ func addDNSToMapStr(m mapstr.M, pbf *pb.Fields, dns *mkdns.Msg, authority bool, m["answers_count"] = len(dns.Answer) if len(dns.Answer) > 0 { var resolvedIPs []string - m["answers"], resolvedIPs = rrsToMapStrs(dns.Answer, true) + m["answers"], resolvedIPs = rrsToMapStrs(dns.Answer, true, logger) if len(resolvedIPs) > 0 { m["resolved_ip"] = resolvedIPs pbf.AddIP(resolvedIPs...) @@ -536,7 +534,7 @@ func addDNSToMapStr(m mapstr.M, pbf *pb.Fields, dns *mkdns.Msg, authority bool, m["authorities_count"] = len(dns.Ns) if authority && len(dns.Ns) > 0 { - m["authorities"], _ = rrsToMapStrs(dns.Ns, false) + m["authorities"], _ = rrsToMapStrs(dns.Ns, false, logger) } if rrOPT != nil { @@ -545,7 +543,7 @@ func addDNSToMapStr(m mapstr.M, pbf *pb.Fields, dns *mkdns.Msg, authority bool, m["additionals_count"] = len(dns.Extra) } if additional && len(dns.Extra) > 0 { - rrsMapStrs, _ := rrsToMapStrs(dns.Extra, false) + rrsMapStrs, _ := rrsToMapStrs(dns.Extra, false, logger) // We do not want OPT RR to appear in the 'additional' section, // that's why rrsMapStrs could be empty even though len(dns.Extra) > 0 if len(rrsMapStrs) > 0 { @@ -590,13 +588,13 @@ func optToMapStr(rrOPT *mkdns.OPT) mapstr.M { // rrsToMapStr converts an slice of RR's to an slice of MapStr's and optionally // returns a list of the IP addresses found in the resource records. -func rrsToMapStrs(records []mkdns.RR, ipList bool) ([]mapstr.M, []string) { +func rrsToMapStrs(records []mkdns.RR, ipList bool, logger *logp.Logger) ([]mapstr.M, []string) { var allIPs []string mapStrSlice := make([]mapstr.M, 0, len(records)) for _, rr := range records { rrHeader := rr.Header() - mapStr, ips := rrToMapStr(rr, ipList) + mapStr, ips := rrToMapStr(rr, ipList, logger) if len(mapStr) == 0 { // OPT pseudo-RR returns an empty MapStr continue } @@ -619,11 +617,11 @@ func rrsToMapStrs(records []mkdns.RR, ipList bool) ([]mapstr.M, []string) { // // TODO An improvement would be to replace 'data' by the real field name // It would require some changes in unit tests -func rrToString(rr mkdns.RR) string { +func rrToString(rr mkdns.RR, logger *logp.Logger) string { var st string var keys []string - mapStr, _ := rrToMapStr(rr, false) + mapStr, _ := rrToMapStr(rr, false, logger) data, ok := mapStr["data"] delete(mapStr, "data") @@ -656,7 +654,7 @@ func rrToString(rr mkdns.RR) string { return b.String() } -func rrToMapStr(rr mkdns.RR, ipList bool) (mapstr.M, []string) { +func rrToMapStr(rr mkdns.RR, ipList bool, logger *logp.Logger) (mapstr.M, []string) { mapStr := mapstr.M{} rrType := rr.Header().Rrtype @@ -671,17 +669,17 @@ func rrToMapStr(rr mkdns.RR, ipList bool) (mapstr.M, []string) { switch x := rr.(type) { default: // We don't have special handling for this type - debugf("No special handling for RR type %s", dnsTypeToString(rrType)) + logger.Debugf("No special handling for RR type %s", dnsTypeToString(rrType)) unsupportedRR := new(mkdns.RFC3597) err := unsupportedRR.ToRFC3597(x) if err == nil { rData, err := hexStringToString(unsupportedRR.Rdata) mapStr["data"] = rData if err != nil { - debugf("%s", err.Error()) + logger.Debugf("%v", err) } } else { - debugf("Rdata for the unhandled RR type %s could not be fetched", dnsTypeToString(rrType)) + logger.Debugf("Rdata for the unhandled RR type %s could not be fetched", dnsTypeToString(rrType)) } // Don't attempt to render IPs for answers that are incomplete. @@ -735,11 +733,11 @@ func rrToMapStr(rr mkdns.RR, ipList bool) (mapstr.M, []string) { mapStr["data"] = trimRightDot(x.Ptr) case *mkdns.RFC3597: // Miekg/dns lib doesn't handle this type - debugf("Unknown RR type %s", dnsTypeToString(rrType)) + logger.Debugf("Unknown RR type %s", dnsTypeToString(rrType)) rData, err := hexStringToString(x.Rdata) mapStr["data"] = rData if err != nil { - debugf("%s", err.Error()) + logger.Debugf("%v", err) } case *mkdns.RRSIG: mapStr["type_covered"] = dnsTypeToString(x.TypeCovered) @@ -781,16 +779,16 @@ func dnsQuestionToString(q mkdns.Question) string { // rrsToString converts an array of RR's to a // string. -func rrsToString(r []mkdns.RR) string { +func rrsToString(r []mkdns.RR, logger *logp.Logger) string { var rrStrs []string for _, rr := range r { - rrStrs = append(rrStrs, rrToString(rr)) + rrStrs = append(rrStrs, rrToString(rr, logger)) } return strings.Join(rrStrs, "; ") } // dnsToString converts a DNS message to a string. -func dnsToString(dns *mkdns.Msg) string { +func dnsToString(dns *mkdns.Msg, logger *logp.Logger) string { var msgType string if dns.Response { msgType = "response" @@ -834,17 +832,17 @@ func dnsToString(dns *mkdns.Msg) string { if len(dns.Answer) > 0 { a = append(a, fmt.Sprintf("ANSWER %s", - rrsToString(dns.Answer))) + rrsToString(dns.Answer, logger))) } if len(dns.Ns) > 0 { a = append(a, fmt.Sprintf("AUTHORITY %s", - rrsToString(dns.Ns))) + rrsToString(dns.Ns, logger))) } if len(dns.Extra) > 0 { a = append(a, fmt.Sprintf("ADDITIONAL %s", - rrsToString(dns.Extra))) + rrsToString(dns.Extra, logger))) } return strings.Join(a, "; ") diff --git a/packetbeat/protos/dns/dns_tcp.go b/packetbeat/protos/dns/dns_tcp.go index 20832ff2479..ac1eacaf88e 100644 --- a/packetbeat/protos/dns/dns_tcp.go +++ b/packetbeat/protos/dns/dns_tcp.go @@ -54,33 +54,31 @@ type dnsConnectionData struct { } func (dns *dnsPlugin) Parse(pkt *protos.Packet, tcpTuple *common.TCPTuple, dir uint8, private protos.ProtocolData) protos.ProtocolData { - defer logp.Recover("Dns ParseTcp") + defer dns.logger.Recover("Dns ParseTcp") - debugf("Parsing packet addressed with %s of length %d.", - pkt.Tuple.String(), len(pkt.Payload)) + dns.logger.Debugf("dns", "Parsing packet addressed with %s of length %d.", &pkt.Tuple, len(pkt.Payload)) - conn := ensureDNSConnection(private) + conn := ensureDNSConnection(private, dns.logger) conn = dns.doParse(conn, pkt, tcpTuple, dir) if conn == nil { return nil } - return conn } -func ensureDNSConnection(private protos.ProtocolData) *dnsConnectionData { +func ensureDNSConnection(private protos.ProtocolData, logger *logp.Logger) *dnsConnectionData { if private == nil { return &dnsConnectionData{} } conn, ok := private.(*dnsConnectionData) if !ok { - logp.Warn("Dns connection data type error, create new one") + logger.Warn("Dns connection data type error, create new one") return &dnsConnectionData{} } if conn == nil { - logp.Warn("Unexpected: dns connection data not set, create new one") + logger.Warn("Unexpected: dns connection data not set, create new one") return &dnsConnectionData{} } @@ -101,16 +99,15 @@ func (dns *dnsPlugin) doParse(conn *dnsConnectionData, pkt *protos.Packet, tcpTu stream.rawData = append(stream.rawData, payload...) if len(stream.rawData) > tcp.TCPMaxDataInStream { - debugf("Stream data too large, dropping DNS stream") + dns.logger.Debugf("dns", "Stream data too large, dropping DNS stream") conn.data[dir] = nil return conn } } decodedData, err := stream.handleTCPRawData() if err != nil { - - if err == incompleteMsg { - debugf("Waiting for more raw data") + if err == incompleteMsg { //nolint:errorlint // incompleteMsg is not wrapped. + dns.logger.Debugf("dns", "Waiting for more raw data") return conn } @@ -118,8 +115,7 @@ func (dns *dnsPlugin) doParse(conn *dnsConnectionData, pkt *protos.Packet, tcpTu dns.publishResponseError(conn, err) } - debugf("%s addresses %s, length %d", err.Error(), - tcpTuple.String(), len(stream.rawData)) + dns.logger.Debugf("dns", "%v addresses %s, length %d", err, tcpTuple, len(stream.rawData)) // This means that malformed requests or responses are being sent... // TODO: publish the situation also if Request @@ -176,7 +172,6 @@ func (dns *dnsPlugin) ReceivedFin(tcpTuple *common.TCPTuple, dir uint8, private return private } stream := conn.data[dir] - if stream == nil || stream.message == nil { return conn } @@ -192,8 +187,7 @@ func (dns *dnsPlugin) ReceivedFin(tcpTuple *common.TCPTuple, dir uint8, private dns.publishResponseError(conn, err) } - debugf("%s addresses %s, length %d", err.Error(), - tcpTuple.String(), len(stream.rawData)) + dns.logger.Debugf("dns", "%v addresses %s, length %d", err, tcpTuple, len(stream.rawData)) return conn } @@ -213,7 +207,6 @@ func (dns *dnsPlugin) GapInStream(tcpTuple *common.TCPTuple, dir uint8, nbytes i } decodedData, err := stream.handleTCPRawData() - if err == nil { dns.messageComplete(conn, tcpTuple, dir, decodedData) return private, true @@ -223,9 +216,8 @@ func (dns *dnsPlugin) GapInStream(tcpTuple *common.TCPTuple, dir uint8, nbytes i dns.publishResponseError(conn, err) } - debugf("%s addresses %s, length %d", err.Error(), - tcpTuple.String(), len(stream.rawData)) - debugf("Dropping the stream %s", tcpTuple.String()) + dns.logger.Debugf("dns", "%v addresses %s, length %d", err, tcpTuple, len(stream.rawData)) + dns.logger.Debugf("dns", "Dropping the stream %s", tcpTuple) // drop the stream because it is binary Data and it would be unexpected to have a decodable message later return private, true @@ -243,26 +235,23 @@ func (dns *dnsPlugin) publishResponseError(conn *dnsConnectionData, err error) { dataOrigin := conn.prevRequest.data dnsTupleOrigin := dnsTupleFromIPPort(&conn.prevRequest.tuple, transportTCP, dataOrigin.Id) - hashDNSTupleOrigin := (&dnsTupleOrigin).hashable() + hashDNSTupleOrigin := dnsTupleOrigin.hashable() trans := dns.deleteTransaction(hashDNSTupleOrigin) - if trans == nil { // happens if Parse, Gap or Fin already published the response error return } - errDNS, ok := err.(*dnsError) - if !ok { - return - } - trans.notes = append(trans.notes, errDNS.responseError()) + if err, ok := err.(*dnsError); ok { //nolint:errorlint // err always comes from handleTCPRawData and is either *dnsError or nil. + trans.notes = append(trans.notes, err.responseError()) - // Should we publish the length (bytes_out) of the failed Response? - // streamReverse.message.Length = len(streamReverse.rawData) - // trans.Response = streamReverse.message + // Should we publish the length (bytes_out) of the failed Response? + // streamReverse.message.Length = len(streamReverse.rawData) + // trans.Response = streamReverse.message - dns.publishTransaction(trans) - dns.deleteTransaction(hashDNSTupleOrigin) + dns.publishTransaction(trans) + dns.deleteTransaction(hashDNSTupleOrigin) + } } // Manages data length prior to decoding the data and manages errors after decoding @@ -298,6 +287,5 @@ func (stream *dnsStream) handleTCPRawData() (*mkdns.Msg, error) { if err != nil { return nil, err } - return decodedData, nil } diff --git a/packetbeat/protos/dns/dns_test.go b/packetbeat/protos/dns/dns_test.go index e4cf5b207c5..c97ee259b65 100644 --- a/packetbeat/protos/dns/dns_test.go +++ b/packetbeat/protos/dns/dns_test.go @@ -85,7 +85,7 @@ type eventStore struct { } func (e *eventStore) publish(event beat.Event) { - publish.MarshalPacketbeatFields(&event, nil, nil) + _, _ = publish.MarshalPacketbeatFields(&event, nil, nil) e.events = append(e.events, event) } @@ -98,7 +98,7 @@ func newDNS(store *eventStore, verbose bool) *dnsPlugin { if verbose { level = logp.DebugLevel } - logp.DevelopmentSetup( + _ = logp.DevelopmentSetup( logp.WithLevel(level), logp.WithSelectors("dns"), ) @@ -327,7 +327,7 @@ func TestRRsToMapStrsWithOPTRecord(t *testing.T) { // The OPT record is a pseudo-record so it doesn't become a real record // in our conversion, and there will be 1 entry instead of 2. - mapStrs, _ := rrsToMapStrs([]mkdns.RR{o, r}, false) + mapStrs, _ := rrsToMapStrs([]mkdns.RR{o, r}, false, logp.NewLogger("dns_test")) assert.Len(t, mapStrs, 1) mapStr := mapStrs[0] diff --git a/packetbeat/protos/dns/dns_udp.go b/packetbeat/protos/dns/dns_udp.go index 5051345494f..89d76dea84c 100644 --- a/packetbeat/protos/dns/dns_udp.go +++ b/packetbeat/protos/dns/dns_udp.go @@ -17,28 +17,23 @@ package dns -import ( - "github.com/elastic/elastic-agent-libs/logp" - - "github.com/elastic/beats/v7/packetbeat/protos" -) +import "github.com/elastic/beats/v7/packetbeat/protos" // Only EDNS packets should have their size beyond this value const maxDNSPacketSize = (1 << 9) // 512 (bytes) func (dns *dnsPlugin) ParseUDP(pkt *protos.Packet) { - defer logp.Recover("Dns ParseUdp") + defer dns.logger.Recover("Dns ParseUdp") packetSize := len(pkt.Payload) - debugf("Parsing packet addressed with %s of length %d.", - pkt.Tuple.String(), packetSize) + dns.logger.Debugf("Parsing packet addressed with %s of length %d.", &pkt.Tuple, packetSize) dnsPkt, err := decodeDNSData(transportUDP, pkt.Payload) if err != nil { // This means that malformed requests or responses are being sent or // that someone is attempting to the DNS port for non-DNS traffic. Both // are issues that a monitoring system should report. - debugf("%s", err.Error()) + dns.logger.Debugf("%v", err) return } diff --git a/packetbeat/protos/dns/names.go b/packetbeat/protos/dns/names.go index afd1080e73e..9961ca6b147 100644 --- a/packetbeat/protos/dns/names.go +++ b/packetbeat/protos/dns/names.go @@ -35,7 +35,7 @@ import ( func dnsOpCodeToString(opCode int) string { s, exists := mkdns.OpcodeToString[opCode] if !exists { - return strconv.Itoa(int(opCode)) + return strconv.Itoa(opCode) } return s } @@ -46,7 +46,7 @@ func dnsOpCodeToString(opCode int) string { func dnsResponseCodeToString(rcode int) string { s, exists := mkdns.RcodeToString[rcode] if !exists { - return fmt.Sprintf("Unknown %d", int(rcode)) + return fmt.Sprintf("Unknown %d", rcode) } return s } @@ -55,7 +55,7 @@ func dnsResponseCodeToString(rcode int) string { // string representation is unknown then the numeric value will be returned // as a string. func dnsTypeToString(t uint16) string { - s, exists := mkdns.TypeToString[uint16(t)] + s, exists := mkdns.TypeToString[t] if !exists { return strconv.Itoa(int(t)) } @@ -66,7 +66,7 @@ func dnsTypeToString(t uint16) string { // string representation is unknown then the numeric value will be returned // as a string. func dnsClassToString(c uint16) string { - s, exists := mkdns.ClassToString[uint16(c)] + s, exists := mkdns.ClassToString[c] if !exists { return strconv.Itoa(int(c)) } @@ -77,7 +77,7 @@ func dnsClassToString(c uint16) string { // string representation is unknown then the numeric value will be returned // as a string. func dnsAlgorithmToString(a uint8) string { - s, exists := mkdns.AlgorithmToString[uint8(a)] + s, exists := mkdns.AlgorithmToString[a] if !exists { return strconv.Itoa(int(a)) } @@ -88,7 +88,7 @@ func dnsAlgorithmToString(a uint8) string { // string representation is unknown then the numeric value will be returned // as a string. func dnsHashToString(h uint8) string { - s, exists := mkdns.HashToString[uint8(h)] + s, exists := mkdns.HashToString[h] if !exists { return strconv.Itoa(int(h)) } diff --git a/packetbeat/protos/dns/names_test.go b/packetbeat/protos/dns/names_test.go index 3e83066e72d..675c50bc06f 100644 --- a/packetbeat/protos/dns/names_test.go +++ b/packetbeat/protos/dns/names_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/packetbeat/pb" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -112,7 +113,7 @@ func assertDNSMessage(t testing.TB, q dnsTestMsg) { } mapStr := mapstr.M{} - addDNSToMapStr(mapStr, pb.NewFields(), dns, true, true) + addDNSToMapStr(mapStr, pb.NewFields(), dns, true, true, logp.NewLogger("dns_test")) if q.question != nil { for k, v := range q.question { assert.NotNil(t, mapStr["question"].(mapstr.M)[k]) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 2635c0a9eaa..57b59c8d4e4 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0-c49fac70-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0-40086bc7-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.6.0-c49fac70-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.6.0-40086bc7-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.6.0-c49fac70-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.6.0-40086bc7-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/functionbeat/docs/page_header.html b/x-pack/functionbeat/docs/page_header.html new file mode 100644 index 00000000000..cec30d66bbf --- /dev/null +++ b/x-pack/functionbeat/docs/page_header.html @@ -0,0 +1,3 @@ +Functionbeat will reach End of Support on October 18, 2023. You should consider +moving your deployments to the more versatile and efficient Elastic Serverless +Forwarder. diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go index a84422b4b82..6411d0d3f09 100644 --- a/x-pack/heartbeat/monitors/browser/browser.go +++ b/x-pack/heartbeat/monitors/browser/browser.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/beats/v7/heartbeat/ecserr" "github.com/elastic/beats/v7/heartbeat/monitors/plugin" ) @@ -24,14 +25,12 @@ func init() { var showExperimentalOnce = sync.Once{} -var ErrNotSyntheticsCapableError = fmt.Errorf("synthetic monitors cannot be created outside the official elastic docker image") - func create(name string, cfg *config.C) (p plugin.Plugin, err error) { // We don't want users running synthetics in environments that don't have the required GUI libraries etc, so we check // this flag. When we're ready to support the many possible configurations of systems outside the docker environment // we can remove this check. if os.Getenv("ELASTIC_SYNTHETICS_CAPABLE") != "true" { - return plugin.Plugin{}, ErrNotSyntheticsCapableError + return plugin.Plugin{}, ecserr.NewNotSyntheticsCapableError() } showExperimentalOnce.Do(func() { From 7d95704002b953f4091ac2474e124b4a0ca2ba56 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Wed, 2 Nov 2022 08:45:07 -0700 Subject: [PATCH 9/9] update elastic-agent-client (#33552) --- NOTICE.txt | 8 ++++---- go.mod | 4 ++-- go.sum | 7 ++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index e0a952bc588..809e663dad9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9867,11 +9867,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.0.0-20220804181728-b0328d2fe484 +Version: v7.0.0-20221028150015-05e494d37ccd Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20220804181728-b0328d2fe484/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20221028150015-05e494d37ccd/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -17875,11 +17875,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/mitchellh/mapstructure -Version: v1.4.3 +Version: v1.5.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.4.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/mitchellh/mapstructure@v1.5.0/LICENSE: The MIT License (MIT) diff --git a/go.mod b/go.mod index 46e0188e28f..1ad43f211b2 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 + github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.3.2 github.com/elastic/go-licenser v0.4.0 @@ -123,7 +123,7 @@ require ( github.com/miekg/dns v1.1.42 github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 - github.com/mitchellh/mapstructure v1.4.3 + github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/osquery/osquery-go v0.0.0-20210622151333-99b4efa62ec5 github.com/otiai10/copy v1.2.0 diff --git a/go.sum b/go.sum index cc02b7355cb..b62c8766d99 100644 --- a/go.sum +++ b/go.sum @@ -615,8 +615,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484 h1:uJIMfLgCenJvxsVmEjBjYGxt0JddCgw2IxgoNfcIXOk= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go.mod h1:fkvyUfFwyAG5OnMF0h+FV9sC0Xn9YLITwQpSuwungQs= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd h1:IuAuac3vcucBrjAXKPQlTJ22H7mBUsSnNWxa7GZYFEg= +github.com/elastic/elastic-agent-client/v7 v7.0.0-20221028150015-05e494d37ccd/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA= github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss= @@ -1364,8 +1364,9 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=