diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6a38197c56f6..0aacb43832a5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -22,6 +22,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add daemonset.name in pods controlled by DaemonSets {pull}26808[26808], {issue}25816[25816] - Kubernetes autodiscover fails in node scope if node name cannot be discovered {pull}26947[26947] - Loading Kibana assets (dashboards, index templates) rely on Saved Object API. So to provide a reliable service, Beats can only import and export dasbhboards using at least Kibana 7.15. {issue}20672[20672] {pull}27220[27220] +- Skip add_kubernetes_metadata processor when kubernetes metadata are already present {pull}27689[27689] *Auditbeat* @@ -82,6 +83,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - threatintel module: Changed the type of `threatintel.indicator.first_seen` from `keyword` to `date`. {pull}26765[26765] - Remove all alias fields pointing to ECS fields from modules. This affects the Suricata and Traefik modules. {issue}10535[10535] {pull}26627[26627] - Add option for S3 input to work without SQS notification {issue}18205[18205] {pull}27332[27332] +- Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623] *Heartbeat* - Remove long deprecated `watch_poll` functionality. {pull}27166[27166] @@ -204,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Do not try to load ILM policy if `check_exists` is `false`. {pull}27508[27508] {issue}26322[26322] - Fix bug with cgroups hierarchy override path in cgroups {pull}27620[27620] - Beat `setup kibana` command may use the elasticsearch API key defined in `output.elasticsearch.api_key`. {issue}24015[24015] {pull}27540[27540] +- Seperate namespaces for V1 and V2 controller paths {pull}27676[27676] *Auditbeat* @@ -306,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update indentation for azure filebeat configuration. {pull}26604[26604] - Auditd: Fix Top Exec Commands dashboard visualization. {pull}27638[27638] - Store offset in `log.offset` field of events from the filestream input. {pull}27688[27688] +- Fix `httpjson` input rate limit processing and documentation. {pull}[] *Heartbeat* @@ -736,6 +740,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update `tags` and `threatintel.indicator.provider` fields in `threatintel.anomali` ingest pipeline {issue}24746[24746] {pull}27141[27141] - Move AWS module and filesets to GA. {pull}27428[27428] - update ecs.version to ECS 1.11.0. {pull}27107[27107] +- Add base64 Encode functionality to httpjson input. {pull}27681[27681] +- Add `join` and `sprintf` functions to `httpjson` input. {pull}27735[27735] *Heartbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 3d552c4a4d04..68ba806b2bf4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -277,14 +277,16 @@ def generateStages(Map args = [:]) { } def cloud(Map args = [:]) { - withNode(labels: args.label, forceWorkspace: true){ - startCloudTestEnv(name: args.directory, dirs: args.dirs) - } - withCloudTestEnv() { - try { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) - } finally { - terraformCleanup(name: args.directory, dir: args.directory) + withGithubNotify(context: args.context) { + withNode(labels: args.label, forceWorkspace: true){ + startCloudTestEnv(name: args.directory, dirs: args.dirs) + } + withCloudTestEnv() { + try { + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) + } finally { + terraformCleanup(name: args.directory, dir: args.directory) + } } } } diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index d988f38386e0..cb644036d251 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -253,11 +253,11 @@ MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard [source,shell] --------------- -./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b >> Filebeat-redis.ndjson +./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -folder module/redis --------------- -This generates a `AV4REOpp5NkDleZmzKkE.ndjson` file inside dashboard directory in the redis module. -It contains all dependencies like visualizations and searches. +This generates an appropriate folder under module/redis for the dashboard, separating assets into dashboards, searches, vizualizations, etc. +Each exported file is a JSON and their names are the IDs of the assets. NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is `app/kibana#/dashboard/AV4REOpp5NkDleZmzKkE?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `AV4REOpp5NkDleZmzKkE`. @@ -289,7 +289,7 @@ By passing the yml file to the `export_dashboards.go` script or to the Beat, you [source,shell] ------------------- -go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml +go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml -folder dashboards ------------------- [source,shell] diff --git a/libbeat/metric/system/cgroup/reader.go b/libbeat/metric/system/cgroup/reader.go index 7b1c72773822..f282008d0d6d 100644 --- a/libbeat/metric/system/cgroup/reader.go +++ b/libbeat/metric/system/cgroup/reader.go @@ -185,10 +185,11 @@ func (r *Reader) GetV1StatsForProcess(pid int) (*StatsV1, error) { if err != nil { return nil, err } + stats := StatsV1{} - stats.Path, stats.ID = getCommonCgroupMetadata(paths) + stats.Path, stats.ID = getCommonCgroupMetadata(paths.V1) stats.Version = CgroupsV1 - for conName, cgPath := range paths { + for conName, cgPath := range paths.V1 { if r.ignoreRootCgroups && (cgPath.ControllerPath == "/" && r.cgroupsHierarchyOverride != cgPath.ControllerPath) { continue } @@ -209,9 +210,9 @@ func (r *Reader) GetV2StatsForProcess(pid int) (*StatsV2, error) { return nil, err } stats := StatsV2{} - stats.Path, stats.ID = getCommonCgroupMetadata(paths) + stats.Path, stats.ID = getCommonCgroupMetadata(paths.V2) stats.Version = CgroupsV2 - for conName, cgPath := range paths { + for conName, cgPath := range paths.V2 { if r.ignoreRootCgroups && (cgPath.ControllerPath == "/" && r.cgroupsHierarchyOverride != cgPath.ControllerPath) { continue } @@ -225,10 +226,10 @@ func (r *Reader) GetV2StatsForProcess(pid int) (*StatsV2, error) { // ProcessCgroupPaths is a wrapper around Reader.ProcessCgroupPaths for libraries that only need the slimmer functionality from // the gosigar cgroups code. This does not have the same function signature, and consumers still need to distinguish between v1 and v2 cgroups. -func ProcessCgroupPaths(hostfs string, pid int) (map[string]ControllerPath, error) { +func ProcessCgroupPaths(hostfs string, pid int) (PathList, error) { reader, err := NewReader(hostfs, false) if err != nil { - return nil, errors.Wrap(err, "error creating cgroups reader") + return PathList{}, errors.Wrap(err, "error creating cgroups reader") } return reader.ProcessCgroupPaths(pid) } @@ -267,7 +268,6 @@ func getStatsV2(path ControllerPath, name string, stats *StatsV2) error { } func getStatsV1(path ControllerPath, name string, stats *StatsV1) error { - id := filepath.Base(path.ControllerPath) switch name { diff --git a/libbeat/metric/system/cgroup/util.go b/libbeat/metric/system/cgroup/util.go index 04a8b9ad6183..cc8d51e5ee58 100644 --- a/libbeat/metric/system/cgroup/util.go +++ b/libbeat/metric/system/cgroup/util.go @@ -60,6 +60,26 @@ type ControllerPath struct { IsV2 bool } +// PathList contains the V1 and V2 controller paths in a process +// Separate the V1 and V2 cgroups so we don't have hybrid cgroups fighting for one namespace +type PathList struct { + V1 map[string]ControllerPath + V2 map[string]ControllerPath +} + +// Flatten combines the V1 and V2 cgroups in cases where we don't need a map with keys +func (pl PathList) Flatten() []ControllerPath { + list := []ControllerPath{} + for _, v1 := range pl.V1 { + list = append(list, v1) + } + for _, v2 := range pl.V2 { + list = append(list, v2) + } + + return list +} + // parseMountinfoLine parses a line from the /proc/[pid]/mountinfo file on // Linux. The format of the line is specified in section 3.5 of // https://www.kernel.org/doc/Documentation/filesystems/proc.txt. @@ -217,14 +237,14 @@ func SubsystemMountpoints(rootfsMountpoint string, subsystems map[string]struct{ // ProcessCgroupPaths returns the cgroups to which a process belongs and the // pathname of the cgroup relative to the mountpoint of the subsystem. -func (r Reader) ProcessCgroupPaths(pid int) (map[string]ControllerPath, error) { +func (r Reader) ProcessCgroupPaths(pid int) (PathList, error) { cgroup, err := os.Open(filepath.Join(r.rootfsMountpoint, "proc", strconv.Itoa(pid), "cgroup")) if err != nil { - return nil, err //return a blank error so other events can use any file not found errors + return PathList{}, err //return a blank error so other events can use any file not found errors } defer cgroup.Close() - cPaths := map[string]ControllerPath{} + cPaths := PathList{V1: map[string]ControllerPath{}, V2: map[string]ControllerPath{}} sc := bufio.NewScanner(cgroup) for sc.Scan() { // http://man7.org/linux/man-pages/man7/cgroups.7.html @@ -265,14 +285,14 @@ the container as /sys/fs/cgroup/unified and start metricbeat with --system.hostf cgpaths, err := ioutil.ReadDir(controllerPath) if err != nil { - return nil, errors.Wrapf(err, "error fetching cgroupV2 controllers for cgroup location '%s' and path line '%s'", r.cgroupMountpoints.V2Loc, line) + return cPaths, errors.Wrapf(err, "error fetching cgroupV2 controllers for cgroup location '%s' and path line '%s'", r.cgroupMountpoints.V2Loc, line) } // In order to produce the same kind of data for cgroups V1 and V2 controllers, // We iterate over the group, and look for controllers, since the V2 unified system doesn't list them under the PID for _, singlePath := range cgpaths { if strings.Contains(singlePath.Name(), "stat") { controllerName := strings.TrimSuffix(singlePath.Name(), ".stat") - cPaths[controllerName] = ControllerPath{ControllerPath: path, FullPath: controllerPath, IsV2: true} + cPaths.V2[controllerName] = ControllerPath{ControllerPath: path, FullPath: controllerPath, IsV2: true} } } // cgroup v1 @@ -280,7 +300,7 @@ the container as /sys/fs/cgroup/unified and start metricbeat with --system.hostf subsystems := strings.Split(fields[1], ",") for _, subsystem := range subsystems { fullPath := filepath.Join(r.cgroupMountpoints.V1Mounts[subsystem], path) - cPaths[subsystem] = ControllerPath{ControllerPath: path, FullPath: fullPath, IsV2: false} + cPaths.V1[subsystem] = ControllerPath{ControllerPath: path, FullPath: fullPath, IsV2: false} } } } diff --git a/libbeat/metric/system/cgroup/util_test.go b/libbeat/metric/system/cgroup/util_test.go index e2feda7f4efb..8766ccb2e043 100644 --- a/libbeat/metric/system/cgroup/util_test.go +++ b/libbeat/metric/system/cgroup/util_test.go @@ -171,17 +171,17 @@ func TestProcessCgroupPaths(t *testing.T) { } path := "/docker/b29faf21b7eff959f64b4192c34d5d67a707fe8561e9eaa608cb27693fba4242" - assert.Equal(t, path, paths["blkio"].ControllerPath) - assert.Equal(t, path, paths["cpu"].ControllerPath) - assert.Equal(t, path, paths["cpuacct"].ControllerPath) - assert.Equal(t, path, paths["cpuset"].ControllerPath) - assert.Equal(t, path, paths["devices"].ControllerPath) - assert.Equal(t, path, paths["freezer"].ControllerPath) - assert.Equal(t, path, paths["memory"].ControllerPath) - assert.Equal(t, path, paths["net_cls"].ControllerPath) - assert.Equal(t, path, paths["net_prio"].ControllerPath) - assert.Equal(t, path, paths["perf_event"].ControllerPath) - assert.Len(t, paths, 10) + assert.Equal(t, path, paths.V1["blkio"].ControllerPath) + assert.Equal(t, path, paths.V1["cpu"].ControllerPath) + assert.Equal(t, path, paths.V1["cpuacct"].ControllerPath) + assert.Equal(t, path, paths.V1["cpuset"].ControllerPath) + assert.Equal(t, path, paths.V1["devices"].ControllerPath) + assert.Equal(t, path, paths.V1["freezer"].ControllerPath) + assert.Equal(t, path, paths.V1["memory"].ControllerPath) + assert.Equal(t, path, paths.V1["net_cls"].ControllerPath) + assert.Equal(t, path, paths.V1["net_prio"].ControllerPath) + assert.Equal(t, path, paths.V1["perf_event"].ControllerPath) + assert.Len(t, paths.Flatten(), 10) } func TestProcessCgroupPathsV2(t *testing.T) { @@ -195,10 +195,10 @@ func TestProcessCgroupPathsV2(t *testing.T) { t.Fatalf("error in ProcessCgroupPaths: %s", err) } - assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths["cgroup"].FullPath) - assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths["cpu"].FullPath) - assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths["io"].FullPath) - assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths["memory"].FullPath) + assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths.V2["cgroup"].FullPath) + assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths.V2["cpu"].FullPath) + assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths.V2["io"].FullPath) + assert.Equal(t, "testdata/docker/sys/fs/cgroup/system.slice/docker-1c8fa019edd4b9d4b2856f4932c55929c5c118c808ed5faee9a135ca6e84b039.scope", paths.V2["memory"].FullPath) } func assertContains(t testing.TB, m map[string]struct{}, key string) { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 6b3e2f07cc01..1756dfcfe311 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -235,7 +235,7 @@ func (d *addDockerMetadata) String() string { // lookupContainerIDByPID finds the container ID based on PID fields contained // in the event. func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.Event) (string, error) { - var cgroups map[string]cgroup.ControllerPath + var cgroups cgroup.PathList for _, field := range d.pidFields { v, err := event.GetValue(field) if err != nil { @@ -264,11 +264,11 @@ func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.Event) (string, e // getProcessCgroups returns a mapping of cgroup subsystem name to path. It // returns an error if it failed to retrieve the cgroup info. -func (d *addDockerMetadata) getProcessCgroups(pid int) (map[string]cgroup.ControllerPath, error) { +func (d *addDockerMetadata) getProcessCgroups(pid int) (cgroup.PathList, error) { // Initialize at time of first use. lazyCgroupCacheInit(d) - cgroups, ok := d.cgroups.Get(pid).(map[string]cgroup.ControllerPath) + cgroups, ok := d.cgroups.Get(pid).(cgroup.PathList) if ok { d.log.Debugf("Using cached cgroups for pid=%v", pid) return cgroups, nil @@ -276,7 +276,7 @@ func (d *addDockerMetadata) getProcessCgroups(pid int) (map[string]cgroup.Contro cgroups, err := processCgroupPaths(d.hostFS, pid) if err != nil { - return nil, errors.Wrapf(err, "failed to read cgroups for pid=%v", pid) + return cgroups, errors.Wrapf(err, "failed to read cgroups for pid=%v", pid) } d.cgroups.Put(pid, cgroups) @@ -287,8 +287,8 @@ func (d *addDockerMetadata) getProcessCgroups(pid int) (map[string]cgroup.Contro // of them are associated with Docker. For cgroups V1, Docker uses /docker/ when // naming cgroups and we use this to determine the container ID. For V2, // it's part of a more complex string. -func getContainerIDFromCgroups(cgroups map[string]cgroup.ControllerPath) (string, error) { - for _, path := range cgroups { +func getContainerIDFromCgroups(cgroups cgroup.PathList) (string, error) { + for _, path := range cgroups.Flatten() { re := regexp.MustCompile(`[\w]{64}`) rs := re.FindStringSubmatch(path.ControllerPath) if rs != nil { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 9df03d052d31..625d5a6ec128 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -38,21 +38,26 @@ import ( func init() { // Stub out the procfs. - processCgroupPaths = func(_ string, pid int) (map[string]cgroup.ControllerPath, error) { + processCgroupPaths = func(_ string, pid int) (cgroup.PathList, error) { + switch pid { case 1000: - return map[string]cgroup.ControllerPath{ - "cpu": {ControllerPath: "/docker/8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b", IsV2: false}, + return cgroup.PathList{ + V1: map[string]cgroup.ControllerPath{ + "cpu": {ControllerPath: "/docker/8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b", IsV2: false}, + }, }, nil case 2000: - return map[string]cgroup.ControllerPath{ - "memory": {ControllerPath: "/user.slice", IsV2: false}, + return cgroup.PathList{ + V1: map[string]cgroup.ControllerPath{ + "memory": {ControllerPath: "/user.slice", IsV2: false}, + }, }, nil case 3000: // Parser error (hopefully this never happens). - return nil, fmt.Errorf("cgroup parse failure") + return cgroup.PathList{}, fmt.Errorf("cgroup parse failure") default: - return nil, os.ErrNotExist + return cgroup.PathList{}, os.ErrNotExist } } } diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index e3ecf49ff35e..d9302bebd75e 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -88,6 +88,14 @@ func isKubernetesAvailableWithRetry(client k8sclient.Interface) bool { } } +// kubernetesMetadataExist checks whether an event is already enriched with kubernetes metadata +func kubernetesMetadataExist(event *beat.Event) bool { + if _, err := event.GetValue("kubernetes"); err != nil { + return false + } + return true +} + // New constructs a new add_kubernetes_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { config, err := newProcessorConfig(cfg, Indexing) @@ -251,6 +259,10 @@ func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) { if !k.kubernetesAvailable { return event, nil } + if kubernetesMetadataExist(event) { + k.log.Debug("Skipping add_kubernetes_metadata processor as kubernetes metadata already exist") + return event, nil + } index := k.matchers.MetadataIndex(event.Fields) if index == "" { k.log.Debug("No container match string, not adding kubernetes data") diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go index ed77c8a5d182..063beedcef1b 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -// Test metadata updates don't replace existing pod metrics -func TestAnnotatorDeepUpdate(t *testing.T) { +// Test Annotator is skipped if kubernetes metadata already exist +func TestAnnotatorSkipped(t *testing.T) { cfg := common.MustNewConfigFrom(map[string]interface{}{ "lookup_fields": []string{"kubernetes.pod.name"}, }) @@ -53,8 +53,7 @@ func TestAnnotatorDeepUpdate(t *testing.T) { "kubernetes": common.MapStr{ "pod": common.MapStr{ "labels": common.MapStr{ - "dont": "replace", - "original": "fields", + "added": "should not", }, }, }, @@ -85,10 +84,6 @@ func TestAnnotatorDeepUpdate(t *testing.T) { "a": 1, "b": 2, }, - "labels": common.MapStr{ - "dont": "replace", - "original": "fields", - }, }, }, }, event.Fields) diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go index 8b0c29e8af97..eef6e91f59fe 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata_test.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata_test.go @@ -74,39 +74,47 @@ func TestAddProcessMetadata(t *testing.T) { } // mock of the cgroup processCgroupPaths - processCgroupPaths = func(_ string, pid int) (map[string]cgroup.ControllerPath, error) { - testMap := map[int]map[string]cgroup.ControllerPath{ + processCgroupPaths = func(_ string, pid int) (cgroup.PathList, error) { + + testMap := map[int]cgroup.PathList{ 1: { - "cpu": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "net_prio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "blkio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "perf_event": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "freezer": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "pids": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "hugetlb": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "cpuacct": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "cpuset": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "net_cls": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "devices": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "memory": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "name=systemd": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + V1: map[string]cgroup.ControllerPath{ + + "cpu": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "net_prio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "blkio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "perf_event": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "freezer": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "pids": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "hugetlb": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "cpuacct": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "cpuset": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "net_cls": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "devices": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "memory": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "name=systemd": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + }, }, 2: { - "cpu": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "net_prio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "blkio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "perf_event": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "freezer": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "pids": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "hugetlb": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "cpuacct": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "cpuset": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "net_cls": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "devices": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "memory": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, - "name=systemd": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + V1: map[string]cgroup.ControllerPath{ + + "cpu": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "net_prio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "blkio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "perf_event": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "freezer": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "pids": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "hugetlb": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "cpuacct": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "cpuset": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "net_cls": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "devices": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "memory": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + "name=systemd": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, + }, }, } + return testMap[pid], nil } @@ -738,9 +746,10 @@ func TestUsingCache(t *testing.T) { selfPID := os.Getpid() // mock of the cgroup processCgroupPaths - processCgroupPaths = func(_ string, pid int) (map[string]cgroup.ControllerPath, error) { - testMap := map[int]map[string]cgroup.ControllerPath{ - selfPID: { + processCgroupPaths = func(_ string, pid int) (cgroup.PathList, error) { + testStruct := cgroup.PathList{ + V1: map[string]cgroup.ControllerPath{ + "cpu": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, "net_prio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, "blkio": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, @@ -756,6 +765,11 @@ func TestUsingCache(t *testing.T) { "name=systemd": {ControllerPath: "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1"}, }, } + testMap := map[int]cgroup.PathList{ + selfPID: testStruct, + } + + //testMap := return testMap[pid], nil } @@ -1031,9 +1045,11 @@ func TestPIDToInt(t *testing.T) { } func TestV2CID(t *testing.T) { - processCgroupPaths = func(_ string, _ int) (map[string]cgroup.ControllerPath, error) { - testMap := map[string]cgroup.ControllerPath{ - "cpu": {IsV2: true, ControllerPath: "system.slice/docker-2dcbab615aebfa9313feffc5cfdacd381543cfa04c6be3f39ac656e55ef34805.scope"}, + processCgroupPaths = func(_ string, _ int) (cgroup.PathList, error) { + testMap := cgroup.PathList{ + V1: map[string]cgroup.ControllerPath{ + "cpu": cgroup.ControllerPath{IsV2: true, ControllerPath: "system.slice/docker-2dcbab615aebfa9313feffc5cfdacd381543cfa04c6be3f39ac656e55ef34805.scope"}, + }, } return testMap, nil } diff --git a/libbeat/processors/add_process_metadata/gosigar_cid_provider.go b/libbeat/processors/add_process_metadata/gosigar_cid_provider.go index 2a1d934117c8..98aafaebd2f2 100644 --- a/libbeat/processors/add_process_metadata/gosigar_cid_provider.go +++ b/libbeat/processors/add_process_metadata/gosigar_cid_provider.go @@ -40,7 +40,7 @@ type gosigarCidProvider struct { cgroupPrefixes []string cgroupRegex string cidRegex *regexp.Regexp - processCgroupPaths func(string, int) (map[string]cgroup.ControllerPath, error) + processCgroupPaths func(string, int) (cgroup.PathList, error) pidCidCache *common.Cache } @@ -72,7 +72,7 @@ func (p gosigarCidProvider) GetCid(pid int) (result string, err error) { return cid, nil } -func newCidProvider(hostPath string, cgroupPrefixes []string, cgroupRegex string, processCgroupPaths func(string, int) (map[string]cgroup.ControllerPath, error), pidCidCache *common.Cache) gosigarCidProvider { +func newCidProvider(hostPath string, cgroupPrefixes []string, cgroupRegex string, processCgroupPaths func(string, int) (cgroup.PathList, error), pidCidCache *common.Cache) gosigarCidProvider { return gosigarCidProvider{ log: logp.NewLogger(providerName), hostPath: hostPath, @@ -86,9 +86,9 @@ func newCidProvider(hostPath string, cgroupPrefixes []string, cgroupRegex string // getProcessCgroups returns a mapping of cgroup subsystem name to path. It // returns an error if it failed to retrieve the cgroup info. -func (p gosigarCidProvider) getProcessCgroups(pid int) (map[string]cgroup.ControllerPath, error) { +func (p gosigarCidProvider) getProcessCgroups(pid int) (cgroup.PathList, error) { - var cgroup map[string]cgroup.ControllerPath + var cgroup cgroup.PathList cgroup, err := p.processCgroupPaths(p.hostPath, pid) switch err.(type) { @@ -111,11 +111,11 @@ func (p gosigarCidProvider) getProcessCgroups(pid int) (map[string]cgroup.Contro // /kubepods/besteffort/pod9b9e44c2-00fd-11ea-95e9-080027421ddf/2bb9fd4de339e5d4f094e78bb87636004acfe53f5668104addc761fe4a93588e // V2 Example: // /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod1f306eaea646903787fd7cc4eb6be515.slice/crio-eac98011dea91157038ca797f2141754106e074b7242721c7170a642a2204a54.scope -func (p gosigarCidProvider) getCid(cgroups map[string]cgroup.ControllerPath) string { +func (p gosigarCidProvider) getCid(cgroups cgroup.PathList) string { // if regex defined use it to find cid if len(p.cgroupRegex) != 0 { re := regexp.MustCompile(p.cgroupRegex) - for _, path := range cgroups { + for _, path := range cgroups.Flatten() { rs := re.FindStringSubmatch(path.ControllerPath) if rs != nil { return rs[1] @@ -126,7 +126,7 @@ func (p gosigarCidProvider) getCid(cgroups map[string]cgroup.ControllerPath) str // In an attempt to not break the user-facing config interface for this processor, // fall back to the config'ed prefixes if we have cgv1, otherwise use regex // This should work with k8s on cgroupsV2, as we're still trying to extract the same container ID - for _, path := range cgroups { + for _, path := range cgroups.Flatten() { if path.IsV2 { rs := p.cidRegex.FindStringSubmatch(path.ControllerPath) if len(rs) > 0 { diff --git a/libbeat/service/service_unix.go b/libbeat/service/service_unix.go index 7d20b04620e6..68851d7e54f3 100644 --- a/libbeat/service/service_unix.go +++ b/libbeat/service/service_unix.go @@ -25,3 +25,6 @@ func ProcessWindowsControlEvents(stopCallback func()) { func notifyWindowsServiceStopped() { } + +// WaitExecutionDone is not used on non-windows platforms. +func WaitExecutionDone() {} diff --git a/libbeat/service/service_windows.go b/libbeat/service/service_windows.go index a81f4fb5a0ff..027011911c20 100644 --- a/libbeat/service/service_windows.go +++ b/libbeat/service/service_windows.go @@ -29,13 +29,15 @@ import ( ) type beatService struct { - stopCallback func() - done chan struct{} + stopCallback func() + done chan struct{} + executeFinished chan struct{} } var serviceInstance = &beatService{ - stopCallback: nil, - done: make(chan struct{}, 0), + stopCallback: nil, + done: make(chan struct{}, 0), + executeFinished: make(chan struct{}, 0), } // Execute runs the beat service with the arguments and manages changes that @@ -85,6 +87,8 @@ const couldNotConnect syscall.Errno = 1063 // stopCallback function is called when the Stop/Shutdown // request is received. func ProcessWindowsControlEvents(stopCallback func()) { + defer close(serviceInstance.executeFinished) + isInteractive, err := svc.IsAnInteractiveSession() if err != nil { logp.Err("IsAnInteractiveSession: %v", err) @@ -125,3 +129,17 @@ func ProcessWindowsControlEvents(stopCallback func()) { logp.Err("Windows service setup failed: %+v", err) } + +// WaitExecutionDone returns only after stop was reported to service manager. +// If response is not retrieved within 500 millisecond wait is aborted. +func WaitExecutionDone() { + if isWinService, err := svc.IsWindowsService(); err != nil || !isWinService { + // not a service, don't wait + return + } + + select { + case <-serviceInstance.executeFinished: + case <-time.After(500 * time.Millisecond): + } +} diff --git a/metricbeat/docs/images/metricbeat-aws-rds-overview.png b/metricbeat/docs/images/metricbeat-aws-rds-overview.png index e7830e44d4a6..99ca7e2fe755 100644 Binary files a/metricbeat/docs/images/metricbeat-aws-rds-overview.png and b/metricbeat/docs/images/metricbeat-aws-rds-overview.png differ diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go index 6d435f5e3fc0..9928f34bdf5b 100644 --- a/metricbeat/module/system/process/process.go +++ b/metricbeat/module/system/process/process.go @@ -21,6 +21,7 @@ package process import ( "fmt" + "os" "runtime" "github.com/pkg/errors" @@ -90,6 +91,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { perCPU: config.IncludePerCPU, IsAgent: systemModule.IsAgent, } + + // If hostfs is set, we may not want to force the hierarchy override, as the user could be expecting a custom path. + if len(paths.Paths.Hostfs) < 2 { + override, isset := os.LookupEnv("LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE") + if isset { + m.stats.CgroupOpts.CgroupsHierarchyOverride = override + } + } + err := m.stats.Init() if err != nil { return nil, err diff --git a/metricbeat/module/system/process/process_test.go b/metricbeat/module/system/process/process_test.go index 76e152a22e47..36cdb291ef80 100644 --- a/metricbeat/module/system/process/process_test.go +++ b/metricbeat/module/system/process/process_test.go @@ -20,11 +20,13 @@ package process import ( + "runtime" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) @@ -36,8 +38,27 @@ func TestFetch(t *testing.T) { if !assert.NotEmpty(t, events) { t.FailNow() } - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), - events[0].BeatEvent("system", "process").Fields.StringToPrint()) + + // We have root cgroups disabled + // This will pick a "populated" event to print + if runtime.GOOS == "linux" { + for _, evt := range events { + field := evt.BeatEvent("system", "process").Fields["system"].(common.MapStr)["process"].(common.MapStr)["cgroup"].(common.MapStr)["cpu"] + if field == nil { + continue + } + if field.(map[string]interface{})["path"].(string) == "/" { + continue + } + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + evt.BeatEvent("system", "process").Fields.StringToPrint()) + return + } + } else { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("system", "process").Fields.StringToPrint()) + } + } func TestData(t *testing.T) { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index c9f02de59fbb..cf24a932a919 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -59,10 +59,14 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { } } -func run(streams *cli.IOStreams, override cfgOverrider) error { // Windows: Mark service as stopped. +func run(streams *cli.IOStreams, override cfgOverrider) error { + // Windows: Mark service as stopped. // After this is run, the service is considered by the OS to be stopped. // This must be the first deferred cleanup task (last to execute). - defer service.NotifyTermination() + defer func() { + service.NotifyTermination() + service.WaitExecutionDone() + }() locker := filelock.NewAppLocker(paths.Data(), paths.AgentLockFileName) if err := locker.TryLock(); err != nil { diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/utils..go b/x-pack/elastic-agent/pkg/agent/transpiler/utils.go similarity index 100% rename from x-pack/elastic-agent/pkg/agent/transpiler/utils..go rename to x-pack/elastic-agent/pkg/agent/transpiler/utils.go diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index dbeb34d1ef2c..10be92ca4100 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -304,12 +304,13 @@ sqs:ChangeMessageVisibility sqs:DeleteMessage ---- -Reduced specific AWS permissions are required for IAM user to access S3 +Reduced specific S3 AWS permissions are required for IAM user to access S3 when using the polling list of S3 bucket objects: ---- s3:GetObject s3:ListBucket +s3:GetBucketLocation ---- [float] diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index b9da841f7213..6b5444be0558 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -103,10 +103,9 @@ The state has the following elements: - `last_response.page`: A number indicating the page number of the last response. - `first_event`: A map representing the first event sent to the output (result from applying transforms to `last_response.body`). - `last_event`: A map representing the last event sent to the output (result from applying transforms to `last_response.body`). -- `url.value`: The full URL with params and fragments. -- `url.params`: A map containing the URL params. -- `header`: A map containing the headers. References request headers when used in <> or <> configuration sections, and to the last response headers when used in <>, <> or <> configuration sections. -- `body`: A map containing the body. References request body when used in <> configuration section, and to the last response body when used in <>, <> or <> configuration sections. +- `url`: The last requested URL as a raw https://pkg.go.dev/net/url#URL[`url.URL`] Go type. +- `header`: A map containing the headers. References the next request headers when used in <> or <> configuration sections, and to the last response headers when used in <>, <>, or <> configuration sections. +- `body`: A map containing the body. References the next request body when used in <> or <> configuration sections, and to the last response body when used in <> or <> configuration sections. - `cursor`: A map containing any data the user configured to be stored between restarts (See <>). All of the mentioned objects are only stored at runtime, except `cursor`, which has values that are persisted between restarts. @@ -204,6 +203,10 @@ Some built-in helper functions are provided to work with the input state inside - `mul`: multiplies two integers. - `div`: does the integer division of two integer values. - `hmac`: calculates the hmac signature of a list of strings concatenated together. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` +- `base64Encode`: Joins and base64 encodes all supplied strings. Example `[[base64Encode "string1" "string2"]]` +- `base64EncodeNoPad`: Joins and base64 encodes all supplied strings without padding. Example `[[base64EncodeNoPad "string1" "string2"]]` +- `join`: joins a list of strings using the specified separator. Example: `[[join .body.arr ","]]` +- `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]` In addition to the provided functions, any of the native functions for https://golang.org/pkg/time/#Time[`time.Time`], https://golang.org/pkg/net/http/#Header[`http.Header`], and https://golang.org/pkg/net/url/#Values[`url.Values`] types can be used on the corresponding objects. Examples: `[[(now).Day]]`, `[[.last_response.header.Get "key"]]` @@ -461,7 +464,7 @@ List of transforms to apply to the request before each execution. Available transforms for request: [`append`, `delete`, `set`]. -Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`]. +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`, `.body.*`]. Can write state to: [`header.*`, `url.params.*`, `body.*`]. @@ -581,7 +584,7 @@ List of transforms that will be applied to the response to every new page reques Available transforms for pagination: [`append`, `delete`, `set`]. -Can read state from: [`.last_response.*`, `.first_event.*`, `.last_event.*`, `.cursor.*`]. +Can read state from: [`.last_response.*`, `.first_event.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`, `.body.*`]. Can write state to: [`body.*`, `header.*`, `url.*`]. diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 6662984e060e..545c24db3221 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -140,9 +140,9 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { if in.config.BucketARN != "" { // Create S3 receiver and S3 notification processor. - poller, err := in.createS3Lister(inputContext, client, persistentStore, states) + poller, err := in.createS3Lister(inputContext, ctx, client, persistentStore, states) if err != nil { - return fmt.Errorf("failed to initialize sqs receiver: %w", err) + return fmt.Errorf("failed to initialize s3 poller: %w", err) } defer poller.metrics.Close() @@ -193,14 +193,23 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsRe return sqsReader, nil } -func (in *s3Input) createS3Lister(ctx v2.Context, client beat.Client, persistentStore *statestore.Store, states *states) (*s3Poller, error) { +func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, client beat.Client, persistentStore *statestore.Store, states *states) (*s3Poller, error) { s3ServiceName := "s3" if in.config.FIPSEnabled { s3ServiceName = "s3-fips" } + s3Client := s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)) + regionName, err := getRegionForBucketARN(cancelCtx, s3Client, in.config.BucketARN) + if err != nil { + return nil, fmt.Errorf("failed to get AWS region for bucket_arn: %w", err) + } + + in.awsConfig.Region = regionName + s3Client = s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)) + s3API := &awsS3API{ - client: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)), + client: s3Client, } log := ctx.Logger.With("bucket_arn", in.config.BucketARN) @@ -246,3 +255,19 @@ func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) { } return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") } + +func getRegionForBucketARN(ctx context.Context, s3Client *s3.Client, bucketARN string) (string, error) { + bucketMetadata := strings.Split(bucketARN, ":") + bucketName := bucketMetadata[len(bucketMetadata)-1] + + req := s3Client.GetBucketLocationRequest(&s3.GetBucketLocationInput{ + Bucket: awssdk.String(bucketName), + }) + + resp, err := req.Send(ctx) + if err != nil { + return "", err + } + + return string(resp.LocationConstraint), nil +} diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 4b8a3115db69..7b157dcfb6dd 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -19,16 +19,16 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/filebeat/beater" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/filebeat/beater" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -36,6 +36,7 @@ import ( pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) const ( @@ -366,3 +367,20 @@ func drainSQS(t *testing.T, tfConfig terraformOutputData) { } t.Logf("Drained %d SQS messages.", deletedCount) } + +func TestGetRegionForBucketARN(t *testing.T) { + logp.TestingSetup() + + // Terraform is used to setup S3 and must be executed manually. + tfConfig := getTerraformOutputs(t) + + awsConfig, err := external.LoadDefaultAWSConfig() + if err != nil { + t.Fatal(err) + } + + s3Client := s3.New(awscommon.EnrichAWSConfigWithEndpoint("", "s3", "", awsConfig)) + + regionName, err := getRegionForBucketARN(context.Background(), s3Client, tfConfig.BucketName) + assert.Equal(t, tfConfig.AWSRegion, regionName) +} diff --git a/x-pack/filebeat/input/awss3/states.go b/x-pack/filebeat/input/awss3/states.go index 6674ee104c10..70db34797dbf 100644 --- a/x-pack/filebeat/input/awss3/states.go +++ b/x-pack/filebeat/input/awss3/states.go @@ -144,7 +144,7 @@ func (s *states) Update(newState state, listingID string) { // No existing state found, add new one s.idx[id] = len(s.states) s.states = append(s.states, newState) - s.log.Debug("input", "New state added for %s", newState.ID) + s.log.Debug("New state added for ", newState.ID) } if listingID == "" || (!newState.Stored && !newState.Error) { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go index bbfdb1e53572..788a3f2d1afa 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go @@ -58,9 +58,9 @@ func TestInput(t *testing.T) { baseConfig: map[string]interface{}{ "interval": 1, "http_method": "GET", - "request.rate_limit.limit": `[[.last_request.header.Get "X-Rate-Limit-Limit"]]`, - "request.rate_limit.remaining": `[[.last_request.header.Get "X-Rate-Limit-Remaining"]]`, - "request.rate_limit.reset": `[[.last_request.header.Get "X-Rate-Limit-Reset"]]`, + "request.rate_limit.limit": `[[.last_response.header.Get "X-Rate-Limit-Limit"]]`, + "request.rate_limit.remaining": `[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`, + "request.rate_limit.reset": `[[.last_response.header.Get "X-Rate-Limit-Reset"]]`, }, handler: rateLimitHandler(), expected: []string{`{"hello":"world"}`}, @@ -320,6 +320,75 @@ func TestInput(t *testing.T) { handler: oauth2Handler, expected: []string{`{"hello": "world"}`}, }, + { + name: "Test request transforms can access state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + "/test-path" + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.url.path", + "value": `[[.url.Path]]`, + }, + }, + }, + }, + handler: defaultHandler("POST", `{"bar":"foo","url":{"path":"/test-path"}}`), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test response transforms can't access request state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + registerResponseTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 10, + "request.method": "GET", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + }, + "response.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, } for _, testCase := range testCases { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go index 5d457dea7ae2..569c0c84f186 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go @@ -102,9 +102,10 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { } tr := transformable{} - tr.setHeader(resp.Header) + ctx := emptyTransformContext() + ctx.updateLastResponse(response{header: resp.Header.Clone()}) - remaining, _ := r.remaining.Execute(emptyTransformContext(), tr, nil, r.log) + remaining, _ := r.remaining.Execute(ctx, tr, nil, r.log) if remaining == "" { return 0, errors.New("remaining value is empty") } @@ -122,7 +123,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { return 0, nil } - reset, _ := r.reset.Execute(emptyTransformContext(), tr, nil, r.log) + reset, _ := r.reset.Execute(ctx, tr, nil, r.log) if reset == "" { return 0, errors.New("reset value is empty") } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go index 7e41272a772b..8e9e0cb0db82 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go @@ -24,9 +24,9 @@ func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, @@ -47,9 +47,9 @@ func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, @@ -74,9 +74,9 @@ func TestGetRateLimitReturnsResetValue(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 0dd4f10e36ac..9f91fcdccce5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -9,8 +9,10 @@ import ( "crypto/hmac" "crypto/sha1" "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" + "fmt" "hash" "reflect" "regexp" @@ -54,6 +56,10 @@ func (t *valueTpl) Unpack(in string) error { "mul": mul, "div": div, "hmac": hmacString, + "base64Encode": base64Encode, + "base64EncodeNoPad": base64EncodeNoPad, + "join": strings.Join, + "sprintf": fmt.Sprintf, }). Delims(leftDelim, rightDelim). Parse(in) @@ -243,6 +249,24 @@ func div(a, b int64) int64 { return a / b } +func base64Encode(values ...string) string { + data := strings.Join(values, "") + if data == "" { + return "" + } + + return base64.StdEncoding.EncodeToString([]byte(data)) +} + +func base64EncodeNoPad(values ...string) string { + data := strings.Join(values, "") + if data == "" { + return "" + } + + return base64.RawStdEncoding.EncodeToString([]byte(data)) +} + func hmacString(hmacType string, hmacKey string, values ...string) string { data := strings.Join(values[:], "") if data == "" { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 2274a25614ae..4586dec77112 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -276,6 +276,61 @@ func TestValueTpl(t *testing.T) { expectedVal: "", expectedError: errEmptyTemplateResult.Error(), }, + { + name: "func base64Encode 2 strings", + value: `[[base64Encode "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "c3RyaW5nMXN0cmluZzI=", + }, + { + name: "func base64Encode no value", + value: `[[base64Encode ""]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "", + expectedError: errEmptyTemplateResult.Error(), + }, + { + name: "func join", + value: `[[join .last_response.body.arr ","]]`, + paramCtx: &transformContext{ + firstEvent: &common.MapStr{}, + lastEvent: &common.MapStr{}, + lastResponse: newTestResponse( + common.MapStr{ + "arr": []string{ + "foo", + "bar", + }, + }, + http.Header{}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "foo,bar", + }, + { + name: "func sprintf", + value: `[[sprintf "%q:%d" (join .last_response.body.arr ",") 1]]`, + paramCtx: &transformContext{ + firstEvent: &common.MapStr{}, + lastEvent: &common.MapStr{}, + lastResponse: newTestResponse( + common.MapStr{ + "arr": []string{ + "foo", + "bar", + }, + }, + http.Header{}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: `"foo,bar":1`, + }, } for _, tc := range cases { diff --git a/x-pack/filebeat/module/crowdstrike/falcon/ingest/pipeline.yml b/x-pack/filebeat/module/crowdstrike/falcon/ingest/pipeline.yml index 7e7efe5cd745..9aeb653b4888 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/ingest/pipeline.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/ingest/pipeline.yml @@ -284,9 +284,10 @@ processors: def args = Arrays.asList(/ /.split(commandLine)); args.removeIf(arg -> arg == ""); - ctx["process.command_line"] = commandLine; - ctx["process.args"] = args; - ctx["process.executable"] = args.get(0); + ctx['process'] = new HashMap(); + ctx.process.command_line = commandLine; + ctx.process.args = args; + ctx.process.executable = args.get(0); } } - pipeline: diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/3367c170-921f-11e9-aa19-159bf182e06f.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/3367c170-921f-11e9-aa19-159bf182e06f.json index d15b3b2f741a..d2a3ac4cbd41 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/3367c170-921f-11e9-aa19-159bf182e06f.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/3367c170-921f-11e9-aa19-159bf182e06f.json @@ -22,33 +22,16 @@ }, "gridData": { "h": 7, - "i": "1", - "w": 17, - "x": 7, - "y": 0 - }, - "panelIndex": "1", - "panelRefName": "panel_1", - "title": "Database Connections", - "type": "visualization", - "version": "7.11.0" - }, - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 8, "i": "5", - "w": 17, - "x": 7, - "y": 7 + "w": 6, + "x": 0, + "y": 8 }, "panelIndex": "5", "panelRefName": "panel_5", "title": "Transaction Blocked", "type": "visualization", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -56,17 +39,35 @@ "hidePanelTitles": false }, "gridData": { - "h": 15, + "h": 8, "i": "6", - "w": 7, + "w": 6, "x": 0, "y": 0 }, "panelIndex": "6", "panelRefName": "panel_6", - "title": "AWS Region Filter", + "title": "Filters", "type": "visualization", - "version": "7.11.0" + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "4abe34b0-e220-4d15-8da1-e32da039bd49", + "w": 18, + "x": 6, + "y": 0 + }, + "panelIndex": "4abe34b0-e220-4d15-8da1-e32da039bd49", + "panelRefName": "panel_4abe34b0-e220-4d15-8da1-e32da039bd49", + "title": "Database Connections", + "type": "lens", + "version": "7.14.0" }, { "embeddableConfig": { @@ -84,7 +85,7 @@ "panelRefName": "panel_4d89e43f-299c-4f43-bde2-0ada0983ff23", "title": "Free Storage", "type": "lens", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -102,7 +103,7 @@ "panelRefName": "panel_d409ab5d-84b5-4ecc-86ae-1f79a882b626", "title": "Read Latency", "type": "lens", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -244,8 +245,7 @@ "visualizationType": "lnsXY" }, "enhancements": {}, - "hidePanelTitles": false, - "type": "lens" + "hidePanelTitles": false }, "gridData": { "h": 15, @@ -274,9 +274,9 @@ }, "panelIndex": "c5476b0e-6a44-43e5-8bb4-0795c4d097c1", "panelRefName": "panel_c5476b0e-6a44-43e5-8bb4-0795c4d097c1", - "title": "Insert Throughput", + "title": "Read Throughput", "type": "lens", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -292,9 +292,9 @@ }, "panelIndex": "bf74bb77-3503-4682-9f0e-6df0994dce5d", "panelRefName": "panel_bf74bb77-3503-4682-9f0e-6df0994dce5d", - "title": "Select Throughput", + "title": "Write Throughput", "type": "lens", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -435,8 +435,7 @@ "visualizationType": "lnsXY" }, "enhancements": {}, - "hidePanelTitles": false, - "type": "lens" + "hidePanelTitles": false }, "gridData": { "h": 15, @@ -591,8 +590,7 @@ "visualizationType": "lnsXY" }, "enhancements": {}, - "hidePanelTitles": false, - "type": "lens" + "hidePanelTitles": false }, "gridData": { "h": 15, @@ -623,7 +621,7 @@ "panelRefName": "panel_29549114-6ebf-4047-aa56-bc035f66d3b4", "title": "Write IOPS", "type": "lens", - "version": "7.11.0" + "version": "7.14.0" }, { "embeddableConfig": { @@ -749,8 +747,7 @@ "visualizationType": "lnsXY" }, "enhancements": {}, - "hidePanelTitles": false, - "type": "lens" + "hidePanelTitles": false }, "gridData": { "h": 15, @@ -770,17 +767,12 @@ "title": "[Metricbeat AWS] RDS Overview", "version": 1 }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "3367c170-921f-11e9-aa19-159bf182e06f", "migrationVersion": { "dashboard": "7.14.0" }, "references": [ - { - "id": "17fcda50-921b-11e9-aa19-159bf182e06f", - "name": "1:panel_1", - "type": "visualization" - }, { "id": "00b29040-921d-11e9-aa19-159bf182e06f", "name": "5:panel_5", @@ -791,6 +783,11 @@ "name": "6:panel_6", "type": "visualization" }, + { + "id": "be8d8d30-09ae-11ec-8d29-f3a1bb757c14", + "name": "4abe34b0-e220-4d15-8da1-e32da039bd49:panel_4abe34b0-e220-4d15-8da1-e32da039bd49", + "type": "lens" + }, { "id": "68f36fb0-f0a0-11eb-b61f-a53cb4913361", "name": "4d89e43f-299c-4f43-bde2-0ada0983ff23:panel_4d89e43f-299c-4f43-bde2-0ada0983ff23", @@ -878,6 +875,6 @@ } ], "type": "dashboard", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTAsMV0=" + "updated_at": "2021-08-30T19:07:33.984Z", + "version": "WzEyMzQyLDJd" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/30fd4e40-f0a1-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/30fd4e40-f0a1-11eb-b61f-a53cb4913361.json index 20096caf1ede..3f19af015013 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/30fd4e40-f0a1-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/30fd4e40-f0a1-11eb-b61f-a53cb4913361.json @@ -124,10 +124,10 @@ "title": "RDS Write IOPS [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "30fd4e40-f0a1-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MjEsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTksMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/41e37710-f09d-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/41e37710-f09d-11eb-b61f-a53cb4913361.json index f9de1b6663af..e654c45d0a84 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/41e37710-f09d-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/41e37710-f09d-11eb-b61f-a53cb4913361.json @@ -17,10 +17,10 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Select Throughput Count/Second", + "label": "Write Throughput MB/Second", "operationType": "formula", "params": { - "formula": "average(aws.rds.throughput.select)", + "formula": "average(aws.rds.throughput.write)", "isFormulaBroken": false }, "references": [ @@ -35,7 +35,7 @@ "label": "Part of Select Throughput Count/Second", "operationType": "average", "scale": "ratio", - "sourceField": "aws.rds.throughput.select" + "sourceField": "aws.rds.throughput.write" }, "bc08fa3e-ce15-4acd-a0fd-c5c5c5452441": { "dataType": "string", @@ -121,13 +121,13 @@ } } }, - "title": "RDS Select Throughput[Metricbeat AWS]", + "title": "RDS Write Throughput[Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "41e37710-f09d-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTgsMV0=" + "updated_at": "2021-08-25T14:42:42.329Z", + "version": "Wzc1NDMsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/68f36fb0-f0a0-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/68f36fb0-f0a0-11eb-b61f-a53cb4913361.json index 4f60c819e780..e59f46f45bc4 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/68f36fb0-f0a0-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/68f36fb0-f0a0-11eb-b61f-a53cb4913361.json @@ -82,10 +82,10 @@ "title": "RDS Free Storage Bytes [Metricbeat AWS]", "visualizationType": "lnsDatatable" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "68f36fb0-f0a0-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -100,6 +100,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTQsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTIsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/8560b400-f096-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/8560b400-f096-11eb-b61f-a53cb4913361.json index cdf5eeb78fc7..99b0af1876b5 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/8560b400-f096-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/8560b400-f096-11eb-b61f-a53cb4913361.json @@ -124,10 +124,10 @@ "title": "RDS Write Latency in Milliseconds [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "8560b400-f096-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTYsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTQsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/b0077d10-f09b-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/b0077d10-f09b-11eb-b61f-a53cb4913361.json index aaf3627aeece..793360c8b918 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/b0077d10-f09b-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/b0077d10-f09b-11eb-b61f-a53cb4913361.json @@ -1,6 +1,6 @@ { "attributes": { - "description": null, + "description": "", "state": { "datasourceStates": { "indexpattern": { @@ -34,10 +34,10 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Insert Throughput Count/Second", + "label": "Read Throughput MB/Second", "operationType": "average", "scale": "ratio", - "sourceField": "aws.rds.throughput.insert" + "sourceField": "aws.rds.throughput.read" }, "e676afd3-ebd5-434a-85d7-a1a708b9a32f": { "dataType": "date", @@ -105,13 +105,13 @@ } } }, - "title": "RDS Insert Throughput [Metricbeat AWS]", + "title": "RDS Read Throughput [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "b0077d10-f09b-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -126,6 +126,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTcsMV0=" + "updated_at": "2021-08-25T14:41:59.837Z", + "version": "Wzc1MjQsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/bbc80900-f09e-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/bbc80900-f09e-11eb-b61f-a53cb4913361.json index 3854dbb889b3..38ea288c7ecf 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/bbc80900-f09e-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/bbc80900-f09e-11eb-b61f-a53cb4913361.json @@ -124,10 +124,10 @@ "title": "RDS Disk Queue Depth [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "bbc80900-f09e-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MjAsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTgsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/be8d8d30-09ae-11ec-8d29-f3a1bb757c14.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/be8d8d30-09ae-11ec-8d29-f3a1bb757c14.json new file mode 100644 index 000000000000..7b7778961c6a --- /dev/null +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/be8d8d30-09ae-11ec-8d29-f3a1bb757c14.json @@ -0,0 +1,130 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e6ccc957-e33e-4fb3-956e-746522e233ef": { + "columnOrder": [ + "4d571997-8dea-491a-b508-6317613db7ac", + "5f3cdc7f-367e-4214-9d88-ee5a27f4d4e4", + "a5d75ee3-a37b-4f64-ab88-701d651a1b24" + ], + "columns": { + "4d571997-8dea-491a-b508-6317613db7ac": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "a5d75ee3-a37b-4f64-ab88-701d651a1b24", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 3 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "5f3cdc7f-367e-4214-9d88-ee5a27f4d4e4": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "a5d75ee3-a37b-4f64-ab88-701d651a1b24": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.rds.database_connections", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.database_connections" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "a5d75ee3-a37b-4f64-ab88-701d651a1b24" + ], + "layerId": "e6ccc957-e33e-4fb3-956e-746522e233ef", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "4d571997-8dea-491a-b508-6317613db7ac", + "xAccessor": "5f3cdc7f-367e-4214-9d88-ee5a27f4d4e4" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Database Connections [Metricbeat AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.14.0", + "id": "be8d8d30-09ae-11ec-8d29-f3a1bb757c14", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-e6ccc957-e33e-4fb3-956e-746522e233ef", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2021-08-30T16:30:52.205Z", + "version": "WzEyMTIyLDJd" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/c4419a90-f091-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/c4419a90-f091-11eb-b61f-a53cb4913361.json index 9934a3313cc6..39fcfddd23d4 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/c4419a90-f091-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/c4419a90-f091-11eb-b61f-a53cb4913361.json @@ -124,10 +124,10 @@ "title": "RDS CPU Total Pct [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "c4419a90-f091-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTksMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTcsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f757eba0-f0a0-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f757eba0-f0a0-11eb-b61f-a53cb4913361.json index e3587494777c..1fc04c8a7811 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f757eba0-f0a0-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f757eba0-f0a0-11eb-b61f-a53cb4913361.json @@ -108,10 +108,10 @@ "title": "RDS Read IOPS [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "f757eba0-f0a0-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -126,6 +126,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MjIsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MjAsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f9ff3450-f094-11eb-b61f-a53cb4913361.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f9ff3450-f094-11eb-b61f-a53cb4913361.json index c03c905c8ed8..44dd2e067565 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f9ff3450-f094-11eb-b61f-a53cb4913361.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/lens/f9ff3450-f094-11eb-b61f-a53cb4913361.json @@ -124,10 +124,10 @@ "title": "RDS Read Latency in Milliseconds [Metricbeat AWS]", "visualizationType": "lnsXY" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "f9ff3450-f094-11eb-b61f-a53cb4913361", "migrationVersion": { - "lens": "7.13.1" + "lens": "7.14.0" }, "references": [ { @@ -142,6 +142,6 @@ } ], "type": "lens", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTUsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTMsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/00b29040-921d-11e9-aa19-159bf182e06f.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/00b29040-921d-11e9-aa19-159bf182e06f.json index ed0363b97494..4ded4c9baa58 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/00b29040-921d-11e9-aa19-159bf182e06f.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/00b29040-921d-11e9-aa19-159bf182e06f.json @@ -34,7 +34,7 @@ }, { "background_color": "rgba(164,221,0,1)", - "id": "6bb4d634-1d42-47fc-9c42-3f8269c8f509", + "id": "9e88cbc5-05c5-4a95-ac8d-2ee74816e8df", "operator": "empty", "value": null } @@ -91,13 +91,13 @@ "type": "metrics" } }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "00b29040-921d-11e9-aa19-159bf182e06f", "migrationVersion": { "visualization": "7.14.0" }, "references": [], "type": "visualization", - "updated_at": "2021-08-04T16:30:34.757Z", - "version": "WzM0MTIsMV0=" + "updated_at": "2021-08-04T21:05:50.194Z", + "version": "WzU1MTEsMl0=" } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/b5308940-7347-11e9-816b-07687310a99a.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/b5308940-7347-11e9-816b-07687310a99a.json index 2394e2459d62..3363af0d8733 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/b5308940-7347-11e9-816b-07687310a99a.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/visualization/b5308940-7347-11e9-816b-07687310a99a.json @@ -31,17 +31,32 @@ }, "parent": "", "type": "list" + }, + { + "fieldName": "aws.dimensions.DBInstanceIdentifier", + "id": "1629902640782", + "indexPatternRefName": "control_1_index_pattern", + "label": "DB Identifier", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" } ], "pinFilters": false, "updateFiltersOnChange": true, "useTimeFilter": false }, - "title": "AWS Region Filter", + "title": "AWS Region Filter [Metricbeat AWS]", "type": "input_control_vis" } }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "7.14.0", "id": "b5308940-7347-11e9-816b-07687310a99a", "migrationVersion": { "visualization": "7.14.0" @@ -51,9 +66,14 @@ "id": "metricbeat-*", "name": "control_0_index_pattern", "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_1_index_pattern", + "type": "index-pattern" } ], "type": "visualization", - "updated_at": "2021-08-04T16:30:39.859Z", - "version": "WzM0NjMsMV0=" + "updated_at": "2021-08-25T14:45:29.404Z", + "version": "Wzc2MDAsMl0=" } \ No newline at end of file