diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 500d7c8c54d..612d486d325 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -317,6 +317,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add filters and pie chart for AWS EC2 dashboard. {pull}10596[10596] - Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] - Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] +- Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a5715bba1a6..a72b37affdf 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1083,6 +1083,43 @@ type: integer The state of the instance (pending | running | shutting-down | terminated | stopping | stopped). +-- + +[float] +== s3_daily_storage fields + +`s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + + + +*`aws.s3_daily_storage.bucket.name`*:: ++ +-- +type: keyword + +Name of a S3 bucket. + + +-- + +*`aws.s3_daily_storage.bucket.size.bytes`*:: ++ +-- +type: scaled_float + +The amount of data in bytes stored in a bucket. + + +-- + +*`aws.s3_daily_storage.number_of_objects`*:: ++ +-- +type: long + +The total number of objects stored in a bucket for all storage classes. + + -- [float] @@ -1172,7 +1209,7 @@ The number of Amazon S3 SELECT Object Content requests made for objects in an Am -- -*`aws.s3_request.requests.select.scanned.bytes`*:: +*`aws.s3_request.requests.select_scanned.bytes`*:: + -- type: scaled_float @@ -1182,7 +1219,7 @@ The number of bytes of data scanned with Amazon S3 SELECT Object Content request -- -*`aws.s3_request.requests.select.returned.bytes`*:: +*`aws.s3_request.requests.select_returned.bytes`*:: + -- type: scaled_float diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 542c4f03792..242b5cc8c62 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs` and `s3_request`. +The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. [float] === Module-specific configuration notes @@ -72,8 +72,11 @@ image::./images/metricbeat-aws-ec2-overview.png[] Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. -=== s3_request metricset -Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +=== s3_request and s3_daily_storage metricset +Daily storage metrics for S3 buckets are reported once per day with no additional cost. Since they are daily metrics, +`period` for `s3_daily_storage` metricset is recommended to be `86400s` or multiples of `86400s`. +Request metrics are available +at 1-minute intervals with additional charges. The s3_request metricset will give more granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see @@ -104,6 +107,7 @@ metricbeat.modules: period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -117,12 +121,16 @@ The following metricsets are available: * <> +* <> + * <> * <> include::aws/ec2.asciidoc[] +include::aws/s3_daily_storage.asciidoc[] + include::aws/s3_request.asciidoc[] include::aws/sqs.asciidoc[] diff --git a/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc new file mode 100644 index 00000000000..d2a730cd656 --- /dev/null +++ b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-s3_daily_storage]] +=== aws s3_daily_storage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 485cab8fb1f..11108798182 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> +.4+| .4+| |<> +|<> beta[] |<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index c20490fb53b..761052d0df7 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 828b34b47d9..37dff29caae 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -161,6 +161,7 @@ metricbeat.modules: period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 3561b58b11f..18a720f84fd 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -11,6 +11,7 @@ period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 5b5ae11018f..8a864df0669 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs` and `s3_request`. +The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. [float] === Module-specific configuration notes @@ -65,8 +65,11 @@ image::./images/metricbeat-aws-ec2-overview.png[] Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. -=== s3_request metricset -Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +=== s3_request and s3_daily_storage metricset +Daily storage metrics for S3 buckets are reported once per day with no additional cost. Since they are daily metrics, +`period` for `s3_daily_storage` metricset is recommended to be `86400s` or multiples of `86400s`. +Request metrics are available +at 1-minute intervals with additional charges. The s3_request metricset will give more granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index caafbc01201..9e670c76d8d 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -11,7 +11,6 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" "github.com/stretchr/testify/assert" @@ -27,11 +26,6 @@ type MockEC2Client struct { ec2iface.EC2API } -// MockCloudWatchClient struct is used for unit tests. -type MockCloudWatchClient struct { - cloudwatchiface.CloudWatchAPI -} - var ( regionName = "us-west-1" instanceID = "i-123" diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index f65009a75f8..c5b0b471c5f 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzEWk1vIzcSvftXFHJKgHFjk5nswYcFJh5jYyCbeGMPctSUyJKaazbZwyIly5gfvyj2hyy5Zcu2WtHBMJot1nuPVcUqUqdwS6szwCWfAEQTLZ3Bd7jk704ANLEKpo7GuzP41wkAwBdc8heovE6WQHlrSUWGj39dQ+WdiT4YN4eKYjCKYRZ8lcfOrU96iVGVxQlAIEvIdAZzPAGYGbKaz/Lsp+Cwog6NfOKqlheDT3X7ZADU5iQPJyL1U/9saLKdEzafL6R++gLKu4jGMcSSem6xxAhLCgSsAtakt9j+JWxhWRpVricY0IjJRZiu8hcvzn8qHtjf1Kn7bFN9SFfVqYg+oi1qFTfe6MizQkt6MrMet194Qgf53JQENQVFLuKcwM8ArfUKI2kBDspXdYoEyZnYyoOBQKUQyEW7AuMgMYF3WUfjOKJTVOwkogJpEyeJcU4jcHGpmlIQHudXn6ExxsB1ux4PMcLMh/xWisaae5Rpn8U9RSvfHRU5YXCkNwg0wrs19hIZUKmQSAMbeWIiLJHBYnKqJA0+AEcMkfRuUpxCbRNPjkiuNbnJrMQFwZTIrVcKHSRnTWXEE3vay5IcyNfOrz6f5xl+aTDDAm0iMAz3FPy+jHmiSgxz0uNSzpwGiUssOR+hRqNB+6UT6o/X/x2g023aiWViME6lIBqh1kZQoIWGyjB1R3Hpw21hXFGjuqXIozJubUAgRWYhzugkr3QwwLhIYYaKeDson4bvUzwq/pzGfYqHwm9cMV1FGhd8tjCK9MfCfijZteFb44tAqEfB/kurNLZlgmDtUxVHHwgW3qaKGHCBxuLUEkS/P/JlMJFGhC7zR3KC6eDYs+q+PjTwc1/VlmRTyLr7mkLeufn1SyA1DEqWVmZmSEs9ZLwWd4ym2meBxmSZLTyk+cq1eg1JjhgTF6okdTuZobE7Nkrr3fxl/P6k2ofIsp/HksImUqltamQmDVMfy83BBhNkTHlXlFFecaRqc8w0FalFjlAZl+L+JCfNfEfmOgaRzs7fQGV4xfYl06cY5YP8SW6485EtYU7hzY2CD8S5IXg+v/WjpsI5FWY4Jm5ptfRhe2wPYJefclAKDJlfuistwdwU9i/Bt+5LC1mDYSd4Fc5Lp430iGtP0BSzxz1shg0DOclFOzqQHmgdzAIjFdrxRIYOK2g7O3z6/Tob7uR9VFXsidLUw564/fgF0C6vFh+klA/EDMjslckd+NK06e/FWNPUGjWWoHnyR3ru6ZUttAOq2AnX4riQ5GIUXF71I9+LwD/A1Cenu43xpZLmECqU18NqvjoR5Xm3NXwH0t/Dj/88nZoIybGZu9wHZyPPII2llEY8qSlMJLEdAS98X5PTEvTfICTnmv+4TDEaNz/Nne03iBQq47Jnf5O6pa7b9+Rf0j8Uj076+P0k0NdEHN904Lee5sG5X/tk1LO/6/eDR39Tivse/k2TtKLFYaP49zZsEa7fdxYGrbcacXP+eKAqQrwoz/dgE/715uaqtwYV6lzAooOPFd57t8b5DgLNMWjbRfyq3hG5PfY5DdcOr0O+hfnfFzdbuKWK8NP/5SN0Kbofc3gGb51GxHv1+eB4NUmrMh7kTxe/XdxcHBp1SXiobmYA868XHz/t5c/P+YLnMZ3hj+ttb3gVSiZLO64m3opzjeT64reL8xv4Iy86nHsXJdEe2CsaJgUrdI7GOTUaOvHyM9AYEVq7TeG3N/W3MA0UU/g7qHaGj8HVmjGjqMeWKwixlWsj1UDnZqN9CqeUR9ajHn8VmiVY28sBs9+2uyylNBJigbj2jqX6UzZpkkp16vVqmFyqj0mts9asRVumAfbFnuB89/JMRyH4wMWHu7vx3OjD3R0oa8Tbs7n+4MRr2muNmkjC9irMz4BM7s3/AT7Aj08S+3lMYj/f3QFTWFA4IjGLkZxaFTMTOE7EOYpq2Ptex7GmcNo5VTQVNc1CE/fN2e3a50jag/5CprncfMQx+uZ2cyPC8g1uPvOdUp8xnyacK+uu3TksZ7JYc3N2vIN7VjuH4ppv22/nI788khuk56Kv7wC/8ttav698xN96XP/3+q0Nn7eaOE4qYsY5TXBOBZM64CpiXQd/Zypp49vffIgsjV1w3p02Bb2GFkN3aPs1UdrRa7VvSiuAq4PdEtxsJpPWCG/gWd+jt7bzpYDz8cFVSLPJYT6fMFVF2mAku2O/6qg4HycLw2Zqx2ltejY9AeNgZs283LEHdcCOAmpbvBgMLdCuI30/ZxA3Ghdo56svAdalpnGR9QXudAUKreUuE/7ZmP9PG12odv/yqEMsOWbkBde6ydX4lIJU1XE1afU75NayRrSlzsery049iRNtmuBuxAXsFRqEK7L1mZTN/Th36flU1NzTM3r+PwAA//+efQFO" + return "eJzsWs1uIzcSvvspCjklwLixmZnswYcFJh5jYyCbeGMPctSUyJKaazbZwyIlazAPvyj2jyy5Zcu2WrlEB8MQW6zvK9Y/+xRuaXUGuOQTgGiipTP4Dpf83QmAJlbB1NF4dwb/OgEA+IxL/gyV18kSKG8tqcjw4c9rqLwz0Qfj5lBRDEYxzIKv8tq59UkvMaqyOAEIZAmZzmCOJwAzQ1bzWd79FBxW1KGRT1zV8mDwqW6/GQC1ucn9jUi97b8b2mznhs3nM6m3n0F5F9E4hlhSzy2WGGFJgYBVwJr0Fts/hS0sS6PK9QYDOmJyEaar/MOL87fFPfmbeuo+21Tv01V1KqKPaItaxY0nOvKs0JKezKzH7Qce0YN8bkqCmoIiF3FO4GeA1nqFkbQAB+WrOkWC5Exs1YOBQKUQyEW7AuMgMYF3WY/GcUSnqNhJRAXSJk4S45xG4OJSNaUgPM6vPkEjjIHr9jzuY4SZD/mpFI01X1G2fRL3FK38dlTkhMGR3iDQKN6tsZfIgEqFRBrYyDcmwhIZLCanStLgA3DEEEnvJsUp1Dbx5IjkWpGbzEpcEEyJ3Pqk0EFy1lRGLLGnvSzJgfzs/OrTed7h5wYzLNAmAsPwlYLflzFPVIlhTnpcypnTIHHxJecj1Gg0aL90Qv3h+b8BdLoNO7FMDMapFERHqLURFGihoTJM3VFc+nBbGFfUqG4p8qiMWxkQSJFZiDE6iSsdDDAuUpihIt52ysfh+xSPij+HcZ/iofAbV0xXkcYFnyWMovpjYT+U2rXhW+OLQKhHwf5zq2lsywTB2ocqjj4QLLxNFTHgAo3FqSWIfn/ky2AijQhd9o/kBNPBsWet+/rQwM99VVuSpJD17msKOXPzy49AahiUKK3MzJCWesh4LeYYTbXPAY3JMku4T/OFZ/USkhwxJi5USep2MkNjdyRK6938efz+oNqHyJLPY0lhE6nUNjUyk4apj+XmYoMJMqacFWWVVxyp2lwzTUVqkSNUxqW4P8lJs9+RuY5BpJPzF1AZPrF9yfQhRvkgf5Ib7nwkJcwpvLpR8IE4NwRPx7d+1VQ4p8IM+8QtrZY+bK/tAezyY3ZKgSH7S3elxZmbwv45+NZ9aSFnMGwEL8J56bSRHnFtCZpitrj7zbBhICexaEcH0gOtg1lgpEI7nsjSYRXa7g4ff7vOgjv1Pqgq9kRp6mFL3P76GdAurxbvpZQPxAzI7JXJHfjStOHv2VjT1Bo1lkLz5g/0uadVttAOqMVOcS2OCwkuRsHlVb/yvSj4B5j65HSXGJ+r0uxChfJ6WJsvDkR5320dvgHp7+HHf55OTYTk2Mxd7oOzkCeQxlJKI57UFCYS2I6AF76vyWlx+m8QknPNf1ymGI2bn+bO9htECpVx2bK/Sd1S1+1z8i/pH4oHkz5+N9Fo7GoiVc7mxOj5Y7/tzbZmgHkN2rVRJ4LX7wYHglOK+44Ep0ka1OKwvv1b68wI1+86CY9JZ/N1nOYkO3UlOV/gaIwodUPbG0q5q5tq9jGMTX6f+NnET/9HasfA4PklkGDLw9h7FUQrYQBbLimkXO/MSlmpkHjQ0gN9ScTxtTbebnPPuttv/rbpR+yl1RE3k/YRjeWXm5urXhpUqHOrhg4+VPjVuzXONxBojkHbLret6h05qsc+p+Eq+WXItzD/++JmC7cYd2f7YvQPOTyBt04j4r36dHC8mqQpHw/yx4tfL24uDo26JDxU3z6A+ZeLDx/3suenbMHzmMbw+/W2NbwIJZOlHZdwr8W5RnJ98evF+Q38ng8dzr2LEmgPbBUNkwkrdI7GmY8OzXa7hN7KbVqcvam/hmmgmMJfQbUTfAyu1ozpRT22XEGIrFw5qwY6N4n2MZzSCFiPevxTaI5gLS87zH5pd1lKaSTEAnHtHUufo2zSJD3Z1OvVMLlUH5NaJ605i7ZMA+yLPcH55vmRjkLwgYv3d3fjmdH7uztQ1oi1Z3H9iNBr2uuMGk/C9tLXz4BMnkL9A3yAHx8l9tOYxH66uwOmsKBwRGIWIzm1KmYmcJyIcRTVIbudmsJpZ1TRVNQ0C43fN7cUa5sjaQ/6q8fmGv8Bx+ibe/wND8vvKuTbjSn1EfNxwrmy7tqdw3ImizU3tyQ7uGdtZ1dc820nS3m4nVdyg/SU9/Ud4Bd+Xev3hY/4VtP1f69f2/B5q4njpCJmnNME51QwqQOeItZ18HemwkjQvt0kamnkgvPutCnoNbQYuuuJL4nSjl6rfVJaAVwd7D7sZjOYtEJ4A8/6jZFWdr7+cj7eu/RrkhzmSZypKtIGI9kd+aqj4nycLAybqR2ntenZ9ASMg5k183JHDuqAHQXUtvJiMLRAu/b0/YxBzGhcoJ2tPgdYF5rGRdYXuNMVKLSWu0j4RyP+P613odr9jl2HWGLMyAeudROr8TENUlXH1aTV3yFTyxrRlnY+XF122hM/0aZx7ka5gL2GBuGK2vpIOupgVjZ/Qp//DwAA//8xd9YQ" } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json new file mode 100644 index 00000000000..0cdfffa3aab --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "aws": { + "s3_daily_storage": { + "bucket": { + "name": "test-s3-ks", + "size": { + "bytes": 1679631 + } + }, + "number_of_objects": 3 + } + }, + "cloud": { + "provider": "aws", + "region": "ap-southeast-1" + }, + "event": { + "dataset": "aws.s3_daily_storage", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "s3_daily_storage" + }, + "service": { + "name": "s3_daily_storage", + "type": "s3_daily_storage" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc new file mode 100644 index 00000000000..fde92cc0964 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc @@ -0,0 +1,10 @@ +The s3_daily_storage metricset of aws module allows you to monitor your AWS S3 buckets. `s3_daily_storage` metricset +fetches Cloudwatch daily storage metrics for each S3 bucket from +https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html[S3 CloudWatch Daily Storage Metrics for Buckets]. + + === AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS s3_daily_storage metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +---- diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml new file mode 100644 index 00000000000..2678c1831bf --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml @@ -0,0 +1,18 @@ +- name: s3_daily_storage + type: group + description: > + `s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + release: beta + fields: + - name: bucket.name + type: keyword + description: > + Name of a S3 bucket. + - name: bucket.size.bytes + type: scaled_float + description: > + The amount of data in bytes stored in a bucket. + - name: number_of_objects + type: long + description: > + The total number of objects stored in a bucket for all storage classes. diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go new file mode 100644 index 00000000000..8fc988e11a3 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_daily_storage + +import ( + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schemaMetricSetFields = s.Schema{ + "bucket": s.Object{ + "size": s.Object{ + "bytes": c.Float("BucketSizeBytes"), + }, + }, + "number_of_objects": c.Int("NumberOfObjects"), + } +) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go new file mode 100644 index 00000000000..7110018bbf0 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -0,0 +1,228 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_daily_storage + +import ( + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +var metricsetName = "s3_daily_storage" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + logger *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The aws s3_daily_storage metricset is beta.") + s3Logger := logp.NewLogger(aws.ModuleName) + + moduleConfig := aws.Config{} + if err := base.Module().UnpackConfig(&moduleConfig); err != nil { + return nil, err + } + + if moduleConfig.Period == "" { + err := errors.New("period is not set in AWS module config") + s3Logger.Error(err) + } + + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + // Check if period is set to be multiple of 86400s + remainder := metricSet.PeriodInSec % 86400 + if remainder != 0 { + err := errors.New("period needs to be set to 86400s (or a multiple of 86400s). " + + "To avoid data missing or extra costs, please make sure period is set correctly " + + "in config.yml") + s3Logger.Info(err) + } + + return &MetricSet{ + MetricSet: metricSet, + logger: s3Logger, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + namespace := "AWS/S3" + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + err = errors.Wrap(err, "Error ParseDuration") + m.logger.Error(err.Error()) + report.Error(err) + return + } + + // GetMetricData for AWS S3 from Cloudwatch + for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + err = errors.Wrap(err, "GetListMetricsOutput failed, skipping region "+regionName) + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + if listMetricsOutputs == nil || len(listMetricsOutputs) == 0 { + continue + } + + metricDataQueries := constructMetricQueries(listMetricsOutputs, m.PeriodInSec) + // Use metricDataQueries to make GetMetricData API calls + metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) + m.logger.Error(err) + report.Error(err) + continue + } + + // Create Cloudwatch Events for s3_daily_storage + bucketNames := getBucketNames(listMetricsOutputs) + for _, bucketName := range bucketNames { + event, err := createCloudWatchEvents(metricDataOutputs, regionName, bucketName) + if err != nil { + err = errors.Wrap(err, "createCloudWatchEvents failed") + m.logger.Error(err) + event.Error = err + report.Event(event) + continue + } + report.Event(event) + } + } +} + +func getBucketNames(listMetricsOutputs []cloudwatch.Metric) (bucketNames []string) { + for _, output := range listMetricsOutputs { + for _, dim := range output.Dimensions { + if *dim.Name == "BucketName" { + if aws.StringInSlice(*dim.Value, bucketNames) { + continue + } + bucketNames = append(bucketNames, *dim.Value) + } + } + } + return +} + +func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery { + metricDataQueries := []cloudwatch.MetricDataQuery{} + metricDataQueryEmpty := cloudwatch.MetricDataQuery{} + metricNames := []string{"NumberOfObjects", "BucketSizeBytes"} + for i, listMetric := range listMetricsOutputs { + if !aws.StringInSlice(*listMetric.MetricName, metricNames) { + continue + } + + metricDataQuery := createMetricDataQuery(listMetric, periodInSec, i) + if metricDataQuery == metricDataQueryEmpty { + continue + } + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} + +func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + period := int64(periodInSec) + id := "s3d" + strconv.Itoa(index) + metricDims := metric.Dimensions + bucketName := "" + storageType := "" + for _, dim := range metricDims { + if *dim.Name == "BucketName" { + bucketName = *dim.Value + } else if *dim.Name == "StorageType" { + storageType = *dim.Value + } + } + metricName := *metric.MetricName + label := bucketName + " " + storageType + " " + metricName + + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} + +func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string) (event mb.Event, err error) { + event.Service = metricsetName + event.RootFields = common.MapStr{} + // Cloud fields in ECS + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + event.RootFields.Put("cloud.provider", "aws") + + // AWS s3_daily_storage metrics + mapOfMetricSetFieldResults := make(map[string]interface{}) + // Find a timestamp for all metrics in output + if len(outputs) > 0 && len(outputs[0].Timestamps) > 0 { + timestamp := outputs[0].Timestamps[0] + for _, output := range outputs { + labels := strings.Split(*output.Label, " ") + // check timestamp to make sure metrics come from the same timestamp + if len(labels) == 3 && labels[0] == bucketName && len(output.Values) > 0 && output.Timestamps[0] == timestamp { + mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + } + } + } + + resultMetricSetFields, err := aws.EventMapping(mapOfMetricSetFieldResults, schemaMetricSetFields) + if err != nil { + err = errors.Wrap(err, "Error trying to apply schema schemaMetricSetFields in AWS s3_daily_storage metricbeat module.") + return + } + + resultMetricSetFields.Put("bucket.name", bucketName) + event.MetricSetFields = resultMetricSetFields + return +} diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go new file mode 100644 index 00000000000..56408a14418 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go @@ -0,0 +1,60 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package s3_daily_storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestFetch(t *testing.T) { + config, info := mtest.GetConfigForTest("s3_daily_storage", "86400s") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } + + s3DailyMetricSet := mbtest.NewReportingMetricSetV2(t, config) + events, err := mbtest.ReportingFetchV2(s3DailyMetricSet) + if err != nil { + t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + + "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") + } + + assert.Empty(t, err) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("Module: %s Metricset: %s", s3DailyMetricSet.Module().Name(), s3DailyMetricSet.Name()) + + for _, event := range events { + // RootField + mtest.CheckEventField("service.name", "string", event, t) + mtest.CheckEventField("cloud.region", "string", event, t) + + // MetricSetField + mtest.CheckEventField("bucket.name", "string", event, t) + mtest.CheckEventField("bucket.size.bytes", "float", event, t) + mtest.CheckEventField("number_of_object", "int", event, t) + } +} + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("s3_daily_storage", "86400s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + ec2MetricSet := mbtest.NewReportingMetricSetV2(t, config) + errs := mbtest.WriteEventsReporterV2(ec2MetricSet, t, "/") + if errs != nil { + t.Fatal("write", errs) + } +} diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json index 4814b96c7db..22306231c82 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json @@ -29,6 +29,7 @@ } }, "cloud": { + "provider": "aws", "region": "ap-southeast-1" }, "event": { @@ -43,4 +44,4 @@ "name": "s3_request", "type": "s3_request" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml index a5469353f96..45ec464f8a3 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml @@ -36,11 +36,11 @@ type: long description: > The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. - - name: requests.select.scanned.bytes + - name: requests.select_scanned.bytes type: scaled_float description: > The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. - - name: requests.select.returned.bytes + - name: requests.select_returned.bytes type: scaled_float description: > The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. diff --git a/x-pack/metricbeat/module/aws/s3_request/data.go b/x-pack/metricbeat/module/aws/s3_request/data.go index 06e6f4cb290..0751ef522f1 100644 --- a/x-pack/metricbeat/module/aws/s3_request/data.go +++ b/x-pack/metricbeat/module/aws/s3_request/data.go @@ -19,8 +19,8 @@ var ( "head": c.Int("HeadRequests"), "post": c.Int("PostRequests"), "select": c.Int("SelectRequests"), - "select.scanned.bytes": c.Float("SelectScannedBytes"), - "select.returned.bytes": c.Float("SelectReturnedBytes"), + "select_scanned.bytes": c.Float("SelectScannedBytes"), + "select_returned.bytes": c.Float("SelectReturnedBytes"), "list": c.Int("ListRequests"), }, "downloaded": s.Object{ diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index eb81ba06b34..67f2cd32425 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -201,6 +201,7 @@ func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName str // Cloud fields in ECS event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.region", regionName) + event.RootFields.Put("cloud.provider", "aws") // AWS s3_request metrics mapOfMetricSetFieldResults := make(map[string]interface{}) diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go index 8a3d369216e..e731e3fb4b0 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go @@ -48,8 +48,8 @@ func TestFetch(t *testing.T) { mtest.CheckEventField("requests.head", "int", event, t) mtest.CheckEventField("requests.post", "int", event, t) mtest.CheckEventField("select.requests", "int", event, t) - mtest.CheckEventField("select.scanned.bytes", "float", event, t) - mtest.CheckEventField("select.returned.bytes", "float", event, t) + mtest.CheckEventField("select_scanned.bytes", "float", event, t) + mtest.CheckEventField("select_returned.bytes", "float", event, t) mtest.CheckEventField("requests.list", "int", event, t) mtest.CheckEventField("downloaded.bytes", "float", event, t) mtest.CheckEventField("uploaded.bytes", "float", event, t) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go index 65497b7fc18..c3d758c82d5 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// +build integration package sqs diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 3561b58b11f..18a720f84fd 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -11,6 +11,7 @@ period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}'