diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 61c87ca1364..168c8eb9f35 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -305,6 +305,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview dashboard to Zookeeper Metricbeat module {pull}10379[10379] - Add Consul Metricbeat module with Agent Metricset {pull}8631[8631] - Add filters and pie chart for AWS EC2 dashboard. {pull}10596[10596] +- Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 53da6d070ff..00080797d30 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1083,6 +1083,103 @@ type: integer The state of the instance (pending | running | shutting-down | terminated | stopping | stopped). +-- + +[float] +== sqs fields + +`sqs` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS SQS. + + + +*`aws.sqs.oldest_message_age.sec`*:: ++ +-- +type: long + +The approximate age of the oldest non-deleted message in the queue. + + +-- + +*`aws.sqs.message.delayed`*:: ++ +-- +type: long + +TThe number of messages in the queue that are delayed and not available for reading immediately. + + +-- + +*`aws.sqs.message.not_visible`*:: ++ +-- +type: long + +The number of messages that are in flight. + + +-- + +*`aws.sqs.message.visible`*:: ++ +-- +type: long + +The number of messages available for retrieval from the queue. + + +-- + +*`aws.sqs.message.deleted`*:: ++ +-- +type: long + +The number of messages deleted from the queue. + + +-- + +*`aws.sqs.message.received`*:: ++ +-- +type: long + +The number of messages returned by calls to the ReceiveMessage action. + + +-- + +*`aws.sqs.message.sent`*:: ++ +-- +type: long + +The number of messages added to a queue. + + +-- + +*`aws.sqs.empty_receives`*:: ++ +-- +type: long + +The number of ReceiveMessage API calls that did not return a message. + + +-- + +*`aws.sqs.sent_message_size.bytes`*:: ++ +-- +type: scaled_float + +The size of messages added to a queue. + + -- [[exported-fields-beat]] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index f6c5f0a40d2..8eae60fd964 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricset is `ec2`. +The default metricsets are `ec2` and `sqs`. [float] === Module-specific configuration notes @@ -35,20 +35,6 @@ aws> sts get-session-token --serial-number arn:aws:iam::1234:mfa/your-email@exam Specific permissions needs to be added into the IAM user's policy to authorize Metricbeat to collect AWS monitoring metrics. Please see documentation under each metricset for required permissions. -By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module -configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics -will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs -$0.01/1000 metrics requested using GetMetricData. Please see https://aws.amazon.com/cloudwatch/pricing/[AWS Cloudwatch Pricing] -for more details. To avoid unnecessary charges, `period` is preferred to be set to `300s` or multiples of `300s`, such as -`600s` and `900s`. - -For more granular monitoring data you can enable detailed monitoring on the instance to get metrics every 1 minute. Please see -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html[Enabling Detailed Monitoring] for instructions -on how to enable detailed monitoring. With detailed monitoring enabled, `period` in aws module configuration can be any number -larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods, setting metricbeat module `period` less -than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is -preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. - Here is an example of aws metricbeat module configuration: [source,yaml] @@ -56,15 +42,29 @@ Here is an example of aws metricbeat module configuration: metricbeat.modules: - module: aws period: 300s - metricsets: ["ec2"] + metricsets: + - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' default_region: '${AWS_REGION:us-west-1}' ---- -This module only collects metrics for EC2 instances that are in `running` state and exist more than 10 minutes to make sure -there are monitoring metrics exist in Cloudwatch already. +By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module +configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics +will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs +$0.01/1000 metrics requested using GetMetricData. Please see https://aws.amazon.com/cloudwatch/pricing/[AWS Cloudwatch Pricing] +for more details. To avoid unnecessary charges, `period` is preferred to be set to `300s` or multiples of `300s`, such as +`600s` and `900s`. For more granular monitoring data you can enable detailed monitoring on the instance to get metrics every 1 minute. Please see +https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html[Enabling Detailed Monitoring] for instructions +on how to enable detailed monitoring. With detailed monitoring enabled, `period` in aws module configuration can be any number +larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods, setting metricbeat module `period` less +than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is +preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. + +Since cloudWatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, +`period` for `sqs` is recommended to set to `300s` or multiples of `300s`. The AWS module comes with a predefined dashboard. For example: @@ -84,6 +84,7 @@ metricbeat.modules: period: 300s metricsets: - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -97,5 +98,9 @@ The following metricsets are available: * <> +* <> + include::aws/ec2.asciidoc[] +include::aws/sqs.asciidoc[] + diff --git a/metricbeat/docs/modules/aws/sqs.asciidoc b/metricbeat/docs/modules/aws/sqs.asciidoc new file mode 100644 index 00000000000..35f0373ab44 --- /dev/null +++ b/metricbeat/docs/modules/aws/sqs.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-sqs]] +=== aws sqs metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index d3285354038..7aef20c7339 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.1+| .1+| |<> +.2+| .2+| |<> +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .7+| .7+| |<> |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 4f662b04b82..4e75b505f43 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index cd90ff182a0..6f5f1327b1e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -152,6 +152,7 @@ metricbeat.modules: period: 300s metricsets: - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index ccb5317ce09..d831e5e094d 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -2,6 +2,7 @@ period: 300s metricsets: - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index a510dfd570c..6706525d8d0 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricset is `ec2`. +The default metricsets are `ec2` and `sqs`. [float] === Module-specific configuration notes @@ -28,20 +28,6 @@ aws> sts get-session-token --serial-number arn:aws:iam::1234:mfa/your-email@exam Specific permissions needs to be added into the IAM user's policy to authorize Metricbeat to collect AWS monitoring metrics. Please see documentation under each metricset for required permissions. -By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module -configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics -will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs -$0.01/1000 metrics requested using GetMetricData. Please see https://aws.amazon.com/cloudwatch/pricing/[AWS Cloudwatch Pricing] -for more details. To avoid unnecessary charges, `period` is preferred to be set to `300s` or multiples of `300s`, such as -`600s` and `900s`. - -For more granular monitoring data you can enable detailed monitoring on the instance to get metrics every 1 minute. Please see -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html[Enabling Detailed Monitoring] for instructions -on how to enable detailed monitoring. With detailed monitoring enabled, `period` in aws module configuration can be any number -larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods, setting metricbeat module `period` less -than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is -preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. - Here is an example of aws metricbeat module configuration: [source,yaml] @@ -49,15 +35,29 @@ Here is an example of aws metricbeat module configuration: metricbeat.modules: - module: aws period: 300s - metricsets: ["ec2"] + metricsets: + - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' default_region: '${AWS_REGION:us-west-1}' ---- -This module only collects metrics for EC2 instances that are in `running` state and exist more than 10 minutes to make sure -there are monitoring metrics exist in Cloudwatch already. +By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module +configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics +will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs +$0.01/1000 metrics requested using GetMetricData. Please see https://aws.amazon.com/cloudwatch/pricing/[AWS Cloudwatch Pricing] +for more details. To avoid unnecessary charges, `period` is preferred to be set to `300s` or multiples of `300s`, such as +`600s` and `900s`. For more granular monitoring data you can enable detailed monitoring on the instance to get metrics every 1 minute. Please see +https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html[Enabling Detailed Monitoring] for instructions +on how to enable detailed monitoring. With detailed monitoring enabled, `period` in aws module configuration can be any number +larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods, setting metricbeat module `period` less +than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is +preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. + +Since cloudWatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, +`period` for `sqs` is recommended to set to `300s` or multiples of `300s`. The AWS module comes with a predefined dashboard. For example: diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 36cc75e590f..15e67526b2a 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzEl0FvGzkPhu/5FURPLdAM8BUf9pDDAtu0h1wWwXaLHh1aoj1ENNJApDxw0R+/oDx2bNdpnKydnYNhjDTU876SKPES7ml5BTjIBYCyBrqCNzjImwsAT+Iy98opXsHvFwAAdzjIHXTJl0DgUgjkVOCPb1+gS5E1ZY5z6EgzO4FZTl1tuw6p+AHVtc0FQKZAKHQFc7wAmDEFL1c1+iVE7GhNY48ue+uYU+nHNwegdoNsByL3YfPuULBHA66eO3If7sClqMhRQFvaaNMWFQbKBOIy9uT31H4ztTC07NqHAAc8EooK02X98PP1h2Zr/F2f1s++1G25ri+NJsXQ9E53eqzFi8NAfjILCfc7/MIHe/5uCXrKjqLinCDNAENIDpW8gYNLXV+UoETW0R7MBK7kTFHDEjhCEYIUq48cRTE6ah4V4jJ51kkRnNMZtMTSTSmbjuvbr7AaTED6cT62GWGWcu1VlAN/Rwv7JPcUg317VnLCHMnvCFgZHx/YWxRA53IhD8L2hhUGFAhYomvJQ8ogilnJPy5KSu5DkckrihuH3FXW4oJgShQfZgojlBi4Y1uJG9lDSxHss+vbr9c1wscVMywwFAIW+E45HatYJq7FPCd/XslV00HhtpdiUuiRPfg0RJP+8/y/B4x+TDvaFgGOrmTzCL1no8AAKymHpUfSIeX7hmPTo7snlbMqHseATI54YYsxWl5ZYwBHpTxDR7K/KX+Nn4q+Kn9N46noqfg5NtOl0nnh6whnsf612E9lu2e559RkQn8W9o+j0zheE4x1k6pEUyZYpFA6EsAFcsBpINB0PPmQWemM6BZfKRrTydmr66k/Nfh16vpAdihU31NPuZ7c8vIpsDsMWpZ2PGPydh/i5G05KnfHTNA5VdYRtmW+cK5eIlIUtUjjWnL3kxlyeOSgDCnOn6fvL+pTVrHzXFvKu6R2t+lRhDxMk7a7jSsmqEz1VLRWWYpSt9vGqxtpQFHoOBY9XuRkFe+VtZ5DyHqc/0DK4Rk7VswmxbiU7afEw5WPHQlzyv+6UEiZpBYET+e3TSt3OKeGD++Je1oOKe+3HQF286luSsOw+FZdedvMq4v9c/ge6tLG5uDwIngR5030bDXiw0rwpHXFbRfDLEDRctEjFcgGtM+8QKXGR5lY02kNHaPDpz+/1IHX9v50qziSkvvDK3H/9TPQbm4X/7erfCYRQJHkuFbgA4/p79msZRrYncvQGvwnP49clSPaCV1cGzdyfLbkwg5ubjctb83gdzBNJfr1wfhcS+sWalzyh918cSKqcfc9fA9W38P/frucskKJwvNY6+A6yBOk2trVSCY95Ykltlfghbc9RW+b/gfkEuPqn7RFleP8sla2P0Apdxzryv5h95a+H/vZX/Lvmot/AgAA//+h+RBa" + return "eJzEWM9v27gSvuevGPTUAo2AVzy8Qw4PaNMecthFtmnRozMmx9YgFKlyhvY66B+/ICU7sSMnTmpldQgCUZ75vo+cXzyFG1qdAS7lBEBZHZ3BG1zKmxMAS2Iit8rBn8H/TwAArnEp19AEmxyBCc6RUYGPP66gCZ41RPZzaEgjG4FZDE1ZO3ch2SWqqasTgEiOUOgM5ngCMGNyVs6K9VPw2NAaTX501eYPY0ht/2YA1LaR+4bIfNi8GzK212D3XJP5cA0meEX2AlrThpvWqLCkSCAmYkt2h+2PzBaWNZv6zsCARkJeYboqP/xy/qG6539bp/WzS/U+XdOmSoOiq1qjW1+syYtBR3YycwF3P3hEh/x8qwlaioa84pwgzACdCwaVbAYOJjRtUoLkWXt5MBKYFCN5dStgD0kIgi86shdFb6jaS8REsqyTJDinEbj41EwpZh7nl9+hcyYgbb8f9zHCLMTyVVJ2fIvZ7JO4p+jyb0dFThg92S0CnfD+DnuNAmhMTGRBOL9hhSUKOEze1GQhRBDFqGT3k5IUW5dk8orkepfbzGpcEEyJ/N1OoYfkHTecT+KG9rImD/ln55ffz4uFTx1mWKBLBCxwSzEcylgmpsY4Jzsu5cJpkHiOJR8UWmQLNix9pv5w/98DetunHa2TAHuTYtYIreWMAh10VIape9JliDcV+6pFc0MqozLufUAkQ7zIh9HnvLKGAeyV4gwNyW5QPg4/JH1V/CWNh6THws++mq6UxgVfPIwi/WthP5bsluWGQxUJ7SjYP/VKY98mZKybVCUaIsEiuNSQAC6QHU4dgYbDkS8jK40IPdtX8hnT0bEX1UN7bODnoWkd5aJQdA8txVK55eVbkHsYzFna8IzJ5n6Ig83HUbk5ZIPGZFk83Kf5wr16CUlR1CSVqcncTGbIbk+hdMHPn8fvK7UhquR6rjXFbaS5t2lRhCxMg9bbix0mKJhKVcyrshKlZnuNu47UoSg07JMeTnLS2XtlrmMQWfv5F6gM79ihZDYpxoSY/yQ/PPnkkjCn+NuDQogkZSB4Or9tVrnBOVU8HBM3tFqGuLt2ALCLzyUoM4xsP09XNgdz19g/B9/dXFrlPRg+BC/CeeEt5xnx7iRY0nLi7g/DLEA+56I9E8gGaBt5gUqV9TLJS8cVtLcOn/+8Ko7X8j7oKg5Eye3wSdx9/QxoF5eL/+ZWPpIIoEgwXCbwJffp79lY09SxGUvQYvyBngeeyh7aEVVcC9fj+JKTCxu4uNysvM0Cv4NpSN6uC+NzJS0hVJlgh9V8cSIqdnc1fA95vof//O90ygrJC899mYOLkyeQap1bI5m0FCc5sb0CXnjbkrc56H9BTN53/0mdVNnPT8tk+wuUYsO+nOxfuW9p2/67/C/Zd9WDmz75Kb910yc/5RVv+q7+uhq86ZuSHnrXF5wl0UlDIjinSS4wQuZI5bsES9vG8Dc3eRP7G78sS+cXfPCnlrrOs8ewLtk/E6U9MdJ/WVlyuDpaj/htu0z3TmQLz90tSu+7tIQ+6L1GOFf1HBClJDUN2Zxb3epxKj7oZMHCU3es3mkPmw0B9jBzPK/1cWCvAmpXPI1MC3RdiBx+GPIxGhfo+qw+B9j6ZmRcZJE0lWu86QoMOifruvO1c/9HH11o9t87rxHnHDPyhlvbdZn4mILUtLqa9PoNj7y/i2hHnY+XF2v1cpxY7oK7Exdwo9DwQET+LpMK345zk1JqIt/SE3r+EwAA//8lh/10" } diff --git a/x-pack/metricbeat/module/aws/sqs/_meta/data.json b/x-pack/metricbeat/module/aws/sqs/_meta/data.json new file mode 100644 index 00000000000..afec23f860d --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 91 + }, + "oldest_message_age": { + "sec": 86404 + }, + "sent_message_size": {} + } + }, + "cloud": { + "region": "us-east-1" + }, + "event": { + "dataset": "aws.sqs", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "sqs" + }, + "service": { + "name": "sqs", + "type": "sqs" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc new file mode 100644 index 00000000000..11ed9f87c50 --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +The sqs metricset of aws module allows you to monitor your AWS SQS queues. `sqs` metricset fetches a set of values from +https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-available-cloudwatch-metrics.html[Amazon SQS Metrics]. +CloudWatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every five minutes. + +=== AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS SQS metrics. +---- +cloudwatch:GetMetricData +ec2:DescribeRegions +---- + +=== Dashboard diff --git a/x-pack/metricbeat/module/aws/sqs/_meta/fields.yml b/x-pack/metricbeat/module/aws/sqs/_meta/fields.yml new file mode 100644 index 00000000000..9a1edd352e6 --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/_meta/fields.yml @@ -0,0 +1,42 @@ +- name: sqs + type: group + description: > + `sqs` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS SQS. + release: beta + fields: + - name: oldest_message_age.sec + type: long + description: > + The approximate age of the oldest non-deleted message in the queue. + - name: message.delayed + type: long + description: > + TThe number of messages in the queue that are delayed and not available for reading immediately. + - name: message.not_visible + type: long + description: > + The number of messages that are in flight. + - name: message.visible + type: long + description: > + The number of messages available for retrieval from the queue. + - name: message.deleted + type: long + description: > + The number of messages deleted from the queue. + - name: message.received + type: long + description: > + The number of messages returned by calls to the ReceiveMessage action. + - name: message.sent + type: long + description: > + The number of messages added to a queue. + - name: empty_receives + type: long + description: > + The number of ReceiveMessage API calls that did not return a message. + - name: sent_message_size.bytes + type: scaled_float + description: > + The size of messages added to a queue. diff --git a/x-pack/metricbeat/module/aws/sqs/data.go b/x-pack/metricbeat/module/aws/sqs/data.go new file mode 100644 index 00000000000..9c45715b081 --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/data.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sqs + +import ( + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schemaRequestFields = s.Schema{ + "oldest_message_age": s.Object{ + "sec": c.Float("ApproximateAgeOfOldestMessage"), + }, + "messages": s.Object{ + "delayed": c.Float("ApproximateNumberOfMessagesDelayed"), + "not_visible": c.Float("ApproximateNumberOfMessagesNotVisible"), + "visible": c.Float("ApproximateNumberOfMessagesVisible"), + "deleted": c.Float("NumberOfMessagesDeleted"), + "received": c.Float("NumberOfMessagesReceived"), + "sent": c.Float("NumberOfMessagesSent"), + }, + "empty_receives": c.Float("NumberOfEmptyReceives"), + "sent_message_size": s.Object{ + "bytes": c.Float("SentMessageSize"), + }, + } +) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go new file mode 100644 index 00000000000..764d3898b7c --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -0,0 +1,179 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sqs + +import ( + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +var metricsetName = "sqs" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + logger *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logger := logp.NewLogger(aws.ModuleName) + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + // Check if period is set to be multiple of 300s + remainder := metricSet.PeriodInSec % 300 + if remainder != 0 { + err := errors.New("period needs to be set to 300s (or a multiple of 300s). " + + "To avoid data missing or extra costs, please make sure period is set correctly in config.yml") + logger.Info(err) + } + + return &MetricSet{ + MetricSet: metricSet, + logger: logger, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + namespace := "AWS/SQS" + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + m.logger.Error(errors.Wrap(err, "Error ParseDuration")) + report.Error(err) + return + } + + for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + + // Get listMetrics output + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + m.logger.Error(err.Error()) + report.Error(err) + continue + } + if listMetricsOutput == nil || len(listMetricsOutput) == 0 { + continue + } + + // Construct metricDataQueries + metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec)) + if len(metricDataQueries) == 0 { + continue + } + + // Use metricDataQueries to make GetMetricData API calls + metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + // Create Cloudwatch Events for SQS + event, err := createSQSEvents(metricDataResults, metricsetName, regionName, schemaRequestFields) + if err != nil { + m.logger.Error(err.Error()) + event.Error = err + report.Event(event) + continue + } + + report.Event(event) + } +} + +func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { + metricDataQueries := []cloudwatch.MetricDataQuery{} + for i, listMetric := range listMetricsOutput { + metricDataQuery := createMetricDataQuery(listMetric, i, period) + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} + +func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + id := "sqs" + strconv.Itoa(index) + metricDims := metric.Dimensions + metricName := *metric.MetricName + queueName := "" + for _, dim := range metricDims { + if *dim.Name == "QueueName" { + queueName = *dim.Value + } + } + label := queueName + " " + metricName + + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} + +func createSQSEvents(getMetricDataResults []cloudwatch.MetricDataResult, metricsetName string, regionName string, schemaMetricFields s.Schema) (event mb.Event, err error) { + event.Service = metricsetName + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + + mapOfMetricSetFieldResults := make(map[string]interface{}) + for _, output := range getMetricDataResults { + if len(output.Values) == 0 { + continue + } + labels := strings.Split(*output.Label, " ") + mapOfMetricSetFieldResults["queue.name"] = labels[0] + mapOfMetricSetFieldResults[labels[1]] = fmt.Sprint(output.Values[0]) + } + + resultMetricSetFields, err := aws.EventMapping(mapOfMetricSetFieldResults, schemaMetricFields) + if err != nil { + err = errors.Wrap(err, "Error trying to apply schemaMetricSetFields in AWS SQS metricbeat module.") + return + } + event.MetricSetFields = resultMetricSetFields + return +} diff --git a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go new file mode 100644 index 00000000000..704242f5211 --- /dev/null +++ b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package sqs + +import ( + "testing" + + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + config, info := mtest.GetConfigForTest("sqs") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } + + sqsMetricSet := mbtest.NewReportingMetricSetV2(t, config) + events, err := mbtest.ReportingFetchV2(sqsMetricSet) + if err != nil { + t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + + "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") + } + + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("Module: %s Metricset: %s", sqsMetricSet.Module().Name(), sqsMetricSet.Name()) + + for _, event := range events { + // RootField + mtest.CheckEventField("service.name", "string", event, t) + mtest.CheckEventField("cloud.region", "string", event, t) + // MetricSetField + mtest.CheckEventField("empty_receives", "float", event, t) + mtest.CheckEventField("messages.delayed", "float", event, t) + mtest.CheckEventField("messages.deleted", "float", event, t) + mtest.CheckEventField("messages.not_visible", "float", event, t) + mtest.CheckEventField("messages.received", "float", event, t) + mtest.CheckEventField("messages.sent", "float", event, t) + mtest.CheckEventField("messages.visible", "float", event, t) + mtest.CheckEventField("oldest_message_age.sec", "float", event, t) + mtest.CheckEventField("sent_message_size", "float", event, t) + } +} + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("sqs") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + sqsMetricSet := mbtest.NewReportingMetricSetV2(t, config) + errs := mbtest.WriteEventsReporterV2(sqsMetricSet, t, "/") + if errs != nil { + t.Fatal("write", errs) + } +} diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index ccb5317ce09..d831e5e094d 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -2,6 +2,7 @@ period: 300s metricsets: - "ec2" + - "sqs" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}'