diff --git a/README.md b/README.md
index e3591b6..05398c8 100644
--- a/README.md
+++ b/README.md
@@ -176,7 +176,7 @@ Amazon Managed Grafana since version 9.4 comes with Prometheus and Cloudwatch pl
1. On the **Default Region** menu, choose your **AWS Region**.
1. Choose **Save & test**.
-#### 3. Add Amazon Athena Data source
+#### 3. Add Amazon Athena Data source for Job metrics
Before starting, you will retrieve the S3 bucket name created to store the AWS Batch jobs data through Amazon Athena.
@@ -206,7 +206,39 @@ In the Amazon Managed Grafana dashboard:
1. On the **Output Location** menu, copy paste the bucket value **s3://DOC-EXAMPLE-BUCKET**.
1. Choose **Save & test**.
-#### 4. Create dashboard
+
+#### 4. Add Amazon Athena Data source for Cost
+
+Before starting, you will retrieve the S3 bucket name created to store the AWS Batch jobs data through Amazon Athena.
+
+In a terminal:
+
+```bash
+sam list stack-outputs --stack-name ${BATCH_DASHBOARD_NAME} \
+ --output json | \
+ jq -r '.[] | select(.OutputKey=="AthenaSpillBucket") | .OutputValue'
+```
+
+Copy the output that you will use in the setup of Amazon Athena data source in Amazon Managed Grafana.
+
+In the Amazon Managed Grafana dashboard:
+
+1. Select the **hamburger** menu on the left pane.
+1. Expand **Administration**
+ 
+1. Choose **Data sources**.
+1. Choose **Add new data source**.
+1. Choose **Amazon Athena**.
+ 
+1. For **Name**, enter your **Amazon Athena Cost Data**.
+1. On the **Default Region** menu, choose your **AWS Region**.
+1. On the **Data source** menu, choose **AwsDataCatalog**.
+1. On the **Database** menu, choose **athenacurcfn_a_w_s_batch_cost_report**.
+1. On the **Workgroup** menu, choose **primary**.
+1. On the **Output Location** menu, copy paste the bucket value **s3://DOC-EXAMPLE-BUCKET**.
+1. Choose **Save & test**.
+
+#### 5. Create AWS Batch job metrics dashboard
To create a dashboard in Amazon Managed Grafana for AWS Batch, you will start from the template provided in this repository to generate dashboard for your environment.
@@ -240,6 +272,26 @@ You will be redirected to the dashboard you imported.
Once you will have your first AWS Batch jobs running.
You will be able to see the data associated with it in the dashboard.
+#### 6. Create AWS Batch compute cost dashboard
+
+Similar to step 3, you will import an template dashboard in Amazon Managed Grafana for AWS Batch that covers Amazon EC2 compute cost.
+You can see an example below.
+
+
+To import the dashboard in Amazon Managed Grafana:
+1. Select the **squares** on the side menu.
+ 
+1. Choose **Import**.
+1. Choose **Upload JSON file**, select the `batch-grafana-cost-dashboard.json` file.
+1. Choose **Load**.
+1. Select the **Athena** and **CloudWatch** data sources your created previously.
+ 
+1. Choose **Import**.
+
+You will be redirected to the dashboard you imported.
+AWS Cost and Usage report takes 24 hours to be populated.
+Split cost data will be able for AWS Batch for jobs that are executed after this deployment.
+
## Clean up
To delete the SAM application deployment, you can use the terminal and enter:
diff --git a/batch-grafana-cost-dashboard.json b/batch-grafana-cost-dashboard.json
new file mode 100644
index 0000000..fc557b5
--- /dev/null
+++ b/batch-grafana-cost-dashboard.json
@@ -0,0 +1,511 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_AMAZON_ATHENA-2",
+ "label": "Amazon Athena CUR",
+ "description": "Contains the Cost and Usage Report data",
+ "type": "datasource",
+ "pluginId": "grafana-athena-datasource",
+ "pluginName": "Amazon Athena"
+ },
+ {
+ "name": "DS_AMAZON_ATHENA",
+ "label": "Amazon Athena",
+ "description": "Contains the AWS Batch data",
+ "type": "datasource",
+ "pluginId": "grafana-athena-datasource",
+ "pluginName": "Amazon Athena"
+ }
+ ],
+ "__elements": [],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "8.4.7"
+ },
+ {
+ "type": "datasource",
+ "id": "grafana-athena-datasource",
+ "name": "Amazon Athena",
+ "version": "2.2.0"
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1688061227855,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 1,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "currencyUSD"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "bill_billing_period_start_date"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "time:MMM-YYYY"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 21,
+ "w": 17,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.7",
+ "targets": [
+ {
+ "connectionArgs": {
+ "catalog": "__default",
+ "database": "__default",
+ "region": "__default"
+ },
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "format": 0,
+ "rawSQL": "SELECT bill_billing_period_start_date, resource_tags_user_project, SUM(split_line_item_split_cost + split_line_item_unused_cost) as Project FROM $__table where $__timeFilter(bill_billing_period_start_date) and\n line_item_product_code in ('AmazonECS') and REGEXP_LIKE(line_item_resource_id, 'arn:aws:ecs:.*:.*:task\\/.*\\/(.*)') GROUP BY 1,2 order by 1",
+ "refId": "A",
+ "table": "awsbatchcostreport"
+ }
+ ],
+ "timeShift": "0M/M",
+ "title": "AWS Batch Project Compute Cost",
+ "transformations": [],
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "currencyUSD"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 17,
+ "x": 0,
+ "y": 21
+ },
+ "id": 4,
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Cost"
+ }
+ ]
+ },
+ "pluginVersion": "8.4.7",
+ "targets": [
+ {
+ "connectionArgs": {
+ "catalog": "__default",
+ "database": "__default",
+ "region": "__default"
+ },
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "format": 1,
+ "rawSQL": "SELECT bill_billing_period_start_date, resource_tags_user_project as Project, SUM(split_line_item_split_cost + split_line_item_unused_cost) as Cost FROM $__table where $__timeFilter(bill_billing_period_start_date) and\n line_item_product_code in ('AmazonECS') and REGEXP_LIKE(line_item_resource_id, 'arn:aws:ecs:.*:.*:task\\/.*\\/(.*)') GROUP BY 1,2 order by 1",
+ "refId": "A",
+ "table": "awsbatchcostreport"
+ }
+ ],
+ "title": "AWS Batch Project Compute Cost",
+ "transformations": [],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "-- Mixed --"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "TaskArn"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 981
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Cost"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 394
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "taskArn"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 557
+ },
+ {
+ "id": "custom.hidden",
+ "value": true
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "jobId A"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 364
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "jobName A"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 174
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "jobStatus A"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 177
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "resource_tags_user_project B"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 224
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 15,
+ "w": 17,
+ "x": 0,
+ "y": 29
+ },
+ "id": 3,
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "8.4.7",
+ "targets": [
+ {
+ "connectionArgs": {
+ "catalog": "__default",
+ "database": "__default",
+ "region": "__default"
+ },
+ "datasource": {
+ "uid": "${DS_AMAZON_ATHENA}"
+ },
+ "format": 1,
+ "hide": false,
+ "rawSQL": "SELECT jobId, jobName, jobStatus, taskArn FROM \"$__table\";",
+ "refId": "A",
+ "table": "{{ TABLE }}"
+ },
+ {
+ "connectionArgs": {
+ "catalog": "__default",
+ "database": "__default",
+ "region": "__default"
+ },
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "format": 1,
+ "hide": false,
+ "rawSQL": "SELECT\n line_item_resource_id as taskArn, \n resource_tags_user_project,\n SUM(split_line_item_split_cost + split_line_item_unused_cost) as Cost \nFROM $__table \nWHERE $__timeFilter(line_item_usage_start_date)\nand\n line_item_product_code in ('AmazonECS')\nAND \n REGEXP_LIKE(line_item_resource_id, 'arn:aws:ecs:.*:.*:task\\/.*\\/(.*)')\nAND resource_tags_user_project in ($Project)\nGROUP BY 1,2",
+ "refId": "B",
+ "table": "awsbatchcostreport"
+ }
+ ],
+ "title": "AWS Batch Jobs Compute Cost",
+ "transformations": [
+ {
+ "id": "seriesToColumns",
+ "options": {
+ "byField": "taskArn"
+ }
+ }
+ ],
+ "type": "table"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 35,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "Project",
+ "options": [],
+ "query": {
+ "column": "resource_tags_user_project",
+ "connectionArgs": {
+ "catalog": "__default",
+ "database": "__default",
+ "region": "__default"
+ },
+ "format": 1,
+ "rawSQL": "select $__column from $__table;",
+ "table": "awsbatchcostreport"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "grafana-athena-datasource",
+ "uid": "${DS_AMAZON_ATHENA-2}"
+ },
+ "definition": "",
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "query0",
+ "options": [],
+ "query": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-30d",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "utc",
+ "title": "AWS Batch Compute Cost",
+ "uid": "5e7i8l_4k",
+ "version": 41,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/docs/images/grafana-athena-cost.png b/docs/images/grafana-athena-cost.png
new file mode 100644
index 0000000..a2d73e7
Binary files /dev/null and b/docs/images/grafana-athena-cost.png differ
diff --git a/docs/images/grafana-batch-cost-dashboard.png b/docs/images/grafana-batch-cost-dashboard.png
new file mode 100644
index 0000000..7ef0760
Binary files /dev/null and b/docs/images/grafana-batch-cost-dashboard.png differ
diff --git a/docs/images/grafana-dashboard-import-cost.png b/docs/images/grafana-dashboard-import-cost.png
new file mode 100644
index 0000000..7b967de
Binary files /dev/null and b/docs/images/grafana-dashboard-import-cost.png differ
diff --git a/generate-grafana-dashboard.py b/generate-grafana-dashboard.py
index 0b8ce99..060d533 100644
--- a/generate-grafana-dashboard.py
+++ b/generate-grafana-dashboard.py
@@ -51,8 +51,8 @@ def container_insights_status(cluster_arn):
return ci_state
-def write_file(data):
- f = open('batch-grafana-dashboard.json', 'w')
+def write_file(filename, data):
+ f = open(filename, 'w')
f.write(data)
f.close()
@@ -82,7 +82,15 @@ def main():
template = jinja2.Template(data)
out = template.render(TABLE=arg.table.lower(), LOG_GROUP=logs)
- write_file(out)
+ write_file('batch-grafana-dashboard.json', out)
+
+ f = open('batch-grafana-cost-dashboard.json')
+ data = f.read()
+ f.close()
+ template = jinja2.Template(data)
+ out = template.render(TABLE=arg.table.lower())
+
+ write_file('batch-grafana-cost.json', out)
if __name__ == '__main__':
diff --git a/template.yaml b/template.yaml
index 1f4e741..dd2a661 100644
--- a/template.yaml
+++ b/template.yaml
@@ -288,6 +288,8 @@ Resources:
Resource:
- !GetAtt AthenaSpillS3Bucket.Arn
- !Join [ '', [ !GetAtt AthenaSpillS3Bucket.Arn, '/*' ]]
+ - !GetAtt BucketCURBatchReport.Arn
+ - !Join [ '', [ !GetAtt BucketCURBatchReport.Arn, '/*' ]]
- Effect: Allow
Action:
- lambda:InvokeFunction
@@ -310,14 +312,15 @@ Resources:
- sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonGrafanaCloudWatchAccess
+ - arn:aws:iam::aws:policy/service-role/AmazonGrafanaAthenaAccess
GrafanaBatch:
Type: AWS::Grafana::Workspace
- Properties:
+ Properties:
AccountAccessType: CURRENT_ACCOUNT
- AuthenticationProviders:
+ AuthenticationProviders:
- AWS_SSO
- DataSources:
+ DataSources:
- ATHENA
- CLOUDWATCH
GrafanaVersion: "9.4"
@@ -326,6 +329,374 @@ Resources:
RoleArn: !Ref GrafanaRole
PermissionType: SERVICE_MANAGED
+#### Cost
+ BucketCURBatchReport:
+ Type: AWS::S3::Bucket
+ DeletionPolicy: Retain
+ Properties:
+ BucketEncryption:
+ ServerSideEncryptionConfiguration:
+ - ServerSideEncryptionByDefault:
+ SSEAlgorithm: 'AES256'
+ PublicAccessBlockConfiguration:
+ BlockPublicAcls: True
+ BlockPublicPolicy: True
+ IgnorePublicAcls: True
+ RestrictPublicBuckets: True
+ Tags:
+ - Key: "Application"
+ Value: "BatchOperationalDashboard"
+ - Key: "Application"
+ Value: "BatchCostReport"
+
+
+ BucketCURBatchReportPolicy:
+ Type: AWS::S3::BucketPolicy
+ Properties:
+ Bucket: !Ref BucketCURBatchReport
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 's3:GetBucketAcl'
+ - 's3:GetBucketPolicy'
+ Resource:
+ - !Join
+ - ''
+ - - 'arn:aws:s3:::'
+ - !Ref BucketCURBatchReport
+ Principal:
+ Service:
+ - 'billingreports.amazonaws.com'
+ Condition:
+ StringEquals:
+ aws:SourceArn: !Sub 'arn:aws:cur:${AWS::Region}:${AWS::AccountId}:definition/*'
+ aws:SourceAccount: !Sub '${AWS::AccountId}'
+ - Effect: Allow
+ Action:
+ - 's3:PutObject'
+ Resource:
+ - !Join
+ - ''
+ - - 'arn:aws:s3:::'
+ - !Ref BucketCURBatchReport
+ - /*
+ Principal:
+ Service:
+ - 'billingreports.amazonaws.com'
+ Condition:
+ StringEquals:
+ aws:SourceArn: !Sub 'arn:aws:cur:${AWS::Region}:${AWS::AccountId}:definition/*'
+ aws:SourceAccount: !Sub '${AWS::AccountId}'
+
+ CURAWSBatch:
+ DependsOn: BucketCURBatchReportPolicy
+ Type: AWS::CUR::ReportDefinition
+ Properties:
+ AdditionalArtifacts:
+ - 'ATHENA'
+ AdditionalSchemaElements:
+ - 'RESOURCES'
+ - 'SPLIT_COST_ALLOCATION_DATA'
+ Compression: 'Parquet'
+ Format: 'Parquet'
+ RefreshClosedReports: True
+ ReportName: 'AWSBatchCostReport'
+ ReportVersioning: 'OVERWRITE_REPORT'
+ S3Bucket: !Ref BucketCURBatchReport
+ S3Prefix: 'aws-batch'
+ S3Region: !Ref 'AWS::Region'
+ TimeUnit: 'HOURLY'
+
+### Glue and Athena for CUR
+ AWSCURDatabase:
+ Type: 'AWS::Glue::Database'
+ Properties:
+ DatabaseInput:
+ Name: 'athenacurcfn_a_w_s_batch_cost_report'
+ CatalogId: !Ref AWS::AccountId
+
+ AWSCURCrawlerComponentFunction:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service:
+ - glue.amazonaws.com
+ Action:
+ - 'sts:AssumeRole'
+ Path: /
+ ManagedPolicyArns:
+ - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole'
+ Policies:
+ - PolicyName: AWSCURCrawlerComponentFunction
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 'logs:CreateLogGroup'
+ - 'logs:CreateLogStream'
+ - 'logs:PutLogEvents'
+ Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
+ - Effect: Allow
+ Action:
+ - 'glue:UpdateDatabase'
+ - 'glue:UpdatePartition'
+ - 'glue:CreateTable'
+ - 'glue:UpdateTable'
+ - 'glue:ImportCatalogToGlue'
+ Resource: '*'
+ - Effect: Allow
+ Action:
+ - 's3:GetObject'
+ - 's3:PutObject'
+ Resource:
+ - !Join
+ - ''
+ - - 'arn:aws:s3:::'
+ - !Ref BucketCURBatchReport
+ - '/aws-batch/AWSBatchCostReport/AWSBatchCostReport*'
+
+ - PolicyName: AWSCURKMSDecryption
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 'kms:Decrypt'
+ Resource: '*'
+
+
+ AWSCURCrawlerLambdaExecutor:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service:
+ - lambda.amazonaws.com
+ Action:
+ - 'sts:AssumeRole'
+ Path: /
+ Policies:
+ - PolicyName: AWSCURCrawlerLambdaExecutor
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 'logs:CreateLogGroup'
+ - 'logs:CreateLogStream'
+ - 'logs:PutLogEvents'
+ Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
+ - Effect: Allow
+ Action:
+ - 'glue:StartCrawler'
+ Resource: '*'
+
+
+ AWSCURCrawler:
+ Type: 'AWS::Glue::Crawler'
+ DependsOn:
+ - AWSCURDatabase
+ - AWSCURCrawlerComponentFunction
+ Properties:
+ Name: AWSCURCrawler-AWSBatchCostReport
+ Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
+ Role: !GetAtt AWSCURCrawlerComponentFunction.Arn
+ DatabaseName: !Ref AWSCURDatabase
+ Targets:
+ S3Targets:
+ - Path: !Join ['', ['s3://', !Ref BucketCURBatchReport,'/aws-batch/AWSBatchCostReport/AWSBatchCostReport']]
+ Exclusions:
+ - '**.json'
+ - '**.yml'
+ - '**.sql'
+ - '**.csv'
+ - '**.gz'
+ - '**.zip'
+ SchemaChangePolicy:
+ UpdateBehavior: UPDATE_IN_DATABASE
+ DeleteBehavior: DELETE_FROM_DATABASE
+
+
+ AWSCURInitializer:
+ Type: 'AWS::Lambda::Function'
+ DependsOn: AWSCURCrawler
+ Properties:
+ Code:
+ ZipFile: >
+ const AWS = require('aws-sdk');
+ const response = require('./cfn-response');
+ exports.handler = function(event, context, callback) {
+ if (event.RequestType === 'Delete') {
+ response.send(event, context, response.SUCCESS);
+ } else {
+ const glue = new AWS.Glue();
+ glue.startCrawler({ Name: 'AWSCURCrawler-AWSBatchCostReport' }, function(err, data) {
+ if (err) {
+ const responseData = JSON.parse(this.httpResponse.body);
+ if (responseData['__type'] == 'CrawlerRunningException') {
+ callback(null, responseData.Message);
+ } else {
+ const responseString = JSON.stringify(responseData);
+ if (event.ResponseURL) {
+ response.send(event, context, response.FAILED,{ msg: responseString });
+ } else {
+ callback(responseString);
+ }
+ }
+ }
+ else {
+ if (event.ResponseURL) {
+ response.send(event, context, response.SUCCESS);
+ } else {
+ callback(null, response.SUCCESS);
+ }
+ }
+ });
+ }
+ };
+ Handler: 'index.handler'
+ Timeout: 30
+ Runtime: nodejs16.x
+ ReservedConcurrentExecutions: 1
+ Role: !GetAtt AWSCURCrawlerLambdaExecutor.Arn
+
+
+ AWSStartCURCrawler:
+ Type: 'Custom::AWSStartCURCrawler'
+ Properties:
+ ServiceToken: !GetAtt AWSCURInitializer.Arn
+
+
+ AWSS3CUREventLambdaPermission:
+ Type: AWS::Lambda::Permission
+ Properties:
+ Action: 'lambda:InvokeFunction'
+ FunctionName: !GetAtt AWSCURInitializer.Arn
+ Principal: 's3.amazonaws.com'
+ SourceAccount: !Ref AWS::AccountId
+ SourceArn: !GetAtt BucketCURBatchReport.Arn
+
+
+ AWSS3CURLambdaExecutor:
+ Type: 'AWS::IAM::Role'
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service:
+ - lambda.amazonaws.com
+ Action:
+ - 'sts:AssumeRole'
+ Path: /
+ Policies:
+ - PolicyName: AWSS3CURLambdaExecutor
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - 'logs:CreateLogGroup'
+ - 'logs:CreateLogStream'
+ - 'logs:PutLogEvents'
+ Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
+ - Effect: Allow
+ Action:
+ - 's3:PutBucketNotification'
+ Resource: !GetAtt BucketCURBatchReport.Arn
+
+
+ AWSS3CURNotification:
+ Type: 'AWS::Lambda::Function'
+ DependsOn:
+ - AWSCURInitializer
+ - AWSS3CUREventLambdaPermission
+ - AWSS3CURLambdaExecutor
+ Properties:
+ Code:
+ ZipFile: >
+ const AWS = require('aws-sdk');
+ const response = require('./cfn-response');
+ exports.handler = function(event, context, callback) {
+ const s3 = new AWS.S3();
+ const putConfigRequest = function(notificationConfiguration) {
+ return new Promise(function(resolve, reject) {
+ s3.putBucketNotificationConfiguration({
+ Bucket: event.ResourceProperties.BucketName,
+ NotificationConfiguration: notificationConfiguration
+ }, function(err, data) {
+ if (err) reject({ msg: this.httpResponse.body.toString(), error: err, data: data });
+ else resolve(data);
+ });
+ });
+ };
+ const newNotificationConfig = {};
+ if (event.RequestType !== 'Delete') {
+ newNotificationConfig.LambdaFunctionConfigurations = [{
+ Events: [ 's3:ObjectCreated:*' ],
+ LambdaFunctionArn: event.ResourceProperties.TargetLambdaArn || 'missing arn',
+ Filter: { Key: { FilterRules: [ { Name: 'prefix', Value: event.ResourceProperties.ReportKey } ] } }
+ }];
+ }
+ putConfigRequest(newNotificationConfig).then(function(result) {
+ response.send(event, context, response.SUCCESS, result);
+ callback(null, result);
+ }).catch(function(error) {
+ response.send(event, context, response.FAILED, error);
+ console.log(error);
+ callback(error);
+ });
+ };
+ Handler: 'index.handler'
+ Timeout: 30
+ Runtime: nodejs16.x
+ ReservedConcurrentExecutions: 1
+ Role: !GetAtt AWSS3CURLambdaExecutor.Arn
+
+
+ AWSPutS3CURNotification:
+ Type: 'Custom::AWSPutS3CURNotification'
+ Properties:
+ ServiceToken: !GetAtt AWSS3CURNotification.Arn
+ TargetLambdaArn: !GetAtt AWSCURInitializer.Arn
+ BucketName: !Ref BucketCURBatchReport
+ ReportKey: 'aws-batch/AWSBatchCostReport/AWSBatchCostReport'
+
+
+ AWSCURReportStatusTable:
+ Type: 'AWS::Glue::Table'
+ DependsOn: AWSCURDatabase
+ Properties:
+ DatabaseName: athenacurcfn_a_w_s_batch_cost_report
+ CatalogId: !Ref AWS::AccountId
+ TableInput:
+ Name: 'cost_and_usage_data_status'
+ TableType: 'EXTERNAL_TABLE'
+ StorageDescriptor:
+ Columns:
+ - Name: status
+ Type: 'string'
+ InputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+ OutputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+ SerdeInfo:
+ SerializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ Location: !Join ['', ['s3://', !Ref BucketCURBatchReport,'/aws-batch/AWSBatchCostReport/cost_and_usage_data_status/']]
+
+
+###
+
+
Outputs:
AthenaDataSource:
Value: !Ref BatchJobData
@@ -334,4 +705,7 @@ Outputs:
Value: !Ref AthenaSpillS3Bucket
GrafanaWorkspaceId:
- Value: !Ref GrafanaBatch
\ No newline at end of file
+ Value: !Ref GrafanaBatch
+
+ CURBatchBucket:
+ Value: !Ref BucketCURBatchReport