Add Data Export creation

README.md

@@ -28,6 +28,8 @@ After we've completed our engagement, you can delete our IAM role and policy res
 
     $ make delete
 
+Note that this won't automatically delete the Data Export we're creating, since you may end up using it yourself.
+
 If you prefer or need to use the AWS console, you can delete the resources manually.
 
 ### Deleting Resources Manually
@@ -36,14 +38,15 @@ Log into the AWS console,
 
  - navigate to `IAM > Policies` and delete: `DuckbillGroupBilling`, `SkywayAccess`, `DuckbillGroupResourceDiscovery`, `DuckbillGroupDenySensitiveAccess`
  - navigate to `IAM > Roles` and delete `DuckbillGroupRole` and `SkywayRole`
+ - navigate to `Data Exports` and delete the export `skyway-export`
 
 ## What this code does
 
 * Creates two roles titled `DuckbillGroupRole` and `SkywayRole`
 * Creates custom policies: `DuckbillGroupBilling`, `DuckbillGroupResourceDiscovery`, `DuckbillGroupDenySensitiveAccess`, `SkywayAccess`
 * Attaches the policies prefixed with `DuckbillGroup` to the `DuckbillGroupRole` role, along with the AWS-managed policy `ViewOnlyAccess`
 * Attaches the policy `SkywayAccess` to the `SkywayRole` role
-
+* Creates a Data Export in the payer account called `skyway-export`
 
 ![Access diagram](access-diagram.png)
 

billing-policy.json

@@ -9,8 +9,8 @@
                 "s3:GetObject"
             ],
             "Resource": [
-                "arn:aws:s3:::CUR-BUCKET-NAME/*",
-                "arn:aws:s3:::CUR-BUCKET-NAME"
+                "arn:aws:s3:::CUR_BUCKET_NAME/*",
+                "arn:aws:s3:::CUR_BUCKET_NAME"
             ]
         },
         {

create-resources.sh

@@ -25,11 +25,27 @@ Please enter the External ID provided to you by Duckbill Cloud Economists
 EOM
 read -rp 'External ID: ' external_id
 
+cat <<EOM
+
+Please enter the name of the S3 bucket where your Cost & Usage Report resides.
+
+EOM
+read -rp 'S3 Bucket Name: ' cur_bucket_name
+
 internal_customer_id=$(echo "${external_id}" | awk -F '-' '{print $1}' | tr -d '\r\n')
 
 sed "s/CUSTOMER_NAME_SLUG/${customer_name_slug}/g;s/INTERNAL_CUSTOMER_ID/${internal_customer_id}/g" \
 	"${this_dir}/deny-sensitive-data-policy.json.template" > "${this_dir}/deny-sensitive-data-policy.json"
 
+sed "s/CUR_BUCKET_NAME/${cur_bucket_name}/g" \
+	"${this_dir}/deny-sensitive-data-policy.json.template" > "${this_dir}/deny-sensitive-data-policy.json"
+
+sed "s/CUR_BUCKET_NAME/${cur_bucket_name}/g" \
+	"${this_dir}/data-export.json" > "${this_dir}/data-export.json"
+
+sed "s/CUR_BUCKET_NAME/${cur_bucket_name}/g" \
+	"${this_dir}/billing-policy.json" > "${this_dir}/billing-policy.json"
+
 sed "s/EXTERNAL_ID/${external_id}/g" \
 	"${this_dir}/dbg-assume-role-trust-policy.json.template" > "${this_dir}/dbg-assume-role-trust-policy.json"
 
@@ -91,4 +107,10 @@ aws iam attach-role-policy \
 	--role-name SkywayRole \
 	--policy-arn "arn:aws:iam::${account_number}:policy/SkywayAccess"
 
+
+# Create CUR config
+data_export_file="file://${this_dir}/data-export.json"
+data_export_content=$(cat "$data_export_file")
+aws bcm-data-exports create-export --export "'$data_export_content'"
+
 echo "Done!"

data-export.json

@@ -0,0 +1,30 @@
+{
+  "DataQuery": {
+    "QueryStatement": "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT",
+    "TableConfigurations": {
+      "COST_AND_USAGE_REPORT": {
+        "INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY": "FALSE",
+        "INCLUDE_RESOURCES": "TRUE",
+        "INCLUDE_SPLIT_COST_ALLOCATION_DATA": "TRUE",
+        "TIME_GRANULARITY": "HOURLY"
+      }
+    }
+  },
+  "DestinationConfigurations": {
+    "S3Destination": {
+      "S3Bucket": "CUR_BUCKET_NAME",
+      "S3OutputConfigurations": {
+        "Compression": "PARQUET",
+        "Format": "PARQUET",
+        "OutputType": "CUSTOM",
+        "Overwrite": "OVERWRITE_REPORT"
+      },
+      "S3Prefix": "skyway",
+      "S3Region": "us-west-2"
+    }
+  },
+  "Name": "skyway-export",
+  "RefreshCadence": {
+    "Frequency": "SYNCHRONOUS"
+  }
+}

deny-sensitive-data-policy.json.template

@@ -153,8 +153,8 @@
                 "workmail:ListUsers"
             ],
             "NotResource": [
-                "arn:aws:s3:::CUR-BUCKET-NAME/*",
-                "arn:aws:s3:::CUR-BUCKET-NAME"
+                "arn:aws:s3:::CUR_BUCKET_NAME/*",
+                "arn:aws:s3:::CUR_BUCKET_NAME"
             ]
         }
     ]