-
Notifications
You must be signed in to change notification settings - Fork 22
/
main.tf
153 lines (124 loc) · 4.81 KB
/
main.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
data "aws_region" "default" {}
resource "aws_kinesis_firehose_delivery_stream" "kinesis_firehose_stream" {
name = var.kinesis_firehose_stream_name
destination = "extended_s3"
extended_s3_configuration {
role_arn = aws_iam_role.kinesis_firehose_stream_role.arn
bucket_arn = aws_s3_bucket.kinesis_firehose_stream_bucket.arn
buffer_size = 128
s3_backup_mode = "Enabled"
prefix = "logs/"
s3_backup_configuration {
role_arn = aws_iam_role.kinesis_firehose_stream_role.arn
bucket_arn = aws_s3_bucket.kinesis_firehose_stream_bucket.arn
prefix = var.kinesis_firehose_stream_backup_prefix
cloudwatch_logging_options {
enabled = true
log_group_name = aws_cloudwatch_log_group.kinesis_firehose_stream_logging_group.name
log_stream_name = aws_cloudwatch_log_stream.kinesis_firehose_stream_logging_stream.name
}
}
processing_configuration {
enabled = true
processors {
type = "Lambda"
parameters {
parameter_name = "LambdaArn"
parameter_value = "${aws_lambda_function.lambda_kinesis_firehose_data_transformation.arn}:$LATEST"
}
}
}
cloudwatch_logging_options {
enabled = true
log_group_name = aws_cloudwatch_log_group.kinesis_firehose_stream_logging_group.name
log_stream_name = aws_cloudwatch_log_stream.kinesis_firehose_stream_logging_stream.name
}
data_format_conversion_configuration {
input_format_configuration {
deserializer {
hive_json_ser_de {}
}
}
output_format_configuration {
serializer {
parquet_ser_de {}
}
}
schema_configuration {
database_name = aws_glue_catalog_database.glue_catalog_database.name
table_name = aws_glue_catalog_table.glue_catalog_table.name
role_arn = aws_iam_role.kinesis_firehose_stream_role.arn
}
}
}
}
resource "aws_cloudwatch_log_group" "kinesis_firehose_stream_logging_group" {
name = "/aws/kinesisfirehose/${var.kinesis_firehose_stream_name}"
}
resource "aws_cloudwatch_log_stream" "kinesis_firehose_stream_logging_stream" {
log_group_name = aws_cloudwatch_log_group.kinesis_firehose_stream_logging_group.name
name = "S3Delivery"
}
resource "aws_s3_bucket" "kinesis_firehose_stream_bucket" {
bucket = var.bucket_name
acl = "private"
}
locals {
path_prefix = "${var.root_path ? path.root : path.module}/functions"
}
data "archive_file" "kinesis_firehose_data_transformation" {
type = "zip"
source_file = format("%s/%s.py", local.path_prefix, var.lambda_function_file_name)
output_path = format("%s/%s.zip", local.path_prefix, var.lambda_function_file_name)
}
resource "aws_cloudwatch_log_group" "lambda_function_logging_group" {
name = "/aws/lambda/${var.lambda_function_name}"
}
resource "aws_lambda_function" "lambda_kinesis_firehose_data_transformation" {
filename = data.archive_file.kinesis_firehose_data_transformation.output_path
function_name = var.lambda_function_name
role = aws_iam_role.lambda.arn
handler = "${var.lambda_function_file_name}.lambda_handler"
source_code_hash = data.archive_file.kinesis_firehose_data_transformation.output_base64sha256
runtime = "python3.6"
timeout = 60
}
resource "aws_glue_catalog_database" "glue_catalog_database" {
name = var.glue_catalog_database_name
}
resource "aws_glue_catalog_table" "glue_catalog_table" {
name = var.glue_catalog_table_name
database_name = aws_glue_catalog_database.glue_catalog_database.name
parameters = {
"classification" = "parquet"
}
storage_descriptor {
input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
location = "s3://${aws_s3_bucket.kinesis_firehose_stream_bucket.bucket}/"
ser_de_info {
name = "JsonSerDe"
serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
parameters = {
"serialization.format" = 1
"explicit.null" = false
"parquet.compression" = "SNAPPY"
}
}
dynamic "columns" {
for_each = var.glue_catalog_table_columns
content {
name = columns.value["name"]
type = columns.value["type"]
}
}
}
}
resource "aws_cloudwatch_log_subscription_filter" "cloudwatch_subscription_filter" {
name = var.cloudwatch_subscription_filter_name
log_group_name = var.cloudwatch_log_group_name
filter_pattern = var.cloudwatch_filter_pattern
destination_arn = aws_kinesis_firehose_delivery_stream.kinesis_firehose_stream.arn
distribution = "ByLogStream"
role_arn = aws_iam_role.cloudwatch_logs_role.arn
}