feat: AWS SNS Sink (#111)

build/config/substation.libsonnet

@@ -150,6 +150,9 @@
         // TODO(v1.0.0): set format and compression defaults
         settings: { bucket: null, prefix: null, prefix_key: null, file_path: null, file_format: null, file_compression: null },
       },
+      aws_sns: {
+        settings: { arn: null },
+      },
       aws_sqs: {
         settings: { queue: null },
       },
@@ -568,6 +571,12 @@
         type: 'aws_s3',
         settings: s,
       },
+      aws_sns(settings=$.defaults.sink.aws_sns.settings): {
+        local s = std.mergePatch($.defaults.sink.aws_sns.settings, settings),
+
+        type: 'aws_sns',
+        settings: s,
+      },
       aws_sqs(settings=$.defaults.sink.aws_sqs.settings): {
         local s = std.mergePatch($.defaults.sink.aws_sqs.settings, settings),
 

build/terraform/aws/dynamodb/main.tf

@@ -25,7 +25,7 @@ resource "aws_dynamodb_table" "table" {
 
   # Streams are only charged for read operations and reads from AWS Lambda are free
   # https://aws.amazon.com/dynamodb/pricing/
-  stream_enabled = true
+  stream_enabled   = true
   stream_view_type = var.stream_view_type
 
   dynamic "attribute" {

build/terraform/aws/dynamodb/output.tf

@@ -1,3 +1,7 @@
 output "arn" {
   value = aws_dynamodb_table.table.arn
 }
+
+output "stream_arn" {
+  value = aws_dynamodb_table.table.stream_arn
+}

build/terraform/aws/iam/main.tf

@@ -181,6 +181,16 @@ data "aws_iam_policy_document" "secretsmanager_read" {
   }
 }
 
+data "aws_iam_policy_document" "sns_write" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "sns:Publish",
+    ]
+    resources = var.resources
+  }
+}
+
 data "aws_iam_policy_document" "sqs_read" {
   statement {
     effect = "Allow"

build/terraform/aws/iam/output.tf

@@ -34,6 +34,10 @@ output "dynamodb_write_policy" {
   value = data.aws_iam_policy_document.dynamodb_write.json
 }
 
+output "dynamodb_stream_read_policy" {
+  value = data.aws_iam_policy_document.dynamodb_stream_read.json
+}
+
 output "kms_read_policy" {
   value = data.aws_iam_policy_document.kms_read.json
 }
@@ -58,6 +62,10 @@ output "secretsmanager_read_policy" {
   value = data.aws_iam_policy_document.secretsmanager_read.json
 }
 
+output "sns_write_policy" {
+  value = data.aws_iam_policy_document.sns_write.json
+}
+
 output "sqs_read_policy" {
   value = data.aws_iam_policy_document.sqs_read.json
 }

build/terraform/aws/sns/_variables.tf

@@ -6,11 +6,6 @@ variable "kms_key_id" {
   type = string
 }
 
-variable "fifo" {
-  type    = bool
-  default = false
-}
-
 variable "tags" {
   type    = map(any)
   default = {}

build/terraform/aws/sns/main.tf

@@ -1,8 +1,8 @@
 resource "aws_sns_topic" "topic" {
   name                        = var.name
   kms_master_key_id           = var.kms_key_id
-  fifo_topic                  = var.fifo ? true : false
-  content_based_deduplication = var.fifo ? true : false
+  fifo_topic                  = endswith(var.name, ".fifo") ? true : false
+  content_based_deduplication = endswith(var.name, ".fifo") ? true : false
 
   tags = var.tags
 }

build/terraform/aws/sqs/_variables.tf

@@ -16,11 +16,6 @@ variable "visibility_timeout_seconds" {
   default = 300
 }
 
-variable "fifo" {
-  type    = bool
-  default = false
-}
-
 variable "tags" {
   type    = map(any)
   default = {}

build/terraform/aws/sqs/main.tf

@@ -5,8 +5,8 @@ resource "aws_sqs_queue" "queue" {
   visibility_timeout_seconds        = var.visibility_timeout_seconds
   kms_master_key_id                 = var.kms_key_id
   kms_data_key_reuse_period_seconds = 300
-  fifo_queue                        = var.fifo ? true : false
-  content_based_deduplication       = var.fifo ? true : false
+  fifo_queue                        = endswith(var.name, ".fifo") ? true : false
+  content_based_deduplication       = endswith(var.name, ".fifo") ? true : false
 
   tags = var.tags
 }

examples/aws/pubsub/README.md

@@ -0,0 +1,25 @@
+# pubsub
+
+This example deploys a data pipeline that uses a [publish/subscribe (pub/sub) pattern](https://aws.amazon.com/what-is/pub-sub-messaging/). The publisher receives change data capture (CDC) events from a DynamoDB table and publishes them to an SNS topic from which three subscribers consume them.
+
+The data pipeline is visualized below:
+```mermaid
+
+flowchart TD
+    %% core infrastructure
+    ddb[DynamoDB\nTable]
+    sns[SNS Topic]
+
+    %% nodes
+    publisher(Publisher)
+    subscriber_x(Subscriber)
+    subscriber_y(Subscriber)
+    subscriber_z(Subscriber)
+
+    %% connections
+    ddb --> publisher
+    publisher -->sns
+    sns-->subscriber_x
+    sns-->subscriber_y
+    sns-->subscriber_z
+```

examples/aws/pubsub/config/publisher/config.jsonnet

@@ -0,0 +1,11 @@
+local sub = import '../../../../../build/config/substation.libsonnet';
+
+{
+  sink: sub.interfaces.sink.aws_sns(
+    // change SNS topic ARN to match the resource created by Terraform
+    settings = { arn: 'arn:aws:sns:us-east-1:123456789012:my-topic'}
+  ),
+  transform: {
+    type: 'noop',
+  },
+}

examples/aws/pubsub/config/subscriber_x/config.jsonnet

@@ -0,0 +1,8 @@
+local sub = import '../../../../../build/config/substation.libsonnet';
+
+{
+  sink: sub.interfaces.sink.stdout,
+  transform: {
+    type: 'noop',
+  },
+}

examples/aws/pubsub/config/subscriber_y/config.jsonnet

@@ -0,0 +1,8 @@
+local sub = import '../../../../../build/config/substation.libsonnet';
+
+{
+  sink: sub.interfaces.sink.stdout,
+  transform: {
+    type: 'noop',
+  },
+}

examples/aws/pubsub/config/subscriber_z/config.jsonnet

@@ -0,0 +1,8 @@
+local sub = import '../../../../../build/config/substation.libsonnet';
+
+{
+  sink: sub.interfaces.sink.stdout,
+  transform: {
+    type: 'noop',
+  },
+}

examples/aws/pubsub/terraform/bootstrap.tf

@@ -0,0 +1,85 @@
+data "aws_caller_identity" "caller" {}
+
+# KMS encryption key that is shared by all Substation infrastructure
+module "kms" {
+  source = "../../../../build/terraform/aws/kms"
+  name   = "alias/substation"
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+    "Effect": "Allow",
+    "Action": [
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ],
+    "Principal": {
+      "Service": "cloudwatch.amazonaws.com"
+    },
+    "Resource": "*"
+    },
+    {
+    "Effect": "Allow",
+    "Action": "kms:*",
+    "Principal": {
+      "AWS": "arn:aws:iam::${data.aws_caller_identity.caller.account_id}:root"
+    },
+    "Resource": "*"
+    }
+  ]
+}
+POLICY
+}
+
+# AppConfig application that is shared by all Substation apps
+resource "aws_appconfig_application" "substation" {
+  name        = "substation"
+  description = "Stores compiled configuration files for Substation"
+}
+
+# use the prod environment for production resources
+resource "aws_appconfig_environment" "prod" {
+  name           = "prod"
+  description    = "Stores production Substation configuration files"
+  application_id = aws_appconfig_application.substation.id
+}
+
+# use the dev environment for development resources
+resource "aws_appconfig_environment" "dev" {
+  name           = "dev"
+  description    = "Stores development Substation configuration files"
+  application_id = aws_appconfig_application.substation.id
+}
+
+# AppConfig doesn't have useful support for non-linear, non-instant deployments on AWS Lambda, so this deployment strategy is used to deploy configurations as quickly as possible
+# todo: add configuration rollback via CloudWatch Lambda monitoring
+resource "aws_appconfig_deployment_strategy" "instant" {
+  name                           = "Instant"
+  description                    = "This strategy deploys the configuration to all targets immediately with zero bake time."
+  deployment_duration_in_minutes = 0
+  final_bake_time_in_minutes     = 0
+  growth_factor                  = 100
+  growth_type                    = "LINEAR"
+  replicate_to                   = "NONE"
+}
+
+# repository for the core Substation app
+module "ecr_substation" {
+  source  = "../../../../build/terraform/aws/ecr"
+  name    = "substation"
+  kms_arn = module.kms.arn
+}
+
+# repository for the validation app
+module "ecr_validation" {
+  source  = "../../../../build/terraform/aws/ecr"
+  name    = "substation_validation"
+  kms_arn = module.kms.arn
+}
+
+module "sns" {
+  source     = "../../../../build/terraform/aws/sns"
+  kms_key_id = module.kms.key_id
+  name       = "substation_sns"
+}

examples/aws/pubsub/terraform/iam_appconfig.tf

@@ -0,0 +1,21 @@
+################################################
+# appconfig permissions
+# all Lambda must have this policy
+################################################
+
+module "iam_appconfig_read" {
+  source    = "../../../../build/terraform/aws/iam"
+  resources = ["${aws_appconfig_application.substation.arn}/*"]
+}
+
+module "iam_appconfig_read_attachment" {
+  source = "../../../../build/terraform/aws/iam_attachment"
+  id     = "substation_appconfig_read"
+  policy = module.iam_appconfig_read.appconfig_read_policy
+  roles = [
+    module.publisher.role,
+    module.subscriber_x.role,
+    module.subscriber_y.role,
+    module.subscriber_z.role,
+  ]
+}

examples/aws/pubsub/terraform/iam_kms.tf

@@ -0,0 +1,47 @@
+################################################
+# KMS read permissions
+# all Lambda must have this policy
+################################################
+
+module "iam_kms_read" {
+  source = "../../../../build/terraform/aws/iam"
+  resources = [
+    module.kms.arn,
+  ]
+}
+
+module "iam_kms_read_attachment" {
+  source = "../../../../build/terraform/aws/iam_attachment"
+  id     = "substation_kms_read"
+  policy = module.iam_kms_read.kms_read_policy
+  roles = [
+    module.publisher.role,
+    module.subscriber_x.role,
+    module.subscriber_y.role,
+    module.subscriber_z.role,
+  ]
+}
+
+################################################
+# KMS write permissions
+# all Lambda must have this policy
+################################################
+
+module "iam_kms_write" {
+  source = "../../../../build/terraform/aws/iam"
+  resources = [
+    module.kms.arn,
+  ]
+}
+
+module "iam_kms_write_attachment" {
+  source = "../../../../build/terraform/aws/iam_attachment"
+  id     = "substation_kms_write"
+  policy = module.iam_kms_write.kms_write_policy
+  roles = [
+    module.publisher.role,
+    module.subscriber_x.role,
+    module.subscriber_y.role,
+    module.subscriber_z.role,
+  ]
+}

examples/aws/pubsub/terraform/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  # profile = "default"
+  region = "us-east-1"
+}