diff --git a/README.md b/README.md index 14a1c63a..53b84a16 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,18 @@ per-grant data records that can be consumed whenever updates to the underlying d This service consists of an event-driven pipeline that uses AWS Lambda for running various compute tasks. Additionally, Amazon S3 is used for archival storage of raw source data, while -DynamoDB provides an active index of processed grant opportunity records. +DynamoDB provides an active index of processed grant opportunity records. At the end of the ingestion +pipeline, newly-created/-updated grant opportunity records are sent to an event bus that delivers +the updates to subscribers. + +
+ High-level architecture + +
+
+ Component-level architecture + +
## Code Organization diff --git a/docs/component-level-architecture.png b/docs/component-level-architecture.png new file mode 100644 index 00000000..9ee9a7ce Binary files /dev/null and b/docs/component-level-architecture.png differ diff --git a/docs/high-level-architecture.png b/docs/high-level-architecture.png new file mode 100644 index 00000000..c0796963 Binary files /dev/null and b/docs/high-level-architecture.png differ diff --git a/terraform/datadog_dashboard.tf b/terraform/datadog_dashboard.tf index 90e9b266..241d10a1 100644 --- a/terraform/datadog_dashboard.tf +++ b/terraform/datadog_dashboard.tf @@ -3,6 +3,8 @@ locals { } resource "datadog_dashboard" "service_dashboard" { + count = var.datadog_dashboards_enabled ? 1 : 0 + title = trimspace("Grants Ingest Service Dashboard ${local.datadog_draft_label}") description = "Dashboard for monitoring the Grants Ingest pipeline service." layout_type = "ordered" diff --git a/terraform/local.tfvars b/terraform/local.tfvars index d3ce3cf9..2069b8bf 100644 --- a/terraform/local.tfvars +++ b/terraform/local.tfvars @@ -2,6 +2,7 @@ namespace = "grants-ingest" environment = "sandbox" version_identifier = "dev" datadog_enabled = false +datadog_dashboards_enabled = false datadog_lambda_extension_version = "38" lambda_default_log_retention_in_days = 7 lambda_default_log_level = "DEBUG" diff --git a/terraform/modules/DownloadGrantsGovDB/main.tf b/terraform/modules/DownloadGrantsGovDB/main.tf index d35d0398..c8286ee3 100644 --- a/terraform/modules/DownloadGrantsGovDB/main.tf +++ b/terraform/modules/DownloadGrantsGovDB/main.tf @@ -19,8 +19,8 @@ locals { lambda_trigger = var.eventbridge_scheduler_enabled ? local.eventbridge_scheduler_trigger : local.cloudwatch_events_trigger dd_tags = merge( { - for item in split(",", try(var.additional_environment_variables.DD_TAGS, "")) : - split(":", trimspace(item))[0] => split(":", trimspace(item))[1] + for item in compact(split(",", try(var.additional_environment_variables.DD_TAGS, ""))) : + split(":", trimspace(item))[0] => try(split(":", trimspace(item))[1], "") }, var.datadog_custom_tags, { handlername = lower(var.function_name), }, diff --git a/terraform/modules/SplitGrantsGovXMLDB/main.tf b/terraform/modules/SplitGrantsGovXMLDB/main.tf index a0065522..e984e050 100644 --- a/terraform/modules/SplitGrantsGovXMLDB/main.tf +++ b/terraform/modules/SplitGrantsGovXMLDB/main.tf @@ -8,8 +8,8 @@ terraform { locals { dd_tags = merge( { - for item in split(",", try(var.additional_environment_variables.DD_TAGS, "")) : - trimspace(split(":", item)[0]) => trimspace(split(":", item)[1]) + for item in compact(split(",", try(var.additional_environment_variables.DD_TAGS, ""))) : + split(":", trimspace(item))[0] => try(split(":", trimspace(item))[1], "") }, var.datadog_custom_tags, { handlername = lower(var.function_name), }, diff --git a/terraform/variables.tf b/terraform/variables.tf index 4e07b393..dfb85e42 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -84,6 +84,12 @@ variable "datadog_enabled" { default = false } +variable "datadog_dashboards_enabled" { + description = "Whether to provision Datadog dashboards." + type = bool + default = true +} + variable "datadog_api_key" { description = "API key to use when provisioning Datadog resources." type = string