From 47edeb3f3c65e1bd8f9ecd927dcdfca5dd0d4d92 Mon Sep 17 00:00:00 2001 From: sdbowen Date: Thu, 2 Nov 2023 09:51:25 -0400 Subject: [PATCH] Updates to account for new grants.gov URL pattern --- cmd/DownloadGrantsGovDB/handler.go | 4 ++-- cmd/DownloadGrantsGovDB/handler_test.go | 7 ++++--- cmd/DownloadGrantsGovDB/main.go | 1 + terraform/modules/DownloadGrantsGovDB/main.tf | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/DownloadGrantsGovDB/handler.go b/cmd/DownloadGrantsGovDB/handler.go index fe6c1917..1fc2b842 100644 --- a/cmd/DownloadGrantsGovDB/handler.go +++ b/cmd/DownloadGrantsGovDB/handler.go @@ -22,8 +22,8 @@ type ScheduledEvent struct { // grantsURL returns the download URL for the Grants.gov database export. func (e *ScheduledEvent) grantsURL() string { - return fmt.Sprintf("%s/extract/GrantsDBExtract%sv2.zip", - env.GrantsGovBaseURL, + return fmt.Sprintf("%sGrantsDBExtract%sv2.zip", + env.GrantsGovBaseURL + env.GrantsGovPathURL, e.Timestamp.Format("20060102"), ) } diff --git a/cmd/DownloadGrantsGovDB/handler_test.go b/cmd/DownloadGrantsGovDB/handler_test.go index 14c39762..4e3a1621 100644 --- a/cmd/DownloadGrantsGovDB/handler_test.go +++ b/cmd/DownloadGrantsGovDB/handler_test.go @@ -34,6 +34,7 @@ func setupLambdaEnvForTesting(t *testing.T) { "GRANTS_SOURCE_DATA_BUCKET_NAME": "test-destination-bucket", "S3_USE_PATH_STYLE": "true", "GRANTS_GOV_BASE_URL": "https://example.gov", + "GRANTS_GOV_PATH_URL": "/extracts/", "MAX_DOWNLOAD_BACKOFF": "1us", }, &env) require.NoError(t, err, "Error configuring lambda environment for testing") @@ -80,11 +81,11 @@ func TestScheduledEventGrantsURL(t *testing.T) { env.GrantsGovBaseURL = "https://example.gov" estTZ := time.FixedZone("America/New_York", -5*3600) - assert.Equal(t, "https://example.gov/extract/GrantsDBExtract20230102v2.zip", + assert.Equal(t, "https://example.gov/extracts/GrantsDBExtract20230102v2.zip", (&ScheduledEvent{time.Date(2023, 1, 2, 3, 4, 5, 6, estTZ)}).grantsURL()) - assert.Equal(t, "https://example.gov/extract/GrantsDBExtract20230203v2.zip", + assert.Equal(t, "https://example.gov/extracts/GrantsDBExtract20230203v2.zip", (&ScheduledEvent{time.Date(2023, 2, 3, 4, 5, 6, 7, time.UTC)}).grantsURL()) - assert.Equal(t, "https://example.gov/extract/GrantsDBExtract20231112v2.zip", + assert.Equal(t, "https://example.gov/extracts/GrantsDBExtract20231112v2.zip", (&ScheduledEvent{time.Date(2023, 11, 12, 0, 0, 0, 0, time.UTC)}).grantsURL()) } diff --git a/cmd/DownloadGrantsGovDB/main.go b/cmd/DownloadGrantsGovDB/main.go index 0d96541c..f90345ae 100644 --- a/cmd/DownloadGrantsGovDB/main.go +++ b/cmd/DownloadGrantsGovDB/main.go @@ -27,6 +27,7 @@ type Environment struct { LogLevel string `env:"LOG_LEVEL,default=INFO"` DestinationBucket string `env:"GRANTS_SOURCE_DATA_BUCKET_NAME,required=true"` GrantsGovBaseURL string `env:"GRANTS_GOV_BASE_URL,required=true"` + GrantsGovPathURL string `env:"GRANTS_GOV_PATH_URL,required=true"` MaxDownloadBackoff time.Duration `env:"MAX_DOWNLOAD_BACKOFF,default=20s"` UsePathStyleS3Opt bool `env:"S3_USE_PATH_STYLE,default=false"` Extras goenv.EnvSet diff --git a/terraform/modules/DownloadGrantsGovDB/main.tf b/terraform/modules/DownloadGrantsGovDB/main.tf index c6d3d9f1..fdf7101f 100644 --- a/terraform/modules/DownloadGrantsGovDB/main.tf +++ b/terraform/modules/DownloadGrantsGovDB/main.tf @@ -88,7 +88,8 @@ module "lambda_function" { timeout = 120 # 2 minutes, in seconds environment_variables = merge(var.additional_environment_variables, { DD_TAGS = join(",", sort([for k, v in local.dd_tags : "${k}:${v}"])) - GRANTS_GOV_BASE_URL = "https://www.grants.gov" + GRANTS_GOV_BASE_URL = "https://prod-grants-gov-chatbot.s3.amazonaws.com" + GRANTS_GOV_PATH_URL = "/extracts/" GRANTS_SOURCE_DATA_BUCKET_NAME = data.aws_s3_bucket.grants_source_data.id LOG_LEVEL = var.log_level })