Added localstack and locale2e

dev/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.8
+
+RUN pip install boto3 requests
+
+COPY ./iam_role.json ./
+COPY ./setup_localstack.py ./
+
+CMD [ "python", "./setup_localstack.py" ]

dev/README.md

@@ -40,6 +40,7 @@ MAPBOX_GEOCODE_RATE_LIMIT_PER_MIN=<Mapbox API rate limit, default = 600>
 REACT_APP_PUBLIC_MAPBOX_TOKEN=<Different Mapbox API token>
 REACT_APP_POLICY_PUBLIC_ID=<Public id for Iubenda service that provides legal policies>
 REACT_APP_COOKIE_CONSENT_PUBLIC_ID=<Public ID for Iubenda service that provides cookie consent banner>
+LOCALSTACK_API_KEY=<Localstack (mock AWS) API key>
 ```
 
 These values are stored in a dedicated secret manager. To request AWS credentials, or for access to

dev/docker-compose.dev.full.yml

@@ -0,0 +1,72 @@
+version: "3.7"
+services:
+  geocoding:
+    environment:
+      DEBUG: "True"
+      ENABLE_FAKE_GEOCODER: "True"
+  mongo:
+    volumes:
+      - ../verification/scripts:/verification/scripts
+  curator:
+    command: "npm run dev"
+    volumes:
+      - ../verification/curator-service/api/src:/usr/src/app/verification/curator-service/api/src
+      - ../verification/curator-service/api/openapi:/usr/src/app/verification/curator-service/api/openapi
+    environment:
+      LOCALHOST_URL: "http://localstack:4566"
+      SERVICE_ENV: "locale2e"
+      AFTER_LOGIN_REDIRECT_URL: "http://localhost:3002"
+  data:
+    command: "npm run dev"
+    volumes:
+      - ../data-serving/data-service/src:/usr/src/app/data-serving/data-service/src
+      - ../data-serving/data-service/api:/usr/src/app/data-serving/data-service/api
+  curatorui:
+    command: "npm run start-noenv"
+    volumes:
+      - ../verification/curator-service/ui/src:/usr/src/app/verification/curator-service/ui/src
+    environment:
+      # We can't use curator:3001 here because that's an internal DNS,
+      # not accessible from the user's browser.
+      # In production simply /auth/google would work.
+      REACT_APP_LOGIN_URL: "http://localhost:3001/auth/google"
+      REACT_APP_PUBLIC_MAPBOX_TOKEN: "${REACT_APP_PUBLIC_MAPBOX_TOKEN}"
+      REACT_APP_POLICY_PUBLIC_ID: "${REACT_APP_POLICY_PUBLIC_ID}"
+      REACT_APP_COOKIE_CONSENT_PUBLIC_ID: "${REACT_APP_COOKIE_CONSENT_PUBLIC_ID}"
+      ENABLE_LOCAL_AUTH: "true"
+  localstack:
+    image: localstack/localstack-full
+    ports:
+      - "53:53"
+      - "443:443"
+      - "4566:4566"
+      - "4571:4571"
+      - "8080:8080"
+    environment:
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
+      LOCALSTACK_API_KEY: "${LOCALSTACK_API_KEY}"
+      SERVICES: s3,ec2,ses,batch,events,iam,lambda
+      DEBUG: 1
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:4566/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+  setup-localstack:
+    build:
+      context: ./
+    depends_on:
+      - localstack
+    environment:
+      AWS_ACCESS_KEY_ID: "fake"
+      AWS_SECRET_ACCESS_KEY: "fake"
+      AWS_DEFAULT_REGION: "us-east-1"
+      AWS_ENDPOINT: "http://localstack:4566"
+      DATA_BUCKET_NAME: "covid-19-data-export"
+      DOWNLOAD_BUCKET_NAME: "covid19-filtered-downloads"
+      BATCH_QUEUE_NAME: "ingestion-queue"
+      SES_EMAIL_ADDRESS: "info@global.health"

dev/iam_role.json

@@ -0,0 +1,9 @@
+{
+    "Effect": "Allow",
+    "Action": [
+        "iam:AttachRolePolicy",
+        "iam:CreateRole",
+        "iam:PutRolePolicy"
+    ],
+    "Resource": "*"
+}

dev/run_full_stack.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+# Store current directory.
+# We have to run docker compose from this directory for it to pick up the .env file.
+pushd `dirname "$0"`
+docker compose -f docker-compose.yml -f docker-compose.dev.full.yml up --build --force-recreate --renew-anon-volumes
+# Restore directory.
+popd

dev/setup_localstack.py

@@ -0,0 +1,192 @@
+from os import environ
+from time import sleep
+
+import boto3
+import requests
+
+
+LOCALSTACK_URL = environ.get("AWS_ENDPOINT", "http://localstack:4566")
+BATCH_READY = '"batch": "running"'
+
+IAM_PROFILE_NAME = "ecsInstanceRole"
+IAM_PROFILE_ARN = "".join(["arn:aws:iam::000000000000:instance-profile/", "ecsInstanceRole"])
+BATCH_SERVICE_ROLE_NAME = "foo"
+ECS_INSTANCE_ROLE_NAME = "ecsInstanceRole"
+IAM_ROLE_POLICY_DOCUMENT = "file://./iam_role.json"
+EC2_SECURITY_GROUP_DESCRIPTION = "bar"
+EC2_SECURITY_GROUP_NAME = "sg-1234abcd"
+BATCH_COMPUTE_ENVIRONMENT_NAME = "M4Spot"
+BATCH_COMPUTE_ENVIRONMENT_ARN = "".join(["arn:aws:batch:us-east-1:000000000000:compute-environment/", BATCH_COMPUTE_ENVIRONMENT_NAME])
+BATCH_COMPUTE_ENVIRONMENT_TYPE = "MANAGED"
+
+BATCH_COMPUTE_RESOURCES = {
+    "type": "EC2",
+    "minvCpus": 1,
+    "maxvCpus": 10,
+    "instanceTypes": [
+        "m4.large",
+    ],
+    "imageId": "string",
+    "subnets": [],
+    "securityGroupIds": [],
+    "instanceRole": IAM_PROFILE_ARN,
+    "ec2Configuration": [
+        {
+            "imageType": "string",
+            "imageIdOverride": "string"
+        },
+    ]
+}
+
+BATCH_JOB_QUEUE_NAME = environ.get("BATCH_QUEUE_NAME", "ingestion-queue")
+BATCH_COMPUTE_ENVIRONMENT_ORDER = [{
+    "order": 1,
+    "computeEnvironment": BATCH_COMPUTE_ENVIRONMENT_ARN
+}]
+
+DATA_BUCKET_NAME = environ.get("DATA_BUCKET_NAME", "covid-19-data-export")
+DOWNLOAD_BUCKET_NAME = environ.get("DOWNLOAD_BUCKET_NAME", "covid19-filtered-downloads")
+SES_EMAIL_ADDRESS = environ.get("SES_EMAIL_ADDRESS", "info@global.health")
+
+class LocalstackWrangler(object):
+
+    def __init__(self):
+        self.iam_client = boto3.client("iam", endpoint_url=LOCALSTACK_URL)
+        self.ec2_client = boto3.client("ec2", endpoint_url=LOCALSTACK_URL)
+        self.s3_client = boto3.client("s3", endpoint_url=LOCALSTACK_URL)
+        self.ses_client = boto3.client("ses", endpoint_url=LOCALSTACK_URL)
+        self.batch_client = boto3.client("batch", endpoint_url=LOCALSTACK_URL)
+
+    def create_iam_role(self, role_name):
+        print(f"Creating IAM role {role_name} for Batch")
+        self.iam_client.create_role(
+            RoleName=role_name,
+            AssumeRolePolicyDocument=IAM_ROLE_POLICY_DOCUMENT 
+        )
+        print("Done creating role")
+
+    def create_iam_profile(self, profile_name):
+        print(f"Creating IAM profile {profile_name} for ECS instance")
+        self.iam_client.create_instance_profile(
+            InstanceProfileName=profile_name
+        )
+        print("Done creating profile")
+
+    def get_iam_role_arn(self, role_name):
+        print(f"Getting ARN for IAM role {role_name}")
+        response = self.iam_client.get_role(
+            RoleName=role_name
+        )
+        role_info = response.get("Role", {})
+        arn = role_info.get("Arn", "")
+        print(f"Got ARN {arn}")
+        return arn
+
+    def create_security_group(self, group_name):
+        print(f"Creating security group {group_name}")
+        self.ec2_client.create_security_group(
+            Description=EC2_SECURITY_GROUP_DESCRIPTION,
+            GroupName=group_name
+        )
+        print("Created security group")
+
+    def get_security_group_id(self):
+        print("Getting security group ID")
+        response = self.ec2_client.describe_security_groups()
+        security_groups = response.get("SecurityGroups", [""])
+        group_id = security_groups[0].get("GroupId", "")
+        print(f"Got group ID {group_id}")
+        return group_id
+
+    def create_subnet(self):
+        print("Creating VPC and subnet")
+        response = self.ec2_client.create_vpc(
+            CidrBlock="10.0.0.0/16"
+        )
+        vpc_info = response.get("Vpc", {})
+        vpc_id = vpc_info.get("VpcId", {})
+        self.ec2_client.create_subnet(
+            CidrBlock="10.0.2.0/24",
+            VpcId=vpc_id
+        )
+        print("Created VPC and subnet")
+
+    def get_subnet_id(self):
+        print("Getting subnet ID")
+        response = self.ec2_client.describe_subnets()
+        subnets = response.get("Subnets", [{}])
+        subnet_id = subnets[0].get("SubnetId", "")
+        print(f"Got subnet ID {subnet_id}")
+        return subnet_id
+
+    def create_compute_environment(self, security_group_id, batch_service_role_arn, subnet_id):
+        print(f"Creating compute environment for security group {security_group_id}")
+        BATCH_COMPUTE_RESOURCES["securityGroupIds"].append(security_group_id)
+        BATCH_COMPUTE_RESOURCES["subnets"].append(subnet_id)
+        self.batch_client.create_compute_environment(
+            computeEnvironmentName=BATCH_COMPUTE_ENVIRONMENT_NAME,
+            type=BATCH_COMPUTE_ENVIRONMENT_TYPE,
+            computeResources=BATCH_COMPUTE_RESOURCES,
+            serviceRole=batch_service_role_arn
+        )
+        print("Created security group")
+
+    def create_batch_job_queue(self, queue_name):
+        print(f"Creating job queue {queue_name}")
+        self.batch_client.create_job_queue(
+            jobQueueName=queue_name,
+            priority=1,
+            state="ENABLED",
+            computeEnvironmentOrder=BATCH_COMPUTE_ENVIRONMENT_ORDER
+        )
+        print("Created job queue")
+
+    def create_s3_bucket(self, bucket_name):
+        print(f"Creating bucket {bucket_name}")
+        self.s3_client.create_bucket(
+            Bucket=bucket_name
+        )
+        print("Created bucket")
+
+    def setup_ses(self, email_address):
+        print(f"Verifying email address {email_address}")
+        self.ses_client.verify_email_identity(
+            EmailAddress=email_address
+        )
+        print("Verified email address")
+
+
+def wait_for_localstack():
+    healthcheck_url = "".join([LOCALSTACK_URL, "/health"])
+    counter = 0
+    while counter < 20:
+        try:
+            response = requests.get(healthcheck_url)
+            if BATCH_READY in response.text:
+                return
+        except requests.exceptions.ConnectionError:
+            pass
+        counter += 1
+        print("Waiting for localstack")
+        sleep(5)
+    raise Exception("Localstack not available")
+
+
+if __name__ == "__main__":
+    print("Setting up localstack resources")
+    wait_for_localstack()
+    lw = LocalstackWrangler()
+    lw.create_iam_role(BATCH_SERVICE_ROLE_NAME)
+    lw.create_iam_profile(IAM_PROFILE_NAME)
+    lw.create_security_group(EC2_SECURITY_GROUP_NAME)
+    security_group_id = lw.get_security_group_id()
+    batch_service_role_arn = lw.get_iam_role_arn(BATCH_SERVICE_ROLE_NAME)
+    ecs_instance_role_arn = None
+    lw.create_subnet()
+    subnet_id = lw.get_subnet_id()
+    lw.create_compute_environment(security_group_id, batch_service_role_arn, subnet_id)
+    lw.create_batch_job_queue(BATCH_JOB_QUEUE_NAME)
+    lw.setup_ses(SES_EMAIL_ADDRESS)
+    lw.create_s3_bucket(DATA_BUCKET_NAME)
+    lw.create_s3_bucket(DOWNLOAD_BUCKET_NAME)
+    print("Done setting up localstack resources")

dev/test_all.sh

@@ -36,7 +36,7 @@ npm --prefix=`dirname "$0"`/../verification/curator-service/ui/ run-script test-
 RUNNING=$(curl localhost:3002 --silent | grep "DOCTYPE" || true)
 if [ -z "$RUNNING" ]; then
   echo "Starting stack..."
-  nohup ./run_stack.sh | tee &
+  nohup ./run_full_stack.sh | tee &
 fi
 
 until $(curl localhost:3002 --silent --fail | grep "DOCTYPE" > /dev/null); do
@@ -45,5 +45,6 @@ until $(curl localhost:3002 --silent --fail | grep "DOCTYPE" > /dev/null); do
 done
 
 echo "Stack running"
+
 echo "Running UI browser tests"
 npm --prefix=`dirname "$0"`/../verification/curator-service/ui/ run-script cypress-run

verification/curator-service/api/src/clients/aws-batch-client.ts

@@ -36,13 +36,22 @@ export default class AwsBatchClient {
     private readonly batchClient: AWS.Batch;
     constructor(
         private readonly serviceEnv: string,
+        private readonly localstackURL: string,
         readonly jobQueueArn: string,
         awsRegion: string,
     ) {
         AWS.config.update({ region: awsRegion });
-        this.batchClient = new AWS.Batch({
-            apiVersion: '2016-08-10',
-        });
+        if (serviceEnv == 'locale2e') {
+            this.batchClient = new AWS.Batch({
+                apiVersion: '2016-08-10',
+                endpoint: localstackURL,
+            });
+        }
+        else {
+            this.batchClient = new AWS.Batch({
+                apiVersion: '2016-08-10',
+            });
+        }
     }
 
     /**

verification/curator-service/api/src/clients/aws-events-client.ts

@@ -17,15 +17,24 @@ import { logger } from '../util/logger';
 export default class AwsEventsClient {
     private readonly cloudWatchEventsClient: AWS.CloudWatchEvents;
     constructor(
+        private readonly serviceEnv: string,
+        private readonly localstackURL: string,
         awsRegion: string,
         readonly batchClient: AwsBatchClient,
         readonly eventRoleArn: string,
-        private readonly serviceEnv: string,
     ) {
         AWS.config.update({ region: awsRegion });
-        this.cloudWatchEventsClient = new AWS.CloudWatchEvents({
-            apiVersion: '2015-10-07',
-        });
+        if (serviceEnv == 'locale2e') {
+            this.cloudWatchEventsClient = new AWS.CloudWatchEvents({
+                apiVersion: '2015-10-07',
+                endpoint: localstackURL,
+            });
+        }
+        else {
+            this.cloudWatchEventsClient = new AWS.CloudWatchEvents({
+                apiVersion: '2015-10-07',
+            });
+        }
     }
 
     /**

verification/curator-service/api/src/clients/aws-lambda-client.ts

@@ -15,12 +15,22 @@ export default class AwsLambdaClient {
     private readonly lambdaClient: AWS.Lambda;
     constructor(
         private readonly serviceEnv: string,
+        private readonly localstackURL: string,
         awsRegion: string,
     ) {
         AWS.config.update({ region: awsRegion });
-        this.lambdaClient = new AWS.Lambda({
-            apiVersion: '2015-03-31',
-        });
+        if (serviceEnv == 'locale2e') {
+            logger.info("using localstack");
+            this.lambdaClient = new AWS.Lambda({
+                apiVersion: '2015-03-31',
+                endpoint: localstackURL,
+            });
+        }
+        else {
+            this.lambdaClient = new AWS.Lambda({
+                apiVersion: '2015-03-31',
+            });
+        }
     }
 
     /**