CFN Tempalte and Lambda to automatically provision and deprovision NAT gateways when Elastio workers are running

.github/workflows/ci.yml

@@ -22,6 +22,7 @@ jobs:
       ci: "${{ steps.filter.outputs.ci }}"
       aws-backup-elastio-integration: "${{ steps.filter.outputs.aws-backup-elastio-integration }}"
       elastio-s3-changelog: "${{ steps.filter.outputs.elastio-s3-changelog }}"
+      elastio-nat-provision-lambda: "${{ steps.filter.outputs.elastio-nat-provision-lambda }}"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -36,6 +37,8 @@ jobs:
               - 'aws-backup-elastio-integration/**'
             elastio-s3-changelog:
               - 'elastio-s3-changelog/**'
+            elastio-nat-provision-lambda:
+              - 'elastio-nat-provision-lambda/**'
 
   typos:
     runs-on: ubuntu-latest
@@ -91,3 +94,22 @@ jobs:
           role-to-assume: ${{ vars.aws_elastio_prod_artifacts_contrib_role_arn }}
 
       - run: ./elastio-s3-changelog/upload.sh
+  upload-elastio-nat-provision-lambda:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: >-
+      github.event_name != 'pull_request' && (
+        needs.changes.outputs.elastio-nat-provision-lambda == 'true' ||
+        needs.changes.outputs.ci == 'true'
+      )
+    env:
+      S3_BUCKET: elastio-prod-artifacts-us-east-2
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ vars.aws_elastio_prod_artifacts_contrib_role_arn }}
+
+      - run: ./elastio-nat-provision-lambda/upload.sh

.gitignore

@@ -5,6 +5,8 @@
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
+.idea/*
+
 .history/
 
 *.vsix

elastio-nat-provision-lambda/README.md

@@ -0,0 +1,34 @@
+# NAT Gateway Provision Lambda
+
+This CloudFormation template deploys a lambda function with EventBridge rules that automatically
+provision a NAT Gateway when Elastio worker EC2 instances are starting, and de-provisions it
+when they are not running anymore.
+
+The lambda will provision one NAT Gateway and Elastic IP per VPC, and configure the route table
+of the subnet where Elastio worker instances are running to route all traffic through the NAT Gateway.
+Note there is a default limit of 5 Elastic IP addresses per AWS region, and there should be at least one
+address available when the lambda deploys the NAT Gateway.
+
+The lambda will only provision a NAT Gateway if Elastio workers are running in a private subnet,
+and there is at least one public subnet in the same availability zone in the same VPC. There must
+be no route `0.0.0.0/0` configured in the route table of the private subnet.
+
+## Deploying the CFN stack
+
+1. Use one of the following quick-create links. Choose the region where your Elastio Cloud Connector is deployed.
+
+    * [us-east-1](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [us-east-2](https://us-east-2.console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [us-west-1](https://us-west-1.console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [us-west-2](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [eu-central-1](https://eu-central-1.console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [eu-west-1](https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [eu-west-2](https://eu-west-2.console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [eu-west-3](https://eu-west-3.console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [ca-central-1](https://ca-central-1.console.aws.amazon.com/cloudformation/home?region=ca-central-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [ap-south-1](https://ap-south-1.console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [ap-southeast-1](https://ap-southeast-1.console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+    * [ap-southeast-2](https://ap-southeast-2.console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?templateURL=https://elastio-prod-artifacts-us-east-2.s3.us-east-2.amazonaws.com/contrib/elastio-nat-provision-lambda/v1/cloudformation-lambda.yaml&stackName=elastio-nat-provision-lambda)
+
+2. Check the box in front of `I acknowledge that AWS CloudFormation might create IAM resources`
+    and click `Create stack`.

elastio-nat-provision-lambda/cloudformation-lambda.yaml

@@ -0,0 +1,242 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/awslabs/goformation/master/schema/cloudformation.schema.json
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >
+  Provisions a NAT Gateway when Elastio worker instances are running, and de-provisions the gateway when
+  the instances are not running.
+Parameters:
+  DeleteQuiescentPeriodSeconds:
+    Type: Number
+    Default: 300
+    MinValue: 0
+    Description: How long to wait for no new EC2 instances to appear before deleting the NAT Gateway
+  CleanupScheduleCron:
+    Type: String
+    Default: '0 1 * * ? *'
+    Description: >
+      A cron expression that defines when and how often to run a cleanup routine.
+      The syntax corresponds to the AWS Scheduler's cron expression syntax:
+      https://docs.aws.amazon.com/scheduler/latest/UserGuide/schedule-types.html#cron-based
+  NatGatewayStackPrefix:
+    Type: String
+    Default: elastio-nat-gateway-
+    MinLength: 1
+    Description: Prefix of the name of the NAT Gateway CFN stack. The name will be <prefix><public-subnet-id>
+  LambdaMemorySize:
+    Type: Number
+    Default: 512
+    MinValue: 128
+    MaxValue: 10240
+    Description: Amount of memory allocated to the lambda function
+  LambdaTimeout:
+    Type: Number
+    Default: 600
+    MinValue: 10
+    MaxValue: 900
+    Description: Max amount of time the lambda function can run
+  LambdaLogsRetention:
+    Type: String
+    Default: '7'
+    AllowedValues: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653]
+    Description: The number of days to retain the log events in the lambda's log group
+
+Resources:
+  # The default log group that AWS Lambda creates has retention disabled.
+  # We don't want to store logs indefinitely, so we create a custom log group with
+  # retention enabled.
+  lambdaLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: /aws/lambda/elastio-nat-gateway-provision
+      RetentionInDays: !Ref LambdaLogsRetention
+
+  lambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: ElastioNatProvisionPolicy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - cloudformation:DescribeStacks
+                  - ec2:AllocateAddress
+                  - ec2:CreateNatGateway
+                  - ec2:CreateRoute
+                  - ec2:CreateTags
+                  - ec2:DeleteRoute
+                  - ec2:DescribeAddresses
+                  - ec2:DescribeInstances
+                  - ec2:DescribeNatGateways
+                  - ec2:DescribeRouteTables
+                  - ec2:DescribeSubnets
+                  - ec2:DescribeVpcs
+                  - ec2:ReleaseAddress
+                  - states:DescribeExecution
+                  - states:ListExecutions
+                Resource: '*'
+              - Effect: Allow
+                Action:
+                  - cloudformation:CreateStack
+                  - cloudformation:DeleteStack
+                  - ec2:DeleteNatGateway
+                Resource: '*'
+                Condition:
+                  StringLike:
+                    aws:ResourceTag/elastio:resource: '*'
+              - Effect: Allow
+                # We don't give the lambda a permission to create log groups
+                # because we pre-create the log group ourselves
+                Action:
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                Resource: arn:aws:logs:*:*:*
+
+  lambdaInvocationRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - states.amazonaws.com
+                - scheduler.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: lambdaInvokePolicy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - lambda:InvokeFunction
+                Resource: !GetAtt lambdaFunction.Arn
+
+  stateMachineExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: events.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: startStateMachinePolicy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - states:StartExecution
+                Resource: !GetAtt natGatewayCleanupStateMachine.Arn
+
+  lambdaFunction:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: elastio-nat-gateway-provision
+      Handler: lambda.lambda_handler
+      Runtime: python3.12
+      MemorySize: !Ref LambdaMemorySize
+      Timeout: !Ref LambdaTimeout
+      Role: !GetAtt lambdaRole.Arn
+      Environment:
+        Variables:
+          NAT_CFN_PREFIX: !Ref NatGatewayStackPrefix
+          NAT_CFN_TEMPLATE_URL: https://{{S3_BUCKET}}.s3.{{AWS_REGION}}.amazonaws.com/{{S3_PREFIX}}/{{VERSION}}/cloudformation-nat.yaml
+          STATE_MACHINE_ARN: !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:elastio-nat-gateway-provision-state-machine'
+      Code:
+        S3Bucket: {{S3_BUCKET}}
+        S3Key: {{S3_PREFIX}}/{{VERSION}}/lambda.zip
+
+  pendingInstancesSubscription:
+    Type: AWS::Events::Rule
+    Properties:
+      Description: Track pending EC2 instances for Elastio NAT Gateway provisioner lambda
+      EventPattern:
+        source: [ aws.ec2 ]
+        detail-type: [ EC2 Instance State-change Notification ]
+        detail:
+          state:
+            - pending
+      Targets:
+        - Arn: !GetAtt lambdaFunction.Arn
+          Id: event-handler
+
+  pendingInstancesLambdaInvokePermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      Action: lambda:InvokeFunction
+      FunctionName: !Ref lambdaFunction
+      Principal: events.amazonaws.com
+      SourceArn: !GetAtt pendingInstancesSubscription.Arn
+
+  stoppedInstancesSubscription:
+    Type: AWS::Events::Rule
+    Properties:
+      Description: Track stopped and terminated EC2 instances for Elastio NAT Gateway provisioner lambda
+      EventPattern:
+        source: [ aws.ec2 ]
+        detail-type: [ EC2 Instance State-change Notification ]
+        detail:
+          state:
+            - stopped
+            - terminated
+      Targets:
+        - Arn: !GetAtt natGatewayCleanupStateMachine.Arn
+          Id: event-handler
+          RoleArn: !GetAtt stateMachineExecutionRole.Arn
+
+  natGatewayCleanupStateMachine:
+    Type: AWS::StepFunctions::StateMachine
+    Properties:
+      StateMachineName: elastio-nat-gateway-provision-state-machine
+      RoleArn: !GetAtt lambdaInvocationRole.Arn
+      Definition:
+        StartAt: Wait
+        States:
+          Wait:
+            Type: Wait
+            Seconds: !Ref DeleteQuiescentPeriodSeconds
+            Next: InvokeLambda
+          InvokeLambda:
+            Type: Task
+            Resource: !GetAtt lambdaFunction.Arn
+            End: true
+
+  cleanupSchedule:
+    Type: AWS::Scheduler::Schedule
+    Properties:
+      Description: A schedule to cleanup unnecessary deployed NAT Gateways
+      ScheduleExpression: !Sub cron(${CleanupScheduleCron})
+      FlexibleTimeWindow:
+        Mode: 'OFF'
+      State: ENABLED
+      Target:
+        Arn: !GetAtt lambdaFunction.Arn
+        RoleArn: !GetAtt lambdaInvocationRole.Arn
+        Input: '{ "elastio_scheduled_cleanup": true }'
+
+Outputs:
+  templateVersion:
+    Value: {{VERSION}}
+  lambdaFunctionArn:
+    Value: !GetAtt lambdaFunction.Arn
+  pendingInstancesSubscriptionArn:
+    Value: !GetAtt pendingInstancesSubscription.Arn
+  stoppedInstancesSubscriptionArn:
+    Value: !GetAtt stoppedInstancesSubscription.Arn
+  natGatewayCleanupStateMachineArn:
+    Value: !GetAtt natGatewayCleanupStateMachine.Arn
+  cleanupScheduleArn:
+    Value: !GetAtt cleanupSchedule.Arn

elastio-nat-provision-lambda/cloudformation-nat.yaml

@@ -0,0 +1,50 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/awslabs/goformation/master/schema/cloudformation.schema.json
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Deploys a single NAT Gateway
+
+Parameters:
+  PublicSubnetId:
+    Type: String
+    Description: ID of a public subnet where the NAT Gateway will be deployed
+  PrivateSubnetRouteTableId:
+    Type: String
+    Description: >
+      ID of a route table associated with the private subnet that will be modified to forward internet
+      traffic through NAT
+
+Resources:
+  eip:
+    Type: AWS::EC2::EIP
+    Properties:
+      Domain: vpc
+      Tags:
+        - Key: elastio:resource
+          Value: 'true'
+        - Key: elastio:nat-provision-stack-id
+          Value: !Ref AWS::StackId
+
+  nat:
+    Type: AWS::EC2::NatGateway
+    Properties:
+      AllocationId: !GetAtt eip.AllocationId
+      SubnetId: !Ref PublicSubnetId
+      Tags:
+        - Key: elastio:resource
+          Value: 'true'
+        - Key: elastio:nat-provision-stack-id
+          Value: !Ref AWS::StackId
+
+  route:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId: !Ref PrivateSubnetRouteTableId
+      DestinationCidrBlock: '0.0.0.0/0'
+      NatGatewayId: !Ref nat
+
+Outputs:
+  templateVersion:
+    Value: {{VERSION}}
+  eipAllocationId:
+    Value: !GetAtt eip.AllocationId
+  natGatewayId:
+    Value: !Ref nat