update cdk code and move env in settings (#254)

developmentseed · Mar 2, 2021 · 54d2b20 · 54d2b20
1 parent f849f83
commit 54d2b20
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 35 deletions.
diff --git a/deployment/aws/cdk/app.py b/deployment/aws/cdk/app.py
@@ -1,8 +1,7 @@
 """Construct App."""
 
 import os
-from copy import deepcopy
-from typing import Any, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from aws_cdk import aws_apigatewayv2 as apigw
 from aws_cdk import aws_apigatewayv2_integrations as apigw_integrations
@@ -16,20 +15,6 @@
 settings = StackSettings()
 
 
-DEFAULT_ENV = dict(
-    CPL_TMPDIR="/tmp",
-    CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".tif",
-    GDAL_CACHEMAX="75%",
-    GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR",
-    GDAL_HTTP_MERGE_CONSECUTIVE_RANGES="YES",
-    GDAL_HTTP_MULTIPLEX="YES",
-    GDAL_HTTP_VERSION="2",
-    PYTHONWARNINGS="ignore",
-    VSI_CACHE="TRUE",
-    VSI_CACHE_SIZE="1000000",
-)
-
-
 class titilerLambdaStack(core.Stack):
     """
     Titiler Lambda Stack
@@ -49,14 +34,15 @@ def __init__(
         runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_8,
         concurrent: Optional[int] = None,
         permissions: Optional[List[iam.PolicyStatement]] = None,
-        env: dict = {},
+        env: Optional[Dict] = None,
         code_dir: str = "./",
         **kwargs: Any,
     ) -> None:
         """Define stack."""
         super().__init__(scope, id, *kwargs)
 
         permissions = permissions or []
+        env = env or {}
 
         lambda_function = aws_lambda.Function(
             self,
@@ -75,7 +61,7 @@ def __init__(
             memory_size=memory,
             reserved_concurrent_executions=concurrent,
             timeout=core.Duration.seconds(timeout),
-            environment={**DEFAULT_ENV, **env},
+            environment=env,
         )
 
         for perm in permissions:
@@ -103,20 +89,21 @@ def __init__(
         mincount: int = 1,
         maxcount: int = 50,
         permissions: Optional[List[iam.PolicyStatement]] = None,
-        env: dict = {},
+        env: Optional[Dict] = None,
         code_dir: str = "./",
         **kwargs: Any,
     ) -> None:
         """Define stack."""
         super().__init__(scope, id, *kwargs)
 
         permissions = permissions or []
+        env = env or {}
 
         vpc = ec2.Vpc(self, f"{id}-vpc", max_azs=2)
 
         cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc)
 
-        task_env = deepcopy(DEFAULT_ENV)
+        task_env = env.copy()
         task_env.update(dict(LOG_LEVEL="error"))
 
         # GUNICORN configuration
@@ -127,8 +114,6 @@ def __init__(
         if settings.web_concurrency:
             task_env.update({"WEB_CONCURRENCY": str(settings.web_concurrency)})
 
-        task_env.update(env)
-
         fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService(
             self,
             f"{id}-service",
@@ -209,7 +194,7 @@ def __init__(
     mincount=settings.min_ecs_instances,
     maxcount=settings.max_ecs_instances,
     permissions=perms,
-    env=settings.additional_env,
+    env=settings.env,
 )
 
 lambda_stackname = f"{settings.name}-lambda-{settings.stage}"
@@ -220,7 +205,7 @@ def __init__(
     timeout=settings.timeout,
     concurrent=settings.max_concurrent,
     permissions=perms,
-    env=settings.additional_env,
+    env=settings.env,
 )
 
 app.synth()
diff --git a/deployment/aws/cdk/config.py b/deployment/aws/cdk/config.py
@@ -14,13 +14,26 @@ class StackSettings(pydantic.BaseSettings):
     owner: Optional[str]
     client: Optional[str]
 
-    additional_env: Dict = {}
+    # Default options are optimized for CloudOptimized GeoTIFF
+    # For more information on GDAL env see: https://gdal.org/user/configoptions.html
+    env: Dict = {
+        "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
+        "GDAL_CACHEMAX": "200",  # 200 mb
+        "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+        "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+        "GDAL_HTTP_MULTIPLEX": "YES",
+        "GDAL_HTTP_VERSION": "2",
+        "PYTHONWARNINGS": "ignore",
+        "VSI_CACHE": "TRUE",
+        "VSI_CACHE_SIZE": "5000000",  # 5 MB (per file-handle)
+    }
 
     # add S3 bucket where TiTiler could do HEAD and GET Requests
     buckets: List = []
 
-    #########
+    ###########################################################################
     # AWS ECS
+    # The following settings only apply to AWS ECS deployment
     min_ecs_instances: int = 5
     max_ecs_instances: int = 50
 
@@ -54,8 +67,9 @@ class StackSettings(pydantic.BaseSettings):
 
     image_version: str = "latest"
 
-    ############
+    ###########################################################################
     # AWS LAMBDA
+    # The following settings only apply to AWS Lambda deployment
     timeout: int = 10
     memory: int = 1536
     # more about lambda config: https://www.sentiatechblog.com/aws-re-invent-2020-day-3-optimizing-lambda-cost-with-multi-threading

diff --git a/docs/deployment/aws/intro.md b/docs/deployment/aws/intro.md
@@ -1,10 +1,10 @@
 # Amazon Web Services deployments
 
-Example of of AWS deployments can be found in https://github.com/developmentseed/titiler/tree/master/deployment/aws. Those examples use [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) to define stacks using python code.
+Examples of AWS deployments can be found in https://github.com/developmentseed/titiler/tree/master/deployment/aws. Those examples use [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) to define stacks using python code.
 
 # Configuration/Settings
 
-Deployment settings are managed via [pydantic.BaseSettings](https://pydantic-docs.helpmanual.io/usage/settings/) and stored in [config.py](https://github.com/developmentseed/titiler/blob/master/stack/config.py). Pydantic BaseSettings can receive input to overwrite the default value from `.env` file or from environment variables.
+Deployment settings are managed via [pydantic.BaseSettings](https://pydantic-docs.helpmanual.io/usage/settings/) and stored in [config.py](https://github.com/developmentseed/titiler/blob/master/stack/config.py). Pydantic BaseSettings can receive input to overwrite the default value from a `.env` file or from environment variables. 
 
 Variables in `.env` or in environment variable need to be prefixed with `TITILER_STACK_`:
 
@@ -14,15 +14,14 @@ TITILER_STACK_NAME="my-tiler"
 TITILER_STACK_STAGE="dev"
 
 TITILER_STACK_BUCKETS='["my-bucket*", "*"]'
-TITILER_STACK_MOSAIC_HOST="my-bucket/mosaics"
 
 TITILER_STACK_MEMORY=3008
 
 # Uncomment to allow lambda to access content on requester-payer buckets
-# TITILER_STACK_ADDITIONAL_ENV='{"AWS_REQUEST_PAYER":"requester"}'
+# TITILER_STACK_ENV='{"AWS_REQUEST_PAYER":"requester"}'
 
 # Uncomment if you only on the /cog endpoint
-# TITILER_STACK_ADDITIONAL_ENV='{"TITILER_API_DISABLE_STAC": "TRUE", "TITILER_API_DISABLE_MOSAIC": "TRUE"}'
+# TITILER_STACK_ENV='{"TITILER_API_DISABLE_STAC": "TRUE", "TITILER_API_DISABLE_MOSAIC": "TRUE"}'
 ```
 
 Default values from [config.py](https://github.com/developmentseed/titiler/blob/master/deployment/aws/cdk/config.py):
@@ -33,13 +32,26 @@ stage: str = "production"
 owner: Optional[str]
 client: Optional[str]
 
-additional_env: Dict = {}
+# Default options are optimized for CloudOptimized GeoTIFF
+# For more information on GDAL env see: https://gdal.org/user/configoptions.html
+env: Dict = {
+    "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
+    "GDAL_CACHEMAX": "200" # 200 mb
+    "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+    "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+    "GDAL_HTTP_MULTIPLEX": "YES",
+    "GDAL_HTTP_VERSION": "2",
+    "PYTHONWARNINGS": "ignore",
+    "VSI_CACHE": "TRUE",
+    "VSI_CACHE_SIZE": "5000000" # 5 MB (per file-handle)      
+}
 
 # add S3 bucket where TiTiler could do HEAD and GET Requests
 buckets: List = []
 
-#########
+###########################################################################
 # AWS ECS
+# The following settings only apply to AWS ECS deployment
 min_ecs_instances: int = 5
 max_ecs_instances: int = 50
 
@@ -73,8 +85,9 @@ web_concurrency: Optional[int]
 
 image_version: str = "latest"
 
-############
+###########################################################################
 # AWS LAMBDA
+# The following settings only apply to AWS Lambda deployment
 timeout: int = 10
 memory: int = 1536
 # more about lambda config: https://www.sentiatechblog.com/aws-re-invent-2020-day-3-optimizing-lambda-cost-with-multi-threading