From c57950e9e69e754ceb1e65802869a27ef6e0979a Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Sun, 29 Oct 2017 13:53:25 -0400 Subject: [PATCH 1/2] add alarms and reduce package size with rasterio wheels --- .gitignore | 1 + Dockerfile | 59 +++++++++++++++++++++++++++++++++++++------------- README.md | 2 +- package.json | 14 ++++++++++++ serverless.yml | 26 +++++++++++++++++++--- 5 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 7da7f17..d66b107 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ ENV/ .serverless package.zip +node_modules/ diff --git a/Dockerfile b/Dockerfile index 2e81291..6cfe824 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,51 @@ -FROM remotepixel/amazonlinux-gdal:latest +# Use the official amazonlinux AMI image +FROM amazonlinux:latest -RUN pip3 install remotepixel --no-binary numpy,rasterio -t /tmp/vendored +# Install apt dependencies +RUN yum install -y \ + gcc gcc-c++ freetype-devel yum-utils findutils openssl-devel -COPY handler.py /tmp/vendored/handler.py +RUN yum -y groupinstall development + +RUN curl https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tar.xz | tar -xJ \ + && cd Python-3.6.1 \ + && ./configure --prefix=/usr/local --enable-shared \ + && make \ + && make install \ + && cd .. \ + && rm -rf Python-3.6.1 + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +RUN pip3 install numpy wheel cython --no-binary numpy + +# Install Python dependencies +RUN pip3 install remotepixel==0.0.2 --no-binary numpy -t /tmp/vendored -U -#Reduce Lambda package size (<250Mb) -RUN echo "package original size $(du -sh /tmp/vendored | cut -f1)" -RUN find /tmp/vendored \ - \( -type d -a -name test -o -name tests \) \ - -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - -print0 | xargs -0 rm -f -RUN echo "package new size $(du -sh /tmp/vendored | cut -f1)" +RUN du -sh /tmp/vendored -RUN cd /tmp \ - && zip -r9q /tmp/package.zip vendored/* +# This is the list of available modules on AWS lambda Python 3 +# ['boto3', 'botocore', 'docutils', 'jmespath', 'pip', 'python-dateutil', 's3transfer', 'setuptools', 'six'] +RUN find /tmp/vendored -name "*-info" -type d -exec rm -rdf {} + +RUN rm -rdf /tmp/vendored/boto3/ +RUN rm -rdf /tmp/vendored/botocore/ +RUN rm -rdf /tmp/vendored/docutils/ +RUN rm -rdf /tmp/vendored/dateutil/ +RUN rm -rdf /tmp/vendored/jmespath/ +RUN rm -rdf /tmp/vendored/s3transfer/ +RUN rm -rdf /tmp/vendored/numpy/doc/ + +# Leave module precompiles for faster Lambda startup +RUN find /tmp/vendored -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-36//'); cp $f $n; done; +RUN find /tmp/vendored -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find /tmp/vendored -type f -a -name '*.py' -print0 | xargs -0 rm -f + +RUN du -sh /tmp/vendored + +COPY handler.py /tmp/vendored/handler.py -RUN cd $APP_DIR/local \ - && zip --symlinks -r9q /tmp/package.zip lib/*.so* \ - && zip -r9q /tmp/package.zip share +# Create archive +RUN cd /tmp/vendored && zip -r9q /tmp/package.zip * +# Cleanup RUN rm -rf /tmp/vendored/ diff --git a/README.md b/README.md index d39b120..c8938b5 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Same for `deploymentBucket` ([here](https://github.com/RemotePixel/remotepixel-a ``` make build && make package -npm install -g serverless +npm install make deploy-us && make deploy-eu ``` diff --git a/package.json b/package.json new file mode 100644 index 0000000..d526617 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "remotepixel-api", + "version": "1.0.0", + "description": "Create a highly customizable `serverless` tile server for Amazon's Landsat Public Dataset", + "devDependencies": { + "serverless-plugin-aws-alerts": "^1.2.4", + "serverless": "^1.23.0" + }, + "repository": { + "type": "git", + "url": "git://github.com/remotepixel/landsat-tiler.git" + }, + "author": "Vincent Sarago" +} diff --git a/serverless.yml b/serverless.yml index 4dd81d8..cbadc46 100644 --- a/serverless.yml +++ b/serverless.yml @@ -3,13 +3,11 @@ service: remotepixel-api provider: name: aws runtime: python3.6 - stage: production + stage: ${opt:stage, 'production'} region: ${opt:region} #us-west-2 or eu-central-1 environment: - PYTHONPATH: /var/task/vendored - GDAL_DATA: /var/task/share/gdal # Needed if build gdal from source GDAL_CACHEMAX: 75% GDAL_TIFF_OVR_BLOCKSIZE: 512 VSI_CACHE: TRUE @@ -40,6 +38,24 @@ provider: deploymentBucket: remotepixel-${opt:region} #Optional Bucket where you store your lambda package +custom: + alerts: + dashboards: true + topics: + alarm: + topic: ${self:service}-${self:provider.stage}-alerts-alarm + notifications: + - protocol: email + endpoint: ${opt:mail} + definitions: + functionInvocations: + threshold: 1000 + functionErrors: + threshold: 10 + +plugins: + - serverless-plugin-aws-alerts + package: artifact: package.zip @@ -56,6 +72,10 @@ functions: l8_full: handler: handler.l8_full_handler + alarms: + - functionInvocations + - functionErrors + - functionThrottles memorySize: 1536 timeout: 45 events: From bb708882683b0e6f5c6cbf7c963b91a621429ad5 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Sun, 29 Oct 2017 14:00:13 -0400 Subject: [PATCH 2/2] update readme --- Makefile | 12 ------------ README.md | 5 +++-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index e39c538..07e65c6 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,6 @@ run: --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ --env AWS_REGION=us-west-2 \ - --env PYTHONPATH=/var/task/vendored \ - --env GDAL_DATA=/var/task/share/gdal \ --env GDAL_CACHEMAX=75% \ --env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \ --env GDAL_TIFF_OVR_BLOCKSIZE=512 \ @@ -43,8 +41,6 @@ shell: --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ --env AWS_REGION=us-west-2 \ - --env PYTHONPATH=/var/task/vendored \ - --env GDAL_DATA=/var/task/share/gdal \ --env GDAL_CACHEMAX=75% \ --env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \ --env GDAL_TIFF_OVR_BLOCKSIZE=512 \ @@ -55,14 +51,6 @@ shell: remotepixel:latest /bin/bash -deploy-us: - sls deploy --region us-west-2 - - -deploy-eu: - sls deploy --region eu-central-1 - - clean: docker stop remotepixel docker rm remotepixel diff --git a/README.md b/README.md index c8938b5..46f2a29 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,12 @@ You need to update `OUTPUT BUCKET` value in https://github.com/RemotePixel/remot Same for `deploymentBucket` ([here](https://github.com/RemotePixel/remotepixel-api/blob/34ffff88f0c6ac5073401443dda0357ffb274370/serverless.yml#L42)) ``` -make build && make package +make all npm install -make deploy-us && make deploy-eu +sls deploy --region us-west-2 --mail {your-email} +sls deploy --region eu-central-1 --mail {your-email} ``` #### Why us & eu ?