Skip to content
This repository has been archived by the owner on Jun 14, 2021. It is now read-only.

Commit

Permalink
Merge pull request #5 from RemotePixel/arlams
Browse files Browse the repository at this point in the history
add alarms and reduce package size with rasterio wheels
  • Loading branch information
vincentsarago authored Oct 29, 2017
2 parents 748b78e + bb70888 commit f80e6ac
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 33 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,4 @@ ENV/
.serverless

package.zip
node_modules/
59 changes: 44 additions & 15 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,22 +1,51 @@
FROM remotepixel/amazonlinux-gdal:latest
# Use the official amazonlinux AMI image
FROM amazonlinux:latest

RUN pip3 install remotepixel --no-binary numpy,rasterio -t /tmp/vendored
# Install apt dependencies
RUN yum install -y \
gcc gcc-c++ freetype-devel yum-utils findutils openssl-devel

COPY handler.py /tmp/vendored/handler.py
RUN yum -y groupinstall development

RUN curl https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tar.xz | tar -xJ \
&& cd Python-3.6.1 \
&& ./configure --prefix=/usr/local --enable-shared \
&& make \
&& make install \
&& cd .. \
&& rm -rf Python-3.6.1

ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH

RUN pip3 install numpy wheel cython --no-binary numpy

# Install Python dependencies
RUN pip3 install remotepixel==0.0.2 --no-binary numpy -t /tmp/vendored -U

#Reduce Lambda package size (<250Mb)
RUN echo "package original size $(du -sh /tmp/vendored | cut -f1)"
RUN find /tmp/vendored \
\( -type d -a -name test -o -name tests \) \
-o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
-print0 | xargs -0 rm -f
RUN echo "package new size $(du -sh /tmp/vendored | cut -f1)"
RUN du -sh /tmp/vendored

RUN cd /tmp \
&& zip -r9q /tmp/package.zip vendored/*
# This is the list of available modules on AWS lambda Python 3
# ['boto3', 'botocore', 'docutils', 'jmespath', 'pip', 'python-dateutil', 's3transfer', 'setuptools', 'six']
RUN find /tmp/vendored -name "*-info" -type d -exec rm -rdf {} +
RUN rm -rdf /tmp/vendored/boto3/
RUN rm -rdf /tmp/vendored/botocore/
RUN rm -rdf /tmp/vendored/docutils/
RUN rm -rdf /tmp/vendored/dateutil/
RUN rm -rdf /tmp/vendored/jmespath/
RUN rm -rdf /tmp/vendored/s3transfer/
RUN rm -rdf /tmp/vendored/numpy/doc/

# Leave module precompiles for faster Lambda startup
RUN find /tmp/vendored -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-36//'); cp $f $n; done;
RUN find /tmp/vendored -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
RUN find /tmp/vendored -type f -a -name '*.py' -print0 | xargs -0 rm -f

RUN du -sh /tmp/vendored

COPY handler.py /tmp/vendored/handler.py

RUN cd $APP_DIR/local \
&& zip --symlinks -r9q /tmp/package.zip lib/*.so* \
&& zip -r9q /tmp/package.zip share
# Create archive
RUN cd /tmp/vendored && zip -r9q /tmp/package.zip *

# Cleanup
RUN rm -rf /tmp/vendored/
12 changes: 0 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ run:
--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
--env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
--env AWS_REGION=us-west-2 \
--env PYTHONPATH=/var/task/vendored \
--env GDAL_DATA=/var/task/share/gdal \
--env GDAL_CACHEMAX=75% \
--env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \
--env GDAL_TIFF_OVR_BLOCKSIZE=512 \
Expand Down Expand Up @@ -43,8 +41,6 @@ shell:
--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
--env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
--env AWS_REGION=us-west-2 \
--env PYTHONPATH=/var/task/vendored \
--env GDAL_DATA=/var/task/share/gdal \
--env GDAL_CACHEMAX=75% \
--env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \
--env GDAL_TIFF_OVR_BLOCKSIZE=512 \
Expand All @@ -55,14 +51,6 @@ shell:
remotepixel:latest /bin/bash


deploy-us:
sls deploy --region us-west-2


deploy-eu:
sls deploy --region eu-central-1


clean:
docker stop remotepixel
docker rm remotepixel
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@ You need to update `OUTPUT BUCKET` value in https://github.com/RemotePixel/remot
Same for `deploymentBucket` ([here](https://github.com/RemotePixel/remotepixel-api/blob/34ffff88f0c6ac5073401443dda0357ffb274370/serverless.yml#L42))

```
make build && make package
make all
npm install -g serverless
npm install
make deploy-us && make deploy-eu
sls deploy --region us-west-2 --mail {your-email}
sls deploy --region eu-central-1 --mail {your-email}
```

#### Why us & eu ?
Expand Down
14 changes: 14 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"name": "remotepixel-api",
"version": "1.0.0",
"description": "Create a highly customizable `serverless` tile server for Amazon's Landsat Public Dataset",
"devDependencies": {
"serverless-plugin-aws-alerts": "^1.2.4",
"serverless": "^1.23.0"
},
"repository": {
"type": "git",
"url": "git://github.com/remotepixel/landsat-tiler.git"
},
"author": "Vincent Sarago"
}
26 changes: 23 additions & 3 deletions serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@ service: remotepixel-api
provider:
name: aws
runtime: python3.6
stage: production
stage: ${opt:stage, 'production'}

region: ${opt:region} #us-west-2 or eu-central-1

environment:
PYTHONPATH: /var/task/vendored
GDAL_DATA: /var/task/share/gdal # Needed if build gdal from source
GDAL_CACHEMAX: 75%
GDAL_TIFF_OVR_BLOCKSIZE: 512
VSI_CACHE: TRUE
Expand Down Expand Up @@ -40,6 +38,24 @@ provider:

deploymentBucket: remotepixel-${opt:region} #Optional Bucket where you store your lambda package

custom:
alerts:
dashboards: true
topics:
alarm:
topic: ${self:service}-${self:provider.stage}-alerts-alarm
notifications:
- protocol: email
endpoint: ${opt:mail}
definitions:
functionInvocations:
threshold: 1000
functionErrors:
threshold: 10

plugins:
- serverless-plugin-aws-alerts

package:
artifact: package.zip

Expand All @@ -56,6 +72,10 @@ functions:

l8_full:
handler: handler.l8_full_handler
alarms:
- functionInvocations
- functionErrors
- functionThrottles
memorySize: 1536
timeout: 45
events:
Expand Down

0 comments on commit f80e6ac

Please sign in to comment.