FIX: automation of lambda creation, upload, and configuration (as muc…

.gitignore

@@ -11,12 +11,14 @@ __pycache__
 *.rdb
 *.egg-info
 podpac/cache/
-
+dist/aws/podpac_dist.zip
+dist/aws/podpac_deps.zip
 
 # test
 .cache
 .coverage
 artifacts
+settings.json
 
 # docs
 doc/build

dist/aws/Dockerfile

@@ -31,15 +31,15 @@ RUN cd /tmp/vendored/ && touch pydap/__init__.py && \
 
 RUN cp -r /podpac/ /tmp/vendored/ && \
     mv /tmp/vendored/podpac/dist/aws/handler.py /tmp/vendored/handler.py && \
-    cp tmp/vendored/podpac/dist/aws/mk_dist.py /tmp/vendored/mk_dist.py && \
+    cp tmp/vendored/podpac/dist/aws/_mk_dist.py /tmp/vendored/_mk_dist.py && \
     rm -rf /tmp/vendored/podpac/dist/ && \
     cp -r /tmp/vendored/podpac/podpac/* /tmp/vendored/podpac/ && \
     rm -rf /tmp/vendored/podpac/podpac/*
 
 RUN cd /tmp/vendored && \
-    find * -maxdepth 0 -type f | grep ".zip" -v | grep -v ".pyc" | xargs zip -9 -rqy podpac_dist_latest.zip
+    find * -maxdepth 0 -type f | grep ".zip" -v | grep -v ".pyc" | xargs zip -9 -rqy podpac_dist.zip
 RUN cd /tmp/vendored && \
     find * -maxdepth 0 -type d -exec zip -9 -rqy {}.zip {} \;
 RUN cd /tmp/vendored && du -s *.zip > zip_package_sizes.txt
 RUN cd /tmp/vendored && du -s * | grep .zip -v > package_sizes.txt
-RUN cd /tmp/vendored && python3 mk_dist.py
+RUN cd /tmp/vendored && python3 _mk_dist.py

dist/aws/_mk_dist.py

@@ -0,0 +1,65 @@
+#!python
+
+# Make dist and deps zip archives
+# for podpac lambda function
+#
+# Used in the Dockerfile
+
+import subprocess
+
+with open("zip_package_sizes.txt", "r") as fid:
+    zps = fid.read()
+
+with open("package_sizes.txt", "r") as fid:
+    ps = fid.read()
+
+
+def parse_ps(ps):
+    lns = ps.split("\n")
+    pkgs = {}
+    for ln in lns:
+        try:
+            parts = ln.split("\t")
+            pkgs[parts[1]] = int(parts[0])
+        except:
+            pass
+    return pkgs
+
+
+pgz = parse_ps(zps)
+pg = parse_ps(ps)
+
+data = {}
+for p, s in pgz.items():
+    os = pg.get(p[:-4], 0)
+    data[p] = {"zip_size": s, "size": os, "ratio": os * 1.0 / s}
+
+sdata = sorted(data.items(), key=lambda t: t[1]["ratio"])
+
+zipsize = data["podpac_dist.zip"]["zip_size"]
+totsize = sum([pg[k] for k in pg if (k + ".zip") not in pgz.keys()])
+pkgs = []
+for val in sdata[::-1]:
+    if val[0] == "podpac_dist.zip" or "rasterio" in val[0] or "pyproj" in val[0]:
+        continue
+    key = val[0]
+    pkgs.append(key)
+
+    zipsize += data[key]["zip_size"]
+    totsize += data[key]["size"]
+
+    if zipsize > 50000 or totsize > 250000:
+        k = pkgs.pop()
+        zipsize -= data[k]["zip_size"]
+        totsize -= data[k]["size"]
+
+core = [k[:-4] for k in pkgs if k != "podpac_dist.zip"]
+deps = [k[:-4] for k in data if k[:-4] not in core and k != "podpac_dist.zip"]
+dep_size = sum([data[k + ".zip"]["size"] for k in deps])
+dep_size_zip = sum([data[k + ".zip"]["zip_size"] for k in deps])
+
+# add core to podpac_dist.zip
+cmd = ["zip", "-9", "-rq", "podpac_dist.zip"] + core
+subprocess.call(cmd)
+cmd = ["zip", "-9", "-rqy", "podpac_deps.zip"] + deps
+subprocess.call(cmd)

dist/aws/build_lambda.sh

@@ -1,45 +1,39 @@
 #!/bin/sh
+#
+# Build podpac lambda distribution and dependencies
+# 
+# Currently, this builds the function using the local 
+# podpac repository,including any outstanding changes.
+# 
+# Usage:
+# 
+# $ bash build_lambda.sh [s3-bucket] [function-name] 
+# 
+# Requires:
+# - Docker
+# - `settings.json` to be copied to the root directory of the podpac repository
+#   This will not be required in the future
+#   
+# Example usage:
+# 
+# $ bash build_lambda.sh
 
-# Set up some variables
-if [ -z "$1" ]
-  then
-    TAG=""
-    COMMIT_SHA="$(git rev-parse HEAD)"
-    VERSION="develop"
-  else
-    TAG=$1
-    VERSION=$TAG
-    COMMIT_SHA=""
-fi
 
-set -e
-aws s3 ls s3://podpac-s3/podpac
-
-# TODO change this to expect tag from args, exit if there is no tag given.
+# variables
+COMMIT_SHA="$(git rev-parse HEAD)"
+TAG="$(git describe --always)"
 DOCKER_NAME="podpac"
-DOCKER_TAG="latest"
-echo "${COMMIT_SHA}"
-echo $DOCKER_NAME:$DOCKER_TAG
+DOCKER_TAG=$TAG
+
+echo "Creating docker image from podpac version ${TAG}"
+echo "${DOCKER_NAME}:${DOCKER_TAG}"
 
-# Build docker, and extract zips
+# Navigate to root, build docker, and extract zips
+pushd ../../
 docker build -f dist/aws/Dockerfile --no-cache --tag $DOCKER_NAME:$DOCKER_TAG --build-arg COMMIT_SHA="${COMMIT_SHA}" --build-arg TAG="${TAG}" .
 docker run --name "${DOCKER_NAME}" -itd $DOCKER_NAME:$DOCKER_TAG
-docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_dist_latest.zip .
-docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_deps_latest.zip .
+docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_dist.zip ./dist/aws
+docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_deps.zip ./dist/aws
 docker stop "${DOCKER_NAME}"
 docker rm "${DOCKER_NAME}"
-
-# Upload zips to S3, according to naming convention
-if [ -z $TAG ]
-  then
-    echo "tag is empty"
-  else
-    aws s3 cp podpac_deps_latest.zip s3://podpac-s3/podpac/podpac_deps_$TAG.zip
-    aws s3 cp podpac_dist_latest.zip s3://podpac-s3/podpac/podpac_dist_$TAG.zip
-fi
-aws s3 cp podpac_deps_latest.zip s3://podpac-s3/podpac/podpac_deps_ESIP3.zip
-aws s3 cp podpac_dist_latest.zip s3://podpac-s3/podpac/podpac_dist_ESIP3.zip
-rm podpac_deps_latest.zip podpac_dist_latest.zip
-
-# Update lambda function to use the zips from S3 (uploaded above).
-aws lambda update-function-code --function-name podpac_lambda_ESIP3 --s3-bucket podpac-s3 --s3-key podpac/podpac_dist_ESIP3.zip
+popd

dist/aws/configure_lambda.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# Configure AWS for podpac lambda function
+# 
+# Usage:
+# 
+# $ bash configure_lambda.sh [s3-bucket] [function-name]
+# 
+# Requires:
+# - AWS CLI: https://docs.aws.amazon.com/cli/
+# - AWS credentials must be configured using the `aws` cli.
+#   See https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration
+# - Dist and Dependencies uploaded using `upload_lambda.sh`
+# - Function must be created from the AWS Dashboard
+# - API Gateway must be created from the AWS Dashboard
+#   - Note down the `rest-api-id` and `resource-id` in parentheses in the top bar of the API gatway dashboard
+#   - You must Select the top level resource '/' and select "Create Method"
+# Example usage:
+# 
+# $ bash configure_lambda.sh podpac-s3 podpac_lambda h827as06ji 1ya7h6
+# 
+
+# TODO: remove this in the future when function generation/update is automated elsewhere
+
+BUCKET=$1
+FUNCTION=$2
+API_ID=$3
+API_RESOURCE_ID=$4
+TAG="$(git describe --always)"
+
+if [ -z "$BUCKET" ]
+  then
+    echo "S3 bucket name required as first cli argument"
+    exit 1
+  else
+    echo "Bucket: ${BUCKET}"
+fi
+
+if [ -z "$FUNCTION" ]
+  then
+    echo "Function name required as second cli argument"
+    exit 1
+  else
+    echo "Function: ${FUNCTION}"
+fi
+
+if [ -z "$API_ID" ]
+  then
+    echo "Rest API ID required as third cli argument"
+    exit 1
+  else
+    echo "REST API ID: ${API_ID}"
+fi
+
+if [ -z "$API_RESOURCE_ID" ]
+  then
+    echo "API Resource ID required as fourth cli argument"
+    exit 1
+  else
+    echo "API Resource ID: ${API_RESOURCE_ID}"
+fi
+
+# Update lambda function to use the zips from S3 (uploaded above)
+# aws lambda update-function-code --function-name $FUNCTION --s3-bucket $BUCKET --s3-key podpac/podpac_dist_$TAG.zip
+# aws lambda update-function-configuration --function-name $FUNCTION --handler handler.handler --timeout 300 --memory-size 2048
+# aws apigateway update-rest-api --rest-api-id $API_ID --patch-operations "op=replace,path=/binaryMediaTypes/*~1*,value='*/*'"
+RESOURCE=$(aws apigateway create-resource --rest-api-id $API_ID --parent-id $API_RESOURCE_ID --path-part 'lambda' --output text)
+RESOURCE_ID=$(echo "$(echo $RESOURCE | cut -d " " -f1)")
+aws apigateway put-method --rest-api-id $API_ID --resource-id $RESOURCE_ID --http-method ANY --authorization-type NONE
+
+echo "Log in to AWS and perform the following steps:"
+echo "1. Navigate to your API in the API Gateway and select the resource /lambda HTTP Method (ANY)."
+echo "2. Select Integration Request -> Lambda Function, Check Use Lambda Proxy Integration, Select your lambda function region and function name."
+echo "3. Press the Actions dropdown and select Deploy API. Select [New Stage] and create a stage name (doesn't matter exact name)"
+echo "4. Navigate to your lambda function console and confirm you see API Gateway as a trigger."
+
+# LAMBDA_URI=`$(aws lambda)
+# aws apigateway put-integration --rest-api-id $API_ID --resource-id $API_RESOURCE_ID --http-method ANY --type AWS --integration-http-method POST --uri