Update template.yaml and make build to support build-time-only re…

…sources

Replace `bin/start-local-api` and `bin/start-with-step` with `make serve` and `make start-with-step`
Update dependencies and test invocation to get rid of some deprecation warnings
Split `make serve` into `make serve-http` and `make serve-https` with HTTPS as the default
Update `README.md` with new startup instructions

.github/workflows/deploy.yml

@@ -51,7 +51,7 @@ jobs:
           aws-region: us-east-1
       - run: ln -s .tfvars/dc-api/samconfig.toml .
       - run: ln -s .tfvars/dc-api/$CONFIG_ENV.parameters .
-      - run: sam build
+      - run: make build
       - run: | 
           sam deploy \
             --no-confirm-changeset \

.gitignore

@@ -224,10 +224,10 @@ $RECYCLE.BIN/
 .venv
 
 .vscode
-/samconfig.toml
-/samconfig.yaml
-/samconfig.*.yaml
-/env.json
-/env.*.json
-/*.parameters
-/schemas
+samconfig.*
+!dev/samconfig.toml
+env.json
+env.*.json
+*.parameters
+/schemas
+.sam-pids

Makefile

@@ -2,35 +2,50 @@ ifndef VERBOSE
 .SILENT:
 endif
 ENV=dev
+SHELL := /bin/bash
 
 help:
-	echo "make build          | build the SAM project"
-	echo "make serve          | run the SAM server locally"
-	echo "make clean          | remove all installed dependencies and build artifacts"
-	echo "make deps           | install all dependencies"
-	echo "make link           | create hard links to allow for hot reloading of a built project"
-	echo "make secrets        | symlink secrets files from ../tfvars"
-	echo "make style          | run all style checks"
-	echo "make test           | run all tests"
-	echo "make cover          | run all tests with coverage"
-	echo "make env ENV=[env]  | activate env.\$$ENV.json file (default: dev)"
-	echo "make deps-node      | install node dependencies"
-	echo "make deps-python    | install python dependencies"
-	echo "make style-node     | run node code style check"
-	echo "make style-python   | run python code style check"
-	echo "make test-node      | run node tests"
-	echo "make test-python    | run python tests"
-	echo "make cover-node     | run node tests with coverage"
-	echo "make cover-python   | run python tests with coverage"
+	echo "make build           | build the SAM project"
+	echo "make serve           | alias for serve-https"
+	echo "make clean           | remove all installed dependencies and build artifacts"
+	echo "make deps            | install all dependencies"
+	echo "make link            | create hard links to allow for hot reloading of a built project"
+	echo "make secrets         | symlink secrets files from ../tfvars"
+	echo "make start-with-step | run the SAM server locally with step function & download lambdas"
+	echo "make style           | run all style checks"
+	echo "make test            | run all tests"
+	echo "make cover           | run all tests with coverage"
+	echo "make env ENV=[env]   | activate env.\$$ENV.json file (default: dev)"
+	echo "make deps-node       | install node dependencies"
+	echo "make deps-python     | install python dependencies"
+	echo "make serve-http      | run the SAM server locally (HTTP on port 3000)"
+	echo "make serve-https     | run the SAM server locally (HTTPS on port 3002)"
+	echo "make style-node      | run node code style check"
+	echo "make style-python    | run python code style check"
+	echo "make test-node       | run node tests"
+	echo "make test-python     | run python tests"
+	echo "make cover-node      | run node tests with coverage"
+	echo "make cover-python    | run python tests with coverage"
 .aws-sam/build.toml: ./template.yaml node/package-lock.json node/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
+	sed -Ei.orig 's/^(\s+)#\*\s/\1/' template.yaml
 	sam build --cached --parallel
+	mv template.yaml.orig template.yaml
 deps-node:
-	cd node && npm ci
-cover-node:
+	cd node/src ;\
+	npm list >/dev/null 2>&1 ;\
+	src_deps=$$? ;\
+	cd .. ;\
+	npm list >/dev/null 2>&1 ;\
+	dev_deps=$$? ;\
+	test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
+
+	cd lambdas ;\
+	npm list >/dev/null 2>&1 || npm ci
+cover-node: deps-node
 	cd node && npm run test:coverage
-style-node:
+style-node: deps-node
 	cd node && npm run prettier
-test-node:
+test-node: deps-node
 	cd node && npm run test
 deps-python:
 	cd chat/src && pip install -r requirements.txt
@@ -43,15 +58,26 @@ style-python: deps-python
 style-python-fix: deps-python
 	cd chat && ruff check --fix . 
 test-python: deps-python
-	cd chat && export SKIP_WEAVIATE_SETUP=True && PYTHONPATH=src:test && python -m unittest discover -v
+	cd chat && SKIP_WEAVIATE_SETUP=True PYTHONPATH=src:test python -m unittest discover -v
 python-version:
 	cd chat && python --version
 build: .aws-sam/build.toml
-link: build
-	cd chat/src && for src in *.py **/*.py; do for target in $$(find ../../.aws-sam/build -maxdepth 1 -type d); do if [[ -f $$target/$$src ]]; then ln -f $$src $$target/$$src; fi; done; done
-	cd node/src && for src in *.js *.json **/*.js **/*.json; do for target in $$(find ../../.aws-sam/build -maxdepth 1 -type d); do if [[ -f $$target/$$src ]]; then ln -f $$src $$target/$$src; fi; done; done
-serve: link
-	sam local start-api --host 0.0.0.0 --log-file dc-api.log
+serve-http: deps-node
+	@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
+	rm -rf .aws-sam
+	sam local start-api --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
+serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem
+serve-https: serve-http
+serve: serve-https
+start-with-step: deps-node
+	sam local start-lambda --host 0.0.0.0 --port 3005 --env-vars env.json --log-file lambda.log & \
+	echo $$! > .sam-pids ;\
+	sam local start-api --host 0.0.0.0 --port 3002 --log-file dc-api.log \
+		--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
+	echo $$! >> .sam-pids ;\
+	docker run --rm -p 8083:8083 -e LAMBDA_ENDPOINT=http://172.17.0.1:3005/ amazon/aws-stepfunctions-local ;\
+	kill $$(cat .sam-pids) ;\
+	rm -f .sam-pids
 deps: deps-node deps-python
 style: style-node style-python
 test: test-node test-python

README.md

@@ -6,7 +6,6 @@
 
 ```
 .
-├── bin/ - utilities for working with API locally
 ├── dev/ - example configs for developers
 ├── docs/ - mkdocs-based API documentation
 ├── events/ - sample HTTP API Lambda events
@@ -61,7 +60,7 @@ asdf install aws-sam-cli
 Then run the following command:
 
 ```shell
-bin/start-local-api
+make serve
 ```
 
 The API will be available at:
@@ -126,7 +125,7 @@ export AWS_PROFILE=staging-admin
 aws sso login
 
 # Start the API + step function and associated lambdas
-bin/start-with-step
+make start-with-step
 
 # Open a second terminal and create the state machine
 aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://state_machines/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole

layers/api_dependencies/package.json

@@ -7,10 +7,11 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@aws-crypto/sha256-browser": "^2.0.1",
-    "@aws-sdk/credential-provider-node": "^3.142.0",
-    "@aws-sdk/node-http-handler": "^3.127.0",
-    "@aws-sdk/protocol-http": "^3.127.0",
-    "@aws-sdk/signature-v4": "^3.130.0",
+    "@aws-sdk/client-sfn": "^3.563.0",
+    "@aws-sdk/credential-provider-node": "^3.563.0",
+    "@smithy/node-http-handler": "^2.5.0",
+    "@smithy/protocol-http": "^3.3.0",
+    "@smithy/signature-v4": "^2.3.0",
     "@honeybadger-io/js": "^4.9.3",
     "axios": ">=0.21.1",
     "cookie": "^0.5.0",