Merge pull request #207 from nulib/deploy/staging

Deploy to production

.github/workflows/deploy.yml

@@ -51,7 +51,7 @@ jobs:
           aws-region: us-east-1
       - run: ln -s .tfvars/dc-api/samconfig.toml .
       - run: ln -s .tfvars/dc-api/$CONFIG_ENV.parameters .
-      - run: sam build
+      - run: make build
       - run: | 
           sam deploy \
             --no-confirm-changeset \
@@ -60,5 +60,6 @@ jobs:
             --config-file ./samconfig.toml \
             --parameter-overrides $(while IFS='=' read -r key value; do params+=" $key=$value"; done < ./$CONFIG_ENV.parameters && echo "$params HoneybadgerRevision=$HONEYBADGER_REVISION") \
           | sed 's/\(Parameter overrides\s*\): .*/\1: ***** REDACTED *****/'
+          exit ${PIPESTATUS[0]}
         env:
           HONEYBADGER_REVISION: ${{ github.sha }}

.github/workflows/production-types.yml

@@ -32,7 +32,7 @@ jobs:
           path: "${{ env.TMP_SRCDIR }}"
       - uses: actions/setup-node@v3
         with:
-          cache-dependency-path: "${{ env.TMP_SRCDIR }}/package-lock.json"
+          cache-dependency-path: "${{ env.TMP_SRCDIR }}/node/package-lock.json"
           node-version: 16.x
           cache: "npm"
       - name: Set short git commit SHA
@@ -46,18 +46,18 @@ jobs:
         with:
           path: "${{ env.TMP_SRCDIR }}"
       - name: Generate Typescript file
-        working-directory: "${{ env.TMP_SRCDIR }}"
+        working-directory: "${{ env.TMP_SRCDIR }}/node"
         run: |
           npm ci
-          npx openapi-typescript docs/docs/spec/data-types.yaml --output schemas.ts
+          npx openapi-typescript ../docs/docs/spec/data-types.yaml --output schemas.ts
       - name: Checkout nulib/dcapi-types
         uses: actions/checkout@v3
         with:
           path: ${{ env.TMP_DSTDIR }}
           ref: ${{ env.TARGET_BRANCH }}
           repository: "nulib/dcapi-types"
           token: ${{env.GH_TOKEN}}
-      - run: mv -f "${{ env.TMP_SRCDIR }}/schemas.ts" "${{ env.TMP_DSTDIR }}/schemas.ts"
+      - run: mv -f "${{ env.TMP_SRCDIR }}/node/schemas.ts" "${{ env.TMP_DSTDIR }}/schemas.ts"
         shell: bash
       - working-directory: "${{ env.TMP_DSTDIR }}"
         run: |

.github/workflows/staging-types.yml

@@ -11,7 +11,7 @@ on:
   workflow_dispatch:
 jobs:
   Push-To-Types-Repo-Staging:
-    if: github.event.pull_request.merged == true
+    if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -31,26 +31,26 @@ jobs:
           path: "${{ env.TMP_SRCDIR }}"
       - uses: actions/setup-node@v3
         with:
-          cache-dependency-path: "${{ env.TMP_SRCDIR }}/package-lock.json"
+          cache-dependency-path: "${{ env.TMP_SRCDIR }}/node/package-lock.json"
           node-version: 16.x
           cache: "npm"
       - name: get-npm-version
         id: package-version
         uses: martinbeentjes/npm-get-version-action@main
         with:
-          path: "${{ env.TMP_SRCDIR }}"
+          path: "${{ env.TMP_SRCDIR }}/node"
       - name: Generate Typescript file
-        working-directory: "${{ env.TMP_SRCDIR }}"
+        working-directory: "${{ env.TMP_SRCDIR }}/node"
         run: |
           npm ci
-          npx openapi-typescript docs/docs/spec/data-types.yaml --output schemas.ts
+          npx openapi-typescript ../docs/docs/spec/data-types.yaml --output schemas.ts
       - name: Checkout dcapi-types
         uses: actions/checkout@v3
         with:
           path: ${{ env.TMP_DSTDIR }}
           ref: ${{ env.TARGET_BRANCH }}
           repository: ${{env.TYPES_REPO}}
-      - run: mv -f "${{ env.TMP_SRCDIR }}/schemas.ts" "${{ env.TMP_DSTDIR }}/schemas.ts"
+      - run: mv -f "${{ env.TMP_SRCDIR }}/node/schemas.ts" "${{ env.TMP_DSTDIR }}/schemas.ts"
         shell: bash
       - name: Set GitHub Deploy Key
         uses: webfactory/ssh-agent@v0.5.3

.gitignore

@@ -224,10 +224,10 @@ $RECYCLE.BIN/
 .venv
 
 .vscode
-/samconfig.toml
-/samconfig.yaml
-/samconfig.*.yaml
-/env.json
-/env.*.json
-/*.parameters
-/schemas
+samconfig.*
+!dev/samconfig.toml
+env.json
+env.*.json
+*.parameters
+/schemas
+.sam-pids

Makefile

@@ -2,35 +2,50 @@ ifndef VERBOSE
 .SILENT:
 endif
 ENV=dev
+SHELL := /bin/bash
 
 help:
-	echo "make build          | build the SAM project"
-	echo "make serve          | run the SAM server locally"
-	echo "make clean          | remove all installed dependencies and build artifacts"
-	echo "make deps           | install all dependencies"
-	echo "make link           | create hard links to allow for hot reloading of a built project"
-	echo "make secrets        | symlink secrets files from ../tfvars"
-	echo "make style          | run all style checks"
-	echo "make test           | run all tests"
-	echo "make cover          | run all tests with coverage"
-	echo "make env ENV=[env]  | activate env.\$$ENV.json file (default: dev)"
-	echo "make deps-node      | install node dependencies"
-	echo "make deps-python    | install python dependencies"
-	echo "make style-node     | run node code style check"
-	echo "make style-python   | run python code style check"
-	echo "make test-node      | run node tests"
-	echo "make test-python    | run python tests"
-	echo "make cover-node     | run node tests with coverage"
-	echo "make cover-python   | run python tests with coverage"
+	echo "make build           | build the SAM project"
+	echo "make serve           | alias for serve-https"
+	echo "make clean           | remove all installed dependencies and build artifacts"
+	echo "make deps            | install all dependencies"
+	echo "make link            | create hard links to allow for hot reloading of a built project"
+	echo "make secrets         | symlink secrets files from ../tfvars"
+	echo "make start-with-step | run the SAM server locally with step function & download lambdas"
+	echo "make style           | run all style checks"
+	echo "make test            | run all tests"
+	echo "make cover           | run all tests with coverage"
+	echo "make env ENV=[env]   | activate env.\$$ENV.json file (default: dev)"
+	echo "make deps-node       | install node dependencies"
+	echo "make deps-python     | install python dependencies"
+	echo "make serve-http      | run the SAM server locally (HTTP on port 3000)"
+	echo "make serve-https     | run the SAM server locally (HTTPS on port 3002)"
+	echo "make style-node      | run node code style check"
+	echo "make style-python    | run python code style check"
+	echo "make test-node       | run node tests"
+	echo "make test-python     | run python tests"
+	echo "make cover-node      | run node tests with coverage"
+	echo "make cover-python    | run python tests with coverage"
 .aws-sam/build.toml: ./template.yaml node/package-lock.json node/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
+	sed -Ei.orig 's/^(\s+)#\*\s/\1/' template.yaml
 	sam build --cached --parallel
+	mv template.yaml.orig template.yaml
 deps-node:
-	cd node && npm ci
-cover-node:
+	cd node/src ;\
+	npm list >/dev/null 2>&1 ;\
+	src_deps=$$? ;\
+	cd .. ;\
+	npm list >/dev/null 2>&1 ;\
+	dev_deps=$$? ;\
+	test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
+
+	cd lambdas ;\
+	npm list >/dev/null 2>&1 || npm ci
+cover-node: deps-node
 	cd node && npm run test:coverage
-style-node:
+style-node: deps-node
 	cd node && npm run prettier
-test-node:
+test-node: deps-node
 	cd node && npm run test
 deps-python:
 	cd chat/src && pip install -r requirements.txt
@@ -43,15 +58,26 @@ style-python: deps-python
 style-python-fix: deps-python
 	cd chat && ruff check --fix . 
 test-python: deps-python
-	cd chat && export SKIP_WEAVIATE_SETUP=True && PYTHONPATH=src:test && python -m unittest discover -v
+	cd chat && SKIP_WEAVIATE_SETUP=True PYTHONPATH=src:test python -m unittest discover -v
 python-version:
 	cd chat && python --version
 build: .aws-sam/build.toml
-link: build
-	cd chat/src && for src in *.py **/*.py; do for target in $$(find ../../.aws-sam/build -maxdepth 1 -type d); do if [[ -f $$target/$$src ]]; then ln -f $$src $$target/$$src; fi; done; done
-	cd node/src && for src in *.js *.json **/*.js **/*.json; do for target in $$(find ../../.aws-sam/build -maxdepth 1 -type d); do if [[ -f $$target/$$src ]]; then ln -f $$src $$target/$$src; fi; done; done
-serve: link
-	sam local start-api --host 0.0.0.0 --log-file dc-api.log
+serve-http: deps-node
+	@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
+	rm -rf .aws-sam
+	sam local start-api --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
+serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem
+serve-https: serve-http
+serve: serve-https
+start-with-step: deps-node
+	sam local start-lambda --host 0.0.0.0 --port 3005 --env-vars env.json --log-file lambda.log & \
+	echo $$! > .sam-pids ;\
+	sam local start-api --host 0.0.0.0 --port 3002 --log-file dc-api.log \
+		--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
+	echo $$! >> .sam-pids ;\
+	docker run --rm -p 8083:8083 -e LAMBDA_ENDPOINT=http://172.17.0.1:3005/ amazon/aws-stepfunctions-local ;\
+	kill $$(cat .sam-pids) ;\
+	rm -f .sam-pids
 deps: deps-node deps-python
 style: style-node style-python
 test: test-node test-python

README.md

@@ -6,7 +6,6 @@
 
 ```
 .
-├── bin/ - utilities for working with API locally
 ├── dev/ - example configs for developers
 ├── docs/ - mkdocs-based API documentation
 ├── events/ - sample HTTP API Lambda events
@@ -61,7 +60,7 @@ asdf install aws-sam-cli
 Then run the following command:
 
 ```shell
-bin/start-local-api
+make serve
 ```
 
 The API will be available at:
@@ -126,7 +125,7 @@ export AWS_PROFILE=staging-admin
 aws sso login
 
 # Start the API + step function and associated lambdas
-bin/start-with-step
+make start-with-step
 
 # Open a second terminal and create the state machine
 aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://state_machines/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole

chat/src/helpers/prompts.py

@@ -2,11 +2,11 @@
 
 
 def prompt_template() -> str:
-    return """Please answer the question based on the documents provided, and include some details about why the documents might be relevant to the particular question. The 'title' field is the document title, and the 'source' field is a UUID that uniquely identifies each document:
+    return """Please provide an answer to the question based on the documents provided. Include specific details from the documents that support your answer. Each document is identified by a 'title' and a unique 'source' UUID:
 
 Documents:
 {context}
-Format the answer as raw markdown. Insert links when referencing documents by title using it's UUID, as in the following guide: [title](https://dc.library.northwestern.edu/items/UUID). Example: [Judy Collins, Jackson Hole Folk Festival](https://dc.library.northwestern.edu/items/f1ca513b-7d13-4af6-ad7b-8c7ffd1d3a37). Suggest keywords searches using the following guide (example: [jazz musicians](https://dc.library.northwestern.edu/search?q=Jazz+musicians)). Offer search terms that vary in scope, highlight specific individuals or groups, or delve deeper into a topic. Remember to include as many direct links to Digital Collections searches as needed for comprehensive study. The `collection` field contains information about the collection the document belongs to. When many of the documents are from the same collection, mention the collection and link to the collection using the collection title and id: [collection['title']](https://dc.library.northwestern.edu/collections/collection['id']), for example [World War II Poster Collection](https://dc.library.northwestern.edu/collections/faf4f60e-78e0-4fbf-96ce-4ca8b4df597a):
+Answer in raw markdown. When referencing a document by title, link to it using its UUID like this: [title](https://dc.library.northwestern.edu/items/UUID). For example: [Judy Collins, Jackson Hole Folk Festival](https://dc.library.northwestern.edu/items/f1ca513b-7d13-4af6-ad7b-8c7ffd1d3a37). Suggest keyword searches using this format: [keyword](https://dc.library.northwestern.edu/search?q=keyword). Offer a variety of search terms that cover different aspects of the topic. Include as many direct links to Digital Collections searches as necessary for a thorough study. The `collection` field contains information about the collection the document belongs to. In the summary, mention the top 1 or 2 collections, explain why they are relevant and link to them using the collection title and id: [collection['title']](https://dc.library.northwestern.edu/collections/collection['id']), for example [World War II Poster Collection](https://dc.library.northwestern.edu/collections/faf4f60e-78e0-4fbf-96ce-4ca8b4df597a):
 
 Question:
 {question}

chat/src/helpers/response.py

@@ -48,7 +48,15 @@ def extract_prompt_value(v):
 
 def prepare_response(config):
     try:
-        subquery = {"match": {"all_text": {"query": config.question}}}
+        subquery = { 
+            "match": {
+                "all_titles":  {
+                    "query": config.question, 
+                    "operator": "AND",
+                    "analyzer": "english"
+                }
+            }
+        }
         docs = config.opensearch.similarity_search(
             query=config.question, k=config.k, subquery=subquery, _source={"excludes": ["embedding"]}
         )

chat/test/helpers/test_metrics.py

@@ -48,7 +48,7 @@ def test_token_usage(self):
 
         expected_result = {
             "answer": 6,
-            "prompt": 328,
+            "prompt": 302,
             "question": 15,
             "source_documents": 1,
         }

docs/docs/spec/data-types.yaml

@@ -132,6 +132,9 @@ components:
         description:
           type: string
           nullable: true
+        download_url:
+          type: string
+          nullable: true
         duration:
           type: number
           nullable: true
@@ -172,6 +175,7 @@ components:
         - id
         - accession_number
         - description
+        - download_url
         - duration
         - height
         - label
@@ -489,6 +493,17 @@ components:
           description: An account of the resource.
           items:
             type: string
+        embedding:
+          type: array
+          description: Vector representation of the resource's location in the repository's semantic space.
+          items:
+            type: number
+        embedding_model:
+          type: string
+          description: The name of the inference model used to generate the `embedding` from the resource's content.
+        download_url:
+          type: string
+          nullable: true
         file_sets:
           type: array
           description: File sets associated with the resource.
@@ -749,6 +764,9 @@ components:
         - cultural_context
         - date_created
         - description
+        - embedding
+        - embedding_model
+        - download_url
         - file_sets
         - folder_name
         - folder_number

lambdas/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lambdas",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Non-API handler lambdas",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"

layers/api_dependencies/package.json

@@ -1,16 +1,17 @@
 {
   "name": "dc-api-dependencies",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "NUL Digital Collections API Dependencies",
   "repository": "https://github.com/nulib/dc-api-v2",
   "author": "nulib",
   "license": "Apache-2.0",
   "dependencies": {
     "@aws-crypto/sha256-browser": "^2.0.1",
-    "@aws-sdk/credential-provider-node": "^3.142.0",
-    "@aws-sdk/node-http-handler": "^3.127.0",
-    "@aws-sdk/protocol-http": "^3.127.0",
-    "@aws-sdk/signature-v4": "^3.130.0",
+    "@aws-sdk/client-sfn": "^3.563.0",
+    "@aws-sdk/credential-provider-node": "^3.563.0",
+    "@smithy/node-http-handler": "^2.5.0",
+    "@smithy/protocol-http": "^3.3.0",
+    "@smithy/signature-v4": "^2.3.0",
     "@honeybadger-io/js": "^4.9.3",
     "axios": ">=0.21.1",
     "cookie": "^0.5.0",