Merge pull request #186 from nulib/prototype-opensearch-vectors

Add chat search to API using OpenSearch

.github/workflows/deploy.yml

@@ -2,24 +2,16 @@ name: Deploy
 on:
   push:
     branches:
-      - deploy/staging
+      - deploy/*
       - main
     paths:
       - ".github/workflows/deploy.yml"
-      - "package.json"
-      - "package-lock.json"
-      - "src/**"
+      - "node/**"
+      - "chat/**"
       - "template.yaml"
   workflow_dispatch:
-    inputs:
-      force_deploy_docs:
-          description: Deploy documentation even if no changes detected
-          type: boolean
-          default: false
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-env:
-  CONFIG_ENV: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
 jobs:
   build-deploy:
     runs-on: ubuntu-latest
@@ -28,6 +20,17 @@ jobs:
       contents: read
     environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
     steps:
+      - name: Set CONFIG_ENV from Branch Name
+        run: |
+          if [[ $BRANCH == 'refs/heads/main' ]]; then
+            echo "CONFIG_ENV=production" >> $GITHUB_ENV
+          else
+            echo "CONFIG_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
+          fi
+        env:
+          BRANCH: ${{ github.ref }}
+      - name: Confirm deploy environment
+        run: echo "Deploying to '$CONFIG_ENV' environment"
       - name: Set GitHub Deploy Key
         uses: webfactory/ssh-agent@v0.5.3
         with:
@@ -39,9 +42,9 @@ jobs:
           ref: main
           path: ".tfvars"
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
       - uses: aws-actions/setup-sam@v1
-      - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
-        run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
       - uses: aws-actions/configure-aws-credentials@master
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
@@ -59,60 +62,3 @@ jobs:
           | sed 's/\(Parameter overrides\s*\): .*/\1: ***** REDACTED *****/'
         env:
           HONEYBADGER_REVISION: ${{ github.sha }}
-  docs-changed:
-    runs-on: ubuntu-latest
-    outputs:
-      result: ${{ steps.changed-files.outputs.any_modified == 'true' }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-      - name: Get changed doc files
-        id: changed-files
-        uses: tj-actions/changed-files@v29.0.2
-        with:
-          files: |
-            .github/workflows/deploy.yaml
-            docs/*
-  publish-docs:
-    needs: docs-changed
-    if: ${{ needs.docs-changed.outputs.result == 'true' || inputs.force_deploy_docs }}
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@master
-        with:
-          role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
-          aws-region: us-east-1
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: 1.4.2
-      - name: Install dependencies
-        run: poetry install
-        working-directory: ./docs
-      - name: Build docs
-        run: poetry run mkdocs build --clean
-        working-directory: ./docs
-      - name: Determine correct deploy domain for environment
-        run: sed -i s/API_HOST/${HOSTNAME}/g docs/site/spec/openapi.*
-        env:
-          HOSTNAME: ${{ secrets.Hostname }}.${{ secrets.HostedZone }}
-      - name: Generate JSON API
-        uses: openapi-generators/openapitools-generator-action@v1
-        with:
-          generator: openapi
-          openapi-file: docs/site/spec/openapi.yaml
-          command-args: -o docs/site/spec
-      - name: Copy to S3
-        run: aws s3 sync --delete docs/site/ s3://${HOST}-docs.${ZONE}/
-        env:
-          HOST: ${{ secrets.Hostname }}
-          ZONE: ${{ secrets.HostedZone }}

.github/workflows/docs.yml

@@ -0,0 +1,53 @@
+name: Build and deploy API documentation
+on:
+  push:
+    branches:
+      - deploy/staging
+      - main
+    paths:
+      - .github/workflows/docs.yaml
+      - docs/*
+  workflow_dispatch:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+jobs:
+  publish-docs:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@master
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
+          aws-region: us-east-1
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: 1.4.2
+      - name: Install dependencies
+        run: poetry install
+        working-directory: ./docs
+      - name: Build docs
+        run: poetry run mkdocs build --clean
+        working-directory: ./docs
+      - name: Determine correct deploy domain for environment
+        run: sed -i s/API_HOST/${HOSTNAME}/g docs/site/spec/openapi.*
+        env:
+          HOSTNAME: ${{ secrets.Hostname }}.${{ secrets.HostedZone }}
+      - name: Generate JSON API
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+          generator: openapi
+          openapi-file: docs/site/spec/openapi.yaml
+          command-args: -o docs/site/spec
+      - name: Copy to S3
+        run: aws s3 sync --delete docs/site/ s3://${HOST}-docs.${ZONE}/
+        env:
+          HOST: ${{ secrets.Hostname }}
+          ZONE: ${{ secrets.HostedZone }}

.github/workflows/test-node.yml

@@ -0,0 +1,30 @@
+name: Run NodeJS Tests
+on:
+  push:
+    paths:
+      - ".github/workflows/test-node.yml"
+      - "node/**"
+  workflow_dispatch:
+defaults:
+  run:
+    working-directory: ./node
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ci
+      AWS_SECRET_ACCESS_KEY: ci
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+          cache: "npm"
+          cache-dependency-path: 'node/package-lock.json'
+      - run: npm ci
+      - name: Check code style
+        run: npm run lint && npm run prettier
+      - name: Run tests
+        run: npm run test:coverage
+      - name: Validate OpenAPI spec
+        run: npm run validate-spec

.github/workflows/test-python.yml

@@ -0,0 +1,31 @@
+name: Run Python Tests
+on:
+  push:
+    paths:
+      - ".github/workflows/test-python.yml"
+      - "chat/**"
+  workflow_dispatch:
+defaults:
+  run:
+    working-directory: ./chat
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ci
+      AWS_SECRET_ACCESS_KEY: ci
+      SKIP_WEAVIATE_SETUP: 'True' 
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache-dependency-path: chat/src/requirements.txt
+      - run: pip install -r requirements.txt
+        working-directory: ./chat/src
+      - name: Check code style
+        run: ruff check .
+      - name: Run tests
+        run: |
+          coverage run --include='src/**/*' -m unittest
+          coverage report

.github/workflows/validate-template.yml

@@ -0,0 +1,28 @@
+name: Validate Template
+on:
+  push:
+    paths:
+      - ".github/workflows/validate-template.yml"
+      - "./template.yml"
+  workflow_dispatch:
+jobs:
+  validate-template:
+      runs-on: ubuntu-latest
+      permissions:
+        id-token: write
+        contents: read
+      environment: test
+      steps:
+        - uses: actions/setup-python@v4
+          with:
+            python-version: '3.9' 
+        - uses: aws-actions/setup-sam@v1
+        - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
+          run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
+        - uses: aws-actions/configure-aws-credentials@master
+          with:
+            role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
+            aws-region: us-east-1
+        - uses: actions/checkout@v3
+        - name: Validate template
+          run: sam build && sam validate

.gitignore

@@ -26,6 +26,16 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+### Python ###
+.coverage
+htmlcov
+__pycache__/
+*.py[cod]
+*$py.class
+pip-log.txt
+pip-delete-this-directory.txt
+*.py,cover
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 
@@ -211,8 +221,12 @@ $RECYCLE.BIN/
 /docs/docs/spec/openapi.json
 /docs/site
 
+.venv
+
 .vscode
 /samconfig.toml
+/samconfig.yaml
+/samconfig.*.yaml
 /env.json
 /env.*.json
 /*.parameters

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
-
-npm run lint && npm run prettier
+cd node && npm run lint && npm run prettier && cd -
+cd chat/src && ruff check . && cd -

.tool-versions

@@ -1,3 +1,4 @@
 nodejs 16.14.0
 java corretto-19.0.1.10.1
 aws-sam-cli 1.107.0
+python 3.10.5