From 10330fb34becd69301b4bf611e2d894db30ef7c4 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 23 Aug 2024 10:33:55 -0700
Subject: [PATCH 01/28] first cut at redis

---
 .../danswer/background/celery/celery_app.py   | 21 ++++++----
 backend/danswer/configs/app_configs.py        | 10 +++++
 backend/requirements/default.txt              |  1 +
 .../docker_compose/docker-compose.dev.yml     | 14 ++++++-
 .../docker_compose/docker-compose.gpu-dev.yml | 13 +++++-
 .../docker-compose.prod-no-letsencrypt.yml    | 11 +++++
 .../docker_compose/docker-compose.prod.yml    | 11 +++++
 .../docker-compose.search-testing.yml         | 11 +++++
 deployment/kubernetes/env-configmap.yaml      |  1 +
 .../kubernetes/redis-service-deployment.yaml  | 41 +++++++++++++++++++
 deployment/kubernetes/secrets.yaml            |  4 +-
 11 files changed, 127 insertions(+), 11 deletions(-)
 create mode 100644 deployment/kubernetes/redis-service-deployment.yaml

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index ffd805c2986..792aaa02dde 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -20,7 +20,10 @@
 from danswer.background.task_utils import name_cc_prune_task
 from danswer.background.task_utils import name_document_set_sync_task
 from danswer.configs.app_configs import JOB_TIMEOUT
-from danswer.configs.constants import POSTGRES_CELERY_APP_NAME
+from danswer.configs.app_configs import REDIS_DB_NUMBER_CELERY
+from danswer.configs.app_configs import REDIS_HOST
+from danswer.configs.app_configs import REDIS_PASSWORD
+from danswer.configs.app_configs import REDIS_PORT
 from danswer.configs.constants import PostgresAdvisoryLocks
 from danswer.connectors.factory import instantiate_connector
 from danswer.connectors.models import InputType
@@ -35,9 +38,7 @@
 from danswer.db.document_set import fetch_documents_for_document_set_paginated
 from danswer.db.document_set import get_document_set_by_id
 from danswer.db.document_set import mark_document_set_as_synced
-from danswer.db.engine import build_connection_string
 from danswer.db.engine import get_sqlalchemy_engine
-from danswer.db.engine import SYNC_DB_API
 from danswer.db.models import DocumentSet
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
@@ -46,11 +47,17 @@
 
 logger = setup_logger()
 
-connection_string = build_connection_string(
-    db_api=SYNC_DB_API, app_name=POSTGRES_CELERY_APP_NAME
+CELERY_PASSWORD_PART = ""
+if REDIS_PASSWORD:
+    CELERY_PASSWORD_PART = f":{REDIS_PASSWORD}@"
+
+# example celery_broker_url: "redis://:password@localhost:6379/15"
+celery_broker_url = (
+    f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}"
+)
+celery_backend_url = (
+    f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}"
 )
-celery_broker_url = f"sqla+{connection_string}"
-celery_backend_url = f"db+{connection_string}"
 celery_app = Celery(__name__, broker=celery_broker_url, backend=celery_backend_url)
 
 
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 16b916f3dec..e41c9c67cd8 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -149,6 +149,16 @@
 except ValueError:
     POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT
 
+REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost"
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") or ""
+
+# Used for general redis things
+REDIS_DB_NUMBER = int(os.environ.get("REDIS_DB_NUMBER", 0))
+
+# Used by celery as broker and backend
+REDIS_DB_NUMBER_CELERY = int(os.environ.get("REDIS_DB_NUMBER_CELERY", 15))
+
 #####
 # Connector Configs
 #####
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index fc36432dfd0..d48c9afc97b 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -54,6 +54,7 @@ python-docx==1.1.0
 python-dotenv==1.0.0
 python-multipart==0.0.7
 pywikibot==9.0.0
+redis==5.0.8
 requests==2.32.2
 requests-oauthlib==1.3.1
 retry==0.9.2  # This pulls in py which is in CVE-2022-42969, must remove py from image
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index d28ccd59f72..cda0c03eb12 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -342,9 +342,19 @@ services:
     # in order to make this work on both Unix-like systems and windows
     command: > 
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
-      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
-        
+      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"  
+
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache:/data
+
 volumes:
+  cache:
   db_volume:
   vespa_volume: # Created by the container itself
 
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index 70120a53944..c8b44390c65 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -355,9 +355,20 @@ services:
     command: > 
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
       && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
-        
+
+
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache:/data
+
 
 volumes:
+  cache:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
index f6c18e1da9b..47f4208b72c 100644
--- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
+++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
@@ -204,7 +204,18 @@ services:
       - .env.nginx
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache:/data
+
+
 volumes:
+  cache:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index 81a067548dc..e4d6881e930 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -221,7 +221,18 @@ services:
     entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache:/data
+
+
 volumes:
+  cache:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index 97c87076a30..6e7ae7b25b3 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -200,7 +200,18 @@ services:
       && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache:/data
+
+
 volumes:
+  cache:
   db_volume:
     driver: local
     driver_opts:
diff --git a/deployment/kubernetes/env-configmap.yaml b/deployment/kubernetes/env-configmap.yaml
index 907fae1c836..3055004e66c 100644
--- a/deployment/kubernetes/env-configmap.yaml
+++ b/deployment/kubernetes/env-configmap.yaml
@@ -38,6 +38,7 @@ data:
   # Other Services
   POSTGRES_HOST: "relational-db-service"
   VESPA_HOST: "document-index-service"
+  REDIS_HOST: "redis-service"
   # Internet Search Tool
   BING_API_KEY: ""
   # Don't change the NLP models unless you know what you're doing
diff --git a/deployment/kubernetes/redis-service-deployment.yaml b/deployment/kubernetes/redis-service-deployment.yaml
new file mode 100644
index 00000000000..ab5113e5f49
--- /dev/null
+++ b/deployment/kubernetes/redis-service-deployment.yaml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-service
+spec:
+  selector:
+    app: redis
+  ports:
+    - name: redis
+      protocol: TCP
+      port: 6379
+      targetPort: 6379
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:7.4-alpine
+        ports:
+        - containerPort: 6379
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
+        command: ["redis-server"]
+        args: ["--requirepass", "$(REDIS_PASSWORD)"]
diff --git a/deployment/kubernetes/secrets.yaml b/deployment/kubernetes/secrets.yaml
index c135a29f676..d4cc9e2a739 100644
--- a/deployment/kubernetes/secrets.yaml
+++ b/deployment/kubernetes/secrets.yaml
@@ -8,4 +8,6 @@ data:
   postgres_user: cG9zdGdyZXM= # "postgres" base64 encoded
   postgres_password: cGFzc3dvcmQ= # "password" base64 encoded
   google_oauth_client_id: ZXhhbXBsZS1jbGllbnQtaWQ= # "example-client-id" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
-  google_oauth_client_secret: example_google_oauth_secret # "example-client-secret" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
+  google_oauth_client_secret: ZXhhbXBsZV9nb29nbGVfb2F1dGhfc2VjcmV0 # "example-client-secret" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
+  redis_password: cGFzc3dvcmQ= # "password" base64 encoded
+  
\ No newline at end of file

From 9dbd29dab098438352ce3418f0ff2c6b85af61e7 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 13:30:53 -0700
Subject: [PATCH 02/28] fix startup dependencies on redis

---
 deployment/docker_compose/docker-compose.dev.yml                | 2 ++
 deployment/docker_compose/docker-compose.gpu-dev.yml            | 2 ++
 .../docker_compose/docker-compose.prod-no-letsencrypt.yml       | 2 ++
 deployment/docker_compose/docker-compose.prod.yml               | 2 ++
 deployment/docker_compose/docker-compose.search-testing.yml     | 2 ++
 5 files changed, 10 insertions(+)

diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index cda0c03eb12..bd014d611b0 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     ports:
@@ -114,6 +115,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index c8b44390c65..cbbd459c6c0 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     ports:
@@ -106,6 +107,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
index 47f4208b72c..673613940f6 100644
--- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
+++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     env_file:
@@ -39,6 +40,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index e4d6881e930..a240cf942b7 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     env_file:
@@ -39,6 +40,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index 6e7ae7b25b3..e29be4b98a0 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
     restart: always
     ports:
       - "8080"
@@ -43,6 +44,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
     restart: always
     env_file:
       - .env_eval

From c388a815fc3533e1b65582a42aaf9ab8bac3be80 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 15:56:55 -0700
Subject: [PATCH 03/28] kombu cleanup - fail silently

---
 backend/danswer/background/celery/celery_app.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 792aaa02dde..5ecfdb365ed 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -6,6 +6,7 @@
 from celery import Celery  # type: ignore
 from celery.contrib.abortable import AbortableTask  # type: ignore
 from celery.exceptions import TaskRevokedError
+from sqlalchemy import inspect
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
@@ -367,6 +368,12 @@ def kombu_message_cleanup_task_helper(ctx: dict, db_session: Session) -> bool:
         bool: Returns True if there are more rows to process, False if not.
     """
 
+    # With the move to redis as celery's broker and backend, kombu tables may not even exist.
+    inspector = inspect(db_session.bind)
+    if not inspector.has_table("kombu_message"):
+        print("kombu_message table does not exist.")
+        return False
+
     query = text(
         """
     SELECT id, timestamp, payload

From f0bdb3970499698b515ed102de40ebde35b2a927 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 16:10:55 -0700
Subject: [PATCH 04/28] mypy

---
 backend/danswer/background/celery/celery_app.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 5ecfdb365ed..c401dde83ca 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -368,10 +368,13 @@ def kombu_message_cleanup_task_helper(ctx: dict, db_session: Session) -> bool:
         bool: Returns True if there are more rows to process, False if not.
     """
 
-    # With the move to redis as celery's broker and backend, kombu tables may not even exist.
     inspector = inspect(db_session.bind)
+    if not inspector:
+        return False
+
+    # With the move to redis as celery's broker and backend, kombu tables may not even exist.
+    # We can fail silently.
     if not inspector.has_table("kombu_message"):
-        print("kombu_message table does not exist.")
         return False
 
     query = text(

From 076ed181ca035ac5d774ab69cb3a6afd09555731 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 19:45:40 -0700
Subject: [PATCH 05/28] add redis_host environment override

---
 .github/workflows/run-it.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-it.yml b/.github/workflows/run-it.yml
index 45d57493b9a..15c0e3b0d2b 100644
--- a/.github/workflows/run-it.yml
+++ b/.github/workflows/run-it.yml
@@ -139,6 +139,7 @@ jobs:
             -e POSTGRES_PASSWORD=password \
             -e POSTGRES_DB=postgres \
             -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
             danswer/integration-test-runner:it

From aa1eec8d462b062c6dcc53e457eb3dbf93267183 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 20:19:33 -0700
Subject: [PATCH 06/28] update REDIS_HOST env var in docker-compose.dev.yml

---
 deployment/docker_compose/docker-compose.dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index ede11744415..cf3d8b4ca58 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -63,6 +63,7 @@ services:
       # Other services
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -139,6 +140,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
       - POSTGRES_DB=${POSTGRES_DB:-}
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose for OAuth2 connectors
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}

From f358acfb0997a6d928ae574370be120d95984203 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Mon, 2 Sep 2024 20:59:29 -0700
Subject: [PATCH 07/28] update the rest of the docker files

---
 deployment/docker_compose/docker-compose.gpu-dev.yml            | 2 ++
 .../docker_compose/docker-compose.prod-no-letsencrypt.yml       | 2 ++
 deployment/docker_compose/docker-compose.prod.yml               | 2 ++
 deployment/docker_compose/docker-compose.search-testing.yml     | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index 6e203f258f9..c45db5d8795 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -59,6 +59,7 @@ services:
       # Other services
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache      
       - WEB_DOMAIN=${WEB_DOMAIN:-}  # For frontend redirect auth purpose
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -131,6 +132,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
       - POSTGRES_DB=${POSTGRES_DB:-}
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-}  # For frontend redirect auth purpose for OAuth2 connectors
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
index 5262625e895..f29162acc61 100644
--- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
+++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
@@ -21,6 +21,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -50,6 +51,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
     extra_hosts:
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index 8cee1c9f35a..4bddd1145af 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -21,6 +21,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -50,6 +51,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
     extra_hosts:
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index 2ce3c165d48..6a557cf2ff8 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -22,6 +22,7 @@ services:
       - AUTH_TYPE=disabled
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - ENV_SEED_CONFIGURATION=${ENV_SEED_CONFIGURATION:-}
@@ -52,6 +53,7 @@ services:
       - AUTH_TYPE=disabled
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}

From 98ec0b791de5ba63b7270dd26c8eb8650b9eeb6e Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 11:43:09 -0700
Subject: [PATCH 08/28] update contributing guide

---
 CONTRIBUTING.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6901fd9c586..f90875b8106 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -135,9 +135,9 @@ ensure it is running before continuing with the following docker commands.
 
 First navigate to `danswer/deployment/docker_compose`, then start up Vespa and Postgres with:
 ```bash
-docker compose -f docker-compose.dev.yml -p danswer-stack up -d index relational_db
+docker compose -f docker-compose.dev.yml -p danswer-stack up -d index relational_db cache
 ```
-(index refers to Vespa and relational_db refers to Postgres)
+(index refers to Vespa, relational_db refers to Postgres, and cache refers to Redis)
 
 #### Running Danswer
 To start the frontend, navigate to `danswer/web` and run:

From dafbe136f11f245a91dffaaaf9fd332b58f91f2a Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 14:16:55 -0700
Subject: [PATCH 09/28] renaming cache to cache_volume

---
 deployment/docker_compose/docker-compose.dev.yml              | 4 ++--
 deployment/docker_compose/docker-compose.gpu-dev.yml          | 4 ++--
 .../docker_compose/docker-compose.prod-no-letsencrypt.yml     | 4 ++--
 deployment/docker_compose/docker-compose.prod.yml             | 4 ++--
 deployment/docker_compose/docker-compose.search-testing.yml   | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index cf3d8b4ca58..d9bdfb53eab 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -341,10 +341,10 @@ services:
       - '6379:6379'
     command: redis-server
     volumes: 
-      - cache:/data
+      - cache_volume:/data
 
 volumes:
-  cache:
+  cache_volume:
   db_volume:
   vespa_volume: # Created by the container itself
 
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index c45db5d8795..e12ef698e9b 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -354,11 +354,11 @@ services:
       - '6379:6379'
     command: redis-server
     volumes: 
-      - cache:/data
+      - cache_volume:/data
 
 
 volumes:
-  cache:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
index f29162acc61..57763c8fca7 100644
--- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
+++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
@@ -215,11 +215,11 @@ services:
       - '6379:6379'
     command: redis-server
     volumes: 
-      - cache:/data
+      - cache_volume:/data
 
 
 volumes:
-  cache:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index 4bddd1145af..85d585a5a88 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -232,11 +232,11 @@ services:
       - '6379:6379'
     command: redis-server
     volumes: 
-      - cache:/data
+      - cache_volume:/data
 
 
 volumes:
-  cache:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index 6a557cf2ff8..7f2ee1d7392 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -211,11 +211,11 @@ services:
       - '6379:6379'
     command: redis-server
     volumes: 
-      - cache:/data
+      - cache_volume:/data
 
 
 volumes:
-  cache:
+  cache_volume:
   db_volume:
     driver: local
     driver_opts:

From bd079080bdc3484aae763da7625fd57be4da8afc Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 14:37:56 -0700
Subject: [PATCH 10/28] add redis password to various deployments

---
 deployment/kubernetes/api_server-service-deployment.yaml | 5 +++++
 deployment/kubernetes/background-deployment.yaml         | 6 ++++++
 deployment/kubernetes/web_server-service-deployment.yaml | 6 ++++++
 3 files changed, 17 insertions(+)

diff --git a/deployment/kubernetes/api_server-service-deployment.yaml b/deployment/kubernetes/api_server-service-deployment.yaml
index eeac5fecc96..ccbbc906d61 100644
--- a/deployment/kubernetes/api_server-service-deployment.yaml
+++ b/deployment/kubernetes/api_server-service-deployment.yaml
@@ -52,6 +52,11 @@ spec:
             secretKeyRef:
               name: danswer-secrets
               key: google_oauth_client_secret
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap
diff --git a/deployment/kubernetes/background-deployment.yaml b/deployment/kubernetes/background-deployment.yaml
index 18521b0f5ad..1a6ef61c104 100644
--- a/deployment/kubernetes/background-deployment.yaml
+++ b/deployment/kubernetes/background-deployment.yaml
@@ -19,6 +19,12 @@ spec:
         command: ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
         # There are some extra values since this is shared between services
         # There are no conflicts though, extra env variables are simply ignored
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap
diff --git a/deployment/kubernetes/web_server-service-deployment.yaml b/deployment/kubernetes/web_server-service-deployment.yaml
index b19b8e37986..b54c1b7f3d0 100644
--- a/deployment/kubernetes/web_server-service-deployment.yaml
+++ b/deployment/kubernetes/web_server-service-deployment.yaml
@@ -33,6 +33,12 @@ spec:
         - containerPort: 3000
         # There are some extra values since this is shared between services
         # There are no conflicts though, extra env variables are simply ignored
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap

From 0bb2e57f5c06b1f1c1bb1a3ab39f64f1ddb3af59 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:09:17 -0700
Subject: [PATCH 11/28] try setting up pr testing for helm

---
 .github/workflows/pr-helm-chart-testing.yml | 58 +++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 .github/workflows/pr-helm-chart-testing.yml

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
new file mode 100644
index 00000000000..9d4615ca9aa
--- /dev/null
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -0,0 +1,58 @@
+name: Helm - Lint and Test Charts
+
+on:
+  merge_group:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Helm
+      uses: azure/setup-helm@v4.2.0
+      with:
+        version: v3.14.4
+      
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: |
+          backend/requirements/default.txt
+          backend/requirements/dev.txt
+          backend/requirements/model_server.txt
+    - run: |
+        python -m pip install --upgrade pip
+        pip install -r backend/requirements/default.txt
+        pip install -r backend/requirements/dev.txt
+        pip install -r backend/requirements/model_server.txt
+
+    - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.7.0
+
+    - name: Run chart-testing (list-changed)
+      id: list-changed
+      run: |
+        changed=$(ct list-changed  --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }})
+        if [[ -n "$changed" ]]; then
+          echo "changed=true" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Run chart-testing (lint)
+      if: steps.list-changed.outputs.changed == 'true'
+      run: ct lint --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+
+    - name: Create kind cluster
+      if: steps.list-changed.outputs.changed == 'true'
+      uses: helm/kind-action@v1.10.0
+
+    - name: Run chart-testing (install)
+      if: steps.list-changed.outputs.changed == 'true'
+      run: ct install --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+      
\ No newline at end of file

From 0b293ddb83a3f80d4dcf9266e24c612b6e8949cf Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:12:11 -0700
Subject: [PATCH 12/28] fix indent

---
 .github/workflows/pr-helm-chart-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 9d4615ca9aa..7013febd459 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -34,7 +34,7 @@ jobs:
         pip install -r backend/requirements/model_server.txt
 
     - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.7.0
+      uses: helm/chart-testing-action@v2.7.0
 
     - name: Run chart-testing (list-changed)
       id: list-changed

From 82bd7f4046a011cd9e342459358340df551c604b Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:17:49 -0700
Subject: [PATCH 13/28] hopefully this release version actually exists

---
 .github/workflows/pr-helm-chart-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 7013febd459..f1400b9ff5c 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -34,7 +34,7 @@ jobs:
         pip install -r backend/requirements/model_server.txt
 
     - name: Set up chart-testing
-      uses: helm/chart-testing-action@v2.7.0
+      uses: helm/chart-testing-action@v2.6.1
 
     - name: Run chart-testing (list-changed)
       id: list-changed

From 09889478d5d01cdefd3efaa670a8b381d2362f52 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:26:54 -0700
Subject: [PATCH 14/28] fix command line option to --chart-dirs

---
 .github/workflows/pr-helm-chart-testing.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index f1400b9ff5c..993cb9ca7af 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -39,14 +39,14 @@ jobs:
     - name: Run chart-testing (list-changed)
       id: list-changed
       run: |
-        changed=$(ct list-changed  --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }})
+        changed=$(ct list-changed --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }})
         if [[ -n "$changed" ]]; then
           echo "changed=true" >> "$GITHUB_OUTPUT"
         fi
 
     - name: Run chart-testing (lint)
       if: steps.list-changed.outputs.changed == 'true'
-      run: ct lint --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+      run: ct lint --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
 
     - name: Create kind cluster
       if: steps.list-changed.outputs.changed == 'true'
@@ -54,5 +54,5 @@ jobs:
 
     - name: Run chart-testing (install)
       if: steps.list-changed.outputs.changed == 'true'
-      run: ct install --charts ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+      run: ct install --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
       
\ No newline at end of file

From c20d92108ce0b1465b43a942819063899f07a74f Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:31:50 -0700
Subject: [PATCH 15/28] fetch-depth 0

---
 .github/workflows/pr-helm-chart-testing.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 993cb9ca7af..ec865c38ba0 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -9,10 +9,13 @@ jobs:
   lint-test:
     runs-on: ubuntu-latest
 
+    # fetch-depth 0 is required for helm/chart-testing-action
     steps:
     - name: Checkout code
       uses: actions/checkout@v3
-
+      with:
+        fetch-depth: 0
+        
     - name: Set up Helm
       uses: azure/setup-helm@v4.2.0
       with:

From d6c3bd34c01dc602dd428c9efeedd7f1b708564d Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 15:39:36 -0700
Subject: [PATCH 16/28] edit values.yaml

---
 deployment/helm/values.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/deployment/helm/values.yaml b/deployment/helm/values.yaml
index 3d425aa6dca..4ba0e375353 100644
--- a/deployment/helm/values.yaml
+++ b/deployment/helm/values.yaml
@@ -84,7 +84,7 @@ postgresql:
   auth:
     existingSecret: danswer-secrets
     secretKeys:
-      adminPasswordKey: postgres_password #overwriting as postgres typically expects 'postgres-password'
+      adminPasswordKey: postgres_password # overwriting as postgres typically expects 'postgres-password'
 
 nginx:
   containerPorts:
@@ -360,6 +360,8 @@ auth:
   # keys are lowercased version of env vars (e.g. SMTP_USER -> smtp_user)
   existingSecret: "" # danswer-secrets
   # optionally override the secret keys to reference in the secret
+  # this is used to populate the env vars in individual deployments
+  # that are read from secrets.yaml
   secretKeys:
     postgres_password: "postgres_password"
     smtp_pass: ""
@@ -369,9 +371,11 @@ auth:
     gen_ai_api_key: ""
     danswer_bot_slack_app_token: ""
     danswer_bot_slack_bot_token: ""
+    redis_password: "password"
   # will be overridden by the existingSecret if set
   secretName: "danswer-secrets"
   # set values as strings, they will be base64 encoded
+  # this is used to populate the secrets yaml
   secrets:
     postgres_password: "postgres"
     smtp_pass: ""
@@ -381,6 +385,7 @@ auth:
     gen_ai_api_key: ""
     danswer_bot_slack_app_token: ""
     danswer_bot_slack_bot_token: ""
+    redis_password: "password"
 
 configMap:
   AUTH_TYPE: "disabled"  # Change this for production uses unless Danswer is only accessible behind VPN

From b8d92056820a4ce31e938082e61c10d17fff514f Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 16:15:01 -0700
Subject: [PATCH 17/28] try setting ct working directory

---
 .github/workflows/pr-helm-chart-testing.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index ec865c38ba0..335b66997e5 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -42,14 +42,16 @@ jobs:
     - name: Run chart-testing (list-changed)
       id: list-changed
       run: |
-        changed=$(ct list-changed --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }})
+        changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
         if [[ -n "$changed" ]]; then
           echo "changed=true" >> "$GITHUB_OUTPUT"
         fi
+      working-directory: ./deployment/helm
 
     - name: Run chart-testing (lint)
       if: steps.list-changed.outputs.changed == 'true'
-      run: ct lint --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+      run: ct lint --target-branch ${{ github.event.repository.default_branch }}
+      working-directory: ./deployment/helm
 
     - name: Create kind cluster
       if: steps.list-changed.outputs.changed == 'true'
@@ -57,5 +59,6 @@ jobs:
 
     - name: Run chart-testing (install)
       if: steps.list-changed.outputs.changed == 'true'
-      run: ct install --chart-dirs ./deployment/helm --target-branch ${{ github.event.repository.default_branch }}
+      run: ct install --target-branch ${{ github.event.repository.default_branch }}
+      working-directory: ./deployment/helm
       
\ No newline at end of file

From f90cfa7cd5ad06c51826cb7c2b4b5bd11217e33b Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 16:24:39 -0700
Subject: [PATCH 18/28] bypass testing only on change for now

---
 .github/workflows/pr-helm-chart-testing.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 335b66997e5..3863781151e 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -49,16 +49,16 @@ jobs:
       working-directory: ./deployment/helm
 
     - name: Run chart-testing (lint)
-      if: steps.list-changed.outputs.changed == 'true'
+#       if: steps.list-changed.outputs.changed == 'true'
       run: ct lint --target-branch ${{ github.event.repository.default_branch }}
       working-directory: ./deployment/helm
 
     - name: Create kind cluster
-      if: steps.list-changed.outputs.changed == 'true'
+#       if: steps.list-changed.outputs.changed == 'true'
       uses: helm/kind-action@v1.10.0
 
     - name: Run chart-testing (install)
-      if: steps.list-changed.outputs.changed == 'true'
+#       if: steps.list-changed.outputs.changed == 'true'
       run: ct install --target-branch ${{ github.event.repository.default_branch }}
       working-directory: ./deployment/helm
       
\ No newline at end of file

From af34d1913169c4284f53a2554b8f4d67342b497a Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 17:54:38 -0700
Subject: [PATCH 19/28] move files and lint them

---
 ct.yaml                                                |  4 ++++
 deployment/helm/{ => charts/danswer}/.gitignore        |  0
 deployment/helm/{ => charts/danswer}/.helmignore       |  0
 deployment/helm/{ => charts/danswer}/Chart.lock        |  0
 deployment/helm/{ => charts/danswer}/Chart.yaml        |  7 ++-----
 .../helm/{ => charts/danswer}/templates/_helpers.tpl   |  0
 .../{ => charts/danswer}/templates/api-deployment.yaml |  0
 .../helm/{ => charts/danswer}/templates/api-hpa.yaml   |  0
 .../{ => charts/danswer}/templates/api-service.yaml    |  0
 .../danswer}/templates/background-deployment.yaml      |  0
 .../{ => charts/danswer}/templates/background-hpa.yaml |  0
 .../helm/{ => charts/danswer}/templates/configmap.yaml |  0
 .../{ => charts/danswer}/templates/danswer-secret.yaml |  0
 .../danswer}/templates/indexing-model-deployment.yaml  |  0
 .../danswer}/templates/indexing-model-pvc.yaml         |  0
 .../danswer}/templates/indexing-model-service.yaml     |  0
 .../danswer}/templates/inference-model-deployment.yaml |  0
 .../danswer}/templates/inference-model-pvc.yaml        |  0
 .../danswer}/templates/inference-model-service.yaml    |  0
 .../{ => charts/danswer}/templates/nginx-conf.yaml     |  0
 .../{ => charts/danswer}/templates/serviceaccount.yaml |  0
 .../danswer}/templates/tests/test-connection.yaml      |  0
 .../danswer}/templates/webserver-deployment.yaml       |  0
 .../{ => charts/danswer}/templates/webserver-hpa.yaml  |  0
 .../danswer}/templates/webserver-service.yaml          |  0
 deployment/helm/{ => charts/danswer}/values.yaml       | 10 +++++-----
 26 files changed, 11 insertions(+), 10 deletions(-)
 create mode 100644 ct.yaml
 rename deployment/helm/{ => charts/danswer}/.gitignore (100%)
 rename deployment/helm/{ => charts/danswer}/.helmignore (100%)
 rename deployment/helm/{ => charts/danswer}/Chart.lock (100%)
 rename deployment/helm/{ => charts/danswer}/Chart.yaml (93%)
 rename deployment/helm/{ => charts/danswer}/templates/_helpers.tpl (100%)
 rename deployment/helm/{ => charts/danswer}/templates/api-deployment.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/api-hpa.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/api-service.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/background-deployment.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/background-hpa.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/configmap.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/danswer-secret.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/indexing-model-deployment.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/indexing-model-pvc.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/indexing-model-service.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/inference-model-deployment.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/inference-model-pvc.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/inference-model-service.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/nginx-conf.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/serviceaccount.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/tests/test-connection.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/webserver-deployment.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/webserver-hpa.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/templates/webserver-service.yaml (100%)
 rename deployment/helm/{ => charts/danswer}/values.yaml (97%)

diff --git a/ct.yaml b/ct.yaml
new file mode 100644
index 00000000000..27e4959342d
--- /dev/null
+++ b/ct.yaml
@@ -0,0 +1,4 @@
+chart-dirs:
+  - deployment/helm/charts
+
+validate-maintainers: false
diff --git a/deployment/helm/.gitignore b/deployment/helm/charts/danswer/.gitignore
similarity index 100%
rename from deployment/helm/.gitignore
rename to deployment/helm/charts/danswer/.gitignore
diff --git a/deployment/helm/.helmignore b/deployment/helm/charts/danswer/.helmignore
similarity index 100%
rename from deployment/helm/.helmignore
rename to deployment/helm/charts/danswer/.helmignore
diff --git a/deployment/helm/Chart.lock b/deployment/helm/charts/danswer/Chart.lock
similarity index 100%
rename from deployment/helm/Chart.lock
rename to deployment/helm/charts/danswer/Chart.lock
diff --git a/deployment/helm/Chart.yaml b/deployment/helm/charts/danswer/Chart.yaml
similarity index 93%
rename from deployment/helm/Chart.yaml
rename to deployment/helm/charts/danswer/Chart.yaml
index 7763f33bec5..96336911ed8 100644
--- a/deployment/helm/Chart.yaml
+++ b/deployment/helm/charts/danswer/Chart.yaml
@@ -22,14 +22,11 @@ dependencies:
     version: 14.3.1
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
-  - name: vespa 
+  - name: vespa
     version: 0.2.3
     repository: https://unoplat.github.io/vespa-helm-charts
     condition: vespa.enabled
   - name: nginx
     version: 15.14.0
     repository: oci://registry-1.docker.io/bitnamicharts
-    condition: nginx.enabled  
-    
-
-    
\ No newline at end of file
+    condition: nginx.enabled
diff --git a/deployment/helm/templates/_helpers.tpl b/deployment/helm/charts/danswer/templates/_helpers.tpl
similarity index 100%
rename from deployment/helm/templates/_helpers.tpl
rename to deployment/helm/charts/danswer/templates/_helpers.tpl
diff --git a/deployment/helm/templates/api-deployment.yaml b/deployment/helm/charts/danswer/templates/api-deployment.yaml
similarity index 100%
rename from deployment/helm/templates/api-deployment.yaml
rename to deployment/helm/charts/danswer/templates/api-deployment.yaml
diff --git a/deployment/helm/templates/api-hpa.yaml b/deployment/helm/charts/danswer/templates/api-hpa.yaml
similarity index 100%
rename from deployment/helm/templates/api-hpa.yaml
rename to deployment/helm/charts/danswer/templates/api-hpa.yaml
diff --git a/deployment/helm/templates/api-service.yaml b/deployment/helm/charts/danswer/templates/api-service.yaml
similarity index 100%
rename from deployment/helm/templates/api-service.yaml
rename to deployment/helm/charts/danswer/templates/api-service.yaml
diff --git a/deployment/helm/templates/background-deployment.yaml b/deployment/helm/charts/danswer/templates/background-deployment.yaml
similarity index 100%
rename from deployment/helm/templates/background-deployment.yaml
rename to deployment/helm/charts/danswer/templates/background-deployment.yaml
diff --git a/deployment/helm/templates/background-hpa.yaml b/deployment/helm/charts/danswer/templates/background-hpa.yaml
similarity index 100%
rename from deployment/helm/templates/background-hpa.yaml
rename to deployment/helm/charts/danswer/templates/background-hpa.yaml
diff --git a/deployment/helm/templates/configmap.yaml b/deployment/helm/charts/danswer/templates/configmap.yaml
similarity index 100%
rename from deployment/helm/templates/configmap.yaml
rename to deployment/helm/charts/danswer/templates/configmap.yaml
diff --git a/deployment/helm/templates/danswer-secret.yaml b/deployment/helm/charts/danswer/templates/danswer-secret.yaml
similarity index 100%
rename from deployment/helm/templates/danswer-secret.yaml
rename to deployment/helm/charts/danswer/templates/danswer-secret.yaml
diff --git a/deployment/helm/templates/indexing-model-deployment.yaml b/deployment/helm/charts/danswer/templates/indexing-model-deployment.yaml
similarity index 100%
rename from deployment/helm/templates/indexing-model-deployment.yaml
rename to deployment/helm/charts/danswer/templates/indexing-model-deployment.yaml
diff --git a/deployment/helm/templates/indexing-model-pvc.yaml b/deployment/helm/charts/danswer/templates/indexing-model-pvc.yaml
similarity index 100%
rename from deployment/helm/templates/indexing-model-pvc.yaml
rename to deployment/helm/charts/danswer/templates/indexing-model-pvc.yaml
diff --git a/deployment/helm/templates/indexing-model-service.yaml b/deployment/helm/charts/danswer/templates/indexing-model-service.yaml
similarity index 100%
rename from deployment/helm/templates/indexing-model-service.yaml
rename to deployment/helm/charts/danswer/templates/indexing-model-service.yaml
diff --git a/deployment/helm/templates/inference-model-deployment.yaml b/deployment/helm/charts/danswer/templates/inference-model-deployment.yaml
similarity index 100%
rename from deployment/helm/templates/inference-model-deployment.yaml
rename to deployment/helm/charts/danswer/templates/inference-model-deployment.yaml
diff --git a/deployment/helm/templates/inference-model-pvc.yaml b/deployment/helm/charts/danswer/templates/inference-model-pvc.yaml
similarity index 100%
rename from deployment/helm/templates/inference-model-pvc.yaml
rename to deployment/helm/charts/danswer/templates/inference-model-pvc.yaml
diff --git a/deployment/helm/templates/inference-model-service.yaml b/deployment/helm/charts/danswer/templates/inference-model-service.yaml
similarity index 100%
rename from deployment/helm/templates/inference-model-service.yaml
rename to deployment/helm/charts/danswer/templates/inference-model-service.yaml
diff --git a/deployment/helm/templates/nginx-conf.yaml b/deployment/helm/charts/danswer/templates/nginx-conf.yaml
similarity index 100%
rename from deployment/helm/templates/nginx-conf.yaml
rename to deployment/helm/charts/danswer/templates/nginx-conf.yaml
diff --git a/deployment/helm/templates/serviceaccount.yaml b/deployment/helm/charts/danswer/templates/serviceaccount.yaml
similarity index 100%
rename from deployment/helm/templates/serviceaccount.yaml
rename to deployment/helm/charts/danswer/templates/serviceaccount.yaml
diff --git a/deployment/helm/templates/tests/test-connection.yaml b/deployment/helm/charts/danswer/templates/tests/test-connection.yaml
similarity index 100%
rename from deployment/helm/templates/tests/test-connection.yaml
rename to deployment/helm/charts/danswer/templates/tests/test-connection.yaml
diff --git a/deployment/helm/templates/webserver-deployment.yaml b/deployment/helm/charts/danswer/templates/webserver-deployment.yaml
similarity index 100%
rename from deployment/helm/templates/webserver-deployment.yaml
rename to deployment/helm/charts/danswer/templates/webserver-deployment.yaml
diff --git a/deployment/helm/templates/webserver-hpa.yaml b/deployment/helm/charts/danswer/templates/webserver-hpa.yaml
similarity index 100%
rename from deployment/helm/templates/webserver-hpa.yaml
rename to deployment/helm/charts/danswer/templates/webserver-hpa.yaml
diff --git a/deployment/helm/templates/webserver-service.yaml b/deployment/helm/charts/danswer/templates/webserver-service.yaml
similarity index 100%
rename from deployment/helm/templates/webserver-service.yaml
rename to deployment/helm/charts/danswer/templates/webserver-service.yaml
diff --git a/deployment/helm/values.yaml b/deployment/helm/charts/danswer/values.yaml
similarity index 97%
rename from deployment/helm/values.yaml
rename to deployment/helm/charts/danswer/values.yaml
index 4ba0e375353..196336dd31a 100644
--- a/deployment/helm/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -84,7 +84,7 @@ postgresql:
   auth:
     existingSecret: danswer-secrets
     secretKeys:
-      adminPasswordKey: postgres_password # overwriting as postgres typically expects 'postgres-password'
+      adminPasswordKey: postgres_password  # overwriting as postgres typically expects 'postgres-password'
 
 nginx:
   containerPorts:
@@ -330,7 +330,7 @@ vespa:
   affinity: {}
 
 
-#ingress:
+# ingress:
 #  enabled: false
 #  className: ""
 #  annotations: {}
@@ -358,7 +358,7 @@ persistence:
 auth:
   # for storing smtp, oauth, slack, and other secrets
   # keys are lowercased version of env vars (e.g. SMTP_USER -> smtp_user)
-  existingSecret: "" # danswer-secrets
+  existingSecret: ""  # danswer-secrets
   # optionally override the secret keys to reference in the secret
   # this is used to populate the env vars in individual deployments
   # that are read from secrets.yaml
@@ -391,8 +391,8 @@ configMap:
   AUTH_TYPE: "disabled"  # Change this for production uses unless Danswer is only accessible behind VPN
   SESSION_EXPIRE_TIME_SECONDS: "86400"  # 1 Day Default
   VALID_EMAIL_DOMAINS: ""  # Can be something like danswer.ai, as an extra double-check
-  SMTP_SERVER: "" # For sending verification emails, if unspecified then defaults to 'smtp.gmail.com'
-  SMTP_PORT: "" # For sending verification emails, if unspecified then defaults to '587'
+  SMTP_SERVER: ""  # For sending verification emails, if unspecified then defaults to 'smtp.gmail.com'
+  SMTP_PORT: ""  # For sending verification emails, if unspecified then defaults to '587'
   SMTP_USER: ""  # 'your-email@company.com'
   # SMTP_PASS: ""  # 'your-gmail-password'
   EMAIL_FROM: ""  # 'your-email@company.com' SMTP_USER missing used instead

From 5d6fe9295db8214c5343a83306d4dfbc1f89fcb3 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 18:03:26 -0700
Subject: [PATCH 20/28] update helm testing

---
 .github/workflows/pr-helm-chart-testing.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 3863781151e..1290ebdadbc 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -46,12 +46,10 @@ jobs:
         if [[ -n "$changed" ]]; then
           echo "changed=true" >> "$GITHUB_OUTPUT"
         fi
-      working-directory: ./deployment/helm
 
     - name: Run chart-testing (lint)
 #       if: steps.list-changed.outputs.changed == 'true'
-      run: ct lint --target-branch ${{ github.event.repository.default_branch }}
-      working-directory: ./deployment/helm
+      run: ct lint --all --target-branch ${{ github.event.repository.default_branch }}
 
     - name: Create kind cluster
 #       if: steps.list-changed.outputs.changed == 'true'
@@ -59,6 +57,6 @@ jobs:
 
     - name: Run chart-testing (install)
 #       if: steps.list-changed.outputs.changed == 'true'
-      run: ct install --target-branch ${{ github.event.repository.default_branch }}
-      working-directory: ./deployment/helm
+      run: ct install --all
+#       run: ct install --target-branch ${{ github.event.repository.default_branch }}
       
\ No newline at end of file

From 8cf40988e2c66ee09596730968da7d5703db97d7 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 18:18:28 -0700
Subject: [PATCH 21/28] some issues suggest using --config works

---
 .github/workflows/pr-helm-chart-testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index 1290ebdadbc..ccc231c47c7 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -49,7 +49,7 @@ jobs:
 
     - name: Run chart-testing (lint)
 #       if: steps.list-changed.outputs.changed == 'true'
-      run: ct lint --all --target-branch ${{ github.event.repository.default_branch }}
+      run: ct lint --all --config ct.yaml --target-branch ${{ github.event.repository.default_branch }}
 
     - name: Create kind cluster
 #       if: steps.list-changed.outputs.changed == 'true'
@@ -57,6 +57,6 @@ jobs:
 
     - name: Run chart-testing (install)
 #       if: steps.list-changed.outputs.changed == 'true'
-      run: ct install --all
+      run: ct install --all --config ct.yaml
 #       run: ct install --target-branch ${{ github.event.repository.default_branch }}
       
\ No newline at end of file

From bc5cb2d95f719dc3b5ede0cbdafea282e90998db Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 18:26:06 -0700
Subject: [PATCH 22/28] add vespa repo

---
 ct.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ct.yaml b/ct.yaml
index 27e4959342d..3a8be65210a 100644
--- a/ct.yaml
+++ b/ct.yaml
@@ -1,4 +1,9 @@
+# helm chart testing configuration file
+
 chart-dirs:
   - deployment/helm/charts
 
+chart-repos:
+  - vespa=https://unoplat.github.io/vespa-helm-charts
+  
 validate-maintainers: false

From ea35879482453e6ddc370aa32c9896ec5021095b Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 18:35:30 -0700
Subject: [PATCH 23/28] add postgresql repo

---
 ct.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ct.yaml b/ct.yaml
index 3a8be65210a..25266ed96ff 100644
--- a/ct.yaml
+++ b/ct.yaml
@@ -5,5 +5,6 @@ chart-dirs:
 
 chart-repos:
   - vespa=https://unoplat.github.io/vespa-helm-charts
+  - postgresql=https://charts.bitnami.com/bitnami
   
 validate-maintainers: false

From b4274395b186b044759dbb42d9f842178284ef02 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 20:16:43 -0700
Subject: [PATCH 24/28] increase timeout

---
 ct.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ct.yaml b/ct.yaml
index 25266ed96ff..764af160daf 100644
--- a/ct.yaml
+++ b/ct.yaml
@@ -1,4 +1,4 @@
-# helm chart testing configuration file
+# See https://github.com/helm/chart-testing#configuration
 
 chart-dirs:
   - deployment/helm/charts
@@ -7,4 +7,6 @@ chart-repos:
   - vespa=https://unoplat.github.io/vespa-helm-charts
   - postgresql=https://charts.bitnami.com/bitnami
   
+helm-extra-args: --timeout 900s
+
 validate-maintainers: false

From 97d36717e2318627beac8d78624e272152362cdd Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 20:38:53 -0700
Subject: [PATCH 25/28] try amd64 runner

---
 .github/workflows/pr-helm-chart-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index ccc231c47c7..c471a59ce4e 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   lint-test:
-    runs-on: ubuntu-latest
+    runs-on: Amd64
 
     # fetch-depth 0 is required for helm/chart-testing-action
     steps:

From 3574c235432d74ea6497e16688ca357e89967303 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 5 Sep 2024 21:37:08 -0700
Subject: [PATCH 26/28] fix redis password reference

---
 deployment/helm/charts/danswer/values.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/helm/charts/danswer/values.yaml b/deployment/helm/charts/danswer/values.yaml
index 196336dd31a..b81ffed8671 100644
--- a/deployment/helm/charts/danswer/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -361,7 +361,7 @@ auth:
   existingSecret: ""  # danswer-secrets
   # optionally override the secret keys to reference in the secret
   # this is used to populate the env vars in individual deployments
-  # that are read from secrets.yaml
+  # the values here reference the keys in secrets below
   secretKeys:
     postgres_password: "postgres_password"
     smtp_pass: ""
@@ -371,7 +371,7 @@ auth:
     gen_ai_api_key: ""
     danswer_bot_slack_app_token: ""
     danswer_bot_slack_bot_token: ""
-    redis_password: "password"
+    redis_password: "redis_password"
   # will be overridden by the existingSecret if set
   secretName: "danswer-secrets"
   # set values as strings, they will be base64 encoded

From f02a2316b51b01c0bfaedd99884567dc303f0e48 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 6 Sep 2024 10:04:34 -0700
Subject: [PATCH 27/28] add comment to helm chart testing workflow

---
 .github/workflows/pr-helm-chart-testing.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml
index c471a59ce4e..7c4903a07f7 100644
--- a/.github/workflows/pr-helm-chart-testing.yml
+++ b/.github/workflows/pr-helm-chart-testing.yml
@@ -1,3 +1,8 @@
+# This workflow is intentionally disabled while we're still working on it
+# It's close to ready, but a race condition needs to be fixed with
+# API server and Vespa startup, and it needs to have a way to build/test against
+# local containers
+
 name: Helm - Lint and Test Charts
 
 on:

From 3323d4c715dc8ab8148d409458398988e09b09df Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 6 Sep 2024 10:05:16 -0700
Subject: [PATCH 28/28] rename helm testing workflow to disable it

---
 ...m-chart-testing.yml => pr-helm-chart-testing.yml.disabled.txt} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/workflows/{pr-helm-chart-testing.yml => pr-helm-chart-testing.yml.disabled.txt} (100%)

diff --git a/.github/workflows/pr-helm-chart-testing.yml b/.github/workflows/pr-helm-chart-testing.yml.disabled.txt
similarity index 100%
rename from .github/workflows/pr-helm-chart-testing.yml
rename to .github/workflows/pr-helm-chart-testing.yml.disabled.txt