From ea2134bcea66cccec724da4ff6d5eeec82f724c0 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Tue, 10 Sep 2024 13:54:50 -0700 Subject: [PATCH 1/2] add SSL parameter support for redis --- .../danswer/background/celery/celeryconfig.py | 23 +++++++++++++------ backend/danswer/configs/app_configs.py | 4 ++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/backend/danswer/background/celery/celeryconfig.py b/backend/danswer/background/celery/celeryconfig.py index cf7e72719fd..7fceddf3e75 100644 --- a/backend/danswer/background/celery/celeryconfig.py +++ b/backend/danswer/background/celery/celeryconfig.py @@ -3,6 +3,9 @@ from danswer.configs.app_configs import REDIS_HOST from danswer.configs.app_configs import REDIS_PASSWORD from danswer.configs.app_configs import REDIS_PORT +from danswer.configs.app_configs import REDIS_SSL +from danswer.configs.app_configs import REDIS_SSL_CA_CERTS +from danswer.configs.app_configs import REDIS_SSL_CERT_REQS from danswer.configs.constants import DanswerCeleryPriority CELERY_SEPARATOR = ":" @@ -11,16 +14,22 @@ if REDIS_PASSWORD: CELERY_PASSWORD_PART = f":{REDIS_PASSWORD}@" +REDIS_SCHEME = "redis" + +# SSL-specific query parameters for Redis URL +SSL_QUERY_PARAMS = "" +if REDIS_SSL: + REDIS_SCHEME == "rediss" + SSL_QUERY_PARAMS = f"?ssl_cert_reqs={REDIS_SSL_CERT_REQS}" + if REDIS_SSL_CA_CERTS: + SSL_QUERY_PARAMS += f"&ssl_ca_certs={REDIS_SSL_CA_CERTS}" + # example celery_broker_url: "redis://:password@localhost:6379/15" -broker_url = ( - f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}" -) +broker_url = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}{SSL_QUERY_PARAMS}" -result_backend = ( - f"redis://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}" -) +result_backend = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}{SSL_QUERY_PARAMS}" -# NOTE: prefetch 4 is significantly faster than prefetch 1 +# NOTE: prefetch 4 is significantly faster than prefetch 1 for small tasks # however, prefetching is bad when tasks are lengthy as those tasks # can stall other tasks. worker_prefetch_multiplier = 4 diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py index d7733fdc0ab..9c531e314ea 100644 --- a/backend/danswer/configs/app_configs.py +++ b/backend/danswer/configs/app_configs.py @@ -149,6 +149,7 @@ except ValueError: POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT +REDIS_SSL = os.getenv("REDIS_SSL", "").lower() == "true" REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost" REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379)) REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") or "" @@ -159,6 +160,9 @@ # Used by celery as broker and backend REDIS_DB_NUMBER_CELERY = int(os.environ.get("REDIS_DB_NUMBER_CELERY", 15)) +REDIS_SSL_CERT_REQS = os.getenv("REDIS_SSL_CERT_REQS", "CERT_NONE") +REDIS_SSL_CA_CERTS = os.getenv("REDIS_SSL_CA_CERTS", "") + ##### # Connector Configs ##### From 61f09fcfcdda84d38be761d076499f634d7467a7 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Wed, 11 Sep 2024 10:08:34 -0700 Subject: [PATCH 2/2] add ssl support to redis pool --- backend/danswer/redis/redis_pool.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py index edea22fc05b..a0751ab6a82 100644 --- a/backend/danswer/redis/redis_pool.py +++ b/backend/danswer/redis/redis_pool.py @@ -9,6 +9,9 @@ from danswer.configs.app_configs import REDIS_HOST from danswer.configs.app_configs import REDIS_PASSWORD from danswer.configs.app_configs import REDIS_PORT +from danswer.configs.app_configs import REDIS_SSL +from danswer.configs.app_configs import REDIS_SSL_CA_CERTS +from danswer.configs.app_configs import REDIS_SSL_CERT_REQS REDIS_POOL_MAX_CONNECTIONS = 10 @@ -27,13 +30,25 @@ def __new__(cls) -> "RedisPool": return cls._instance def _init_pool(self) -> None: - self._pool = redis.ConnectionPool( - host=REDIS_HOST, - port=REDIS_PORT, - db=REDIS_DB_NUMBER, - password=REDIS_PASSWORD, - max_connections=REDIS_POOL_MAX_CONNECTIONS, - ) + if REDIS_SSL: + self._pool = redis.ConnectionPool( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DB_NUMBER, + password=REDIS_PASSWORD, + max_connections=REDIS_POOL_MAX_CONNECTIONS, + ssl=True, + ssl_ca_certs=REDIS_SSL_CA_CERTS, + ssl_cert_reqs=REDIS_SSL_CERT_REQS, + ) + else: + self._pool = redis.ConnectionPool( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DB_NUMBER, + password=REDIS_PASSWORD, + max_connections=REDIS_POOL_MAX_CONNECTIONS, + ) def get_client(self) -> Redis: return redis.Redis(connection_pool=self._pool)