Reorganize settings

Signed-off-by: Olivier Léobal <olivier.leobal@owkin.com>

CHANGELOG.md

@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - BREAKING: Support for multiple API tokens with expanded functionality ([#639](https://github.com/Substra/substra-backend/pull/639))
-- New `JWT_SECRET_PATH` and `JWT_SECRET_NEEDED` environment variables ([#657](https://github.com/Substra/substra-backend/pull/657))
+- New `SECRET_KEY_PATH` and `SECRET_KEY_STORE` environment variables ([#657](https://github.com/Substra/substra-backend/pull/657))
 
 ### Changed
 

backend/backend/settings/common.py

@@ -12,62 +12,24 @@
 
 import json
 import os
-import sys
 from datetime import timedelta
 
 import structlog
 from django.core.files.storage import FileSystemStorage
 
-from substrapp.compute_tasks.errors import CeleryRetryError
-
+from .deps.celery import *
 from .deps.jwt import *
 from .deps.org import *
-
-TRUE_VALUES = {"t", "T", "y", "Y", "yes", "YES", "true", "True", "TRUE", "on", "On", "ON", "1", 1, True}
-
-
-def to_bool(value):
-    return value in TRUE_VALUES
-
-
-def build_broker_url(user: str, password: str, host: str, port: str) -> str:
-    """Builds a redis connection string
-
-    Args:
-        user (str): redis user
-        password (str): redis password
-        host (str): redis hostname
-        port (str): redis port
-
-    Returns:
-        str: a connection string of the form "redis://user:password@hostname:port//"
-    """
-    conn_info = ""
-    conn_port = ""
-    if user and password:
-        conn_info = f"{user}:{password}@"
-    if port:
-        conn_port = f":{port}"
-    return f"redis://{conn_info}{host}{conn_port}//"
-
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-PROJECT_ROOT = os.path.dirname(BASE_DIR)
-
-sys.path.append(PROJECT_ROOT)
-sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, "libs")))
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
-
+from .deps.path import *
+from .deps.utils import to_bool
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-SUBPATH = ""
-if os.environ.get("SUBPATH"):
-    SUBPATH = os.environ.get("SUBPATH").strip("/") + "/"
+
+SUBPATH = os.environ.get("SUBPATH", "")  # prefix for backend endpoints
+if SUBPATH:
+    SUBPATH = SUBPATH.strip("/") + "/"
 
 ALLOWED_HOSTS = ["127.0.0.1", "::1", "localhost"] + json.loads(os.environ.get("ALLOWED_HOSTS", "[]"))
 if os.environ.get("HOST_IP"):
@@ -231,40 +193,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str:
     "COMPUTE_POD_STARTUP_TIMEOUT_SECONDS": int(os.environ.get("COMPUTE_POD_STARTUP_TIMEOUT_SECONDS", 300)),
 }
 
-CELERY_BROKER_USER = os.environ.get("CELERY_BROKER_USER")
-CELERY_BROKER_PASSWORD = os.environ.get("CELERY_BROKER_PASSWORD")
-CELERY_BROKER_HOST = os.environ.get("CELERY_BROKER_HOST", "localhost")
-CELERY_BROKER_PORT = os.environ.get("CELERY_BROKER_PORT", "5672")
-CELERY_BROKER_URL = build_broker_url(CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOST, CELERY_BROKER_PORT)
-
-CELERY_ACCEPT_CONTENT = ["application/json"]
-CELERY_RESULT_SERIALIZER = "json"
-CELERY_TASK_SERIALIZER = "json"
-CELERY_TASK_TRACK_STARTED = True  # since 4.0
-
-# With these settings, tasks will be retried for up to a maximum of 127 minutes.
-#
-# max_wait = CELERY_TASK_RETRY_BACKOFF * sum(2 ** n for n in range(CELERY_TASK_MAX_RETRIES))
-#          = 60 * (1 + 2 + 4 + 8 + 16 + 32 + 64)
-#          = 127 minutes
-#
-# Since jitter is enabled, the actual cumulative wait can be much less than max_wait. From the doc
-# (https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.retry_jitter):
-#
-# > If this option is set to True, the delay value calculated by retry_backoff is treated as a maximum, and the actual
-# > delay value will be a random number between zero and that maximum.
-CELERY_TASK_AUTORETRY_FOR = (CeleryRetryError,)
-CELERY_TASK_MAX_RETRIES = int(os.environ.get("CELERY_TASK_MAX_RETRIES", 7))
-CELERY_TASK_RETRY_BACKOFF = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF", 60))  # time in seconds
-CELERY_TASK_RETRY_BACKOFF_MAX = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF_MAX", 64 * 60))
-CELERY_TASK_RETRY_JITTER = True
-
-CELERY_WORKER_CONCURRENCY = int(os.environ.get("CELERY_WORKER_CONCURRENCY", 1))
-CELERY_BROADCAST = f"{ORG_NAME}.broadcast"
-
-CELERYBEAT_MAXIMUM_IMAGES_TTL = os.environ.get("CELERYBEAT_MAXIMUM_IMAGES_TTL", 7 * 24 * 3600)
-CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD = os.environ.get("CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD", 24 * 3600)
-
 WORKER_PVC_IS_HOSTPATH = to_bool(os.environ.get("WORKER_PVC_IS_HOSTPATH"))
 WORKER_PVC_DOCKER_CACHE = os.environ.get("WORKER_PVC_DOCKER_CACHE")
 WORKER_PVC_SUBTUPLE = os.environ.get("WORKER_PVC_SUBTUPLE")
@@ -406,9 +334,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str:
 
 CONTENT_DISPOSITION_HEADER = {}
 
-# To encode unique jwt token generated with reset password request
-RESET_JWT_SIGNATURE_ALGORITHM = "HS256"
-
 # Username of additional Django user representing user external to organization
 EXTERNAL_USERNAME = "external"
 

backend/backend/settings/deps/__init__.py

@@ -0,0 +1,3 @@
+"""
+self-contained categorized settings
+"""

backend/backend/settings/deps/celery.py

@@ -0,0 +1,65 @@
+"""
+Task broker settings
+"""
+
+import os
+
+from substrapp.compute_tasks.errors import CeleryRetryError
+
+from .org import ORG_NAME
+
+
+def build_broker_url(user: str, password: str, host: str, port: str) -> str:
+    """Builds a redis connection string
+
+    Args:
+        user (str): redis user
+        password (str): redis password
+        host (str): redis hostname
+        port (str): redis port
+
+    Returns:
+        str: a connection string of the form "redis://user:password@hostname:port//"
+    """
+    conn_info = ""
+    conn_port = ""
+    if user and password:
+        conn_info = f"{user}:{password}@"
+    if port:
+        conn_port = f":{port}"
+    return f"redis://{conn_info}{host}{conn_port}//"
+
+
+CELERY_BROKER_USER = os.environ.get("CELERY_BROKER_USER")
+CELERY_BROKER_PASSWORD = os.environ.get("CELERY_BROKER_PASSWORD")
+CELERY_BROKER_HOST = os.environ.get("CELERY_BROKER_HOST", "localhost")
+CELERY_BROKER_PORT = os.environ.get("CELERY_BROKER_PORT", "5672")
+CELERY_BROKER_URL = build_broker_url(CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOST, CELERY_BROKER_PORT)
+
+CELERY_ACCEPT_CONTENT = ["application/json"]
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TASK_SERIALIZER = "json"
+CELERY_TASK_TRACK_STARTED = True  # since 4.0
+
+# With these settings, tasks will be retried for up to a maximum of 127 minutes.
+#
+# max_wait = CELERY_TASK_RETRY_BACKOFF * sum(2 ** n for n in range(CELERY_TASK_MAX_RETRIES))
+#          = 60 * (1 + 2 + 4 + 8 + 16 + 32 + 64)
+#          = 127 minutes
+#
+# Since jitter is enabled, the actual cumulative wait can be much less than max_wait. From the doc
+# (https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.retry_jitter):
+#
+# > If this option is set to True, the delay value calculated by retry_backoff is treated as a maximum, and the actual
+# > delay value will be a random number between zero and that maximum.
+CELERY_TASK_AUTORETRY_FOR = (CeleryRetryError,)
+CELERY_TASK_MAX_RETRIES = int(os.environ.get("CELERY_TASK_MAX_RETRIES", 7))
+CELERY_TASK_RETRY_BACKOFF = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF", 60))  # time in seconds
+CELERY_TASK_RETRY_BACKOFF_MAX = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF_MAX", 64 * 60))
+CELERY_TASK_RETRY_JITTER = True
+
+CELERY_WORKER_CONCURRENCY = int(os.environ.get("CELERY_WORKER_CONCURRENCY", 1))
+CELERY_BROADCAST = f"{ORG_NAME}.broadcast"
+
+CELERYBEAT_MAXIMUM_IMAGES_TTL = os.environ.get("CELERYBEAT_MAXIMUM_IMAGES_TTL", 7 * 24 * 3600)
+CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD = os.environ.get("CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD", 24 * 3600)

backend/backend/settings/deps/jwt.py

@@ -1,22 +1,17 @@
-import os
-import pathlib
-import secrets
+"""
+JSON web tokens
+"""
 
-from .. import common
+import os
+from datetime import timedelta
 
-# SECURITY WARNING: keep the secret key used in production secret!
-JWT_SECRET_PATH = os.environ.get("JWT_SECRET_PATH", os.path.normpath(os.path.join(common.PROJECT_ROOT, "SECRET")))
+SIMPLE_JWT = {
+    "ACCESS_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("ACCESS_TOKEN_LIFETIME", 24 * 60))),
+    "REFRESH_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("REFRESH_TOKEN_LIFETIME", 24 * 60 * 7))),
+    "ROTATE_REFRESH_TOKENS": True,
+    "AUTH_HEADER_TYPES": ("JWT",),
+    "BLACKLIST_AFTER_ROTATION": True,
+}
 
-# Key configuration for JSON web tokens (JWT) authentication
-if common.to_bool(os.environ.get("JWT_SECRET_NEEDED", "False")):
-    try:
-        JWT_SECRET_KEY = pathlib.Path(JWT_SECRET_PATH).read_text().strip()
-    except IOError:
-        try:
-            JWT_SECRET_KEY = secrets.token_urlsafe()  # uses a "reasonable default" length
-            with open(JWT_SECRET_PATH, "w") as fp:
-                fp.write(JWT_SECRET_KEY)
-        except IOError:
-            raise Exception(f"Cannot open file `{JWT_SECRET_PATH}` for writing.")
-else:
-    JWT_SECRET_KEY = "unused default value " + secrets.token_urlsafe()
+# To encode unique jwt token generated with reset password request
+RESET_JWT_SIGNATURE_ALGORITHM = "HS256"

backend/backend/settings/deps/orchestrator.py

@@ -1,11 +1,11 @@
 import os
 
-from .. import common
+from .utils import to_bool
 
 ORCHESTRATOR_HOST = os.environ.get("ORCHESTRATOR_HOST")
 ORCHESTRATOR_PORT = os.environ.get("ORCHESTRATOR_PORT")
-ORCHESTRATOR_TLS_ENABLED = common.to_bool(os.environ.get("ORCHESTRATOR_TLS_ENABLED"))
-ORCHESTRATOR_MTLS_ENABLED = common.to_bool(os.environ.get("ORCHESTRATOR_MTLS_ENABLED"))
+ORCHESTRATOR_TLS_ENABLED = to_bool(os.environ.get("ORCHESTRATOR_TLS_ENABLED"))
+ORCHESTRATOR_MTLS_ENABLED = to_bool(os.environ.get("ORCHESTRATOR_MTLS_ENABLED"))
 ORCHESTRATOR_TLS_SERVER_CACERT_PATH = os.environ.get("ORCHESTRATOR_TLS_SERVER_CACERT_PATH")
 ORCHESTRATOR_TLS_CLIENT_CERT_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_CERT_PATH")
 ORCHESTRATOR_TLS_CLIENT_KEY_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_KEY_PATH")

backend/backend/settings/deps/path.py

@@ -0,0 +1,9 @@
+import os
+import sys
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROJECT_ROOT = os.path.dirname(BASE_DIR)
+
+sys.path.append(PROJECT_ROOT)
+sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, "libs")))

backend/backend/settings/deps/restframework.py

@@ -1,5 +1,4 @@
 import os
-from datetime import timedelta
 
 REST_FRAMEWORK = {
     "TEST_REQUEST_DEFAULT_FORMAT": "json",
@@ -36,14 +35,6 @@
     "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",),
 }
 
-SIMPLE_JWT = {
-    "ACCESS_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("ACCESS_TOKEN_LIFETIME", 24 * 60))),
-    "REFRESH_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("REFRESH_TOKEN_LIFETIME", 24 * 60 * 7))),
-    "ROTATE_REFRESH_TOKENS": True,
-    "AUTH_HEADER_TYPES": ("JWT",),
-    "BLACKLIST_AFTER_ROTATION": True,
-}
-
 SPECTACULAR_SETTINGS = {
     "SERVE_PERMISSIONS": ["rest_framework.permissions.IsAuthenticated"],
 }

backend/backend/settings/deps/secret_key.py

@@ -0,0 +1,34 @@
+"""
+SECRET_KEY is built in Django, but also used for signing JWTs
+"""
+
+import os
+import pathlib
+import secrets
+
+from . import path
+from .utils import to_bool
+
+SECRET_KEY_PATH = os.environ.get("SECRET_KEY_PATH", os.path.normpath(os.path.join(path.PROJECT_ROOT, "SECRET")))
+
+
+def _generate_secret_key():
+    return secrets.token_urlsafe()  # uses a "reasonable default" length
+
+
+_SECRET_KEY_LOAD_AND_STORE = to_bool(
+    os.environ.get("SECRET_KEY_LOAD_AND_STORE", "False")
+)  # Whether to load the secret key from file (and write it there if it doesn't exist)
+
+if _SECRET_KEY_LOAD_AND_STORE:
+    try:
+        SECRET_KEY = pathlib.Path(SECRET_KEY_PATH).read_text().strip()
+    except IOError:
+        try:
+            SECRET_KEY = _generate_secret_key()
+            with open(SECRET_KEY_PATH, "w") as fp:
+                fp.write(SECRET_KEY)
+        except IOError:
+            raise Exception(f"Cannot open file `{SECRET_KEY_PATH}` for writing.")
+else:
+    SECRET_KEY = _generate_secret_key()

backend/backend/settings/deps/utils.py

@@ -0,0 +1,5 @@
+TRUE_VALUES = {"t", "T", "y", "Y", "yes", "YES", "true", "True", "TRUE", "on", "On", "ON", "1", 1, True}
+
+
+def to_bool(value):
+    return value in TRUE_VALUES

backend/backend/settings/dev.py

@@ -3,11 +3,11 @@
 from substrapp.storages.minio import MinioStorage
 
 from .common import *
-from .deps.cors import *
 from .deps.ledger import *
-from .deps.oidc import *
 from .deps.org import *
 from .deps.restframework import *
+from .mods.cors import *
+from .mods.oidc import *
 
 DEBUG = True
 

backend/backend/settings/localdev.py

@@ -1,4 +1,4 @@
-from .deps.oidc import *
+from .mods.oidc import *
 from .test import *
 
 # Enable Browsable API

backend/backend/settings/mods/__init__.py

@@ -0,0 +1,3 @@
+"""
+settings that modify common settings
+"""

backend/backend/settings/deps/cors.py → backend/backend/settings/mods/cors.py

@@ -1,7 +1,12 @@
+"""
+Add Cross-origin site requests (CORS) restrictions
+"""
+
 import json
 import os
 
 from .. import common
+from ..deps.utils import to_bool
 
 common.INSTALLED_APPS += ("corsheaders",)
 
@@ -11,7 +16,7 @@
 CORS_ALLOWED_ORIGINS = json.loads(os.environ.get("CORS_ORIGIN_WHITELIST", "[]"))
 CORS_ALLOW_ALL_ORIGINS = False
 # @CORS_ALLOW_CREDENTIALS: If True cookies can be included in cross site requests. Set this to `True` for frontend auth.
-CORS_ALLOW_CREDENTIALS = common.to_bool(os.environ.get("CORS_ALLOW_CREDENTIALS", False))
+CORS_ALLOW_CREDENTIALS = to_bool(os.environ.get("CORS_ALLOW_CREDENTIALS", False))
 CORS_ALLOW_HEADERS = (
     "accept",
     "accept-encoding",