From aad6230074ab96531721aa8b5d9ffc597b0330fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20L=C3=A9obal?= Date: Tue, 13 Jun 2023 15:35:30 +0200 Subject: [PATCH] Reorganize settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Olivier Léobal --- CHANGELOG.md | 2 +- backend/backend/settings/common.py | 90 ++----------------- backend/backend/settings/deps/__init__.py | 3 + backend/backend/settings/deps/celery.py | 65 ++++++++++++++ backend/backend/settings/deps/jwt.py | 33 +++---- backend/backend/settings/deps/orchestrator.py | 6 +- backend/backend/settings/deps/path.py | 9 ++ .../backend/settings/deps/restframework.py | 9 -- backend/backend/settings/deps/secret_key.py | 34 +++++++ backend/backend/settings/deps/utils.py | 5 ++ backend/backend/settings/dev.py | 4 +- backend/backend/settings/localdev.py | 2 +- backend/backend/settings/mods/__init__.py | 3 + .../backend/settings/{deps => mods}/cors.py | 7 +- .../backend/settings/{deps => mods}/oidc.py | 13 ++- backend/backend/settings/prod.py | 4 +- backend/backend/settings/test.py | 4 +- backend/users/views/user.py | 6 +- docs/settings.md | 47 ++++++---- tools/build_settings_doc.py | 23 +++-- 20 files changed, 218 insertions(+), 151 deletions(-) create mode 100644 backend/backend/settings/deps/celery.py create mode 100644 backend/backend/settings/deps/path.py create mode 100644 backend/backend/settings/deps/secret_key.py create mode 100644 backend/backend/settings/deps/utils.py create mode 100644 backend/backend/settings/mods/__init__.py rename backend/backend/settings/{deps => mods}/cors.py (80%) rename backend/backend/settings/{deps => mods}/oidc.py (91%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fb64f989..36f9f815a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - BREAKING: Support for multiple API tokens with expanded functionality ([#639](https://github.com/Substra/substra-backend/pull/639)) -- New `JWT_SECRET_PATH` and `JWT_SECRET_NEEDED` environment variables ([#657](https://github.com/Substra/substra-backend/pull/657)) +- New `SECRET_KEY_PATH` and `SECRET_KEY_STORE` environment variables ([#657](https://github.com/Substra/substra-backend/pull/657)) ### Changed diff --git a/backend/backend/settings/common.py b/backend/backend/settings/common.py index f56b31e2f..1097ad53e 100644 --- a/backend/backend/settings/common.py +++ b/backend/backend/settings/common.py @@ -12,62 +12,25 @@ import json import os -import sys from datetime import timedelta import structlog from django.core.files.storage import FileSystemStorage -from substrapp.compute_tasks.errors import CeleryRetryError - +from .deps.celery import * from .deps.jwt import * from .deps.org import * - -TRUE_VALUES = {"t", "T", "y", "Y", "yes", "YES", "true", "True", "TRUE", "on", "On", "ON", "1", 1, True} - - -def to_bool(value): - return value in TRUE_VALUES - - -def build_broker_url(user: str, password: str, host: str, port: str) -> str: - """Builds a redis connection string - - Args: - user (str): redis user - password (str): redis password - host (str): redis hostname - port (str): redis port - - Returns: - str: a connection string of the form "redis://user:password@hostname:port//" - """ - conn_info = "" - conn_port = "" - if user and password: - conn_info = f"{user}:{password}@" - if port: - conn_port = f":{port}" - return f"redis://{conn_info}{host}{conn_port}//" - - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PROJECT_ROOT = os.path.dirname(BASE_DIR) - -sys.path.append(PROJECT_ROOT) -sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, "libs"))) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ - +from .deps.path import * +from .deps.secret_key import * +from .deps.utils import to_bool # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -SUBPATH = "" -if os.environ.get("SUBPATH"): - SUBPATH = os.environ.get("SUBPATH").strip("/") + "/" + +SUBPATH = os.environ.get("SUBPATH", "") # prefix for backend endpoints +if SUBPATH: + SUBPATH = SUBPATH.strip("/") + "/" ALLOWED_HOSTS = ["127.0.0.1", "::1", "localhost"] + json.loads(os.environ.get("ALLOWED_HOSTS", "[]")) if os.environ.get("HOST_IP"): @@ -231,40 +194,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str: "COMPUTE_POD_STARTUP_TIMEOUT_SECONDS": int(os.environ.get("COMPUTE_POD_STARTUP_TIMEOUT_SECONDS", 300)), } -CELERY_BROKER_USER = os.environ.get("CELERY_BROKER_USER") -CELERY_BROKER_PASSWORD = os.environ.get("CELERY_BROKER_PASSWORD") -CELERY_BROKER_HOST = os.environ.get("CELERY_BROKER_HOST", "localhost") -CELERY_BROKER_PORT = os.environ.get("CELERY_BROKER_PORT", "5672") -CELERY_BROKER_URL = build_broker_url(CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOST, CELERY_BROKER_PORT) - -CELERY_ACCEPT_CONTENT = ["application/json"] -CELERY_RESULT_SERIALIZER = "json" -CELERY_TASK_SERIALIZER = "json" -CELERY_TASK_TRACK_STARTED = True # since 4.0 - -# With these settings, tasks will be retried for up to a maximum of 127 minutes. -# -# max_wait = CELERY_TASK_RETRY_BACKOFF * sum(2 ** n for n in range(CELERY_TASK_MAX_RETRIES)) -# = 60 * (1 + 2 + 4 + 8 + 16 + 32 + 64) -# = 127 minutes -# -# Since jitter is enabled, the actual cumulative wait can be much less than max_wait. From the doc -# (https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.retry_jitter): -# -# > If this option is set to True, the delay value calculated by retry_backoff is treated as a maximum, and the actual -# > delay value will be a random number between zero and that maximum. -CELERY_TASK_AUTORETRY_FOR = (CeleryRetryError,) -CELERY_TASK_MAX_RETRIES = int(os.environ.get("CELERY_TASK_MAX_RETRIES", 7)) -CELERY_TASK_RETRY_BACKOFF = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF", 60)) # time in seconds -CELERY_TASK_RETRY_BACKOFF_MAX = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF_MAX", 64 * 60)) -CELERY_TASK_RETRY_JITTER = True - -CELERY_WORKER_CONCURRENCY = int(os.environ.get("CELERY_WORKER_CONCURRENCY", 1)) -CELERY_BROADCAST = f"{ORG_NAME}.broadcast" - -CELERYBEAT_MAXIMUM_IMAGES_TTL = os.environ.get("CELERYBEAT_MAXIMUM_IMAGES_TTL", 7 * 24 * 3600) -CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD = os.environ.get("CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD", 24 * 3600) - WORKER_PVC_IS_HOSTPATH = to_bool(os.environ.get("WORKER_PVC_IS_HOSTPATH")) WORKER_PVC_DOCKER_CACHE = os.environ.get("WORKER_PVC_DOCKER_CACHE") WORKER_PVC_SUBTUPLE = os.environ.get("WORKER_PVC_SUBTUPLE") @@ -406,9 +335,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str: CONTENT_DISPOSITION_HEADER = {} -# To encode unique jwt token generated with reset password request -RESET_JWT_SIGNATURE_ALGORITHM = "HS256" - # Username of additional Django user representing user external to organization EXTERNAL_USERNAME = "external" diff --git a/backend/backend/settings/deps/__init__.py b/backend/backend/settings/deps/__init__.py index e69de29bb..39b71fc5f 100644 --- a/backend/backend/settings/deps/__init__.py +++ b/backend/backend/settings/deps/__init__.py @@ -0,0 +1,3 @@ +""" +self-contained categorized settings +""" diff --git a/backend/backend/settings/deps/celery.py b/backend/backend/settings/deps/celery.py new file mode 100644 index 000000000..bdb9639ee --- /dev/null +++ b/backend/backend/settings/deps/celery.py @@ -0,0 +1,65 @@ +""" +Task broker settings +""" + +import os + +from substrapp.compute_tasks.errors import CeleryRetryError + +from .org import ORG_NAME + + +def build_broker_url(user: str, password: str, host: str, port: str) -> str: + """Builds a redis connection string + + Args: + user (str): redis user + password (str): redis password + host (str): redis hostname + port (str): redis port + + Returns: + str: a connection string of the form "redis://user:password@hostname:port//" + """ + conn_info = "" + conn_port = "" + if user and password: + conn_info = f"{user}:{password}@" + if port: + conn_port = f":{port}" + return f"redis://{conn_info}{host}{conn_port}//" + + +CELERY_BROKER_USER = os.environ.get("CELERY_BROKER_USER") +CELERY_BROKER_PASSWORD = os.environ.get("CELERY_BROKER_PASSWORD") +CELERY_BROKER_HOST = os.environ.get("CELERY_BROKER_HOST", "localhost") +CELERY_BROKER_PORT = os.environ.get("CELERY_BROKER_PORT", "5672") +CELERY_BROKER_URL = build_broker_url(CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOST, CELERY_BROKER_PORT) + +CELERY_ACCEPT_CONTENT = ["application/json"] +CELERY_RESULT_SERIALIZER = "json" +CELERY_TASK_SERIALIZER = "json" +CELERY_TASK_TRACK_STARTED = True # since 4.0 + +# With these settings, tasks will be retried for up to a maximum of 127 minutes. +# +# max_wait = CELERY_TASK_RETRY_BACKOFF * sum(2 ** n for n in range(CELERY_TASK_MAX_RETRIES)) +# = 60 * (1 + 2 + 4 + 8 + 16 + 32 + 64) +# = 127 minutes +# +# Since jitter is enabled, the actual cumulative wait can be much less than max_wait. From the doc +# (https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.retry_jitter): +# +# > If this option is set to True, the delay value calculated by retry_backoff is treated as a maximum, and the actual +# > delay value will be a random number between zero and that maximum. +CELERY_TASK_AUTORETRY_FOR = (CeleryRetryError,) +CELERY_TASK_MAX_RETRIES = int(os.environ.get("CELERY_TASK_MAX_RETRIES", 7)) +CELERY_TASK_RETRY_BACKOFF = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF", 60)) # time in seconds +CELERY_TASK_RETRY_BACKOFF_MAX = int(os.environ.get("CELERY_TASK_RETRY_BACKOFF_MAX", 64 * 60)) +CELERY_TASK_RETRY_JITTER = True + +CELERY_WORKER_CONCURRENCY = int(os.environ.get("CELERY_WORKER_CONCURRENCY", 1)) +CELERY_BROADCAST = f"{ORG_NAME}.broadcast" + +CELERYBEAT_MAXIMUM_IMAGES_TTL = os.environ.get("CELERYBEAT_MAXIMUM_IMAGES_TTL", 7 * 24 * 3600) +CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD = os.environ.get("CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD", 24 * 3600) diff --git a/backend/backend/settings/deps/jwt.py b/backend/backend/settings/deps/jwt.py index f797cbca9..78ad9139c 100644 --- a/backend/backend/settings/deps/jwt.py +++ b/backend/backend/settings/deps/jwt.py @@ -1,22 +1,17 @@ -import os -import pathlib -import secrets +""" +JSON web tokens +""" -from .. import common +import os +from datetime import timedelta -# SECURITY WARNING: keep the secret key used in production secret! -JWT_SECRET_PATH = os.environ.get("JWT_SECRET_PATH", os.path.normpath(os.path.join(common.PROJECT_ROOT, "SECRET"))) +SIMPLE_JWT = { + "ACCESS_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("ACCESS_TOKEN_LIFETIME", 24 * 60))), + "REFRESH_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("REFRESH_TOKEN_LIFETIME", 24 * 60 * 7))), + "ROTATE_REFRESH_TOKENS": True, + "AUTH_HEADER_TYPES": ("JWT",), + "BLACKLIST_AFTER_ROTATION": True, +} -# Key configuration for JSON web tokens (JWT) authentication -if common.to_bool(os.environ.get("JWT_SECRET_NEEDED", "False")): - try: - JWT_SECRET_KEY = pathlib.Path(JWT_SECRET_PATH).read_text().strip() - except IOError: - try: - JWT_SECRET_KEY = secrets.token_urlsafe() # uses a "reasonable default" length - with open(JWT_SECRET_PATH, "w") as fp: - fp.write(JWT_SECRET_KEY) - except IOError: - raise Exception(f"Cannot open file `{JWT_SECRET_PATH}` for writing.") -else: - JWT_SECRET_KEY = "unused default value " + secrets.token_urlsafe() +# To encode unique jwt token generated with reset password request +RESET_JWT_SIGNATURE_ALGORITHM = "HS256" diff --git a/backend/backend/settings/deps/orchestrator.py b/backend/backend/settings/deps/orchestrator.py index 9f755edcb..90aa724da 100644 --- a/backend/backend/settings/deps/orchestrator.py +++ b/backend/backend/settings/deps/orchestrator.py @@ -1,11 +1,11 @@ import os -from .. import common +from .utils import to_bool ORCHESTRATOR_HOST = os.environ.get("ORCHESTRATOR_HOST") ORCHESTRATOR_PORT = os.environ.get("ORCHESTRATOR_PORT") -ORCHESTRATOR_TLS_ENABLED = common.to_bool(os.environ.get("ORCHESTRATOR_TLS_ENABLED")) -ORCHESTRATOR_MTLS_ENABLED = common.to_bool(os.environ.get("ORCHESTRATOR_MTLS_ENABLED")) +ORCHESTRATOR_TLS_ENABLED = to_bool(os.environ.get("ORCHESTRATOR_TLS_ENABLED")) +ORCHESTRATOR_MTLS_ENABLED = to_bool(os.environ.get("ORCHESTRATOR_MTLS_ENABLED")) ORCHESTRATOR_TLS_SERVER_CACERT_PATH = os.environ.get("ORCHESTRATOR_TLS_SERVER_CACERT_PATH") ORCHESTRATOR_TLS_CLIENT_CERT_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_CERT_PATH") ORCHESTRATOR_TLS_CLIENT_KEY_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_KEY_PATH") diff --git a/backend/backend/settings/deps/path.py b/backend/backend/settings/deps/path.py new file mode 100644 index 000000000..a49009cfc --- /dev/null +++ b/backend/backend/settings/deps/path.py @@ -0,0 +1,9 @@ +import os +import sys + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROJECT_ROOT = os.path.dirname(BASE_DIR) + +sys.path.append(PROJECT_ROOT) +sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, "libs"))) diff --git a/backend/backend/settings/deps/restframework.py b/backend/backend/settings/deps/restframework.py index 07b2a81be..3425c11d0 100644 --- a/backend/backend/settings/deps/restframework.py +++ b/backend/backend/settings/deps/restframework.py @@ -1,5 +1,4 @@ import os -from datetime import timedelta REST_FRAMEWORK = { "TEST_REQUEST_DEFAULT_FORMAT": "json", @@ -36,14 +35,6 @@ "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), } -SIMPLE_JWT = { - "ACCESS_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("ACCESS_TOKEN_LIFETIME", 24 * 60))), - "REFRESH_TOKEN_LIFETIME": timedelta(minutes=int(os.environ.get("REFRESH_TOKEN_LIFETIME", 24 * 60 * 7))), - "ROTATE_REFRESH_TOKENS": True, - "AUTH_HEADER_TYPES": ("JWT",), - "BLACKLIST_AFTER_ROTATION": True, -} - SPECTACULAR_SETTINGS = { "SERVE_PERMISSIONS": ["rest_framework.permissions.IsAuthenticated"], } diff --git a/backend/backend/settings/deps/secret_key.py b/backend/backend/settings/deps/secret_key.py new file mode 100644 index 000000000..3ad15fcec --- /dev/null +++ b/backend/backend/settings/deps/secret_key.py @@ -0,0 +1,34 @@ +""" +SECRET_KEY is built in Django, but also used for signing JWTs +""" + +import os +import pathlib +import secrets + +from . import path +from .utils import to_bool + +SECRET_KEY_PATH = os.environ.get("SECRET_KEY_PATH", os.path.normpath(os.path.join(path.PROJECT_ROOT, "SECRET"))) + + +def _generate_secret_key(): + return secrets.token_urlsafe() # uses a "reasonable default" length + + +_SECRET_KEY_LOAD_AND_STORE = to_bool( + os.environ.get("SECRET_KEY_LOAD_AND_STORE", "False") +) # Whether to load the secret key from file (and write it there if it doesn't exist) + +if _SECRET_KEY_LOAD_AND_STORE: + try: + SECRET_KEY = pathlib.Path(SECRET_KEY_PATH).read_text().strip() + except IOError: + try: + SECRET_KEY = _generate_secret_key() + with open(SECRET_KEY_PATH, "w") as fp: + fp.write(SECRET_KEY) + except IOError: + raise Exception(f"Cannot open file `{SECRET_KEY_PATH}` for writing.") +else: + SECRET_KEY = _generate_secret_key() diff --git a/backend/backend/settings/deps/utils.py b/backend/backend/settings/deps/utils.py new file mode 100644 index 000000000..4fe8a419b --- /dev/null +++ b/backend/backend/settings/deps/utils.py @@ -0,0 +1,5 @@ +TRUE_VALUES = {"t", "T", "y", "Y", "yes", "YES", "true", "True", "TRUE", "on", "On", "ON", "1", 1, True} + + +def to_bool(value): + return value in TRUE_VALUES diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/dev.py index 3f5c833b2..bdbccf8d4 100644 --- a/backend/backend/settings/dev.py +++ b/backend/backend/settings/dev.py @@ -3,11 +3,11 @@ from substrapp.storages.minio import MinioStorage from .common import * -from .deps.cors import * from .deps.ledger import * -from .deps.oidc import * from .deps.org import * from .deps.restframework import * +from .mods.cors import * +from .mods.oidc import * DEBUG = True diff --git a/backend/backend/settings/localdev.py b/backend/backend/settings/localdev.py index 2a94a2dbc..98ea995be 100644 --- a/backend/backend/settings/localdev.py +++ b/backend/backend/settings/localdev.py @@ -1,4 +1,4 @@ -from .deps.oidc import * +from .mods.oidc import * from .test import * # Enable Browsable API diff --git a/backend/backend/settings/mods/__init__.py b/backend/backend/settings/mods/__init__.py new file mode 100644 index 000000000..0a5878a6c --- /dev/null +++ b/backend/backend/settings/mods/__init__.py @@ -0,0 +1,3 @@ +""" +settings that modify common settings +""" diff --git a/backend/backend/settings/deps/cors.py b/backend/backend/settings/mods/cors.py similarity index 80% rename from backend/backend/settings/deps/cors.py rename to backend/backend/settings/mods/cors.py index f3801a11b..773aed81c 100644 --- a/backend/backend/settings/deps/cors.py +++ b/backend/backend/settings/mods/cors.py @@ -1,7 +1,12 @@ +""" +Add Cross-origin site requests (CORS) restrictions +""" + import json import os from .. import common +from ..deps.utils import to_bool common.INSTALLED_APPS += ("corsheaders",) @@ -11,7 +16,7 @@ CORS_ALLOWED_ORIGINS = json.loads(os.environ.get("CORS_ORIGIN_WHITELIST", "[]")) CORS_ALLOW_ALL_ORIGINS = False # @CORS_ALLOW_CREDENTIALS: If True cookies can be included in cross site requests. Set this to `True` for frontend auth. -CORS_ALLOW_CREDENTIALS = common.to_bool(os.environ.get("CORS_ALLOW_CREDENTIALS", False)) +CORS_ALLOW_CREDENTIALS = to_bool(os.environ.get("CORS_ALLOW_CREDENTIALS", False)) CORS_ALLOW_HEADERS = ( "accept", "accept-encoding", diff --git a/backend/backend/settings/deps/oidc.py b/backend/backend/settings/mods/oidc.py similarity index 91% rename from backend/backend/settings/deps/oidc.py rename to backend/backend/settings/mods/oidc.py index dcd7c98a0..eaee681ba 100644 --- a/backend/backend/settings/deps/oidc.py +++ b/backend/backend/settings/mods/oidc.py @@ -1,10 +1,15 @@ +""" +OpenID Connect (for SSO) +""" + import logging import os import requests from .. import common -from . import ledger +from ..deps import ledger +from ..deps.utils import to_bool _LOGGER = logging.getLogger(__name__) @@ -13,7 +18,7 @@ # we'll use an "OIDC" dict for any setting added by us OIDC = { - "ENABLED": common.to_bool(os.environ.get("OIDC_ENABLED", "false")), + "ENABLED": to_bool(os.environ.get("OIDC_ENABLED", "false")), "USERS": {}, "OP": {}, } @@ -26,7 +31,7 @@ "propagate": False, } - OIDC["USERS"]["APPEND_DOMAIN"] = common.to_bool(os.environ.get("OIDC_USERS_APPEND_DOMAIN", "false")) + OIDC["USERS"]["APPEND_DOMAIN"] = to_bool(os.environ.get("OIDC_USERS_APPEND_DOMAIN", "false")) OIDC["USERS"]["DEFAULT_CHANNEL"] = os.environ.get("OIDC_USERS_DEFAULT_CHANNEL") if not OIDC["USERS"]["DEFAULT_CHANNEL"]: @@ -53,7 +58,7 @@ OIDC["OP"]["DISPLAY_NAME"] = os.environ.get("OIDC_OP_DISPLAY_NAME", OIDC["OP"]["URL"]) OIDC["OP"]["SUPPORTS_REFRESH"] = False - OIDC["USERS"]["USE_REFRESH_TOKEN"] = common.to_bool(os.environ.get("OIDC_USERS_USE_REFRESH_TOKEN", "false")) + OIDC["USERS"]["USE_REFRESH_TOKEN"] = to_bool(os.environ.get("OIDC_USERS_USE_REFRESH_TOKEN", "false")) op_settings = None try: diff --git a/backend/backend/settings/prod.py b/backend/backend/settings/prod.py index e744e979b..d428249c3 100644 --- a/backend/backend/settings/prod.py +++ b/backend/backend/settings/prod.py @@ -3,11 +3,11 @@ from substrapp.storages.minio import MinioStorage from .common import * -from .deps.cors import * from .deps.ledger import * -from .deps.oidc import * from .deps.org import * from .deps.restframework import * +from .mods.cors import * +from .mods.oidc import * DEBUG = False diff --git a/backend/backend/settings/test.py b/backend/backend/settings/test.py index 13bac6402..d32833c02 100644 --- a/backend/backend/settings/test.py +++ b/backend/backend/settings/test.py @@ -2,9 +2,9 @@ import tempfile from .common import * -from .deps.cors import * -from .deps.oidc import * from .deps.restframework import * +from .mods.cors import * +from .mods.oidc import * logging.disable(logging.CRITICAL) diff --git a/backend/users/views/user.py b/backend/users/views/user.py index c83a88e8d..78a355bfa 100644 --- a/backend/users/views/user.py +++ b/backend/users/views/user.py @@ -187,7 +187,7 @@ def set_password(self, request, *args, **kwargs): username = unquote(kwargs.get("username")) instance = self.user_model.objects.get(username=username) - secret = _xor_secrets(instance.password, force_str(settings.JWT_SECRET_KEY)) + secret = _xor_secrets(instance.password, force_str(settings.SECRET_KEY)) token_validation = _validate_token(token, secret) if token_validation.get("is_valid"): @@ -207,7 +207,7 @@ def verify_token(self, request, *args, **kwargs): username = unquote(kwargs.get("username")) instance = self.user_model.objects.get(username=username) - secret = _xor_secrets(instance.password, force_str(settings.JWT_SECRET_KEY)) + secret = _xor_secrets(instance.password, force_str(settings.SECRET_KEY)) token_validation = _validate_token(token, secret) if token_validation.get("is_valid"): return ApiResponse(data={}, status=status.HTTP_200_OK, headers=self.get_success_headers({})) @@ -221,7 +221,7 @@ def verify_token(self, request, *args, **kwargs): def generate_reset_password_token(self, request, *args, **kwargs): """Returns reset password token. Restricted to Admin request""" instance = self.get_object() - secret = _xor_secrets(instance.password, force_str(settings.JWT_SECRET_KEY)) + secret = _xor_secrets(instance.password, force_str(settings.SECRET_KEY)) jwt_token = jwt.encode( payload={"exp": datetime.datetime.now() + datetime.timedelta(days=7)}, diff --git a/docs/settings.md b/docs/settings.md index ebe3f313b..6d694e396 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -14,16 +14,6 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T |------|---------|---------------|---------| | json | `ALLOWED_HOSTS` | `[]` | | | string | `BACKEND_VERSION` | `dev` | | -| string | `CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD` | `86400` (`24 * 3600`) | | -| string | `CELERYBEAT_MAXIMUM_IMAGES_TTL` | `604800` (`7 * 24 * 3600`) | | -| string | `CELERY_BROKER_HOST` | `localhost` | | -| string | `CELERY_BROKER_PASSWORD` | nil | | -| string | `CELERY_BROKER_PORT` | `5672` | | -| string | `CELERY_BROKER_USER` | nil | | -| int | `CELERY_TASK_MAX_RETRIES` | `7` | | -| int | `CELERY_TASK_RETRY_BACKOFF` | `60` | time in seconds | -| int | `CELERY_TASK_RETRY_BACKOFF_MAX` | `3840` (`64 * 60`) | | -| int | `CELERY_WORKER_CONCURRENCY` | `1` | | | string | `COMMON_HOST_DOMAIN` | nil | | | string | `COMPUTE_POD_FS_GROUP` | nil | | | int | `COMPUTE_POD_GKE_GPUS_LIMITS` | `0` | | @@ -31,7 +21,7 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T | string | `COMPUTE_POD_RUN_AS_USER` | nil | | | int | `COMPUTE_POD_STARTUP_TIMEOUT_SECONDS` | `300` | | | json | `CSRF_TRUSTED_ORIGINS` | `[]` | A list of origins that are allowed to use unsafe HTTP methods | -| string | `DATABASE_DATABASE` | `?` (`f'backend_{ORG_NAME}'`) | | +| string | `DATABASE_DATABASE` | `f'backend_{ORG_NAME}'` | | | string | `DATABASE_HOSTNAME` | `localhost` | | | string | `DATABASE_PASSWORD` | `backend` | | | int | `DATABASE_PORT` | `5432` | | @@ -47,8 +37,6 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T | string | `HOST_IP` | nil | | | int | `HTTP_CLIENT_TIMEOUT_SECONDS` | `30` | | | bool | `ISOLATED` | nil | | -| bool | `JWT_SECRET_NEEDED` | `False` | | -| string | `JWT_SECRET_PATH` | `?` (`os.path.normpath(os.path.join(PROJECT_ROOT, 'SECRET'))`) | | | string | `K8S_SECRET_NAMESPACE` | `default` | | | string | `KANIKO_DOCKER_CONFIG_SECRET_NAME` | nil | | | string | `KANIKO_IMAGE` | nil | | @@ -66,7 +54,7 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T | string | `REGISTRY_PULL_DOMAIN` | nil | | | string | `REGISTRY_SCHEME` | nil | | | string | `REGISTRY_SERVICE_NAME` | nil | | -| string | `SUBPATH` | nil | | +| string | `SUBPATH` | empty string | prefix for backend endpoints | | bool | `TASK_CACHE_DOCKER_IMAGES` | `False` | | | bool | `TASK_CHAINKEYS_ENABLED` | `False` | | | bool | `TASK_LIST_WORKSPACE` | `True` | | @@ -75,6 +63,20 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T | string | `WORKER_PVC_SUBTUPLE` | nil | | | string | `WORKER_REPLICA_SET_NAME` | nil | | +## Secret key settings + +| Type | Setting | Default value | Comment | +|------|---------|---------------|---------| +| bool | `SECRET_KEY_LOAD_AND_STORE` | `False` | Whether to load the secret key from file (and write it there if it doesn't exist) | +| string | `SECRET_KEY_PATH` | `os.path.normpath(os.path.join(path.PROJECT_ROOT, 'SECRET'))` | | + +## JWT settings + +| Type | Setting | Default value | Comment | +|------|---------|---------------|---------| +| int | `ACCESS_TOKEN_LIFETIME` | `1440` (`24 * 60`) | | +| int | `REFRESH_TOKEN_LIFETIME` | `10080` (`24 * 60 * 7`) | | + ## Orchestrator settings | Type | Setting | Default value | Comment | @@ -91,6 +93,21 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T | bool | `ORCHESTRATOR_TLS_ENABLED` | nil | | | string | `ORCHESTRATOR_TLS_SERVER_CACERT_PATH` | nil | | +## Task broker settings + +| Type | Setting | Default value | Comment | +|------|---------|---------------|---------| +| string | `CELERYBEAT_FLUSH_EXPIRED_TOKENS_TASK_PERIOD` | `86400` (`24 * 3600`) | | +| string | `CELERYBEAT_MAXIMUM_IMAGES_TTL` | `604800` (`7 * 24 * 3600`) | | +| string | `CELERY_BROKER_HOST` | `localhost` | | +| string | `CELERY_BROKER_PASSWORD` | nil | | +| string | `CELERY_BROKER_PORT` | `5672` | | +| string | `CELERY_BROKER_USER` | nil | | +| int | `CELERY_TASK_MAX_RETRIES` | `7` | | +| int | `CELERY_TASK_RETRY_BACKOFF` | `60` | time in seconds | +| int | `CELERY_TASK_RETRY_BACKOFF_MAX` | `3840` (`64 * 60`) | | +| int | `CELERY_WORKER_CONCURRENCY` | `1` | | + ## Org settings | Type | Setting | Default value | Comment | @@ -104,7 +121,7 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T |------|---------|---------------|---------| | bool | `OIDC_ENABLED` | `false` | | | string | `OIDC_OP_AUTHORIZATION_ENDPOINT` | nil | | -| string | `OIDC_OP_DISPLAY_NAME` | `?` (`OIDC['OP']['URL']`) | | +| string | `OIDC_OP_DISPLAY_NAME` | `OIDC['OP']['URL']` | | | string | `OIDC_OP_JWKS_URI` | nil | | | string | `OIDC_OP_TOKEN_ENDPOINT` | nil | | | string | `OIDC_OP_URL` | nil | | diff --git a/tools/build_settings_doc.py b/tools/build_settings_doc.py index 4f2c31f53..ba38efe07 100644 --- a/tools/build_settings_doc.py +++ b/tools/build_settings_doc.py @@ -98,17 +98,17 @@ def is_env_query(node: ast.AST) -> bool: try: setting.default_value = ast.literal_eval(node.args[1]) except ValueError: + setting.default_value_comment = ast.unparse(node.args[1]) # default value is an expression, evaluate it to get a valid input try: setting.default_value = eval(ast.unparse(node.args[1])) # nosec except Exception as e: - setting.default_value = "?" + setting.default_value = setting.default_value_comment + setting.default_value_comment = None print( f"Warning: Couldn't eval {filename.relative_to(ROOT_PATH)} line {node.lineno}: {ast.unparse(node.args[1])} ({e}), leaving it as-is in the documentation." ) - setting.default_value_comment = ast.unparse(node.args[1]) - # get same-line comment for line in range(node_to_consider_for_comments.lineno, node_to_consider_for_comments.end_lineno + 1): if "#" in settings_file_lines[line - 1]: @@ -164,7 +164,13 @@ def generate_doc(settings_by_section: dict[str, list[Setting]], true_values: Col for setting in sorted(settings, key=lambda setting: setting.name): # Replace None with nil since None has a meaning in python - default = "nil" if setting.default_value is None else f"`{setting.default_value}`" + + if setting.default_value is None: + default = "nil" + elif isinstance(setting.default_value, str) and len(setting.default_value) == 0: + default = "empty string" + else: + default = f"`{setting.default_value}`" default_value_comment = f" (`{setting.default_value_comment}`)" if setting.default_value_comment else "" settings_doc.write( f"| {setting.type} | `{setting.name}` | {default}{default_value_comment} | {setting.comment} |\n" @@ -201,15 +207,18 @@ def compare_content(generated: pathlib.Path, committed: pathlib.Path) -> bool: args = parse_arguments() settings = {} settings["Global"] = load_settings_from_file(SETTINGS_FOLDER / "common.py") + settings["Secret key"] = load_settings_from_file(SETTINGS_FOLDER / "deps/secret_key.py") + settings["JWT"] = load_settings_from_file(SETTINGS_FOLDER / "deps/jwt.py") settings["Orchestrator"] = load_settings_from_file(SETTINGS_FOLDER / "deps/orchestrator.py") + settings["Task broker"] = load_settings_from_file(SETTINGS_FOLDER / "deps/celery.py") settings["Org"] = load_settings_from_file(SETTINGS_FOLDER / "deps/org.py") - settings["OpenID Connect"] = load_settings_from_file(SETTINGS_FOLDER / "deps/oidc.py") - settings["CORS"] = load_settings_from_file(SETTINGS_FOLDER / "deps/cors.py") + settings["OpenID Connect"] = load_settings_from_file(SETTINGS_FOLDER / "mods/oidc.py") + settings["CORS"] = load_settings_from_file(SETTINGS_FOLDER / "mods/cors.py") settings["Ledger"] = load_settings_from_file(SETTINGS_FOLDER / "deps/ledger.py") settings["Worker event app"] = load_settings_from_file(SETTINGS_FOLDER / "worker/events/common.py") settings["API event app"] = load_settings_from_file(SETTINGS_FOLDER / "api/events/common.py") - true_values = load_true_values_from_file(SETTINGS_FOLDER / "common.py") + true_values = load_true_values_from_file(SETTINGS_FOLDER / "deps/utils.py") if args["check"]: with tempfile.TemporaryDirectory() as tmpdir: