From 8253c0cd6a97d4b2f259de944d7153038bea8ae8 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 5 Nov 2024 14:42:50 +0100 Subject: [PATCH 01/21] Initial config addition --- app.py | 5 +++ base_config.json | 14 ++++++ config/.gitkeep | 0 config_handler.py | 98 +++++++++++++++++++++++++++++++++++++++++ fast_api/models.py | 4 ++ fast_api/routes/misc.py | 26 ++++++++++- notify_handler.py | 25 +++++++++++ 7 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 base_config.json create mode 100644 config/.gitkeep create mode 100644 config_handler.py create mode 100644 notify_handler.py diff --git a/app.py b/app.py index fb2ebc69..7dc01c4e 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,7 @@ CognitionImport, CognitionPrepareProject, ) +from config_handler import SERVICES_TO_NOTIFY, init_config from fast_api.routes.organization import router as org_router from fast_api.routes.project import router as project_router from fast_api.routes.project_setting import router as project_setting_router @@ -40,6 +41,7 @@ from starlette.routing import Route, Mount from controller.project.manager import check_in_deletion_projects +from notify_handler import notify_others_about_change_thread from route_prefix import ( PREFIX_ORGANIZATION, PREFIX_PROJECT, @@ -154,5 +156,8 @@ clean_up.clean_up_database() clean_up.clean_up_disk() +init_config() +notify_others_about_change_thread(SERVICES_TO_NOTIFY) + session.start_session_cleanup_thread() log_storage.start_persist_thread() diff --git a/base_config.json b/base_config.json new file mode 100644 index 00000000..b82e8f59 --- /dev/null +++ b/base_config.json @@ -0,0 +1,14 @@ +{ + "is_managed": null, + "is_demo": null, + "allow_data_tracking": true, + "s3_region": null, + "spacy_downloads": [ + "en_core_web_sm", + "de_core_news_sm" + ], + "tokens": { + "INTERCOM": "" + }, + "KERN_S3_ENDPOINT": null +} \ No newline at end of file diff --git a/config/.gitkeep b/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/config_handler.py b/config_handler.py new file mode 100644 index 00000000..c49e61c7 --- /dev/null +++ b/config_handler.py @@ -0,0 +1,98 @@ +from typing import Dict, Any +import os +import json + +__blacklist_base_config = ["is_managed", "is_demo"] +__config = None + +BASE_CONFIG_PATH = "base_config.json" +CURRENT_CONFIG_PATH = "/config/current_config.json" + +SERVICES_TO_NOTIFY = { + "EMBEDDER": "http://refinery-embedder:80", + "UPDATER": "http://refinery-updater:80", + "TOKENIZER": "http://refinery-tokenizer:80", +} + + +def __read_and_change_base_config(): + print("reading base config file", flush=True) + global __config + f = open(BASE_CONFIG_PATH) + __config = json.load(f) + + print("transfer os variables", flush=True) + __config["is_managed"] = os.getenv("IS_MANAGED") == "1" + __config["is_demo"] = os.getenv("IS_DEMO") == "1" + __config["tokens"]["INTERCOM"] = os.getenv("INTERCOM", "") + __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1") + + __save_current_config() + + +def change_config(changes: Dict[str, Any]) -> bool: + global __config + something_changed = False + for key in changes: + if key == "KERN_S3_ENDPOINT": + continue + if key in __config: + if isinstance(changes[key], dict): + for subkey in changes[key]: + if subkey in __config[key]: + __config[key][subkey] = changes[key][subkey] + something_changed = True + else: + __config[key] = changes[key] + something_changed = True + if something_changed: + __save_current_config() + else: + print("nothing was changed with input", changes, flush=True) + return something_changed + + +def __save_current_config() -> None: + print("saving config file", flush=True) + with open(CURRENT_CONFIG_PATH, "w") as f: + json.dump(__config, f, indent=4) + + +def init_config() -> None: + if not os.path.exists(CURRENT_CONFIG_PATH): + __read_and_change_base_config() + else: + __load_and_remove_outdated_config_keys() + # this one is to be set on every start to ensure its up to date + print("setting s3 endpoint", flush=True) + __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT") + + +def __load_and_remove_outdated_config_keys(): + if not os.path.exists(CURRENT_CONFIG_PATH): + return + + global __config + with open(CURRENT_CONFIG_PATH) as f: + __config = json.load(f) + + with open(BASE_CONFIG_PATH) as f: + base_config = json.load(f) + + to_remove = [key for key in __config if key not in base_config] + + if len(to_remove) > 0: + print("removing outdated config keys", to_remove, flush=True) + for key in to_remove: + del __config[key] + __save_current_config() + + +def get_config(basic: bool = True) -> Dict[str, Any]: + global __config, __blacklist_base_config + if not basic: + return __config + + return { + key: __config[key] for key in __config if key not in __blacklist_base_config + } diff --git a/fast_api/models.py b/fast_api/models.py index bbd54529..b7a72e56 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -448,3 +448,7 @@ class UpdateCustomerButton(BaseModel): location: Optional[CustomerButtonLocation] = None visible: Optional[StrictBool] = None config: Optional[Dict[StrictStr, Any]] = None + + +class ChangeRequest(BaseModel): + dict_string: str diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index d503c25f..34d56b3f 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,6 +1,7 @@ import json -from fastapi import APIRouter, Body, Request, status +from fastapi import APIRouter, Body, Request, status, responses from fastapi.responses import PlainTextResponse +from config_handler import SERVICES_TO_NOTIFY, change_config, get_config from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, @@ -8,6 +9,7 @@ ModelProviderDownloadModelBody, CreateCustomerButton, UpdateCustomerButton, + ChangeRequest, ) from fast_api.routes.client_response import pack_json_result, SILENT_SUCCESS_RESPONSE from typing import Dict, Optional @@ -17,6 +19,7 @@ from controller.monitor import manager as controller_manager from controller.model_provider import manager as model_provider_manager from controller.task_master import manager as task_master_manager +from notify_handler import notify_others_about_change_thread from submodules.model import enums from submodules.model.global_objects import customer_button as customer_button_db_go import util.user_activity @@ -47,6 +50,27 @@ def get_is_demo(request: Request) -> Dict: return pack_json_result({"data": {"isDemo": is_demo}}) +@router.post("/change_config") +def change(request: ChangeRequest) -> responses.PlainTextResponse: + if change_config(json.loads(request.dict_string)): + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + return responses.PlainTextResponse(status_code=status.HTTP_200_OK) + + +@router.get("/full_config") +def full_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(False) + ) + + +@router.get("/base_config") +def base_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(True) + ) + + @router.get("/version-overview") def get_version_overview(request: Request) -> Dict: data = manager.get_version_overview() diff --git a/notify_handler.py b/notify_handler.py new file mode 100644 index 00000000..64064f2e --- /dev/null +++ b/notify_handler.py @@ -0,0 +1,25 @@ +from typing import Dict + +import requests +import traceback + +from submodules.model import daemon + + +def notify_others_about_change(notify: Dict[str, str]) -> None: + + for key in notify: + url = f"{notify[key]}/config_changed" + try: + response = requests.put(url) + if response.status_code != 200: + print(f"couldn't notify - code:{response.status_code}", key, flush=True) + except requests.exceptions.ConnectionError: + print("couldn't notify", key, flush=True) + except Exception: + print(traceback.format_exc(), flush=True) + + +def notify_others_about_change_thread(notify: Dict[str, str]) -> None: + # needs to be threaded to work with circular requests (this notifies about changes -> service requests full config from this) + daemon.run_without_db_token(notify_others_about_change, notify) From 1dee431f3d3c32d1e4cbcb4aa19aea95e8e58f9c Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 5 Nov 2024 16:18:58 +0100 Subject: [PATCH 02/21] adjust services --- check_config_service | 31 ------------------------------- controller/misc/config_service.py | 13 ++++--------- 2 files changed, 4 insertions(+), 40 deletions(-) delete mode 100644 check_config_service diff --git a/check_config_service b/check_config_service deleted file mode 100644 index 00bd9d4c..00000000 --- a/check_config_service +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -CONFIG_EXISTS=0 -if [ -n "$(docker ps -q -f name=dev-setup_refinery-config_1)" ] -then - if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup_refinery-config_1 )" == "running" ]; - then - CONFIG_EXISTS=1 - fi -elif [ -n "$(docker ps -q -f name=dev-setup-refinery-config-1)" ] -then - if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup-refinery-config-1 )" == "running" ]; - then - CONFIG_EXISTS=1 - fi -else - if [ -n "$(docker ps -q -f name=refinery-config)" ]; - then - if [ "$( docker container inspect -f '{{.State.Status}}' refinery-config )" == "running" ]; - then - CONFIG_EXISTS=1 - fi - fi -fi -if [ $CONFIG_EXISTS -eq 0 ] -then - echo "refinery-config couldn't be found - exit" - exit 1 -else - echo "refinery-config found -> proceeding" -fi - diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py index 19be9be9..e28ee7b1 100644 --- a/controller/misc/config_service.py +++ b/controller/misc/config_service.py @@ -1,15 +1,10 @@ from typing import Dict, Any, Optional, Union -import requests import time +from fast_api.routes.misc import change, full_config from submodules.model import daemon -from util import service_requests __config = None -# these are ment to be constant values since os variables will sooner or later be removed for addresses (and used with values from config-service) -REQUEST_URL = "http://refinery-config:80/full_config" -CHANGE_URL = "http://refinery-config:80/change_config" - def __get_config() -> Dict[str, Any]: global __config @@ -20,14 +15,14 @@ def __get_config() -> Dict[str, Any]: def refresh_config(): - response = requests.get(REQUEST_URL) + response = full_config() if response.status_code != 200: raise ValueError( f"Config service cant be reached -- response.code{response.status_code}" ) global __config __config = response.json() - daemon.run_without_db_token(invalidate_after, 3600) # one hour as failsave + daemon.run_without_db_token(invalidate_after, 3600) # one hour as fail safe def get_config_value( @@ -55,4 +50,4 @@ def invalidate_after(sec: int) -> None: def change_config(dict_str: str) -> None: data = {"dict_string": dict_str} - service_requests.post_call_or_raise(CHANGE_URL, data) + return change(data).text From 8f349d767473383ac024accb0a6bc125c4206da6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 5 Nov 2024 16:19:42 +0100 Subject: [PATCH 03/21] edits --- fast_api/routes/organization.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fast_api/routes/organization.py b/fast_api/routes/organization.py index 277429b0..cceb8fc7 100644 --- a/fast_api/routes/organization.py +++ b/fast_api/routes/organization.py @@ -1,5 +1,5 @@ import json -from fastapi import APIRouter, Depends, Request, Body +from fastapi import APIRouter, Request, Body from controller.misc import config_service from fast_api.models import ( AddUserToOrganizationBody, @@ -27,7 +27,6 @@ from controller.misc import manager as misc from fast_api.routes.client_response import get_silent_success, pack_json_result -from submodules.model import events from submodules.model.business_objects import organization from submodules.model.util import sql_alchemy_to_dict from util import notification From 626656c9b6e55c276af0c45ef1a4d0177d41111e Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 5 Nov 2024 17:25:02 +0100 Subject: [PATCH 04/21] Adjust config route --- app.py | 36 +++++++++++++++++++++++++++++++++--- fast_api/routes/misc.py | 26 +------------------------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/app.py b/app.py index 7dc01c4e..6af8efc9 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,6 @@ import logging - -from fastapi import FastAPI +import json +from fastapi import FastAPI, status, responses from api.healthcheck import Healthcheck from starlette.middleware import Middleware from api.misc import IsDemoRest, IsManagedRest @@ -16,7 +16,7 @@ CognitionImport, CognitionPrepareProject, ) -from config_handler import SERVICES_TO_NOTIFY, init_config +from config_handler import SERVICES_TO_NOTIFY, init_config, change_config, get_config from fast_api.routes.organization import router as org_router from fast_api.routes.project import router as project_router from fast_api.routes.project_setting import router as project_setting_router @@ -65,12 +65,41 @@ from util import security, clean_up from middleware import log_storage from submodules.model import session +from fast_api.models import ( + ChangeRequest, +) logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) fastapi_app = FastAPI() +""" +Config routes (un-routed exposure) +""" + + +@fastapi_app.post("/change_config") +def change(request: ChangeRequest) -> responses.PlainTextResponse: + if change_config(json.loads(request.dict_string)): + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + return responses.PlainTextResponse(status_code=status.HTTP_200_OK) + + +@fastapi_app.get("/full_config") +def full_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(False) + ) + + +@fastapi_app.get("/base_config") +def base_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(True) + ) + + fastapi_app.include_router( org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"] ) @@ -117,6 +146,7 @@ fastapi_app_internal.include_router( task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"] ) + routes = [ Route("/notify/{path:path}", Notify), Route("/healthcheck", Healthcheck), diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index 34d56b3f..d503c25f 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,7 +1,6 @@ import json -from fastapi import APIRouter, Body, Request, status, responses +from fastapi import APIRouter, Body, Request, status from fastapi.responses import PlainTextResponse -from config_handler import SERVICES_TO_NOTIFY, change_config, get_config from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, @@ -9,7 +8,6 @@ ModelProviderDownloadModelBody, CreateCustomerButton, UpdateCustomerButton, - ChangeRequest, ) from fast_api.routes.client_response import pack_json_result, SILENT_SUCCESS_RESPONSE from typing import Dict, Optional @@ -19,7 +17,6 @@ from controller.monitor import manager as controller_manager from controller.model_provider import manager as model_provider_manager from controller.task_master import manager as task_master_manager -from notify_handler import notify_others_about_change_thread from submodules.model import enums from submodules.model.global_objects import customer_button as customer_button_db_go import util.user_activity @@ -50,27 +47,6 @@ def get_is_demo(request: Request) -> Dict: return pack_json_result({"data": {"isDemo": is_demo}}) -@router.post("/change_config") -def change(request: ChangeRequest) -> responses.PlainTextResponse: - if change_config(json.loads(request.dict_string)): - notify_others_about_change_thread(SERVICES_TO_NOTIFY) - return responses.PlainTextResponse(status_code=status.HTTP_200_OK) - - -@router.get("/full_config") -def full_config() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(False) - ) - - -@router.get("/base_config") -def base_config() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(True) - ) - - @router.get("/version-overview") def get_version_overview(request: Request) -> Dict: data = manager.get_version_overview() From 12ec0f5ccf2928f97b1207ccbf30479ec6745254 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Wed, 6 Nov 2024 16:09:46 +0100 Subject: [PATCH 05/21] remove check_config_service --- start | 2 -- 1 file changed, 2 deletions(-) diff --git a/start b/start index 486b7016..268a66ac 100755 --- a/start +++ b/start @@ -1,8 +1,6 @@ #!/bin/bash trap "echo -ne '\nstopping container...' && docker stop refinery-gateway > /dev/null 2>&1 && echo -ne '\t\t [done]\n'" EXIT -source check_config_service - HOST_IP=$(docker network inspect bridge --format='{{json .IPAM.Config}}' | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | tail -1) echo -ne 'stopping old container...' From a04fde3a2ec0ef9227ff9fdb280c3b5922eef424 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 7 Nov 2024 16:49:33 +0100 Subject: [PATCH 06/21] move implementation --- app.py | 25 ++++++++++++------------- config_handler.py | 23 +++++++++++++++++++++++ controller/misc/config_service.py | 6 +++--- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/app.py b/app.py index 6af8efc9..1dadaf78 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,5 @@ import logging -import json -from fastapi import FastAPI, status, responses +from fastapi import FastAPI, responses from api.healthcheck import Healthcheck from starlette.middleware import Middleware from api.misc import IsDemoRest, IsManagedRest @@ -16,7 +15,13 @@ CognitionImport, CognitionPrepareProject, ) -from config_handler import SERVICES_TO_NOTIFY, init_config, change_config, get_config +from config_handler import ( + SERVICES_TO_NOTIFY, + base_config_json, + change_json, + full_config_json, + init_config, +) from fast_api.routes.organization import router as org_router from fast_api.routes.project import router as project_router from fast_api.routes.project_setting import router as project_setting_router @@ -75,29 +80,23 @@ fastapi_app = FastAPI() """ -Config routes (un-routed exposure) +Config routes """ @fastapi_app.post("/change_config") def change(request: ChangeRequest) -> responses.PlainTextResponse: - if change_config(json.loads(request.dict_string)): - notify_others_about_change_thread(SERVICES_TO_NOTIFY) - return responses.PlainTextResponse(status_code=status.HTTP_200_OK) + return change_json(request) @fastapi_app.get("/full_config") def full_config() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(False) - ) + return full_config_json() @fastapi_app.get("/base_config") def base_config() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(True) - ) + return base_config_json() fastapi_app.include_router( diff --git a/config_handler.py b/config_handler.py index c49e61c7..513e9012 100644 --- a/config_handler.py +++ b/config_handler.py @@ -1,6 +1,11 @@ from typing import Dict, Any import os import json +from notify_handler import notify_others_about_change_thread +from fastapi import responses, status +from fast_api.models import ( + ChangeRequest, +) __blacklist_base_config = ["is_managed", "is_demo"] __config = None @@ -96,3 +101,21 @@ def get_config(basic: bool = True) -> Dict[str, Any]: return { key: __config[key] for key in __config if key not in __blacklist_base_config } + + +def change_json(request: ChangeRequest) -> responses.PlainTextResponse: + if change_config(json.loads(request.dict_string)): + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + return responses.PlainTextResponse(status_code=status.HTTP_200_OK) + + +def full_config_json() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(False) + ) + + +def base_config_json() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(True) + ) diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py index e28ee7b1..36da3de4 100644 --- a/controller/misc/config_service.py +++ b/controller/misc/config_service.py @@ -1,6 +1,6 @@ from typing import Dict, Any, Optional, Union import time -from fast_api.routes.misc import change, full_config +from config_handler import change_json, full_config_json from submodules.model import daemon __config = None @@ -15,7 +15,7 @@ def __get_config() -> Dict[str, Any]: def refresh_config(): - response = full_config() + response = full_config_json() if response.status_code != 200: raise ValueError( f"Config service cant be reached -- response.code{response.status_code}" @@ -50,4 +50,4 @@ def invalidate_after(sec: int) -> None: def change_config(dict_str: str) -> None: data = {"dict_string": dict_str} - return change(data).text + return change_json(data).text From b82c6e63c0fe25d4030a2ec7f8ef665bb0b95f61 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 7 Nov 2024 17:01:51 +0100 Subject: [PATCH 07/21] Change order of config --- app.py | 5 ++--- start | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 1dadaf78..070bf4b1 100644 --- a/app.py +++ b/app.py @@ -77,7 +77,9 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) +init_config() fastapi_app = FastAPI() +notify_others_about_change_thread(SERVICES_TO_NOTIFY) """ Config routes @@ -185,8 +187,5 @@ def base_config() -> responses.JSONResponse: clean_up.clean_up_database() clean_up.clean_up_disk() -init_config() -notify_others_about_change_thread(SERVICES_TO_NOTIFY) - session.start_session_cleanup_thread() log_storage.start_persist_thread() diff --git a/start b/start index 268a66ac..e52d12cd 100755 --- a/start +++ b/start @@ -31,6 +31,20 @@ fi INFERENCE_DIR=${DEV_SETUP_DIR}inference/ LOG_DIR=${DEV_SETUP_DIR}logs/ +CONFIG_DIR=${PWD%/*}/dev-setup/config/ +if [ ! -d "$CONFIG_DIR" ] +then + CONFIG_DIR=${PWD%/*/*}/dev-setup/config/ + if [ ! -d "$CONFIG_DIR" ] + then + # to include volume for local development, use the dev-setup inference folder: + # alternative use manual logic with + # -v /path/to/dev-setup/config:/config \ + echo "Can't find config directory: $CONFIG_DIR -> stopping" + exit 1 + fi +fi + echo -ne 'building container...' docker build -t graphql-dev -f dev.Dockerfile . > /dev/null 2>&1 echo -ne '\t\t [done]\n' @@ -74,6 +88,7 @@ docker run -d --rm \ -e POSTGRES_POOL_USE_LIFO=x \ -v "$INFERENCE_DIR":/inference \ -v "$LOG_DIR":/logs \ +-v "$CONFIG_DIR":/config \ --mount type=bind,source="$(pwd)"/,target=/app \ -v /var/run/docker.sock:/var/run/docker.sock \ --network dev-setup_default \ From 1368d06320f5103d4f07ea1b21695a0a4292af9e Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 7 Nov 2024 17:18:44 +0100 Subject: [PATCH 08/21] Access values directly --- controller/misc/config_service.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py index 36da3de4..2a6ba18d 100644 --- a/controller/misc/config_service.py +++ b/controller/misc/config_service.py @@ -1,6 +1,6 @@ from typing import Dict, Any, Optional, Union import time -from config_handler import change_json, full_config_json +from config_handler import change_json, get_config from submodules.model import daemon __config = None @@ -15,13 +15,9 @@ def __get_config() -> Dict[str, Any]: def refresh_config(): - response = full_config_json() - if response.status_code != 200: - raise ValueError( - f"Config service cant be reached -- response.code{response.status_code}" - ) + response = get_config(False) global __config - __config = response.json() + __config = response daemon.run_without_db_token(invalidate_after, 3600) # one hour as fail safe @@ -50,4 +46,4 @@ def invalidate_after(sec: int) -> None: def change_config(dict_str: str) -> None: data = {"dict_string": dict_str} - return change_json(data).text + return change_json(data) From 8b06114bf353f4f1bc8cc8e9a4756a2e0614fe0c Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 8 Nov 2024 10:04:10 +0100 Subject: [PATCH 09/21] adjust fastapi --- app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 070bf4b1..f9d08263 100644 --- a/app.py +++ b/app.py @@ -78,6 +78,7 @@ logger = logging.getLogger(__name__) init_config() +fastapi_config_app = FastAPI() fastapi_app = FastAPI() notify_others_about_change_thread(SERVICES_TO_NOTIFY) @@ -86,17 +87,17 @@ """ -@fastapi_app.post("/change_config") +@fastapi_config_app.post("/change_config") def change(request: ChangeRequest) -> responses.PlainTextResponse: return change_json(request) -@fastapi_app.get("/full_config") +@fastapi_config_app.get("/full_config") def full_config() -> responses.JSONResponse: return full_config_json() -@fastapi_app.get("/base_config") +@fastapi_config_app.get("/base_config") def base_config() -> responses.JSONResponse: return base_config_json() From 14ba26737a58aeec0cc8db6716c54cb6329b29e3 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 8 Nov 2024 11:30:29 +0100 Subject: [PATCH 10/21] Add new routes for config --- api/misc.py | 37 +++++++++++++++++++++++++++++ app.py | 39 +++++++++---------------------- config_handler.py | 22 +++++++++++------ controller/misc/config_service.py | 4 ++-- submodules/model | 2 +- 5 files changed, 66 insertions(+), 38 deletions(-) diff --git a/api/misc.py b/api/misc.py index 476d26f0..667f69c1 100644 --- a/api/misc.py +++ b/api/misc.py @@ -2,6 +2,18 @@ from starlette.endpoints import HTTPEndpoint from starlette.responses import JSONResponse from starlette import status +from fastapi import Request, responses +import json + +from config_handler import ( + base_config_json, + change_json, + full_config_json, +) + +from fast_api.models import ( + ChangeRequest, +) class IsManagedRest(HTTPEndpoint): @@ -9,7 +21,32 @@ def get(self, request) -> JSONResponse: is_managed = config_service.get_config_value("is_managed") return JSONResponse(is_managed, status_code=status.HTTP_200_OK) + class IsDemoRest(HTTPEndpoint): def get(self, request) -> JSONResponse: is_managed = config_service.get_config_value("is_demo") return JSONResponse(is_managed, status_code=status.HTTP_200_OK) + + +class ChangeConfigRest(HTTPEndpoint): + async def post(self, request: Request) -> JSONResponse: + try: + data = await request.json() + change_request = ChangeRequest(**data) + config_data = json.loads(change_request.dict_string) + return await change_json(config_data) + except Exception as e: + return responses.PlainTextResponse( + f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + +class FullConfigRest(HTTPEndpoint): + def get(self, request: Request) -> JSONResponse: + return full_config_json() + + +class BaseConfigRest(HTTPEndpoint): + def get(self, request: Request) -> JSONResponse: + print("REACHED BASE CONFIG", flush=True) + return base_config_json() diff --git a/app.py b/app.py index f9d08263..ccbe12a6 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,14 @@ import logging -from fastapi import FastAPI, responses +from fastapi import FastAPI from api.healthcheck import Healthcheck from starlette.middleware import Middleware -from api.misc import IsDemoRest, IsManagedRest +from api.misc import ( + BaseConfigRest, + ChangeConfigRest, + FullConfigRest, + IsDemoRest, + IsManagedRest, +) from api.project import ProjectDetails from api.transfer import ( AssociationsImport, @@ -17,9 +23,6 @@ ) from config_handler import ( SERVICES_TO_NOTIFY, - base_config_json, - change_json, - full_config_json, init_config, ) from fast_api.routes.organization import router as org_router @@ -70,37 +73,14 @@ from util import security, clean_up from middleware import log_storage from submodules.model import session -from fast_api.models import ( - ChangeRequest, -) logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) init_config() -fastapi_config_app = FastAPI() fastapi_app = FastAPI() notify_others_about_change_thread(SERVICES_TO_NOTIFY) -""" -Config routes -""" - - -@fastapi_config_app.post("/change_config") -def change(request: ChangeRequest) -> responses.PlainTextResponse: - return change_json(request) - - -@fastapi_config_app.get("/full_config") -def full_config() -> responses.JSONResponse: - return full_config_json() - - -@fastapi_config_app.get("/base_config") -def base_config() -> responses.JSONResponse: - return base_config_json() - fastapi_app.include_router( org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"] @@ -150,6 +130,9 @@ def base_config() -> responses.JSONResponse: ) routes = [ + Route("/change_config", ChangeConfigRest), + Route("/base_config", BaseConfigRest), + Route("/full_config", FullConfigRest), Route("/notify/{path:path}", Notify), Route("/healthcheck", Healthcheck), Route("/project/{project_id:str}", ProjectDetails), diff --git a/config_handler.py b/config_handler.py index 513e9012..4f2d599f 100644 --- a/config_handler.py +++ b/config_handler.py @@ -3,9 +3,6 @@ import json from notify_handler import notify_others_about_change_thread from fastapi import responses, status -from fast_api.models import ( - ChangeRequest, -) __blacklist_base_config = ["is_managed", "is_demo"] __config = None @@ -103,10 +100,21 @@ def get_config(basic: bool = True) -> Dict[str, Any]: } -def change_json(request: ChangeRequest) -> responses.PlainTextResponse: - if change_config(json.loads(request.dict_string)): - notify_others_about_change_thread(SERVICES_TO_NOTIFY) - return responses.PlainTextResponse(status_code=status.HTTP_200_OK) +async def change_json(config_data) -> responses.PlainTextResponse: + try: + has_changed = change_config(config_data) + + if has_changed: + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + + return responses.PlainTextResponse( + f"Did update: {has_changed}", status_code=status.HTTP_200_OK + ) + + except Exception as e: + return responses.PlainTextResponse( + f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) def full_config_json() -> responses.JSONResponse: diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py index 2a6ba18d..84c44d08 100644 --- a/controller/misc/config_service.py +++ b/controller/misc/config_service.py @@ -45,5 +45,5 @@ def invalidate_after(sec: int) -> None: def change_config(dict_str: str) -> None: - data = {"dict_string": dict_str} - return change_json(data) + config_data = {"dict_string": dict_str} + return change_json(config_data) diff --git a/submodules/model b/submodules/model index 67a33b99..afe67cc3 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 67a33b99a200f18202feb7bc5222d2e3d3177972 +Subproject commit afe67cc3b6ecaa79fa5582d44ea137cfa61545a4 From ec5f005cbef434da28d3ef59fde174d55041dbe2 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 8 Nov 2024 14:13:09 +0100 Subject: [PATCH 11/21] edit --- .gitignore | 4 +++- start | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 73edd7ff..62673f27 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,6 @@ project_export* tmp/* !tmp/.gitkeep -logs/* \ No newline at end of file +logs/* + +current_config.json \ No newline at end of file diff --git a/start b/start index e52d12cd..32a07afc 100755 --- a/start +++ b/start @@ -28,6 +28,8 @@ then fi fi +MINIO_ENDPOINT="http://$HOST_IP:7053" + INFERENCE_DIR=${DEV_SETUP_DIR}inference/ LOG_DIR=${DEV_SETUP_DIR}logs/ @@ -86,6 +88,7 @@ docker run -d --rm \ -e INFERENCE_DIR=$INFERENCE_DIR \ -e SECRET_KEY=default \ -e POSTGRES_POOL_USE_LIFO=x \ +-e KERN_S3_ENDPOINT=${MINIO_ENDPOINT} \ -v "$INFERENCE_DIR":/inference \ -v "$LOG_DIR":/logs \ -v "$CONFIG_DIR":/config \ From fcef6c0b84b68dea9e5e3802729309588e795589 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 8 Nov 2024 14:13:29 +0100 Subject: [PATCH 12/21] edit --- api/misc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/misc.py b/api/misc.py index 667f69c1..c1087711 100644 --- a/api/misc.py +++ b/api/misc.py @@ -48,5 +48,4 @@ def get(self, request: Request) -> JSONResponse: class BaseConfigRest(HTTPEndpoint): def get(self, request: Request) -> JSONResponse: - print("REACHED BASE CONFIG", flush=True) return base_config_json() From 9bc067781ca740fd185643405e07d44d654bfe1d Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 8 Nov 2024 14:22:47 +0100 Subject: [PATCH 13/21] add direct default route for base_config --- fast_api/routes/misc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index d503c25f..71fc3ef5 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,6 +1,7 @@ import json from fastapi import APIRouter, Body, Request, status from fastapi.responses import PlainTextResponse +from config_handler import base_config_json from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, @@ -37,6 +38,11 @@ def get_is_admin(request: Request) -> Dict: return pack_json_result({"data": {"isAdmin": data}}) +@router.get("/base-config-default") +def get_base_config() -> Dict: + return base_config_json() + + @router.get("/is-demo") def get_is_demo(request: Request) -> Dict: is_demo = False From ab63c348fefc8fec988c0c72aa9b4ffe5b5cac50 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 12 Nov 2024 14:08:36 +0100 Subject: [PATCH 14/21] Fix: outdated lines --- config_handler.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config_handler.py b/config_handler.py index 4f2d599f..8d5fe62e 100644 --- a/config_handler.py +++ b/config_handler.py @@ -22,13 +22,7 @@ def __read_and_change_base_config(): global __config f = open(BASE_CONFIG_PATH) __config = json.load(f) - - print("transfer os variables", flush=True) - __config["is_managed"] = os.getenv("IS_MANAGED") == "1" - __config["is_demo"] = os.getenv("IS_DEMO") == "1" - __config["tokens"]["INTERCOM"] = os.getenv("INTERCOM", "") __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1") - __save_current_config() From e54a1135b28293e07a625c29891427d540de36ed Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 12 Nov 2024 14:15:01 +0100 Subject: [PATCH 15/21] Fix: unnecessary async --- api/misc.py | 2 +- config_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/misc.py b/api/misc.py index c1087711..a10eee7e 100644 --- a/api/misc.py +++ b/api/misc.py @@ -34,7 +34,7 @@ async def post(self, request: Request) -> JSONResponse: data = await request.json() change_request = ChangeRequest(**data) config_data = json.loads(change_request.dict_string) - return await change_json(config_data) + return change_json(config_data) except Exception as e: return responses.PlainTextResponse( f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/config_handler.py b/config_handler.py index 8d5fe62e..a8c5d41f 100644 --- a/config_handler.py +++ b/config_handler.py @@ -94,7 +94,7 @@ def get_config(basic: bool = True) -> Dict[str, Any]: } -async def change_json(config_data) -> responses.PlainTextResponse: +def change_json(config_data) -> responses.PlainTextResponse: try: has_changed = change_config(config_data) From 4f0a715c7ca314df3bd2921b11f933d2222f361c Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 12 Nov 2024 14:53:34 +0100 Subject: [PATCH 16/21] Remove unused service --- api/misc.py | 6 +-- config_handler.py | 18 ++++++++- controller/auth/manager.py | 8 ++-- controller/information_source/manager.py | 4 +- controller/misc/config_service.py | 49 ------------------------ controller/misc/manager.py | 13 +++---- controller/organization/manager.py | 4 +- controller/payload/payload_scheduler.py | 4 +- controller/transfer/checks.py | 3 +- fast_api/routes/organization.py | 9 ++--- 10 files changed, 39 insertions(+), 79 deletions(-) delete mode 100644 controller/misc/config_service.py diff --git a/api/misc.py b/api/misc.py index a10eee7e..173f645d 100644 --- a/api/misc.py +++ b/api/misc.py @@ -1,4 +1,3 @@ -from controller.misc import config_service from starlette.endpoints import HTTPEndpoint from starlette.responses import JSONResponse from starlette import status @@ -9,6 +8,7 @@ base_config_json, change_json, full_config_json, + get_config_value, ) from fast_api.models import ( @@ -18,13 +18,13 @@ class IsManagedRest(HTTPEndpoint): def get(self, request) -> JSONResponse: - is_managed = config_service.get_config_value("is_managed") + is_managed = get_config_value("is_managed") return JSONResponse(is_managed, status_code=status.HTTP_200_OK) class IsDemoRest(HTTPEndpoint): def get(self, request) -> JSONResponse: - is_managed = config_service.get_config_value("is_demo") + is_managed = get_config_value("is_demo") return JSONResponse(is_managed, status_code=status.HTTP_200_OK) diff --git a/config_handler.py b/config_handler.py index a8c5d41f..3c4d9f0d 100644 --- a/config_handler.py +++ b/config_handler.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Dict, Any, Optional, Union import os import json from notify_handler import notify_others_about_change_thread @@ -17,6 +17,22 @@ } +def get_config_value( + key: str, subkey: Optional[str] = None +) -> Union[str, Dict[str, str]]: + if key not in __config: + raise ValueError(f"Key {key} coudn't be found in config") + value = __config[key] + + if not subkey: + return value + + if isinstance(value, dict) and subkey in value: + return value[subkey] + else: + raise ValueError(f"Subkey {subkey} coudn't be found in config[{key}]") + + def __read_and_change_base_config(): print("reading base config file", flush=True) global __config diff --git a/controller/auth/manager.py b/controller/auth/manager.py index b200c75d..84464290 100644 --- a/controller/auth/manager.py +++ b/controller/auth/manager.py @@ -1,7 +1,7 @@ from typing import Any, Dict from fastapi import Request -from controller.misc import config_service +from config_handler import get_config_value from exceptions.exceptions import ( AuthManagerError, NotAllowedInDemoError, @@ -128,9 +128,7 @@ def check_is_admin(request: Any) -> bool: def check_demo_access(info: Any) -> None: - if not check_is_admin(info.context["request"]) and config_service.get_config_value( - "is_demo" - ): + if not check_is_admin(info.context["request"]) and get_config_value("is_demo"): check_black_white(info) @@ -145,7 +143,7 @@ def check_black_white(info: Any): def check_is_demo_without_info() -> None: - if config_service.get_config_value("is_demo"): + if get_config_value("is_demo"): raise NotAllowedInDemoError diff --git a/controller/information_source/manager.py b/controller/information_source/manager.py index 5ff81be6..41b81e53 100644 --- a/controller/information_source/manager.py +++ b/controller/information_source/manager.py @@ -1,5 +1,6 @@ import os from typing import List +from config_handler import get_config_value from controller.information_source.util import resolve_source_return_type from submodules.model import InformationSource, LabelingTask, enums from submodules.model.business_objects import ( @@ -7,7 +8,6 @@ information_source, payload, ) -from controller.misc import config_service from controller.labeling_access_link import manager as link_manager from submodules.model import daemon @@ -86,7 +86,7 @@ def delete_information_source(project_id: str, source_id: str) -> None: if ( information_source_item.type == enums.InformationSourceType.ACTIVE_LEARNING.value - and config_service.get_config_value("is_managed") + and get_config_value("is_managed") ): daemon.run_without_db_token( __delete_active_learner_from_inference_dir, project_id, source_id diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py deleted file mode 100644 index 84c44d08..00000000 --- a/controller/misc/config_service.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Dict, Any, Optional, Union -import time -from config_handler import change_json, get_config -from submodules.model import daemon - -__config = None - - -def __get_config() -> Dict[str, Any]: - global __config - if __config: - return __config - refresh_config() - return __config - - -def refresh_config(): - response = get_config(False) - global __config - __config = response - daemon.run_without_db_token(invalidate_after, 3600) # one hour as fail safe - - -def get_config_value( - key: str, subkey: Optional[str] = None -) -> Union[str, Dict[str, str]]: - config = __get_config() - if key not in config: - raise ValueError(f"Key {key} coudn't be found in config") - value = config[key] - - if not subkey: - return value - - if isinstance(value, dict) and subkey in value: - return value[subkey] - else: - raise ValueError(f"Subkey {subkey} coudn't be found in config[{key}]") - - -def invalidate_after(sec: int) -> None: - time.sleep(sec) - global __config - __config = None - - -def change_config(dict_str: str) -> None: - config_data = {"dict_string": dict_str} - return change_json(config_data) diff --git a/controller/misc/manager.py b/controller/misc/manager.py index a2c7b99b..38c3bfcf 100644 --- a/controller/misc/manager.py +++ b/controller/misc/manager.py @@ -1,5 +1,5 @@ from typing import Any, Dict, List -from controller.misc import config_service +from config_handler import change_json, get_config_value from controller.misc import black_white_demo from fast_api.types import ServiceVersionResult from submodules.model.global_objects import customer_button @@ -20,19 +20,16 @@ def check_is_managed() -> bool: - return config_service.get_config_value("is_managed") + return get_config_value("is_managed") def check_is_demo() -> bool: - return config_service.get_config_value("is_demo") + return get_config_value("is_demo") def update_config(dict_str: str) -> None: - return config_service.change_config(dict_str) - - -def refresh_config() -> None: - config_service.refresh_config() + config_data = {"dict_string": dict_str} + return change_json(config_data) def get_black_white_demo() -> Dict[str, List[str]]: diff --git a/controller/organization/manager.py b/controller/organization/manager.py index 99b0451c..4ec72419 100644 --- a/controller/organization/manager.py +++ b/controller/organization/manager.py @@ -1,5 +1,5 @@ from typing import Any, List, Dict, Optional, Union -from controller.misc import config_service +from config_handler import get_config_value from submodules.model import enums from submodules.model.business_objects import organization, general, user @@ -102,7 +102,7 @@ def get_overview_stats(org_id: str) -> List[Dict[str, Union[str, int]]]: def can_create_local(org: bool = True) -> bool: - if config_service.get_config_value("is_managed"): + if get_config_value("is_managed"): return False existing_orgs = organization.get_all() checkvalue = 0 if org else 1 diff --git a/controller/payload/payload_scheduler.py b/controller/payload/payload_scheduler.py index 5d1fc3ec..2a1db438 100644 --- a/controller/payload/payload_scheduler.py +++ b/controller/payload/payload_scheduler.py @@ -12,6 +12,7 @@ from dateutil import parser import datetime +from config_handler import get_config_value from exceptions.exceptions import PayloadSchedulerError from submodules.model import enums from submodules.model.business_objects import ( @@ -49,7 +50,6 @@ from util import notification from submodules.s3 import controller as s3 from controller.knowledge_base import util as knowledge_base -from controller.misc import config_service from util.notification import create_notification from util.miscellaneous_functions import chunk_dict from controller.weak_supervision import weak_supervision_service as weak_supervision @@ -509,7 +509,7 @@ def read_container_logs_thread( def get_inference_dir() -> str: - if config_service.get_config_value("is_managed"): + if get_config_value("is_managed"): return os.getenv("INFERENCE_DIR") return None diff --git a/controller/transfer/checks.py b/controller/transfer/checks.py index 05b7fcfe..bff43101 100644 --- a/controller/transfer/checks.py +++ b/controller/transfer/checks.py @@ -1,6 +1,5 @@ from typing import Tuple, List, Union, Dict from controller.auth import manager as auth_manager -from controller.misc.config_service import get_config_value from controller.transfer import util as transfer_util from controller.transfer.valid_arguments import valid_arguments @@ -239,7 +238,7 @@ def get_update_amount(df: pd.DataFrame, project_id: str) -> int: if sql: sql_df = pd.read_sql(sql, con=general.get_bind()) for column in keys: - if not column in df.columns: + if column not in df.columns: return 0 type_name = df[column].dtype.name if type_name in ["int64", "float64", "bool"]: diff --git a/fast_api/routes/organization.py b/fast_api/routes/organization.py index cceb8fc7..54755443 100644 --- a/fast_api/routes/organization.py +++ b/fast_api/routes/organization.py @@ -1,6 +1,6 @@ import json from fastapi import APIRouter, Request, Body -from controller.misc import config_service +from config_handler import get_config_value from fast_api.models import ( AddUserToOrganizationBody, ArchiveAdminMessageBody, @@ -145,7 +145,7 @@ def can_create_local_org(request: Request): @router.post("/create-organization") def create_organization(request: Request, body: CreateOrganizationBody = Body(...)): - if config_service.get_config_value("is_managed"): + if get_config_value("is_managed"): auth_manager.check_admin_access(request.state.info) else: if not organization_manager.can_create_local(): @@ -158,7 +158,7 @@ def create_organization(request: Request, body: CreateOrganizationBody = Body(.. def add_user_to_organization( request: Request, body: AddUserToOrganizationBody = Body(...) ): - if config_service.get_config_value("is_managed"): + if get_config_value("is_managed"): auth_manager.check_admin_access(request.state.info) else: if not organization_manager.can_create_local(False): @@ -178,7 +178,7 @@ def remove_user_from_organization( @router.post("/change-organization") def change_organization(request: Request, body: ChangeOrganizationBody = Body(...)): - if config_service.get_config_value("is_managed"): + if get_config_value("is_managed"): auth_manager.check_admin_access(request.state.info) organization_manager.change_organization(body.org_id, json.loads(body.changes)) return pack_json_result({"data": {"changeOrganization": {"ok": True}}}) @@ -192,7 +192,6 @@ def update_config(request: Request, body: UpdateConfigBody = Body(...)): ) return misc.update_config(body.dict_str) - misc.refresh_config() orgs = organization.get_all() if not orgs or len(orgs) != 1: print("local version should only have one organization") From 9ccc21dd793a901ee6c9794e686205da6d6460ef Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 12 Nov 2024 16:19:19 +0100 Subject: [PATCH 17/21] Moves change config to misc routing --- api/misc.py | 21 +-------------------- app.py | 2 -- fast_api/routes/misc.py | 16 ++++++++++++++-- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/api/misc.py b/api/misc.py index 173f645d..fcdd3c86 100644 --- a/api/misc.py +++ b/api/misc.py @@ -1,20 +1,14 @@ from starlette.endpoints import HTTPEndpoint from starlette.responses import JSONResponse from starlette import status -from fastapi import Request, responses -import json +from fastapi import Request from config_handler import ( base_config_json, - change_json, full_config_json, get_config_value, ) -from fast_api.models import ( - ChangeRequest, -) - class IsManagedRest(HTTPEndpoint): def get(self, request) -> JSONResponse: @@ -28,19 +22,6 @@ def get(self, request) -> JSONResponse: return JSONResponse(is_managed, status_code=status.HTTP_200_OK) -class ChangeConfigRest(HTTPEndpoint): - async def post(self, request: Request) -> JSONResponse: - try: - data = await request.json() - change_request = ChangeRequest(**data) - config_data = json.loads(change_request.dict_string) - return change_json(config_data) - except Exception as e: - return responses.PlainTextResponse( - f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR - ) - - class FullConfigRest(HTTPEndpoint): def get(self, request: Request) -> JSONResponse: return full_config_json() diff --git a/app.py b/app.py index ccbe12a6..7abe5560 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,6 @@ from starlette.middleware import Middleware from api.misc import ( BaseConfigRest, - ChangeConfigRest, FullConfigRest, IsDemoRest, IsManagedRest, @@ -130,7 +129,6 @@ ) routes = [ - Route("/change_config", ChangeConfigRest), Route("/base_config", BaseConfigRest), Route("/full_config", FullConfigRest), Route("/notify/{path:path}", Notify), diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index 71fc3ef5..44c741c9 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,10 +1,11 @@ import json -from fastapi import APIRouter, Body, Request, status +from fastapi import APIRouter, Body, Request, status, responses from fastapi.responses import PlainTextResponse -from config_handler import base_config_json +from config_handler import base_config_json, change_json from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, + ChangeRequest, ModelProviderDeleteModelBody, ModelProviderDownloadModelBody, CreateCustomerButton, @@ -38,6 +39,17 @@ def get_is_admin(request: Request) -> Dict: return pack_json_result({"data": {"isAdmin": data}}) +@router.post("/change-config") +def change_config_r(body: ChangeRequest = Body(...)): + try: + config_data = json.loads(body.dict_string) + return change_json(config_data) + except Exception as e: + return responses.PlainTextResponse( + f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + @router.get("/base-config-default") def get_base_config() -> Dict: return base_config_json() From 0b9393bae304734ec08965899eb64af22feda009 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Thu, 14 Nov 2024 10:22:09 +0100 Subject: [PATCH 18/21] remove org createino endpoint for os version --- fast_api/routes/organization.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fast_api/routes/organization.py b/fast_api/routes/organization.py index 54755443..076dfb75 100644 --- a/fast_api/routes/organization.py +++ b/fast_api/routes/organization.py @@ -137,12 +137,6 @@ def all_admin_messages(request: Request, limit: int = 100) -> str: return pack_json_result({"data": {"allAdminMessages": data_dict}}) -@router.get("/can-create-local-org") -def can_create_local_org(request: Request): - data = manager.can_create_local() - return pack_json_result({"data": {"canCreateLocalOrg": data}}) - - @router.post("/create-organization") def create_organization(request: Request, body: CreateOrganizationBody = Body(...)): if get_config_value("is_managed"): From de0af28e93d610c70a873ccc7f558f77ad306e34 Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Thu, 14 Nov 2024 11:46:00 +0100 Subject: [PATCH 19/21] Removes unused config parts --- api/misc.py | 20 ------ api/transfer.py | 4 -- app.py | 6 -- base_config.json | 10 +-- config_handler.py | 22 ++---- controller/auth/manager.py | 23 ------ controller/embedding/manager.py | 9 +-- controller/information_source/manager.py | 2 - controller/misc/black_white_demo.py | 89 ------------------------ controller/misc/manager.py | 19 ----- controller/organization/manager.py | 13 ---- controller/payload/payload_scheduler.py | 5 +- exceptions/exceptions.py | 8 --- fast_api/models.py | 8 --- fast_api/routes/embedding.py | 4 +- fast_api/routes/misc.py | 46 ++---------- fast_api/routes/organization.py | 37 +--------- fast_api/routes/project.py | 5 -- middleware/database_session.py | 37 ---------- 19 files changed, 19 insertions(+), 348 deletions(-) delete mode 100644 controller/misc/black_white_demo.py diff --git a/api/misc.py b/api/misc.py index fcdd3c86..6c9602d5 100644 --- a/api/misc.py +++ b/api/misc.py @@ -1,32 +1,12 @@ from starlette.endpoints import HTTPEndpoint from starlette.responses import JSONResponse -from starlette import status from fastapi import Request from config_handler import ( - base_config_json, full_config_json, - get_config_value, ) -class IsManagedRest(HTTPEndpoint): - def get(self, request) -> JSONResponse: - is_managed = get_config_value("is_managed") - return JSONResponse(is_managed, status_code=status.HTTP_200_OK) - - -class IsDemoRest(HTTPEndpoint): - def get(self, request) -> JSONResponse: - is_managed = get_config_value("is_demo") - return JSONResponse(is_managed, status_code=status.HTTP_200_OK) - - class FullConfigRest(HTTPEndpoint): def get(self, request: Request) -> JSONResponse: return full_config_json() - - -class BaseConfigRest(HTTPEndpoint): - def get(self, request: Request) -> JSONResponse: - return base_config_json() diff --git a/api/transfer.py b/api/transfer.py index 72eae0db..15fd5f55 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -28,7 +28,6 @@ from controller.upload_task import manager as upload_task_manager from controller.auth import manager as auth_manager from controller.transfer import association_transfer_manager -from controller.auth import manager as auth from controller.project import manager as project_manager from controller.attribute import manager as attribute_manager @@ -135,7 +134,6 @@ def get(self, request) -> JSONResponse: class PrepareFileImport(HTTPEndpoint): async def post(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] request_body = await request.json() @@ -168,7 +166,6 @@ async def post(self, request) -> JSONResponse: class JSONImport(HTTPEndpoint): async def post(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] request_body = await request.json() user_id = request_body["user_id"] @@ -272,7 +269,6 @@ async def post(self, request) -> JSONResponse: class UploadTaskInfo(HTTPEndpoint): def get(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] task_id = request.path_params["task_id"] user_id = request.query_params["user_id"] diff --git a/app.py b/app.py index 54d50b18..c8ca9687 100644 --- a/app.py +++ b/app.py @@ -3,10 +3,7 @@ from api.healthcheck import Healthcheck from starlette.middleware import Middleware from api.misc import ( - BaseConfigRest, FullConfigRest, - IsDemoRest, - IsManagedRest, ) from api.project import ProjectDetails from api.transfer import ( @@ -124,7 +121,6 @@ ) routes = [ - Route("/base_config", BaseConfigRest), Route("/full_config", FullConfigRest), Route("/notify/{path:path}", Notify), Route("/healthcheck", Healthcheck), @@ -145,8 +141,6 @@ CognitionPrepareProject, ), Route("/project/{project_id:str}/import/task/{task_id:str}", UploadTaskInfo), - Route("/is_managed", IsManagedRest), - Route("/is_demo", IsDemoRest), Mount("/api", app=fastapi_app, name="REST API"), Mount( "/internal/api", app=fastapi_app_internal, name="INTERNAL REST API" diff --git a/base_config.json b/base_config.json index b82e8f59..8df4b168 100644 --- a/base_config.json +++ b/base_config.json @@ -1,14 +1,8 @@ { - "is_managed": null, - "is_demo": null, - "allow_data_tracking": true, "s3_region": null, + "KERN_S3_ENDPOINT": null, "spacy_downloads": [ "en_core_web_sm", "de_core_news_sm" - ], - "tokens": { - "INTERCOM": "" - }, - "KERN_S3_ENDPOINT": null + ] } \ No newline at end of file diff --git a/config_handler.py b/config_handler.py index 3c4d9f0d..0d236d05 100644 --- a/config_handler.py +++ b/config_handler.py @@ -4,7 +4,6 @@ from notify_handler import notify_others_about_change_thread from fastapi import responses, status -__blacklist_base_config = ["is_managed", "is_demo"] __config = None BASE_CONFIG_PATH = "base_config.json" @@ -100,14 +99,9 @@ def __load_and_remove_outdated_config_keys(): __save_current_config() -def get_config(basic: bool = True) -> Dict[str, Any]: - global __config, __blacklist_base_config - if not basic: - return __config - - return { - key: __config[key] for key in __config if key not in __blacklist_base_config - } +def get_config() -> Dict[str, Any]: + global __config + return __config def change_json(config_data) -> responses.PlainTextResponse: @@ -128,12 +122,4 @@ def change_json(config_data) -> responses.PlainTextResponse: def full_config_json() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(False) - ) - - -def base_config_json() -> responses.JSONResponse: - return responses.JSONResponse( - status_code=status.HTTP_200_OK, content=get_config(True) - ) + return responses.JSONResponse(status_code=status.HTTP_200_OK, content=get_config()) diff --git a/controller/auth/manager.py b/controller/auth/manager.py index 84464290..ed285527 100644 --- a/controller/auth/manager.py +++ b/controller/auth/manager.py @@ -1,10 +1,8 @@ from typing import Any, Dict from fastapi import Request -from config_handler import get_config_value from exceptions.exceptions import ( AuthManagerError, - NotAllowedInDemoError, ProjectAccessError, ) import jwt @@ -14,7 +12,6 @@ from submodules.model import enums, exceptions from submodules.model.business_objects import organization from submodules.model.models import Organization, Project, User -from controller.misc import manager as misc_manager import sqlalchemy DEV_USER_ID = "741df1c2-a531-43b6-b259-df23bc78e9a2" @@ -127,26 +124,6 @@ def check_is_admin(request: Any) -> bool: return False -def check_demo_access(info: Any) -> None: - if not check_is_admin(info.context["request"]) and get_config_value("is_demo"): - check_black_white(info) - - -def check_black_white(info: Any): - black_white = misc_manager.get_black_white_demo() - if str(info.parent_type) == "Mutation": - if info.field_name not in black_white["mutations"]: - raise NotAllowedInDemoError - elif str(info.parent_type) == "Query": - if info.field_name in black_white["queries"]: - raise NotAllowedInDemoError - - -def check_is_demo_without_info() -> None: - if get_config_value("is_demo"): - raise NotAllowedInDemoError - - def check_is_single_organization() -> bool: return len(organization_manager.get_all_organizations()) == 1 diff --git a/controller/embedding/manager.py b/controller/embedding/manager.py index 8183459a..38bb6a2a 100644 --- a/controller/embedding/manager.py +++ b/controller/embedding/manager.py @@ -42,14 +42,11 @@ def get_current_terms_text( return term_text -def get_recommended_encoders(is_managed: bool) -> List[Any]: - # only use is_managed if it is really managed +def get_recommended_encoders() -> List[Any]: # can run into circular import problems if directly resolved here by helper method recommendations = connector.request_listing_recommended_encoders() - if is_managed: - existing_models = model_manager.get_model_provider_info() - else: - existing_models = [] + existing_models = model_manager.get_model_provider_info() + for model in existing_models: not_yet_known = ( len( diff --git a/controller/information_source/manager.py b/controller/information_source/manager.py index 41b81e53..79e05986 100644 --- a/controller/information_source/manager.py +++ b/controller/information_source/manager.py @@ -1,6 +1,5 @@ import os from typing import List -from config_handler import get_config_value from controller.information_source.util import resolve_source_return_type from submodules.model import InformationSource, LabelingTask, enums from submodules.model.business_objects import ( @@ -86,7 +85,6 @@ def delete_information_source(project_id: str, source_id: str) -> None: if ( information_source_item.type == enums.InformationSourceType.ACTIVE_LEARNING.value - and get_config_value("is_managed") ): daemon.run_without_db_token( __delete_active_learner_from_inference_dir, project_id, source_id diff --git a/controller/misc/black_white_demo.py b/controller/misc/black_white_demo.py deleted file mode 100644 index b57e5296..00000000 --- a/controller/misc/black_white_demo.py +++ /dev/null @@ -1,89 +0,0 @@ -from typing import Dict, List - - -def get_black_white_demo() -> Dict[str, List[str]]: - global __whitelist_mutation_demo_parsed - global __blacklist_query_demo_parsed - - if not __whitelist_mutation_demo_parsed: - __parse_black_white_list() - toReturn = {"mutations": [], "queries": []} - for v in __whitelist_mutation_demo_parsed: - toReturn["mutations"].append(v) - for v in __blacklist_query_demo_parsed: - toReturn["queries"].append(v) - - return toReturn - - -def __snake_case_to_camel_case(str: str): - return "".join( - [ - word.title() if idx > 0 else word.lower() - for idx, word in enumerate(str.split("_")) - ] - ) - - -def __parse_black_white_list(): - global __whitelist_mutation_demo - global __blacklist_query_demo - global __whitelist_mutation_demo_parsed - global __blacklist_query_demo_parsed - - tmp = {} - for v in __whitelist_mutation_demo: - tmp[v[0].lower() + v[1:]] = v - __whitelist_mutation_demo_parsed = tmp - tmp = {} - for v in __blacklist_query_demo: - tmp[__snake_case_to_camel_case(v)] = v - __blacklist_query_demo_parsed = tmp - - -# CAUTION: this is not necessarily the class name but often times similar or equal. This is the schema name we would use in gql -__whitelist_mutation_demo = { - "CreateDataSlice", - "UpdateDataSlice", - "DeleteDataSlice", - "CreateOutlierSlice", - "UpdateSliceTypeManual", - "CreateInformationSource", - # "DeleteInformationSource", - "ToggleInformationSource", - "SetAllInformationSourceSelected", - # "UpdateInformationSource", - "CreateKnowledgeBase", - # "DeleteKnowledgeBase", - # "UpdateKnowledgeBase", - "AddTermToKnowledgeBase", - "PasteKnowledgeTerms", - "UpdateKnowledgeTerm", - "DeleteKnowledgeTerm", - "BlacklistTerm", - # "CreateLabelingTaskLabel", - "UpdateLabelColor", - "UpdateLabelHotkey", - # "DeleteLabelingTaskLabel", - "PostEvent", - "CreateNotification", - "AddClassificationLabelsToRecord", - "AddExtractionLabelToRecord", - "SetGoldStarAnnotationForTask", - "DeleteRecordLabelAssociationByIds", - "DeleteGoldStarAssociationForTask", - "UpdateRlaIsValidManual", -} - - -# Function names without resolve_ -__blacklist_query_demo = { - "upload_credentials_and_id", - "upload_task_by_id", - "prepare_project_export", -} - - -# Parsed to a dict with gql query names -__whitelist_mutation_demo_parsed = None -__blacklist_query_demo_parsed = None diff --git a/controller/misc/manager.py b/controller/misc/manager.py index 38c3bfcf..c8db7771 100644 --- a/controller/misc/manager.py +++ b/controller/misc/manager.py @@ -1,6 +1,4 @@ from typing import Any, Dict, List -from config_handler import change_json, get_config_value -from controller.misc import black_white_demo from fast_api.types import ServiceVersionResult from submodules.model.global_objects import customer_button from datetime import datetime @@ -19,23 +17,6 @@ BASE_URI_UPDATER = os.getenv("UPDATER") -def check_is_managed() -> bool: - return get_config_value("is_managed") - - -def check_is_demo() -> bool: - return get_config_value("is_demo") - - -def update_config(dict_str: str) -> None: - config_data = {"dict_string": dict_str} - return change_json(config_data) - - -def get_black_white_demo() -> Dict[str, List[str]]: - return black_white_demo.get_black_white_demo() - - def get_version_overview() -> List[ServiceVersionResult]: updater_version_overview = __updater_version_overview() date_format = "%Y-%m-%dT%H:%M:%S.%f" # 2022-09-06T12:10:39.167397 diff --git a/controller/organization/manager.py b/controller/organization/manager.py index 4ec72419..5bb805e0 100644 --- a/controller/organization/manager.py +++ b/controller/organization/manager.py @@ -1,5 +1,4 @@ from typing import Any, List, Dict, Optional, Union -from config_handler import get_config_value from submodules.model import enums from submodules.model.business_objects import organization, general, user @@ -101,18 +100,6 @@ def get_overview_stats(org_id: str) -> List[Dict[str, Union[str, int]]]: return organization.get_organization_overview_stats(org_id) -def can_create_local(org: bool = True) -> bool: - if get_config_value("is_managed"): - return False - existing_orgs = organization.get_all() - checkvalue = 0 if org else 1 - if len(existing_orgs) != checkvalue: - return False - if user.get_count_assigned() != 0: - return False - return True - - def __check_notification(org_id: str, key: str, value: Any): if key in ["gdpr_compliant"]: notification.send_organization_update( diff --git a/controller/payload/payload_scheduler.py b/controller/payload/payload_scheduler.py index 2a1db438..420b8f2c 100644 --- a/controller/payload/payload_scheduler.py +++ b/controller/payload/payload_scheduler.py @@ -12,7 +12,6 @@ from dateutil import parser import datetime -from config_handler import get_config_value from exceptions.exceptions import PayloadSchedulerError from submodules.model import enums from submodules.model.business_objects import ( @@ -509,9 +508,7 @@ def read_container_logs_thread( def get_inference_dir() -> str: - if get_config_value("is_managed"): - return os.getenv("INFERENCE_DIR") - return None + return os.getenv("INFERENCE_DIR") def update_records( diff --git a/exceptions/exceptions.py b/exceptions/exceptions.py index ce743be7..fa7a4102 100644 --- a/exceptions/exceptions.py +++ b/exceptions/exceptions.py @@ -18,14 +18,6 @@ class MissingArgumentsException(Exception): pass -class NotAllowedInDemoError(Exception): - pass - - -class NotAllowedInOpenSourceError(Exception): - pass - - class BadPasswordError(Exception): pass diff --git a/fast_api/models.py b/fast_api/models.py index baaeb25a..6385027e 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -210,10 +210,6 @@ class ArchiveAdminMessageBody(BaseModel): archived_reason: StrictStr -class UpdateConfigBody(BaseModel): - dict_str: StrictStr - - class ChangeUserRoleBody(BaseModel): user_id: StrictStr role: StrictStr @@ -444,7 +440,3 @@ class UpdateCustomerButton(BaseModel): location: Optional[CustomerButtonLocation] = None visible: Optional[StrictBool] = None config: Optional[Dict[StrictStr, Any]] = None - - -class ChangeRequest(BaseModel): - dict_string: str diff --git a/fast_api/routes/embedding.py b/fast_api/routes/embedding.py index 9f722f86..5c7d36d0 100644 --- a/fast_api/routes/embedding.py +++ b/fast_api/routes/embedding.py @@ -2,7 +2,6 @@ from fast_api.models import CreateEmbeddingBody, UpdateEmbeddingBody from fast_api.routes.client_response import pack_json_result -from controller.misc import manager as misc from fastapi import APIRouter, Body, Depends, Request from controller.embedding import manager from controller.task_master import manager as task_master_manager @@ -30,8 +29,7 @@ def get_embedding_platforms(): @router.get("/recommended-encoders") def data_slices(request: Request, project_id: Optional[str] = None) -> List: - is_managed = misc.check_is_managed() - data = manager.get_recommended_encoders(is_managed) + data = manager.get_recommended_encoders() for v in data: v["applicability"] = json.dumps(v["applicability"]) return pack_json_result({"data": {"recommendedEncoders": data}}) diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index 44c741c9..54adf984 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,11 +1,8 @@ import json -from fastapi import APIRouter, Body, Request, status, responses +from fastapi import APIRouter, Body, Request, status from fastapi.responses import PlainTextResponse -from config_handler import base_config_json, change_json -from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, - ChangeRequest, ModelProviderDeleteModelBody, ModelProviderDownloadModelBody, CreateCustomerButton, @@ -15,7 +12,6 @@ from typing import Dict, Optional from controller.auth import manager as auth from controller.misc import manager -from controller.misc import manager as misc from controller.monitor import manager as controller_manager from controller.model_provider import manager as model_provider_manager from controller.task_master import manager as task_master_manager @@ -39,32 +35,6 @@ def get_is_admin(request: Request) -> Dict: return pack_json_result({"data": {"isAdmin": data}}) -@router.post("/change-config") -def change_config_r(body: ChangeRequest = Body(...)): - try: - config_data = json.loads(body.dict_string) - return change_json(config_data) - except Exception as e: - return responses.PlainTextResponse( - f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR - ) - - -@router.get("/base-config-default") -def get_base_config() -> Dict: - return base_config_json() - - -@router.get("/is-demo") -def get_is_demo(request: Request) -> Dict: - is_demo = False - try: - auth.check_demo_access(request.state.info) - except Exception: - is_demo = True - return pack_json_result({"data": {"isDemo": is_demo}}) - - @router.get("/version-overview") def get_version_overview(request: Request) -> Dict: data = manager.get_version_overview() @@ -81,11 +51,8 @@ def has_updates(request: Request) -> Dict: def model_provider_delete_model( request: Request, body: ModelProviderDeleteModelBody = Body(...) ): - if misc.check_is_managed(): - if not auth.check_is_single_organization(): - auth.check_admin_access(request.state.info) - else: - raise ProjectAccessError("Not allowed in open source version.") + if not auth.check_is_single_organization(): + auth.check_admin_access(request.state.info) model_provider_manager.model_provider_delete_model(body.model_name) return pack_json_result({"data": {"modelProviderDeleteModel": {"ok": True}}}) @@ -95,11 +62,8 @@ def model_provider_delete_model( def model_provider_download_model( request: Request, body: ModelProviderDownloadModelBody = Body(...) ): - if misc.check_is_managed(): - if not auth.check_is_single_organization(): - auth.check_admin_access(request.state.info) - else: - raise ProjectAccessError("Not allowed in open source version.") + if not auth.check_is_single_organization(): + auth.check_admin_access(request.state.info) model_provider_manager.model_provider_download_model(body.model_name) return pack_json_result({"data": {"modelProviderDownloadModel": {"ok": True}}}) diff --git a/fast_api/routes/organization.py b/fast_api/routes/organization.py index 076dfb75..8314d8e2 100644 --- a/fast_api/routes/organization.py +++ b/fast_api/routes/organization.py @@ -1,6 +1,5 @@ import json from fastapi import APIRouter, Request, Body -from config_handler import get_config_value from fast_api.models import ( AddUserToOrganizationBody, ArchiveAdminMessageBody, @@ -12,7 +11,6 @@ DeleteUserBody, MappedSortedPaginatedUsers, RemoveUserToOrganizationBody, - UpdateConfigBody, UserLanguageDisplay, ) from controller.auth import manager as auth_manager @@ -24,7 +22,6 @@ from controller.admin_message import manager as admin_message_manager from controller.organization import manager as organization_manager from controller.user import manager as user_manager -from controller.misc import manager as misc from fast_api.routes.client_response import get_silent_success, pack_json_result from submodules.model.business_objects import organization @@ -139,11 +136,7 @@ def all_admin_messages(request: Request, limit: int = 100) -> str: @router.post("/create-organization") def create_organization(request: Request, body: CreateOrganizationBody = Body(...)): - if get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) - else: - if not organization_manager.can_create_local(): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) organization = organization_manager.create_organization(body.name) return {"data": {"createOrganization": {"organization": organization}}} @@ -152,11 +145,7 @@ def create_organization(request: Request, body: CreateOrganizationBody = Body(.. def add_user_to_organization( request: Request, body: AddUserToOrganizationBody = Body(...) ): - if get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) - else: - if not organization_manager.can_create_local(False): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) user_manager.update_organization_of_user(body.organization_name, body.user_mail) return pack_json_result({"data": {"addUserToOrganization": {"ok": True}}}) @@ -172,31 +161,11 @@ def remove_user_from_organization( @router.post("/change-organization") def change_organization(request: Request, body: ChangeOrganizationBody = Body(...)): - if get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) organization_manager.change_organization(body.org_id, json.loads(body.changes)) return pack_json_result({"data": {"changeOrganization": {"ok": True}}}) -@router.post("/update-config") -def update_config(request: Request, body: UpdateConfigBody = Body(...)): - if misc.check_is_managed(): - print( - "config should only be changed for open source/local version to prevent limit issues" - ) - return - misc.update_config(body.dict_str) - orgs = organization.get_all() - if not orgs or len(orgs) != 1: - print("local version should only have one organization") - return - - for org in orgs: - # send to all so all are notified about the change - notification.send_organization_update(None, "config_updated", True, str(org.id)) - return pack_json_result({"data": {"updateConfig": {"ok": True}}}) - - @router.get("/user-roles") def get_user_roles(request: Request): auth_manager.check_admin_access(request.state.info) diff --git a/fast_api/routes/project.py b/fast_api/routes/project.py index ad1a5ccd..3b0425ce 100644 --- a/fast_api/routes/project.py +++ b/fast_api/routes/project.py @@ -31,8 +31,6 @@ to_frontend_obj_raw, ) from util import notification -from controller.misc import manager as misc -from exceptions.exceptions import NotAllowedInOpenSourceError from submodules.model.business_objects import notification as notification_model from submodules.model.business_objects import tokenization, task_queue @@ -292,9 +290,6 @@ def record_export_by_project_id(project_id: str) -> str: @router.get("/model-provider-info") def get_model_provider_info(request: Request) -> Dict: - if not misc.check_is_managed(): - raise NotAllowedInOpenSourceError - data = model_manager.get_model_provider_info() return pack_json_result({"data": {"modelProviderInfo": data}}) diff --git a/middleware/database_session.py b/middleware/database_session.py index 8b44548a..cb3f52dc 100644 --- a/middleware/database_session.py +++ b/middleware/database_session.py @@ -1,10 +1,5 @@ import logging from fastapi import Request -from fastapi.responses import JSONResponse -from exceptions.exceptions import ( - DatabaseSessionError, - NotAllowedInDemoError, -) from fast_api.routes.fastapi_resolve_info import FastAPIResolveInfo from middleware.query_mapping import path_query_map from submodules.model.business_objects import general @@ -26,11 +21,6 @@ async def handle_db_session(request: Request, call_next): request.state.info = info request.state.parsed = {} - if request.url.hostname != "localhost" or request.url.port != 7051: - access_response = _check_access(request, info) - if access_response is not None: - return access_response - log_request = auth_manager.extract_state_info(request, "log_request") length = request.headers.get("content-length") @@ -50,33 +40,6 @@ async def handle_db_session(request: Request, call_next): general.remove_and_refresh_session(session_token) -def _check_access(request, info): - try: - auth_manager.check_demo_access(info) - except NotAllowedInDemoError: - return JSONResponse( - status_code=401, - content={"message": "Unauthorized access"}, - ) - except DatabaseSessionError as e: - return JSONResponse( - status_code=400, - content={"message": e.message}, - ) - except ValueError as e: - return JSONResponse( - status_code=400, - content={"message": str(e)}, - ) - except Exception: - return JSONResponse( - status_code=500, - content={"message": "Internal server error"}, - ) - - return None - - def _prepare_info(request): field_name = None parent_type = None From 8610e3c38c9a1a2290a77719fb9e0e2a5007714c Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Thu, 14 Nov 2024 11:58:18 +0100 Subject: [PATCH 20/21] remove unused notify services --- config_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/config_handler.py b/config_handler.py index 0d236d05..f81ca1c4 100644 --- a/config_handler.py +++ b/config_handler.py @@ -10,8 +10,6 @@ CURRENT_CONFIG_PATH = "/config/current_config.json" SERVICES_TO_NOTIFY = { - "EMBEDDER": "http://refinery-embedder:80", - "UPDATER": "http://refinery-updater:80", "TOKENIZER": "http://refinery-tokenizer:80", } From fd94ddc673fe83585d21ecf44417110dc3ecdb5b Mon Sep 17 00:00:00 2001 From: JWittmeyer Date: Thu, 14 Nov 2024 15:34:28 +0100 Subject: [PATCH 21/21] Removes auto notify on startup --- app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app.py b/app.py index c8ca9687..57b3fb04 100644 --- a/app.py +++ b/app.py @@ -18,7 +18,6 @@ CognitionPrepareProject, ) from config_handler import ( - SERVICES_TO_NOTIFY, init_config, ) from fast_api.routes.organization import router as org_router @@ -44,7 +43,6 @@ from starlette.routing import Route, Mount from controller.project.manager import check_in_deletion_projects -from notify_handler import notify_others_about_change_thread from route_prefix import ( PREFIX_ORGANIZATION, PREFIX_PROJECT, @@ -73,7 +71,6 @@ init_config() fastapi_app = FastAPI() -notify_others_about_change_thread(SERVICES_TO_NOTIFY) fastapi_app.include_router(