Remove refinery-config

.gitignore

@@ -143,4 +143,6 @@ project_export*
 tmp/*
 !tmp/.gitkeep
 
-logs/*
+logs/*
+
+current_config.json

api/misc.py

@@ -2,14 +2,50 @@
 from starlette.endpoints import HTTPEndpoint
 from starlette.responses import JSONResponse
 from starlette import status
+from fastapi import Request, responses
+import json
+
+from config_handler import (
+    base_config_json,
+    change_json,
+    full_config_json,
+)
+
+from fast_api.models import (
+    ChangeRequest,
+)
 
 
 class IsManagedRest(HTTPEndpoint):
     def get(self, request) -> JSONResponse:
         is_managed = config_service.get_config_value("is_managed")
         return JSONResponse(is_managed, status_code=status.HTTP_200_OK)
 
+
 class IsDemoRest(HTTPEndpoint):
     def get(self, request) -> JSONResponse:
         is_managed = config_service.get_config_value("is_demo")
         return JSONResponse(is_managed, status_code=status.HTTP_200_OK)
+
+
+class ChangeConfigRest(HTTPEndpoint):
+    async def post(self, request: Request) -> JSONResponse:
+        try:
+            data = await request.json()
+            change_request = ChangeRequest(**data)
+            config_data = json.loads(change_request.dict_string)
+            return await change_json(config_data)
+        except Exception as e:
+            return responses.PlainTextResponse(
+                f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+
+
+class FullConfigRest(HTTPEndpoint):
+    def get(self, request: Request) -> JSONResponse:
+        return full_config_json()
+
+
+class BaseConfigRest(HTTPEndpoint):
+    def get(self, request: Request) -> JSONResponse:
+        return base_config_json()

app.py

@@ -1,9 +1,14 @@
 import logging
-
 from fastapi import FastAPI
 from api.healthcheck import Healthcheck
 from starlette.middleware import Middleware
-from api.misc import IsDemoRest, IsManagedRest
+from api.misc import (
+    BaseConfigRest,
+    ChangeConfigRest,
+    FullConfigRest,
+    IsDemoRest,
+    IsManagedRest,
+)
 from api.project import ProjectDetails
 from api.transfer import (
     AssociationsImport,
@@ -16,6 +21,10 @@
     CognitionImport,
     CognitionPrepareProject,
 )
+from config_handler import (
+    SERVICES_TO_NOTIFY,
+    init_config,
+)
 from fast_api.routes.organization import router as org_router
 from fast_api.routes.project import router as project_router
 from fast_api.routes.project_setting import router as project_setting_router
@@ -40,6 +49,7 @@
 from starlette.routing import Route, Mount
 
 from controller.project.manager import check_in_deletion_projects
+from notify_handler import notify_others_about_change_thread
 from route_prefix import (
     PREFIX_ORGANIZATION,
     PREFIX_PROJECT,
@@ -67,7 +77,10 @@
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
+init_config()
 fastapi_app = FastAPI()
+notify_others_about_change_thread(SERVICES_TO_NOTIFY)
+
 
 fastapi_app.include_router(
     org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"]
@@ -115,7 +128,11 @@
 fastapi_app_internal.include_router(
     task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"]
 )
+
 routes = [
+    Route("/change_config", ChangeConfigRest),
+    Route("/base_config", BaseConfigRest),
+    Route("/full_config", FullConfigRest),
     Route("/notify/{path:path}", Notify),
     Route("/healthcheck", Healthcheck),
     Route("/project/{project_id:str}", ProjectDetails),

base_config.json

@@ -0,0 +1,14 @@
+{
+    "is_managed": null,
+    "is_demo": null,
+    "allow_data_tracking": true,
+    "s3_region": null,
+    "spacy_downloads": [
+        "en_core_web_sm",
+        "de_core_news_sm"
+    ],
+    "tokens": {
+        "INTERCOM": ""
+    },
+    "KERN_S3_ENDPOINT": null
+}

config_handler.py

@@ -0,0 +1,129 @@
+from typing import Dict, Any
+import os
+import json
+from notify_handler import notify_others_about_change_thread
+from fastapi import responses, status
+
+__blacklist_base_config = ["is_managed", "is_demo"]
+__config = None
+
+BASE_CONFIG_PATH = "base_config.json"
+CURRENT_CONFIG_PATH = "/config/current_config.json"
+
+SERVICES_TO_NOTIFY = {
+    "EMBEDDER": "http://refinery-embedder:80",
+    "UPDATER": "http://refinery-updater:80",
+    "TOKENIZER": "http://refinery-tokenizer:80",
+}
+
+
+def __read_and_change_base_config():
+    print("reading base config file", flush=True)
+    global __config
+    f = open(BASE_CONFIG_PATH)
+    __config = json.load(f)
+
+    print("transfer os variables", flush=True)
+    __config["is_managed"] = os.getenv("IS_MANAGED") == "1"
+    __config["is_demo"] = os.getenv("IS_DEMO") == "1"
+    __config["tokens"]["INTERCOM"] = os.getenv("INTERCOM", "")
+    __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1")
+
+    __save_current_config()
+
+
+def change_config(changes: Dict[str, Any]) -> bool:
+    global __config
+    something_changed = False
+    for key in changes:
+        if key == "KERN_S3_ENDPOINT":
+            continue
+        if key in __config:
+            if isinstance(changes[key], dict):
+                for subkey in changes[key]:
+                    if subkey in __config[key]:
+                        __config[key][subkey] = changes[key][subkey]
+                        something_changed = True
+            else:
+                __config[key] = changes[key]
+                something_changed = True
+    if something_changed:
+        __save_current_config()
+    else:
+        print("nothing was changed with input", changes, flush=True)
+    return something_changed
+
+
+def __save_current_config() -> None:
+    print("saving config file", flush=True)
+    with open(CURRENT_CONFIG_PATH, "w") as f:
+        json.dump(__config, f, indent=4)
+
+
+def init_config() -> None:
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        __read_and_change_base_config()
+    else:
+        __load_and_remove_outdated_config_keys()
+    # this one is to be set on every start to ensure its up to date
+    print("setting s3 endpoint", flush=True)
+    __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT")
+
+
+def __load_and_remove_outdated_config_keys():
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        return
+
+    global __config
+    with open(CURRENT_CONFIG_PATH) as f:
+        __config = json.load(f)
+
+    with open(BASE_CONFIG_PATH) as f:
+        base_config = json.load(f)
+
+    to_remove = [key for key in __config if key not in base_config]
+
+    if len(to_remove) > 0:
+        print("removing outdated config keys", to_remove, flush=True)
+        for key in to_remove:
+            del __config[key]
+        __save_current_config()
+
+
+def get_config(basic: bool = True) -> Dict[str, Any]:
+    global __config, __blacklist_base_config
+    if not basic:
+        return __config
+
+    return {
+        key: __config[key] for key in __config if key not in __blacklist_base_config
+    }
+
+
+async def change_json(config_data) -> responses.PlainTextResponse:
+    try:
+        has_changed = change_config(config_data)
+
+        if has_changed:
+            notify_others_about_change_thread(SERVICES_TO_NOTIFY)
+
+        return responses.PlainTextResponse(
+            f"Did update: {has_changed}", status_code=status.HTTP_200_OK
+        )
+
+    except Exception as e:
+        return responses.PlainTextResponse(
+            f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+
+
+def full_config_json() -> responses.JSONResponse:
+    return responses.JSONResponse(
+        status_code=status.HTTP_200_OK, content=get_config(False)
+    )
+
+
+def base_config_json() -> responses.JSONResponse:
+    return responses.JSONResponse(
+        status_code=status.HTTP_200_OK, content=get_config(True)
+    )

controller/misc/config_service.py

@@ -1,15 +1,10 @@
 from typing import Dict, Any, Optional, Union
-import requests
 import time
+from config_handler import change_json, get_config
 from submodules.model import daemon
-from util import service_requests
 
 __config = None
 
-# these are ment to be constant values since os variables will sooner or later be removed for addresses (and used with values from config-service)
-REQUEST_URL = "http://refinery-config:80/full_config"
-CHANGE_URL = "http://refinery-config:80/change_config"
-
 
 def __get_config() -> Dict[str, Any]:
     global __config
@@ -20,14 +15,10 @@ def __get_config() -> Dict[str, Any]:
 
 
 def refresh_config():
-    response = requests.get(REQUEST_URL)
-    if response.status_code != 200:
-        raise ValueError(
-            f"Config service cant be reached -- response.code{response.status_code}"
-        )
+    response = get_config(False)
     global __config
-    __config = response.json()
-    daemon.run_without_db_token(invalidate_after, 3600)  # one hour as failsave
+    __config = response
+    daemon.run_without_db_token(invalidate_after, 3600)  # one hour as fail safe
 
 
 def get_config_value(
@@ -54,5 +45,5 @@ def invalidate_after(sec: int) -> None:
 
 
 def change_config(dict_str: str) -> None:
-    data = {"dict_string": dict_str}
-    service_requests.post_call_or_raise(CHANGE_URL, data)
+    config_data = {"dict_string": dict_str}
+    return change_json(config_data)

fast_api/models.py

@@ -448,3 +448,7 @@ class UpdateCustomerButton(BaseModel):
     location: Optional[CustomerButtonLocation] = None
     visible: Optional[StrictBool] = None
     config: Optional[Dict[StrictStr, Any]] = None
+
+
+class ChangeRequest(BaseModel):
+    dict_string: str

fast_api/routes/misc.py

@@ -1,6 +1,7 @@
 import json
 from fastapi import APIRouter, Body, Request, status
 from fastapi.responses import PlainTextResponse
+from config_handler import base_config_json
 from exceptions.exceptions import ProjectAccessError
 from fast_api.models import (
     CancelTaskBody,
@@ -37,6 +38,11 @@ def get_is_admin(request: Request) -> Dict:
     return pack_json_result({"data": {"isAdmin": data}})
 
 
+@router.get("/base-config-default")
+def get_base_config() -> Dict:
+    return base_config_json()
+
+
 @router.get("/is-demo")
 def get_is_demo(request: Request) -> Dict:
     is_demo = False

fast_api/routes/organization.py

@@ -1,5 +1,5 @@
 import json
-from fastapi import APIRouter, Depends, Request, Body
+from fastapi import APIRouter, Request, Body
 from controller.misc import config_service
 from fast_api.models import (
     AddUserToOrganizationBody,
@@ -27,7 +27,6 @@
 from controller.misc import manager as misc
 
 from fast_api.routes.client_response import get_silent_success, pack_json_result
-from submodules.model import events
 from submodules.model.business_objects import organization
 from submodules.model.util import sql_alchemy_to_dict
 from util import notification